mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
1843ef9a02
Eight of nine group-3 tasks were rejected. Each defect was hiding a number, and the numbers are why the loop was not learning: - The fixed side was judged against the *vulnerable* side's function ranges. 191 of 227 pairs with a resolvable labeled span (84%) have different spans on the two sides, so a leak was recorded as silence, always in the direction that flatters Youden. - Round zero measured every family with the generalist. `family:unknown` earns no credit against any real family, so every family but `unknown` baselined at a structural zero and every noise floor with it. - The cross-family sweep counted absolute failures instead of negative flips. On a corpus where most pairs fail at round zero that rejects every round for damage it did not do. Unscorable rows voted too. - K runs was declared and never threaded; `pairs --k-runs` was parsed and discarded, and its only test grepped `--help`. - The meta-agent proposer was a stub, so `learning round` could never propose anything. It now composes MetaAgent.evolve against a scratch copy of the family directory, and `--proposals FILE` runs a recorded sequence offline. - `refused_holdout` was a vocabulary word no code path could produce, and the covering test accepted either outcome. - Cost was metered once, after everything had been paid for; a stage that raised left no record at all. - The corrected lever number was published beside the earlier one rather than in place of it, and the test named for that property never checked. - Refusals were `continue` statements: `Refusal.degradation()` existed and nothing called it. A train-split identical twin was reported as `split:train`, which is untrue. Test repairs that each changed a number: two split tests measured an `x = 1` fixture that teaches nothing whatever the rule says; two CLI scoreboard tests asserted against all-zero output produced by a stub that answers twice and puts its finding outside the labeled function. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
149 lines
7.4 KiB
Python
149 lines
7.4 KiB
Python
"""learning-harness task 3.1: one teacher rule across every learning path (offline)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored
|
|
|
|
VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n"
|
|
FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n"
|
|
|
|
|
|
def _teaching_case(tmp_path: Path, name: str, *, split: str = "train") -> PairCase:
|
|
"""A pair that really carries a mechanism shape.
|
|
|
|
The trivial `x = 1` fixture below teaches nothing whatever the split says, so a teacher-rule assertion made
|
|
against it passes with the rule deleted. This one is the difference between measuring the rule and measuring
|
|
an empty corpus."""
|
|
(tmp_path / f"{name}-v.py").write_text(VULN)
|
|
(tmp_path / f"{name}-f.py").write_text(FIXED)
|
|
return PairCase(
|
|
name=name,
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / f"{name}-v.py",
|
|
fixed_file=tmp_path / f"{name}-f.py",
|
|
relpath="app.py",
|
|
expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
split=split,
|
|
review_tier="seeded",
|
|
)
|
|
|
|
|
|
def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase:
|
|
(tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
|
|
return PairCase(
|
|
name=name,
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / f"{name}.py",
|
|
fixed_file=tmp_path / f"{name}.py",
|
|
relpath="app.py",
|
|
expected=(ExpectedFinding(cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", function="f", mechanism="other"),),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
split=split,
|
|
review_tier=tier,
|
|
reviewer="m" if tier == "reviewed" else "",
|
|
unscorable=unscorable,
|
|
)
|
|
|
|
|
|
def test_only_vendored_trusted_scorable_train_pairs_may_teach(tmp_path: Path) -> None:
|
|
from openultrasast.learning.split import teachers
|
|
|
|
cases = [
|
|
_case(tmp_path, "train-seeded"),
|
|
_case(tmp_path, "train-reviewed", tier="reviewed"),
|
|
_case(tmp_path, "holdout-seeded", split="holdout"),
|
|
_case(tmp_path, "train-untrusted", tier="title"),
|
|
_case(tmp_path, "train-twin", unscorable="identical_twin"),
|
|
]
|
|
assert [case.name for case in teachers(cases)] == ["train-seeded", "train-reviewed"]
|
|
|
|
|
|
def test_a_candidate_taught_by_a_holdout_pair_is_refused_by_name(tmp_path: Path) -> None:
|
|
from openultrasast.learning.split import refuse_if_holdout
|
|
|
|
cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
|
|
assert refuse_if_holdout(["train-one"], cases) is None
|
|
refusal = refuse_if_holdout(["train-one", "holdout-one"], cases)
|
|
assert refusal is not None and refusal.pairs == ("holdout-one",)
|
|
assert refusal.reason == "holdout_pair_refused"
|
|
degradation = refusal.degradation()
|
|
assert degradation["stage"] == "learning" and degradation["pairs"] == ["holdout-one"]
|
|
assert "holdout-one" in str(degradation["detail"])
|
|
|
|
|
|
def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None:
|
|
from openultrasast.semantic.mechanisms import MechanismStore, corpus_mechanisms
|
|
from openultrasast.semantic.seed import export_mechanisms
|
|
|
|
cases = select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py"))
|
|
holdout = {case.name for case in cases if case.split == "holdout"}
|
|
store = MechanismStore(tmp_path / "candidates.jsonl")
|
|
report = export_mechanisms(cases, store)
|
|
taught_by_holdout = [record.id for record in corpus_mechanisms(store.load()) if set(record.pairs) & holdout]
|
|
assert taught_by_holdout == []
|
|
assert report.records > 0 and any(reason.startswith("split:") for _pair, reason in report.skipped)
|
|
|
|
|
|
def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None:
|
|
"""Both arms, because `teaching_pairs <= 1` on a two-pair corpus is satisfied by a rule that does nothing.
|
|
|
|
The same two excerpts teach twice when both are on the train split and once when one is held out, and the run
|
|
names the pair it refused (Req 5.1, 5.2)."""
|
|
from openultrasast.semantic.loo import evaluate_loo
|
|
|
|
both_train = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "train-two")])
|
|
assert both_train.teaching_pairs == 2 and both_train.degradations == ()
|
|
result = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")])
|
|
assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"} # every pair is still scored
|
|
assert result.teaching_pairs == 1
|
|
refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"]
|
|
assert [item["pairs"] for item in refusals] == [["holdout-one"]]
|
|
|
|
|
|
def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None:
|
|
"""Both arms. The shape is the one the corpus really teaches, taken from the pair itself.
|
|
|
|
Asserting only the refusal would pass against a lever that admits nothing at all, and a hand-written shape that
|
|
matches no excerpt can never be admitted either, so neither would measure the split rule (Req 5.2)."""
|
|
from openultrasast.improve.evolve import propose_mechanism_edits
|
|
from openultrasast.semantic.loo import _facts
|
|
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
|
from openultrasast.semantic.seed import pair_lessons
|
|
|
|
cases = [_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")]
|
|
(lesson,) = pair_lessons(cases[0], _facts())
|
|
|
|
def store(name: str, pair: str) -> MechanismStore:
|
|
path = MechanismStore(tmp_path / name)
|
|
append_from_pair(path, lesson.shape, summary=lesson.summary, cwe=lesson.cwe, pair=pair, provenance="human", tier="seeded")
|
|
return path
|
|
|
|
reasons: list[dict[str, object]] = []
|
|
refused = propose_mechanism_edits(cases, store("refused.jsonl", "holdout-one"), tmp_path / "scan.jsonl", degradations=reasons)
|
|
assert [edit.mechanism_id for edit in refused if edit.action == "admit"] == []
|
|
assert [item["pairs"] for item in reasons] == [["holdout-one"]]
|
|
admitted = propose_mechanism_edits(cases, store("allowed.jsonl", "train-one"), tmp_path / "scan2.jsonl")
|
|
assert [edit.mechanism_id for edit in admitted if edit.action == "admit"], "a train-taught shape is still admissible"
|
|
|
|
|
|
def test_the_corrected_lever_number_is_measured_and_committed() -> None:
|
|
report = Path("benchmarks/measurements/2026-09-06-mechanism-lever-split.json")
|
|
assert report.is_file()
|
|
payload = json.loads(report.read_text())
|
|
assert payload["slice"] == "vibe-py" and payload["holdout_pairs"] > 0
|
|
assert set(payload["before"]) >= {"detected", "silent", "youden", "shapes"}
|
|
assert set(payload["after"]) >= {"detected", "silent", "youden", "shapes"}
|
|
assert payload["after"]["shapes"] < payload["before"]["shapes"] # holdout pairs no longer teach
|
|
assert payload["after"]["youden"] <= payload["before"]["youden"]
|