mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
86ed54c2e4
Task 3.1. A teacher is a vendored pair, at a review tier a maintainer stands behind, on the train split, and scorable. Every learning path now asks the same module that one question, so the split cannot be enforced in one place and forgotten in another. A refusal is data rather than an exception: the path records which holdout pairs a candidate touched and moves on, so a leak is visible in the run instead of silently accepted. The exporter refuses to seed from a holdout pair, leave-one-out teaches from the train split while still scoring every pair, and the mechanism lever refuses a candidate a holdout pair taught, because such a shape can only recover the pair that produced it. Measured and committed: on the vibe-py holdout the lever's own number falls from seven detected and a Youden of +0.059 to five detected and -0.059 once holdout pairs stop teaching, and the candidate set shrinks from ten shapes to six. That gap is the inflation this rule removes. Two existing tests encoded the leak and now encode the truth. The documented export-to-improve flow had a single holdout pair teaching itself, and now needs a train pair to teach and a holdout pair to recover. The obligation export claimed the corpus seeds at least one obligation shape, which was true only because holdout pairs were teaching: every vibe-py absence pair that yields an obligation lesson sits on the holdout side, so the honest number today is zero. The derivation stays covered by fixtures; the corpus gap belongs to the absence re-harvest and to a maintainer pass over the splits. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
110 lines
5.0 KiB
Python
110 lines
5.0 KiB
Python
"""learning-harness task 3.1: one teacher rule across every learning path (offline)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored
|
|
|
|
|
|
def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase:
|
|
(tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n")
|
|
return PairCase(
|
|
name=name,
|
|
slice="vibe-py",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=tmp_path / f"{name}.py",
|
|
fixed_file=tmp_path / f"{name}.py",
|
|
relpath="app.py",
|
|
expected=(ExpectedFinding(cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", function="f", mechanism="other"),),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
split=split,
|
|
review_tier=tier,
|
|
reviewer="m" if tier == "reviewed" else "",
|
|
unscorable=unscorable,
|
|
)
|
|
|
|
|
|
def test_only_vendored_trusted_scorable_train_pairs_may_teach(tmp_path: Path) -> None:
|
|
from openultrasast.learning.split import teachers
|
|
|
|
cases = [
|
|
_case(tmp_path, "train-seeded"),
|
|
_case(tmp_path, "train-reviewed", tier="reviewed"),
|
|
_case(tmp_path, "holdout-seeded", split="holdout"),
|
|
_case(tmp_path, "train-untrusted", tier="title"),
|
|
_case(tmp_path, "train-twin", unscorable="identical_twin"),
|
|
]
|
|
assert [case.name for case in teachers(cases)] == ["train-seeded", "train-reviewed"]
|
|
|
|
|
|
def test_a_candidate_taught_by_a_holdout_pair_is_refused_by_name(tmp_path: Path) -> None:
|
|
from openultrasast.learning.split import refuse_if_holdout
|
|
|
|
cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
|
|
assert refuse_if_holdout(["train-one"], cases) is None
|
|
refusal = refuse_if_holdout(["train-one", "holdout-one"], cases)
|
|
assert refusal is not None and refusal.pairs == ("holdout-one",)
|
|
assert refusal.reason == "holdout_pair_refused"
|
|
degradation = refusal.degradation()
|
|
assert degradation["stage"] == "learning" and degradation["pairs"] == ["holdout-one"]
|
|
assert "holdout-one" in str(degradation["detail"])
|
|
|
|
|
|
def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None:
|
|
from openultrasast.semantic.mechanisms import MechanismStore, corpus_mechanisms
|
|
from openultrasast.semantic.seed import export_mechanisms
|
|
|
|
cases = select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py"))
|
|
holdout = {case.name for case in cases if case.split == "holdout"}
|
|
store = MechanismStore(tmp_path / "candidates.jsonl")
|
|
report = export_mechanisms(cases, store)
|
|
taught_by_holdout = [record.id for record in corpus_mechanisms(store.load()) if set(record.pairs) & holdout]
|
|
assert taught_by_holdout == []
|
|
assert report.records > 0 and any(reason.startswith("split:") for _pair, reason in report.skipped)
|
|
|
|
|
|
def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None:
|
|
from openultrasast.semantic.loo import evaluate_loo
|
|
|
|
cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
|
|
result = evaluate_loo(cases)
|
|
assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"} # every pair is still scored
|
|
assert result.teaching_pairs <= 1 # only the train pair could ever have taught
|
|
|
|
|
|
def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None:
|
|
from openultrasast.improve.evolve import propose_mechanism_edits
|
|
from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair
|
|
from openultrasast.semantic.variants import Shape
|
|
|
|
shape = Shape(
|
|
language="python",
|
|
sink_name="system",
|
|
arity=1,
|
|
source_positions=(0,),
|
|
source_kinds=("request",),
|
|
guard="none",
|
|
mechanism="source_reaches_sink",
|
|
)
|
|
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
|
append_from_pair(candidates, shape, summary="s", cwe="CWE-78", pair="holdout-one", provenance="human", tier="seeded")
|
|
cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")]
|
|
edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl")
|
|
assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == []
|
|
|
|
|
|
def test_the_corrected_lever_number_is_measured_and_committed() -> None:
|
|
report = Path("benchmarks/measurements/2026-09-06-mechanism-lever-split.json")
|
|
assert report.is_file()
|
|
payload = json.loads(report.read_text())
|
|
assert payload["slice"] == "vibe-py" and payload["holdout_pairs"] > 0
|
|
assert set(payload["before"]) >= {"detected", "silent", "youden", "shapes"}
|
|
assert set(payload["after"]) >= {"detected", "silent", "youden", "shapes"}
|
|
assert payload["after"]["shapes"] < payload["before"]["shapes"] # holdout pairs no longer teach
|
|
assert payload["after"]["youden"] <= payload["before"]["youden"]
|