mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(corpus-seeded-mechanisms): the mechanisms lever runs from ousast improve
Review rounds 3-5 remediations (Req 5, 7). - ousast improve resolves candidates from the target's calibration dir, then the working directory's (where `mechanisms export` writes), and admits into <target>/.openultrasast/calibration/mechanisms.jsonl, where a scan of that target reads; --mechanism-candidates, --mechanism-store, --no-mechanisms; --dry-run redirects the store to scratch; accepted rounds print admit/retract <id>. - run_improvement threads the store paths; RoundOutcome carries the mechanism edits; reverted mechanism keys are blocked outright; a no-gain admission is rejected as mechanism_no_gain; a retraction may name an admitted record that left the candidate set. - leave-one-out forwards variant-search degradations and pins the containment and label rules; mechanisms export defaults to the candidates file so it never re-admits a tombstone. - design §Lever rewritten to the implemented gate; README and docs show export -> improve -> scan. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
624024e72a
commit
b3b47ad757
@@ -105,3 +105,54 @@ def test_cli_pairs_loo_writes_the_artifact_offline(tmp_path: Path, monkeypatch:
|
||||
payload = json.loads(buf.getvalue())
|
||||
assert payload["per_slice"]["vibe-py"]["pairs"] == 35 and "found_by" in payload["outcomes"][0]
|
||||
assert json.loads((tmp_path / "loo.json").read_text()) == payload
|
||||
|
||||
|
||||
HELPER_SINK = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n\n\ndef ping(host):\n return run('ping ' + host)\n"
|
||||
HELPER_SINK_FIX = (
|
||||
"import subprocess\n\n\ndef run(cmd):\n subprocess.run(cmd.split())\n\n\ndef ping(host):\n return run('ping ' + host)\n"
|
||||
)
|
||||
|
||||
|
||||
def test_hits_outside_the_labeled_function_or_off_label_never_count(tmp_path: Path) -> None:
|
||||
"""Round-3 mutants: a hit must sit inside the labeled function and name the labeled mechanism or sink."""
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
teacher = _case(tmp_path, "t", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78")
|
||||
outside = _case(
|
||||
tmp_path, "outside", HELPER_SINK, HELPER_SINK_FIX, function="ping", cwe="CWE-78"
|
||||
) # sink call lives in `run`, not `ping`
|
||||
off_label = PairCase(
|
||||
**{
|
||||
**_case(tmp_path, "offlabel", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78").__dict__,
|
||||
"expected": (
|
||||
ExpectedFinding(
|
||||
cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="ping", mechanism="missing_auth_guard"
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
result = evaluate_loo([teacher, outside, off_label])
|
||||
by_name = {outcome.pair: outcome for outcome in result.outcomes}
|
||||
assert by_name["outside"].hits_vuln >= 1 and not by_name["outside"].detected
|
||||
assert by_name["offlabel"].hits_vuln >= 1 and not by_name["offlabel"].detected
|
||||
|
||||
|
||||
def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.semantic.loo import evaluate_loo
|
||||
|
||||
monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
|
||||
js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n"
|
||||
(tmp_path / "j-v.js").write_text(js)
|
||||
(tmp_path / "j-f.js").write_text(js)
|
||||
case = PairCase(
|
||||
**{
|
||||
**_toy(tmp_path)[0].__dict__,
|
||||
"name": "j",
|
||||
"language": "javascript",
|
||||
"vuln_file": tmp_path / "j-v.js",
|
||||
"fixed_file": tmp_path / "j-f.js",
|
||||
"relpath": "app.js",
|
||||
}
|
||||
)
|
||||
result = evaluate_loo([*_toy(tmp_path), case])
|
||||
assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations)
|
||||
|
||||
@@ -83,3 +83,30 @@ def test_apply_mechanism_edits_admits_into_the_scan_store_and_reverts_byte_for_b
|
||||
snapshot.restore()
|
||||
assert scan_store.read_bytes() == before # byte-for-byte revert of the whole round
|
||||
assert b.id != a.id
|
||||
|
||||
|
||||
def test_retract_is_allowed_for_an_admitted_record_that_left_the_candidate_set(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError
|
||||
|
||||
scan = MechanismStore(tmp_path / "scan.jsonl")
|
||||
stale = _record(scan, sink_name="popen")
|
||||
validator = EvolveValidator()
|
||||
with pytest.raises(StrictValidationError, match="unknown mechanism"):
|
||||
validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=stale.id), {}, admitted={stale.id: stale})
|
||||
validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=stale.id), {}, admitted={stale.id: stale})
|
||||
|
||||
|
||||
def test_export_defaults_to_a_candidates_file_and_never_writes_the_scan_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "benchmarks").symlink_to(Path(__file__).resolve().parents[1] / "benchmarks")
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
assert main(["mechanisms", "export", "--slice", "vibe-py", "--json"]) == 0
|
||||
payload = __import__("json").loads(buf.getvalue())
|
||||
assert payload["store"].endswith("mechanism-candidates.jsonl")
|
||||
assert not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists()
|
||||
|
||||
@@ -157,5 +157,162 @@ def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for
|
||||
mechanism_store=scan_store,
|
||||
scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")],
|
||||
)
|
||||
assert not forced.accepted and forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression")
|
||||
assert not forced.accepted and (forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression"))
|
||||
assert scan_store.read_bytes() == before # byte-for-byte revert
|
||||
|
||||
|
||||
def test_no_gain_admission_is_rejected_and_reverted_mechanism_edits_are_not_reproposed(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import run_round
|
||||
from openultrasast.improve.journal import reverted_edit_keys
|
||||
from openultrasast.improve.validator import MechanismEdit
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
unrelated = append_from_pair(
|
||||
candidates, _shape("popen"), summary="s", cwe="CWE-78", pair="teacher-c", provenance="human", tier="seeded"
|
||||
)
|
||||
holdout = [_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89")]
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
journal = tmp_path / "journal.json"
|
||||
kwargs: dict[str, object] = {
|
||||
"ledger_path": ledger,
|
||||
"journal_path": journal,
|
||||
"ruleset_dir": env["ruleset_dir"],
|
||||
"pair_cases": holdout,
|
||||
"min_holdout_pairs": 1,
|
||||
"mechanism_candidates": candidates.log_path,
|
||||
"mechanism_store": scan_store,
|
||||
}
|
||||
edit = MechanismEdit(action="admit", mechanism_id=unrelated.id, source="export", rationale="scripted: recovers nothing")
|
||||
outcome = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type]
|
||||
assert not outcome.accepted and outcome.reason == "mechanism_no_gain"
|
||||
assert not scan_store.exists() or scan_store.read_bytes() == b""
|
||||
assert edit.key() in reverted_edit_keys(__import__("json").loads(journal.read_text()))
|
||||
# the same reverted edit, offered again with a rationale, is filtered before validation (novelty gate)
|
||||
again = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type]
|
||||
assert again.reason == "no_proposals" and again.mechanism_edits == []
|
||||
|
||||
|
||||
def _holdout_catalog(tmp_path: Path) -> Path:
|
||||
(tmp_path / "x-v.py").write_text(EXEC_VULN)
|
||||
(tmp_path / "x-f.py").write_text(EXEC_FIX)
|
||||
catalog = tmp_path / "pairs.toml"
|
||||
catalog.write_text(
|
||||
'[[pair]]\nname = "x"\nslice = "github"\nlanguage = "python"\nvuln = "x-v.py"\nfixed = "x-f.py"\nrelpath = "app.py"\n'
|
||||
'split = "holdout"\nreview_tier = "reviewed"\nreviewer = "t"\n\n'
|
||||
'[[pair.expected]]\ncwe = "CWE-89"\nclass = "sql injection"\npath = "app.py"\nfunction = "lookup"\n'
|
||||
'sink = "execute"\nmechanism = "source_reaches_sink"\n'
|
||||
)
|
||||
return catalog
|
||||
|
||||
|
||||
def test_ousast_improve_admits_candidates_into_the_scan_store_by_default(tmp_path: Path) -> None:
|
||||
"""Req 5: the lever is reachable from the operator command; export -> candidates -> improve -> mechanisms.jsonl the scan reads."""
|
||||
from openultrasast.cli import main
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
repo = Path(str(env["target"]))
|
||||
calibration = repo / ".openultrasast" / "calibration"
|
||||
candidates = MechanismStore(calibration / "mechanism-candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
catalog = _holdout_catalog(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
assert (
|
||||
main(
|
||||
["improve", str(manifest), "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
admitted = MechanismStore(calibration / "mechanisms.jsonl").load()
|
||||
assert [r.id for r in admitted] == [exec_record.id]
|
||||
journal = json.loads((calibration / "improve_journal.json").read_text())
|
||||
assert any(e["lever"] == "mechanisms" and e["mechanism_id"] == exec_record.id for entry in journal for e in entry["edits"])
|
||||
|
||||
|
||||
def test_run_improvement_threads_mechanism_paths(tmp_path: Path) -> None:
|
||||
from openultrasast.improve.evolve import run_improvement
|
||||
from openultrasast.pairs import load_pair_catalog
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
candidates = MechanismStore(tmp_path / "candidates.jsonl")
|
||||
exec_record = append_from_pair(
|
||||
candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded"
|
||||
)
|
||||
cases = load_pair_catalog(_holdout_catalog(tmp_path))
|
||||
scan_store = tmp_path / "scan" / "mechanisms.jsonl"
|
||||
outcomes = run_improvement(
|
||||
env["target"],
|
||||
env["manifest"],
|
||||
ledger_path=tmp_path / "ledger.json",
|
||||
journal_path=tmp_path / "journal.json", # type: ignore[arg-type]
|
||||
ruleset_dir=env["ruleset_dir"],
|
||||
pair_cases=cases,
|
||||
min_holdout_pairs=1, # type: ignore[arg-type]
|
||||
mechanism_candidates=candidates.log_path,
|
||||
mechanism_store=scan_store,
|
||||
max_rounds=3,
|
||||
)
|
||||
assert any(o.accepted and o.mechanism_edits for o in outcomes)
|
||||
assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id]
|
||||
assert outcomes[-1].reason == "no_proposals" # converges once the candidate is admitted
|
||||
|
||||
|
||||
def _cli_env(tmp_path: Path) -> tuple[dict[str, object], Path, Path]:
|
||||
env = _round_env(tmp_path)
|
||||
calibration = Path(str(env["target"])) / ".openultrasast" / "calibration"
|
||||
candidates = MechanismStore(calibration / "mechanism-candidates.jsonl")
|
||||
append_from_pair(candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded")
|
||||
return env, calibration, _holdout_catalog(tmp_path)
|
||||
|
||||
|
||||
def test_dry_run_and_no_mechanisms_never_write_the_scan_store(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
from openultrasast.cli import main
|
||||
|
||||
env, calibration, catalog = _cli_env(tmp_path)
|
||||
base = [
|
||||
"improve",
|
||||
str(tmp_path / "bench.toml"),
|
||||
"--ruleset-dir",
|
||||
str(env["ruleset_dir"]),
|
||||
"--pair-catalog",
|
||||
str(catalog),
|
||||
"--min-holdout-pairs",
|
||||
"1",
|
||||
]
|
||||
assert main([*base, "--dry-run"]) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists() # a dry run mutates nothing under the target
|
||||
assert main([*base, "--no-mechanisms"]) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists()
|
||||
(calibration / "mechanism-candidates.jsonl").unlink()
|
||||
capsys.readouterr() # the dry run legitimately reports its scratch admission; judge only the candidate-less round below
|
||||
assert main(base) == 0
|
||||
assert not (calibration / "mechanisms.jsonl").exists() # no candidates file: the lever is inert, the round is a plain rule round
|
||||
out = capsys.readouterr().out
|
||||
assert "admit " not in out and "retract " not in out
|
||||
|
||||
|
||||
def test_documented_default_flow_connects_export_to_improve(tmp_path: Path, monkeypatch, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
"""README flow: `ousast mechanisms export` (cwd default) then `ousast improve --pair-catalog` admits into the target's store."""
|
||||
from openultrasast.cli import main
|
||||
|
||||
env = _round_env(tmp_path)
|
||||
catalog = _holdout_catalog(tmp_path)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# export a candidates file exactly as documented (cwd default), seeded from a tiny trusted catalog of our own
|
||||
assert main(["mechanisms", "export", "--catalog", str(catalog), "--slice", "all"]) == 0
|
||||
exported = tmp_path / ".openultrasast" / "calibration" / "mechanism-candidates.jsonl"
|
||||
assert exported.is_file() and not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists()
|
||||
# the target is a different directory (the benchmark fixture); improve must still find the cwd candidates by default
|
||||
assert (
|
||||
main(
|
||||
["improve", "bench.toml", "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
target_store = Path(str(env["target"])) / ".openultrasast" / "calibration" / "mechanisms.jsonl"
|
||||
assert MechanismStore(target_store).load() # admitted where the scan of that target reads
|
||||
out = capsys.readouterr().out
|
||||
assert "admit corpus:" in out # the accepted round names its mechanism edits
|
||||
|
||||
Reference in New Issue
Block a user