mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(propose-adjudicate-prove): overlay adjudicates inventory before prove
Regex inventory stays the proposer. MAP attaches promote/demote/unadjudicated/coverage from intra-file taint (tree-sitter when a grammar parses, Python ast as last resort). Deep sandboxes promotions only, ordered by the existing heuristic ranker plus optional OpenRouter mechanism recall. The SAST pair slice scores overlay outcomes; the stage-1 detection gate and local pair gate stay inventory-only.
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,343 @@
|
||||
"""MAP overlay, prove filter, reports, pair scoring, mechanism memory."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import ExpectedFinding
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.complexity.map import Hotspot
|
||||
from openultrasast.findings import StaticFinding, quick_scan_findings
|
||||
from openultrasast.gate import LANGUAGE_MANIFESTS
|
||||
from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.rank import rank_targets
|
||||
from openultrasast.semantic import MechanismStore, OverlayRecord, append_mechanism, filter_promoted_hotspots, order_promotions
|
||||
from openultrasast.semantic.prove_filter import promoted_findings
|
||||
|
||||
|
||||
def _latest_run(repo: Path) -> Path:
|
||||
return sorted((repo / ".runs").iterdir())[-1]
|
||||
|
||||
|
||||
def _finding(finding_id: str, path: str = "app.py", line: int = 1) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=finding_id,
|
||||
path=path,
|
||||
title="test",
|
||||
severity="high",
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="Static pattern python-unsafe-eval (CWE-95) matched",
|
||||
line=line,
|
||||
function_name="run",
|
||||
reachability_status="reachable",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["syscall_entry"],
|
||||
ranking_priority=1.0,
|
||||
)
|
||||
|
||||
|
||||
def _record(proposal_id: str, disposition: str, **kwargs: object) -> OverlayRecord:
|
||||
dominating = kwargs.get("dominating_fact")
|
||||
sanitizers = kwargs.get("sanitizers", ())
|
||||
return OverlayRecord(
|
||||
proposal_id=proposal_id,
|
||||
path=str(kwargs.get("path", "app.py")),
|
||||
line=int(kwargs.get("line", 1)),
|
||||
disposition=disposition,
|
||||
reason=str(kwargs.get("reason", disposition)),
|
||||
cwe=str(kwargs.get("cwe", "CWE-95")),
|
||||
sources=tuple(kwargs.get("sources", ())), # type: ignore[arg-type]
|
||||
sinks=tuple(kwargs.get("sinks", ("eval",))), # type: ignore[arg-type]
|
||||
sanitizers=tuple(sanitizers) if isinstance(sanitizers, tuple | list) else (),
|
||||
evidence_level="static_corroboration",
|
||||
dominating_fact=str(dominating) if dominating else ("constant" if disposition == "demote" else None),
|
||||
language=str(kwargs.get("language", "python")),
|
||||
)
|
||||
|
||||
|
||||
def _hotspot(*finding_ids: str, score: float = 5.0, path: str = "app.py") -> Hotspot:
|
||||
return Hotspot(
|
||||
path=path,
|
||||
function_name="run",
|
||||
score=score,
|
||||
band="high",
|
||||
signals={},
|
||||
rationale="test",
|
||||
test_hint=None,
|
||||
inventory_finding_ids=finding_ids,
|
||||
)
|
||||
|
||||
|
||||
def test_quick_scan_writes_no_overlay(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
assert not (run_dir / "overlay.json").exists()
|
||||
report = (run_dir / "report.md").read_text()
|
||||
assert "## Overlay" not in report
|
||||
|
||||
|
||||
def test_standard_scan_writes_overlay_without_executing_target(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
payload = json.loads((run_dir / "overlay.json").read_text())
|
||||
assert payload["records"]
|
||||
assert payload["records"][0]["disposition"] == "promote"
|
||||
report = (run_dir / "report.md").read_text()
|
||||
assert "promoted" in report
|
||||
assert "Disposition: `promote`" in report
|
||||
sarif = json.loads((run_dir / "report.sarif").read_text())
|
||||
assert sarif["runs"][0]["results"][0]["properties"]["disposition"] == "promote"
|
||||
assert (run_dir / "manifest.json").read_text()
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert manifest["artifacts"]["overlay"] == "overlay.json"
|
||||
|
||||
|
||||
def test_prove_filter_drops_demoted_keeps_promoted() -> None:
|
||||
promoted_id = "python-unsafe-eval:app.py:2"
|
||||
demoted_id = "python-unsafe-eval:app.py:1"
|
||||
hotspots = (
|
||||
_hotspot(demoted_id, score=9.0),
|
||||
_hotspot(promoted_id, score=3.0),
|
||||
)
|
||||
records = (
|
||||
_record(demoted_id, "demote", dominating_fact="constant"),
|
||||
_record(promoted_id, "promote", sources=("request",)),
|
||||
)
|
||||
selected = filter_promoted_hotspots(hotspots, records)
|
||||
assert len(selected) == 1
|
||||
assert selected[0].inventory_finding_ids == (promoted_id,)
|
||||
findings = [_finding(demoted_id, line=1), _finding(promoted_id, line=2)]
|
||||
assert [item.finding_id for item in promoted_findings(findings, records)] == [promoted_id]
|
||||
|
||||
|
||||
def test_deep_scan_offers_only_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast import cli
|
||||
from openultrasast.sandbox import FakeSandboxRunner, SandboxResult
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def safe():\n exec('ls')\n\ndef bad():\n return eval(request.args)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
|
||||
fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False))
|
||||
monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake)
|
||||
assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
|
||||
dispositions = {item["proposal_id"]: item["disposition"] for item in overlay}
|
||||
demoted = [key for key, value in dispositions.items() if value == "demote"]
|
||||
promoted = [key for key, value in dispositions.items() if value == "promote"]
|
||||
assert demoted
|
||||
assert promoted
|
||||
verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
|
||||
offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]}
|
||||
assert offered <= set(promoted)
|
||||
assert not (offered & set(demoted))
|
||||
|
||||
|
||||
def test_fail_on_worth_fixing_ignores_unadjudicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
||||
assert main(["scan", str(repo), "--mode", "standard", "--fail-on", "worth-fixing"]) == 0
|
||||
run_dir = _latest_run(repo)
|
||||
overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
|
||||
assert overlay
|
||||
assert all(item["disposition"] == "unadjudicated" for item in overlay)
|
||||
report = (run_dir / "report.md").read_text()
|
||||
assert "unadjudicated" in report
|
||||
|
||||
|
||||
def test_unlabeled_messy_tree_has_both_dispositions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
|
||||
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
overlay = json.loads((_latest_run(repo) / "overlay.json").read_text())["records"]
|
||||
by_path = {}
|
||||
for item in overlay:
|
||||
by_path.setdefault(item["path"], set()).add(item["disposition"])
|
||||
assert "unadjudicated" in by_path["broken.py"]
|
||||
assert "promote" in by_path["app.py"]
|
||||
|
||||
|
||||
def test_sast_slice_scores_overlay_not_inventory(tmp_path: Path) -> None:
|
||||
vuln = tmp_path / "vuln.py"
|
||||
fixed = tmp_path / "fixed.py"
|
||||
vuln.write_text("def bad(x):\n exec(x)\n")
|
||||
fixed.write_text("def goodG2B():\n exec('ls')\n")
|
||||
case = PairCase(
|
||||
name="synth-overlay-exec",
|
||||
slice="sast",
|
||||
language="python",
|
||||
origin="test",
|
||||
vuln_file=vuln,
|
||||
fixed_file=fixed,
|
||||
relpath="app.py",
|
||||
expected=(
|
||||
ExpectedFinding(
|
||||
cwe="CWE-95",
|
||||
vulnerability_class="code injection",
|
||||
path="app.py",
|
||||
evidence="exec",
|
||||
rule_id="python-unsafe-eval",
|
||||
sink="exec",
|
||||
),
|
||||
),
|
||||
min_recall=1.0,
|
||||
fix_policy="silent",
|
||||
)
|
||||
fix_root = tmp_path / "fix-tree"
|
||||
fix_root.mkdir()
|
||||
(fix_root / "app.py").write_text(fixed.read_text())
|
||||
_, fix_targets = preprocess_repository(fix_root)
|
||||
assert quick_scan_findings(fix_root, fix_targets, rank_targets(fix_targets)), "inventory still fires on goodG2B exec"
|
||||
outcome = evaluate_pair(case)
|
||||
assert outcome.detected_vuln
|
||||
assert outcome.silent_fix
|
||||
assert outcome.pair_correct
|
||||
|
||||
|
||||
def test_sast_pairs_stay_out_of_language_manifests() -> None:
|
||||
listed = {name for names in LANGUAGE_MANIFESTS.values() for name in names}
|
||||
leaked = [case.name for case in load_pair_catalog() if case.name in listed]
|
||||
assert leaked == []
|
||||
|
||||
|
||||
def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None:
|
||||
from openultrasast.reports import write_markdown_report
|
||||
|
||||
finding = _finding("python-unsafe-eval:app.py:1")
|
||||
records = (
|
||||
_record("python-unsafe-eval:app.py:1", "promote", sources=("request",)),
|
||||
_record("python-unsafe-eval:b.py:1", "demote", path="b.py", dominating_fact="constant"),
|
||||
OverlayRecord(
|
||||
proposal_id="python-unsafe-eval:c.py:1",
|
||||
path="c.py",
|
||||
line=1,
|
||||
disposition="unadjudicated",
|
||||
reason="parse_failed",
|
||||
cwe="CWE-95",
|
||||
sources=(),
|
||||
sinks=(),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
),
|
||||
OverlayRecord(
|
||||
proposal_id="overlay-coverage:d.py:1:hashlib",
|
||||
path="d.py",
|
||||
line=1,
|
||||
disposition="coverage",
|
||||
reason="uninventoried sink hashlib",
|
||||
cwe="CWE-327",
|
||||
sources=("parameter",),
|
||||
sinks=("hashlib",),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
origin="overlay",
|
||||
),
|
||||
)
|
||||
output = tmp_path / "report.md"
|
||||
write_markdown_report([finding], output, overlay=records)
|
||||
text = output.read_text()
|
||||
assert "promoted" in text
|
||||
assert "demoted" in text
|
||||
assert "unadjudicated" in text
|
||||
assert "coverage" in text
|
||||
|
||||
|
||||
def test_mechanism_order_boosts_paraphrased_match(tmp_path: Path) -> None:
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
append_mechanism(
|
||||
store,
|
||||
summary="request parameter reaches eval",
|
||||
cwe="CWE-95",
|
||||
language="python",
|
||||
tags=["syscall_entry"],
|
||||
what_made_it_exploitable="dynamic execution of attacker input",
|
||||
)
|
||||
eval_id = "python-unsafe-eval:app.py:1"
|
||||
hash_id = "python-weak-hash:hash.py:1"
|
||||
hotspots = (_hotspot(hash_id, score=5.0, path="hash.py"), _hotspot(eval_id, score=5.0, path="app.py"))
|
||||
records = (
|
||||
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",), language="python"),
|
||||
_record(eval_id, "promote", sources=("request",), cwe="CWE-95", sinks=("eval",), language="python"),
|
||||
)
|
||||
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
||||
|
||||
class _Client:
|
||||
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
||||
del model, timeout_seconds
|
||||
vectors: list[list[float]] = []
|
||||
for text in inputs:
|
||||
if "eval" in text.lower() or "CWE-95" in text:
|
||||
vectors.append([1.0, 0.0])
|
||||
else:
|
||||
vectors.append([0.0, 1.0])
|
||||
return vectors
|
||||
|
||||
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=_Client(), model="test-embed")
|
||||
assert degradation is None
|
||||
assert ordered[0].inventory_finding_ids == (eval_id,)
|
||||
assert {item.inventory_finding_ids[0] for item in ordered} == {eval_id, hash_id}
|
||||
|
||||
|
||||
def test_mechanism_degrades_to_heuristic_without_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
append_mechanism(
|
||||
store,
|
||||
summary="request parameter reaches eval",
|
||||
cwe="CWE-95",
|
||||
language="python",
|
||||
tags=["syscall_entry"],
|
||||
what_made_it_exploitable="dynamic execution of attacker input",
|
||||
)
|
||||
eval_id = "python-unsafe-eval:app.py:1"
|
||||
hash_id = "python-weak-hash:hash.py:1"
|
||||
hotspots = (_hotspot(hash_id, score=6.0, path="hash.py"), _hotspot(eval_id, score=3.0, path="app.py"))
|
||||
records = (
|
||||
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",)),
|
||||
_record(eval_id, "promote", sources=("request",)),
|
||||
)
|
||||
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
||||
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=None, model=None)
|
||||
assert degradation == "embeddings_unavailable"
|
||||
assert [item.inventory_finding_ids[0] for item in ordered] == [hash_id, eval_id]
|
||||
|
||||
|
||||
def test_mechanisms_are_not_written_from_unadjudicated(tmp_path: Path) -> None:
|
||||
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
||||
record = OverlayRecord(
|
||||
proposal_id="python-unsafe-eval:app.py:1",
|
||||
path="app.py",
|
||||
line=1,
|
||||
disposition="unadjudicated",
|
||||
reason="parse_failed",
|
||||
cwe="CWE-95",
|
||||
sources=(),
|
||||
sinks=(),
|
||||
sanitizers=(),
|
||||
evidence_level="static_corroboration",
|
||||
)
|
||||
assert record.disposition != "promote"
|
||||
assert store.load() == ()
|
||||
Reference in New Issue
Block a user