mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius Ciepluch
1ae41eea19
Regex inventory stays the proposer. MAP attaches promote/demote/unadjudicated/coverage from intra-file taint (tree-sitter when a grammar parses, Python ast as last resort). Deep sandboxes promotions only, ordered by the existing heuristic ranker plus optional OpenRouter mechanism recall. The SAST pair slice scores overlay outcomes; the stage-1 detection gate and local pair gate stay inventory-only.
344 lines
14 KiB
Python
344 lines
14 KiB
Python
"""MAP overlay, prove filter, reports, pair scoring, mechanism memory."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from openultrasast.benchmark import ExpectedFinding
|
|
from openultrasast.cli import main
|
|
from openultrasast.complexity.map import Hotspot
|
|
from openultrasast.findings import StaticFinding, quick_scan_findings
|
|
from openultrasast.gate import LANGUAGE_MANIFESTS
|
|
from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog
|
|
from openultrasast.preprocess import preprocess_repository
|
|
from openultrasast.rank import rank_targets
|
|
from openultrasast.semantic import MechanismStore, OverlayRecord, append_mechanism, filter_promoted_hotspots, order_promotions
|
|
from openultrasast.semantic.prove_filter import promoted_findings
|
|
|
|
|
|
def _latest_run(repo: Path) -> Path:
|
|
return sorted((repo / ".runs").iterdir())[-1]
|
|
|
|
|
|
def _finding(finding_id: str, path: str = "app.py", line: int = 1) -> StaticFinding:
|
|
return StaticFinding(
|
|
finding_id=finding_id,
|
|
path=path,
|
|
title="test",
|
|
severity="high",
|
|
confidence="medium",
|
|
evidence_level="static_corroboration",
|
|
rationale="Static pattern python-unsafe-eval (CWE-95) matched",
|
|
line=line,
|
|
function_name="run",
|
|
reachability_status="reachable",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=["syscall_entry"],
|
|
ranking_priority=1.0,
|
|
)
|
|
|
|
|
|
def _record(proposal_id: str, disposition: str, **kwargs: object) -> OverlayRecord:
|
|
dominating = kwargs.get("dominating_fact")
|
|
sanitizers = kwargs.get("sanitizers", ())
|
|
return OverlayRecord(
|
|
proposal_id=proposal_id,
|
|
path=str(kwargs.get("path", "app.py")),
|
|
line=int(kwargs.get("line", 1)),
|
|
disposition=disposition,
|
|
reason=str(kwargs.get("reason", disposition)),
|
|
cwe=str(kwargs.get("cwe", "CWE-95")),
|
|
sources=tuple(kwargs.get("sources", ())), # type: ignore[arg-type]
|
|
sinks=tuple(kwargs.get("sinks", ("eval",))), # type: ignore[arg-type]
|
|
sanitizers=tuple(sanitizers) if isinstance(sanitizers, tuple | list) else (),
|
|
evidence_level="static_corroboration",
|
|
dominating_fact=str(dominating) if dominating else ("constant" if disposition == "demote" else None),
|
|
language=str(kwargs.get("language", "python")),
|
|
)
|
|
|
|
|
|
def _hotspot(*finding_ids: str, score: float = 5.0, path: str = "app.py") -> Hotspot:
|
|
return Hotspot(
|
|
path=path,
|
|
function_name="run",
|
|
score=score,
|
|
band="high",
|
|
signals={},
|
|
rationale="test",
|
|
test_hint=None,
|
|
inventory_finding_ids=finding_ids,
|
|
)
|
|
|
|
|
|
def test_quick_scan_writes_no_overlay(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
|
run_dir = _latest_run(repo)
|
|
assert not (run_dir / "overlay.json").exists()
|
|
report = (run_dir / "report.md").read_text()
|
|
assert "## Overlay" not in report
|
|
|
|
|
|
def test_standard_scan_writes_overlay_without_executing_target(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
|
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
|
run_dir = _latest_run(repo)
|
|
payload = json.loads((run_dir / "overlay.json").read_text())
|
|
assert payload["records"]
|
|
assert payload["records"][0]["disposition"] == "promote"
|
|
report = (run_dir / "report.md").read_text()
|
|
assert "promoted" in report
|
|
assert "Disposition: `promote`" in report
|
|
sarif = json.loads((run_dir / "report.sarif").read_text())
|
|
assert sarif["runs"][0]["results"][0]["properties"]["disposition"] == "promote"
|
|
assert (run_dir / "manifest.json").read_text()
|
|
manifest = json.loads((run_dir / "manifest.json").read_text())
|
|
assert manifest["artifacts"]["overlay"] == "overlay.json"
|
|
|
|
|
|
def test_prove_filter_drops_demoted_keeps_promoted() -> None:
|
|
promoted_id = "python-unsafe-eval:app.py:2"
|
|
demoted_id = "python-unsafe-eval:app.py:1"
|
|
hotspots = (
|
|
_hotspot(demoted_id, score=9.0),
|
|
_hotspot(promoted_id, score=3.0),
|
|
)
|
|
records = (
|
|
_record(demoted_id, "demote", dominating_fact="constant"),
|
|
_record(promoted_id, "promote", sources=("request",)),
|
|
)
|
|
selected = filter_promoted_hotspots(hotspots, records)
|
|
assert len(selected) == 1
|
|
assert selected[0].inventory_finding_ids == (promoted_id,)
|
|
findings = [_finding(demoted_id, line=1), _finding(promoted_id, line=2)]
|
|
assert [item.finding_id for item in promoted_findings(findings, records)] == [promoted_id]
|
|
|
|
|
|
def test_deep_scan_offers_only_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from openultrasast import cli
|
|
from openultrasast.sandbox import FakeSandboxRunner, SandboxResult
|
|
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "app.py").write_text("def safe():\n exec('ls')\n\ndef bad():\n return eval(request.args)\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
|
|
fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False))
|
|
monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake)
|
|
assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 0
|
|
run_dir = _latest_run(repo)
|
|
overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
|
|
dispositions = {item["proposal_id"]: item["disposition"] for item in overlay}
|
|
demoted = [key for key, value in dispositions.items() if value == "demote"]
|
|
promoted = [key for key, value in dispositions.items() if value == "promote"]
|
|
assert demoted
|
|
assert promoted
|
|
verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
|
|
offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]}
|
|
assert offered <= set(promoted)
|
|
assert not (offered & set(demoted))
|
|
|
|
|
|
def test_fail_on_worth_fixing_ignores_unadjudicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
|
|
assert main(["scan", str(repo), "--mode", "standard", "--fail-on", "worth-fixing"]) == 0
|
|
run_dir = _latest_run(repo)
|
|
overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
|
|
assert overlay
|
|
assert all(item["disposition"] == "unadjudicated" for item in overlay)
|
|
report = (run_dir / "report.md").read_text()
|
|
assert "unadjudicated" in report
|
|
|
|
|
|
def test_unlabeled_messy_tree_has_both_dispositions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
(repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
|
|
(repo / "app.py").write_text("def run():\n return eval(request.args)\n")
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
|
overlay = json.loads((_latest_run(repo) / "overlay.json").read_text())["records"]
|
|
by_path = {}
|
|
for item in overlay:
|
|
by_path.setdefault(item["path"], set()).add(item["disposition"])
|
|
assert "unadjudicated" in by_path["broken.py"]
|
|
assert "promote" in by_path["app.py"]
|
|
|
|
|
|
def test_sast_slice_scores_overlay_not_inventory(tmp_path: Path) -> None:
|
|
vuln = tmp_path / "vuln.py"
|
|
fixed = tmp_path / "fixed.py"
|
|
vuln.write_text("def bad(x):\n exec(x)\n")
|
|
fixed.write_text("def goodG2B():\n exec('ls')\n")
|
|
case = PairCase(
|
|
name="synth-overlay-exec",
|
|
slice="sast",
|
|
language="python",
|
|
origin="test",
|
|
vuln_file=vuln,
|
|
fixed_file=fixed,
|
|
relpath="app.py",
|
|
expected=(
|
|
ExpectedFinding(
|
|
cwe="CWE-95",
|
|
vulnerability_class="code injection",
|
|
path="app.py",
|
|
evidence="exec",
|
|
rule_id="python-unsafe-eval",
|
|
sink="exec",
|
|
),
|
|
),
|
|
min_recall=1.0,
|
|
fix_policy="silent",
|
|
)
|
|
fix_root = tmp_path / "fix-tree"
|
|
fix_root.mkdir()
|
|
(fix_root / "app.py").write_text(fixed.read_text())
|
|
_, fix_targets = preprocess_repository(fix_root)
|
|
assert quick_scan_findings(fix_root, fix_targets, rank_targets(fix_targets)), "inventory still fires on goodG2B exec"
|
|
outcome = evaluate_pair(case)
|
|
assert outcome.detected_vuln
|
|
assert outcome.silent_fix
|
|
assert outcome.pair_correct
|
|
|
|
|
|
def test_sast_pairs_stay_out_of_language_manifests() -> None:
|
|
listed = {name for names in LANGUAGE_MANIFESTS.values() for name in names}
|
|
leaked = [case.name for case in load_pair_catalog() if case.name in listed]
|
|
assert leaked == []
|
|
|
|
|
|
def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None:
|
|
from openultrasast.reports import write_markdown_report
|
|
|
|
finding = _finding("python-unsafe-eval:app.py:1")
|
|
records = (
|
|
_record("python-unsafe-eval:app.py:1", "promote", sources=("request",)),
|
|
_record("python-unsafe-eval:b.py:1", "demote", path="b.py", dominating_fact="constant"),
|
|
OverlayRecord(
|
|
proposal_id="python-unsafe-eval:c.py:1",
|
|
path="c.py",
|
|
line=1,
|
|
disposition="unadjudicated",
|
|
reason="parse_failed",
|
|
cwe="CWE-95",
|
|
sources=(),
|
|
sinks=(),
|
|
sanitizers=(),
|
|
evidence_level="static_corroboration",
|
|
),
|
|
OverlayRecord(
|
|
proposal_id="overlay-coverage:d.py:1:hashlib",
|
|
path="d.py",
|
|
line=1,
|
|
disposition="coverage",
|
|
reason="uninventoried sink hashlib",
|
|
cwe="CWE-327",
|
|
sources=("parameter",),
|
|
sinks=("hashlib",),
|
|
sanitizers=(),
|
|
evidence_level="static_corroboration",
|
|
origin="overlay",
|
|
),
|
|
)
|
|
output = tmp_path / "report.md"
|
|
write_markdown_report([finding], output, overlay=records)
|
|
text = output.read_text()
|
|
assert "promoted" in text
|
|
assert "demoted" in text
|
|
assert "unadjudicated" in text
|
|
assert "coverage" in text
|
|
|
|
|
|
def test_mechanism_order_boosts_paraphrased_match(tmp_path: Path) -> None:
|
|
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
|
append_mechanism(
|
|
store,
|
|
summary="request parameter reaches eval",
|
|
cwe="CWE-95",
|
|
language="python",
|
|
tags=["syscall_entry"],
|
|
what_made_it_exploitable="dynamic execution of attacker input",
|
|
)
|
|
eval_id = "python-unsafe-eval:app.py:1"
|
|
hash_id = "python-weak-hash:hash.py:1"
|
|
hotspots = (_hotspot(hash_id, score=5.0, path="hash.py"), _hotspot(eval_id, score=5.0, path="app.py"))
|
|
records = (
|
|
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",), language="python"),
|
|
_record(eval_id, "promote", sources=("request",), cwe="CWE-95", sinks=("eval",), language="python"),
|
|
)
|
|
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
|
|
|
class _Client:
|
|
def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]:
|
|
del model, timeout_seconds
|
|
vectors: list[list[float]] = []
|
|
for text in inputs:
|
|
if "eval" in text.lower() or "CWE-95" in text:
|
|
vectors.append([1.0, 0.0])
|
|
else:
|
|
vectors.append([0.0, 1.0])
|
|
return vectors
|
|
|
|
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=_Client(), model="test-embed")
|
|
assert degradation is None
|
|
assert ordered[0].inventory_finding_ids == (eval_id,)
|
|
assert {item.inventory_finding_ids[0] for item in ordered} == {eval_id, hash_id}
|
|
|
|
|
|
def test_mechanism_degrades_to_heuristic_without_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
|
append_mechanism(
|
|
store,
|
|
summary="request parameter reaches eval",
|
|
cwe="CWE-95",
|
|
language="python",
|
|
tags=["syscall_entry"],
|
|
what_made_it_exploitable="dynamic execution of attacker input",
|
|
)
|
|
eval_id = "python-unsafe-eval:app.py:1"
|
|
hash_id = "python-weak-hash:hash.py:1"
|
|
hotspots = (_hotspot(hash_id, score=6.0, path="hash.py"), _hotspot(eval_id, score=3.0, path="app.py"))
|
|
records = (
|
|
_record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",)),
|
|
_record(eval_id, "promote", sources=("request",)),
|
|
)
|
|
findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)]
|
|
ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=None, model=None)
|
|
assert degradation == "embeddings_unavailable"
|
|
assert [item.inventory_finding_ids[0] for item in ordered] == [hash_id, eval_id]
|
|
|
|
|
|
def test_mechanisms_are_not_written_from_unadjudicated(tmp_path: Path) -> None:
|
|
store = MechanismStore(tmp_path / "mechanisms.jsonl")
|
|
record = OverlayRecord(
|
|
proposal_id="python-unsafe-eval:app.py:1",
|
|
path="app.py",
|
|
line=1,
|
|
disposition="unadjudicated",
|
|
reason="parse_failed",
|
|
cwe="CWE-95",
|
|
sources=(),
|
|
sinks=(),
|
|
sanitizers=(),
|
|
evidence_level="static_corroboration",
|
|
)
|
|
assert record.disposition != "promote"
|
|
assert store.load() == ()
|