Open_Ultra_SAST/tests/test_semantic_pipeline.py

"""MAP overlay, prove filter, reports, pair scoring."""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from openultrasast.benchmark import ExpectedFinding
from openultrasast.cli import main
from openultrasast.complexity.map import Hotspot
from openultrasast.findings import StaticFinding, quick_scan_findings
from openultrasast.gate import LANGUAGE_MANIFESTS
from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
from openultrasast.semantic import OverlayRecord, filter_promoted_hotspots
from openultrasast.semantic.prove_filter import promoted_findings


def _latest_run(repo: Path) -> Path:
    return sorted((repo / ".runs").iterdir())[-1]


def _finding(finding_id: str, path: str = "app.py", line: int = 1) -> StaticFinding:
    return StaticFinding(
        finding_id=finding_id,
        path=path,
        title="test",
        severity="high",
        confidence="medium",
        evidence_level="static_corroboration",
        rationale="Static pattern python-unsafe-eval (CWE-95) matched",
        line=line,
        function_name="run",
        reachability_status="reachable",
        reachability_evidence=[],
        reachability_conditions=[],
        tags=["syscall_entry"],
        ranking_priority=1.0,
    )


def _record(proposal_id: str, disposition: str, **kwargs: object) -> OverlayRecord:
    dominating = kwargs.get("dominating_fact")
    sanitizers = kwargs.get("sanitizers", ())
    return OverlayRecord(
        proposal_id=proposal_id,
        path=str(kwargs.get("path", "app.py")),
        line=int(kwargs.get("line", 1)),
        disposition=disposition,
        reason=str(kwargs.get("reason", disposition)),
        cwe=str(kwargs.get("cwe", "CWE-95")),
        sources=tuple(kwargs.get("sources", ())),  # type: ignore[arg-type]
        sinks=tuple(kwargs.get("sinks", ("eval",))),  # type: ignore[arg-type]
        sanitizers=tuple(sanitizers) if isinstance(sanitizers, tuple | list) else (),
        evidence_level="static_corroboration",
        dominating_fact=str(dominating) if dominating else ("constant" if disposition == "demote" else None),
        language=str(kwargs.get("language", "python")),
    )


def _hotspot(*finding_ids: str, score: float = 5.0, path: str = "app.py") -> Hotspot:
    return Hotspot(
        path=path,
        function_name="run",
        score=score,
        band="high",
        signals={},
        rationale="test",
        test_hint=None,
        inventory_finding_ids=finding_ids,
    )


def test_quick_scan_writes_no_overlay(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    (repo / "app.py").write_text("def run():\n    return eval(request.args)\n")
    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
    assert main(["scan", str(repo), "--mode", "quick"]) == 0
    run_dir = _latest_run(repo)
    assert not (run_dir / "overlay.json").exists()
    report = (run_dir / "report.md").read_text()
    assert "## Overlay" not in report


def test_standard_scan_writes_overlay_without_executing_target(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    (repo / "app.py").write_text("def run():\n    return eval(request.args)\n")
    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
    assert main(["scan", str(repo), "--mode", "standard"]) == 0
    run_dir = _latest_run(repo)
    payload = json.loads((run_dir / "overlay.json").read_text())
    assert payload["records"]
    assert payload["records"][0]["disposition"] == "promote"
    report = (run_dir / "report.md").read_text()
    assert "promoted" in report
    assert "Disposition: `promote`" in report
    sarif = json.loads((run_dir / "report.sarif").read_text())
    assert sarif["runs"][0]["results"][0]["properties"]["disposition"] == "promote"
    assert (run_dir / "manifest.json").read_text()
    manifest = json.loads((run_dir / "manifest.json").read_text())
    assert manifest["artifacts"]["overlay"] == "overlay.json"


def test_prove_filter_drops_demoted_keeps_promoted() -> None:
    promoted_id = "python-unsafe-eval:app.py:2"
    demoted_id = "python-unsafe-eval:app.py:1"
    hotspots = (
        _hotspot(demoted_id, score=9.0),
        _hotspot(promoted_id, score=3.0),
    )
    records = (
        _record(demoted_id, "demote", dominating_fact="constant"),
        _record(promoted_id, "promote", sources=("request",)),
    )
    selected = filter_promoted_hotspots(hotspots, records)
    assert len(selected) == 1
    assert selected[0].inventory_finding_ids == (promoted_id,)
    findings = [_finding(demoted_id, line=1), _finding(promoted_id, line=2)]
    assert [item.finding_id for item in promoted_findings(findings, records)] == [promoted_id]


def test_deep_scan_offers_only_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    from openultrasast import cli
    from openultrasast.sandbox import FakeSandboxRunner, SandboxResult

    repo = tmp_path / "repo"
    repo.mkdir()
    (repo / "app.py").write_text("def safe():\n    exec('ls')\n\ndef bad():\n    return eval(request.args)\n")
    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
    fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False))
    monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake)
    assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 0
    run_dir = _latest_run(repo)
    overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
    dispositions = {item["proposal_id"]: item["disposition"] for item in overlay}
    demoted = [key for key, value in dispositions.items() if value == "demote"]
    promoted = [key for key, value in dispositions.items() if value == "promote"]
    assert demoted
    assert promoted
    verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"]
    offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]}
    assert offered <= set(promoted)
    assert not (offered & set(demoted))


def test_fail_on_worth_fixing_ignores_unadjudicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    (repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0")
    assert main(["scan", str(repo), "--mode", "standard", "--fail-on", "worth-fixing"]) == 0
    run_dir = _latest_run(repo)
    overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
    assert overlay
    assert all(item["disposition"] == "unadjudicated" for item in overlay)
    report = (run_dir / "report.md").read_text()
    assert "unadjudicated" in report


def test_unlabeled_messy_tree_has_both_dispositions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    (repo / "broken.py").write_text("eval(request.args)\ndef oops(\n")
    (repo / "app.py").write_text("def run():\n    return eval(request.args)\n")
    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
    assert main(["scan", str(repo), "--mode", "standard"]) == 0
    overlay = json.loads((_latest_run(repo) / "overlay.json").read_text())["records"]
    by_path = {}
    for item in overlay:
        by_path.setdefault(item["path"], set()).add(item["disposition"])
    assert "unadjudicated" in by_path["broken.py"]
    assert "promote" in by_path["app.py"]


def test_sast_slice_scores_overlay_not_inventory(tmp_path: Path) -> None:
    vuln = tmp_path / "vuln.py"
    fixed = tmp_path / "fixed.py"
    vuln.write_text("def bad(x):\n    exec(x)\n")
    fixed.write_text("def goodG2B():\n    exec('ls')\n")
    case = PairCase(
        name="synth-overlay-exec",
        slice="sast",
        language="python",
        origin="test",
        vuln_file=vuln,
        fixed_file=fixed,
        relpath="app.py",
        expected=(
            ExpectedFinding(
                cwe="CWE-95",
                vulnerability_class="code injection",
                path="app.py",
                evidence="exec",
                rule_id="python-unsafe-eval",
                sink="exec",
            ),
        ),
        min_recall=1.0,
        fix_policy="silent",
    )
    fix_root = tmp_path / "fix-tree"
    fix_root.mkdir()
    (fix_root / "app.py").write_text(fixed.read_text())
    _, fix_targets = preprocess_repository(fix_root)
    assert quick_scan_findings(fix_root, fix_targets, rank_targets(fix_targets)), "inventory still fires on goodG2B exec"
    outcome = evaluate_pair(case)
    assert outcome.detected_vuln
    assert outcome.silent_fix
    assert outcome.pair_correct


def test_sast_pairs_stay_out_of_language_manifests() -> None:
    listed = {name for names in LANGUAGE_MANIFESTS.values() for name in names}
    leaked = [case.name for case in load_pair_catalog() if case.name in listed]
    assert leaked == []


def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None:
    from openultrasast.reports import write_markdown_report

    finding = _finding("python-unsafe-eval:app.py:1")
    records = (
        _record("python-unsafe-eval:app.py:1", "promote", sources=("request",)),
        _record("python-unsafe-eval:b.py:1", "demote", path="b.py", dominating_fact="constant"),
        OverlayRecord(
            proposal_id="python-unsafe-eval:c.py:1",
            path="c.py",
            line=1,
            disposition="unadjudicated",
            reason="parse_failed",
            cwe="CWE-95",
            sources=(),
            sinks=(),
            sanitizers=(),
            evidence_level="static_corroboration",
        ),
        OverlayRecord(
            proposal_id="overlay-coverage:d.py:1:hashlib",
            path="d.py",
            line=1,
            disposition="coverage",
            reason="uninventoried sink hashlib",
            cwe="CWE-327",
            sources=("parameter",),
            sinks=("hashlib",),
            sanitizers=(),
            evidence_level="static_corroboration",
            origin="overlay",
        ),
    )
    output = tmp_path / "report.md"
    write_markdown_report([finding], output, overlay=records)
    text = output.read_text()
    assert "promoted" in text
    assert "demoted" in text
    assert "unadjudicated" in text
    assert "coverage" in text