feat(verification): add evidence ladder verifier

tests/test_verification.py

@@ -0,0 +1,141 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.cli import main
+from openultrasast.findings import StaticFinding
+from openultrasast.verification import (
+    EvidenceLevel,
+    VerificationStatus,
+    build_independent_verifier_context,
+    is_report_verified,
+    validate_evidence_transition,
+    verify_finding,
+    write_verification_results,
+)
+
+
+def test_evidence_transition_rejects_backward_and_skipped_levels() -> None:
+    assert validate_evidence_transition("suspicion", "static_corroboration") == EvidenceLevel.STATIC_CORROBORATION
+
+    with pytest.raises(ValueError, match="backward"):
+        validate_evidence_transition("root_cause_explained", "static_corroboration")
+
+    with pytest.raises(ValueError, match="skip"):
+        validate_evidence_transition("suspicion", "root_cause_explained")
+
+
+def test_verified_status_requires_static_corroboration_and_acceptance() -> None:
+    assert is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.ACCEPTED)
+    assert not is_report_verified(EvidenceLevel.SUSPICION, VerificationStatus.ACCEPTED)
+    assert not is_report_verified(EvidenceLevel.STATIC_CORROBORATION, VerificationStatus.REJECTED)
+
+
+def test_independent_verifier_context_excludes_hunter_reasoning() -> None:
+    finding = _finding(evidence_level="static_corroboration", reachability_status="reachable")
+
+    context = build_independent_verifier_context(finding)
+
+    assert "hunter_reasoning" not in context
+    assert "chain_of_thought" not in context
+    assert context["finding_id"] == finding.finding_id
+    assert context["reachability_status"] == "reachable"
+
+
+def test_verifier_accepts_reachable_static_corroborated_finding() -> None:
+    finding = _finding(evidence_level="static_corroboration", reachability_status="reachable")
+
+    result = verify_finding(finding)
+
+    assert result.status == VerificationStatus.ACCEPTED
+    assert result.verified
+    assert result.evidence_level == EvidenceLevel.STATIC_CORROBORATION
+    assert result.pro_case
+    assert result.counter_case
+    assert result.tie_breaker
+    assert result.required_next_step
+
+
+def test_verifier_rejects_unknown_reachability_static_finding() -> None:
+    finding = _finding(evidence_level="static_corroboration", reachability_status="unknown")
+
+    result = verify_finding(finding)
+
+    assert result.status == VerificationStatus.REJECTED
+    assert not result.verified
+    assert "reach" in result.counter_case.lower()
+    assert "call-graph" in result.tie_breaker
+
+
+def test_verifier_does_not_verify_suspicion_only_finding() -> None:
+    finding = _finding(evidence_level="suspicion", reachability_status="reachable")
+
+    result = verify_finding(finding)
+
+    assert result.status == VerificationStatus.NEEDS_EVIDENCE
+    assert not result.verified
+    assert result.required_next_step == "Collect static corroboration before reporting as verified."
+
+
+def test_write_verification_results_artifact(tmp_path: Path) -> None:
+    result = verify_finding(_finding(evidence_level="static_corroboration", reachability_status="reachable"))
+    output = tmp_path / "verification.json"
+
+    write_verification_results([result], output)
+
+    payload = json.loads(output.read_text())
+    assert payload["verifications"][0]["finding_id"] == "python-unsafe-eval:app.py:3"
+    assert payload["verifications"][0]["status"] == "accepted"
+
+
+def test_cli_scan_writes_accepted_and_rejected_verifications(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "@app.route('/admin')\n"
+        "def admin():\n"
+        "    return eval(request.data)\n"
+        "\n"
+        "def helper(value):\n"
+        "    return exec(value)\n"
+    )
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+
+    assert main(["scan", str(repo), "--mode", "quick"]) == 0
+
+    run_dirs = sorted((repo / ".runs").iterdir())
+    verification_path = run_dirs[-1] / "verification.json"
+    payload = json.loads(verification_path.read_text())
+    statuses = {item["status"] for item in payload["verifications"]}
+
+    assert statuses == {"accepted", "rejected"}
+
+
+def _finding(*, evidence_level: str, reachability_status: str) -> StaticFinding:
+    evidence = [
+        {
+            "kind": "route",
+            "access_level": "public",
+            "line": 1,
+            "end_line": 3,
+            "function_name": "admin",
+            "conditions": [],
+        }
+    ] if reachability_status == "reachable" else []
+    return StaticFinding(
+        finding_id="python-unsafe-eval:app.py:3",
+        path="app.py",
+        title="Dynamic Python execution needs review",
+        severity="high",
+        confidence="medium",
+        evidence_level=evidence_level,
+        rationale="Static pattern matched.",
+        line=3,
+        function_name="admin" if reachability_status == "reachable" else None,
+        reachability_status=reachability_status,
+        reachability_evidence=evidence,
+        reachability_conditions=[],
+        tags=["syscall_entry"],
+        ranking_priority=3.0,
+    )