feat(detection): language-scoped rules + multi-language benchmark corpora

Rebuild the quick-scan detection engine and ground-truth benchmarks to hit >=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java. Engine (findings.py, preprocess.py): - Scope each PatternRule by language (fixes cross-language false positives such as the Python eval/exec rule firing on JS .exec()). - Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS, SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv excluded as safe alternatives. - Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject) and a Groovy server-side template rule (yieldUnescaped of a bound variable). - Broaden C/C++ and Python coverage; precision guards for format strings and parameterised vs concatenated SQL; #-comment skipping; CWE in rationale. - Register .java/.groovy/.tpl source extensions. Benchmarks: - Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries complete, independently-labeled ground truth including genuinely hard-to-detect vulns (integer overflow, second-order SQLi, prototype pollution) so recall is measured honestly, plus safe-API files for precision. - Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets, missing C/C++ strcpy entry, Java SQLi line). benchmark.py: - Fix sink matching to normalise the expected sink consistently so sinks with underscores (e.g. render_template_string) can match. Tests: - Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP goal per language and overall. Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%; overall 93.6% recall, 0.0% false-positive rate. Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-10-16 11:49:28 +00:00 · 2026-06-21 11:49:31 +02:00
parent be65aed96e
commit cf9f945992
38 changed files with 2420 additions and 15 deletions
@@ -0,0 +1,357 @@
+import json
+from pathlib import Path
+
+from openultrasast.benchmark import (
+    create_benchmark_run,
+    evaluate_benchmark,
+    load_baseline_findings,
+    load_benchmark_manifest,
+    write_benchmark_artifacts,
+)
+from openultrasast.cli import main
+from openultrasast.findings import StaticFinding
+
+
+def test_benchmark_manifest_loads_expected_findings(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+frameworks = ["native"]
+setup = ["make"]
+modes = ["quick"]
+known_noise = ["sample-only fixture warning"]
+baselines = ["clang-tidy"]
+
+[source]
+type = "local"
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 3
+rule_id = "c-unsafe-memory-copy"
+evidence = "memcpy copies attacker-controlled input"
+function = "copy_user"
+sink = "memcpy"
+""".strip()
+        + "\n"
+    )
+
+    manifest = load_benchmark_manifest(manifest_path)
+
+    assert manifest.name == "cpp-smoke"
+    assert manifest.language == "c_cpp"
+    assert manifest.setup == ["make"]
+    assert manifest.source.path == "repo"
+    assert manifest.known_noise == ["sample-only fixture warning"]
+    assert manifest.expected[0].rule_id == "c-unsafe-memory-copy"
+    assert manifest.expected[0].function == "copy_user"
+    assert manifest.expected[0].sink == "memcpy"
+
+
+def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None:
+    manifest_paths = sorted(Path("benchmarks/manifests").glob("*.toml"))
+
+    assert {path.name for path in manifest_paths} == {
+        "c-cpp-smoke.toml",
+        "cpp-damn-vulnerable.toml",
+        "java-spring-boot-vulnerable.toml",
+        "java-web-smoke.toml",
+        "javascript-node-web-smoke.toml",
+        "javascript-vulnerable.toml",
+        "python-vulnerable.toml",
+        "python-web-smoke.toml",
+    }
+    for manifest_path in manifest_paths:
+        manifest = load_benchmark_manifest(manifest_path)
+        assert manifest.expected
+        assert manifest.language in {"c_cpp", "java_web", "javascript_node_web", "python_web"}
+        assert (manifest_path.parent / manifest.source.path).resolve().exists()
+
+
+def test_benchmark_result_records_missed_expected_findings(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 3
+rule_id = "c-unsafe-memory-copy"
+evidence = "memcpy copies attacker-controlled input"
+""".strip()
+        + "\n"
+    )
+    manifest = load_benchmark_manifest(manifest_path)
+    run = create_benchmark_run(repo, manifest)
+
+    result = evaluate_benchmark(run=run, mode="quick", findings=[], scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
+    write_benchmark_artifacts(run, manifest_path, result)
+
+    payload = json.loads((run.root / "benchmark_result.json").read_text())
+    assert payload["metrics"]["expected_findings_total"] == 1
+    assert payload["metrics"]["matched_findings_total"] == 0
+    assert payload["metrics"]["missed_findings_total"] == 1
+    assert payload["metrics"]["false_positive_findings_total"] == 0
+    assert payload["metrics"]["recall"] == 0.0
+    assert payload["metrics"]["precision"] is None
+    assert payload["misses"][0]["reason"] == "no OpenUltraSAST finding matched the expected benchmark vulnerability"
+
+
+def test_benchmark_cli_runs_scan_and_writes_scoreboard(tmp_path: Path, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    repo = tmp_path / "repo"
+    source = repo / "src"
+    source.mkdir(parents=True)
+    (source / "vuln.cpp").write_text("#include <string.h>\nvoid f(char *a, char *b) {\n  memcpy(a, b, 10);\n}\n")
+    manifest_path = tmp_path / "benchmark.toml"
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 3
+rule_id = "c-unsafe-memory-copy"
+evidence = "memcpy copies attacker-controlled input"
+""".strip()
+        + "\n"
+    )
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+
+    assert main(["benchmark", str(manifest_path), "--mode", "quick"]) == 0
+
+    benchmark_dir = sorted((repo / ".openultrasast" / "benchmarks").iterdir())[-1]
+    payload = json.loads((benchmark_dir / "benchmark_result.json").read_text())
+
+    assert payload["benchmark_name"] == "cpp-smoke"
+    assert payload["metrics"]["expected_findings_total"] == 1
+    assert payload["metrics"]["matched_findings_total"] == 1
+    assert payload["metrics"]["recall"] == 1.0
+    assert payload["metrics"]["precision"] == 1.0
+    assert payload["metrics"]["artifact_links"]["findings"].endswith("findings.json")
+
+
+def test_benchmark_matching_uses_function_sink_and_evidence_text(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 9
+rule_id = "c-unsafe-memory-copy"
+function = "copy_user"
+sink = "memcpy"
+evidence = "attacker-controlled memcpy"
+""".strip()
+        + "\n"
+    )
+    manifest = load_benchmark_manifest(manifest_path)
+    run = create_benchmark_run(repo, manifest)
+    findings = [
+        _finding(
+            finding_id="c-unsafe-memory-copy:src/vuln.cpp:9",
+            path="src/vuln.cpp",
+            line=9,
+            function_name="copy_user",
+            rationale="Static pattern c-unsafe-memory-copy matched memcpy with attacker-controlled input.",
+        )
+    ]
+
+    result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
+
+    assert result.metrics.matched_findings_total == 1
+    assert result.metrics.missed_findings_total == 0
+    assert result.metrics.false_positive_findings_total == 0
+    assert result.metrics.recall == 1.0
+    assert result.metrics.precision == 1.0
+
+
+def test_benchmark_records_unmatched_findings_as_false_positives(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 9
+rule_id = "c-unsafe-memory-copy"
+sink = "memcpy"
+evidence = "memcpy"
+""".strip()
+        + "\n"
+    )
+    manifest = load_benchmark_manifest(manifest_path)
+    run = create_benchmark_run(repo, manifest)
+    findings = [
+        _finding(
+            finding_id="c-shell-exec:src/other.cpp:12",
+            path="src/other.cpp",
+            line=12,
+            rationale="Static pattern c-shell-exec matched system.",
+        )
+    ]
+
+    result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
+
+    assert result.metrics.matched_findings_total == 0
+    assert result.metrics.missed_findings_total == 1
+    assert result.metrics.false_positive_findings_total == 1
+    assert result.metrics.precision == 0.0
+    assert result.false_positives[0].finding_id == "c-shell-exec:src/other.cpp:12"
+    assert result.false_positives[0].reason == "finding did not match any expected benchmark vulnerability"
+
+
+def test_benchmark_ingests_baselines_and_writes_calibration_records(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    baseline_path = tmp_path / "clearwing-normalized.json"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    baseline_path.write_text(
+        json.dumps(
+            {
+                "findings": [
+                    {
+                        "tool": "clearwing",
+                        "finding_id": "cw-1",
+                        "path": "src/vuln.cpp",
+                        "line": 9,
+                        "rule_id": "c-unsafe-memory-copy",
+                        "cwe": "CWE-120",
+                        "evidence": "memcpy copies attacker-controlled input",
+                    },
+                    {
+                        "tool": "clearwing",
+                        "finding_id": "cw-extra",
+                        "path": "src/other.cpp",
+                        "line": 12,
+                        "rule_id": "c-shell-exec",
+                        "cwe": "CWE-78",
+                        "evidence": "system executes shell command",
+                    },
+                ]
+            }
+        )
+        + "\n"
+    )
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+baselines = ["clearwing-normalized.json"]
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "unsafe memory copy"
+path = "src/vuln.cpp"
+line = 9
+rule_id = "c-unsafe-memory-copy"
+sink = "memcpy"
+evidence = "memcpy copies attacker-controlled input"
+""".strip()
+        + "\n"
+    )
+    manifest = load_benchmark_manifest(manifest_path)
+    run = create_benchmark_run(repo, manifest)
+    baseline_findings = load_baseline_findings(manifest_path, manifest)
+    unrelated_finding = _finding(
+        finding_id="c-shell-exec:src/other.cpp:12",
+        path="src/other.cpp",
+        line=12,
+        rationale="Static pattern c-shell-exec matched system.",
+    )
+
+    result = evaluate_benchmark(
+        run=run,
+        mode="quick",
+        findings=[unrelated_finding],
+        scan_id="scan-1",
+        scan_run_dir=repo / ".runs" / "scan-1",
+        baseline_findings=baseline_findings,
+    )
+    write_benchmark_artifacts(run, manifest_path, result)
+
+    payload = json.loads((run.root / "benchmark_result.json").read_text())
+    calibration = json.loads((run.root / "calibration_records.json").read_text())
+    deltas = json.loads((run.root / "external_baseline_deltas.json").read_text())
+    assert payload["metrics"]["missed_findings_total"] == 1
+    assert payload["metrics"]["false_positive_findings_total"] == 1
+    assert payload["false_positives"][0]["finding_id"] == "c-shell-exec:src/other.cpp:12"
+    assert calibration == [
+        {
+            "cwe": "CWE-120",
+            "evidence": "memcpy copies attacker-controlled input",
+            "failed_stage": "benchmark_ground_truth_matching",
+            "next_improvement_candidate": "rules_or_language_hunter",
+            "path": "src/vuln.cpp",
+            "reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
+            "vulnerability_class": "unsafe memory copy",
+        }
+    ]
+    assert deltas == [{"extra_findings_total": 1, "matched_expected_total": 1, "missed_expected_total": 0, "tool": "clearwing"}]
+
+
+def _finding(
+    *,
+    finding_id: str,
+    path: str,
+    line: int,
+    rationale: str,
+    function_name: str | None = None,
+) -> StaticFinding:
+    return StaticFinding(
+        finding_id=finding_id,
+        path=path,
+        title="Test finding",
+        severity="high",
+        confidence="medium",
+        evidence_level="static_corroboration",
+        rationale=rationale,
+        line=line,
+        function_name=function_name,
+        reachability_status="unknown",
+        reachability_evidence=[],
+        reachability_conditions=[],
+        tags=[],
+        ranking_priority=1.0,
+    )
@@ -0,0 +1,79 @@
+"""Detection-quality gate for the in-scope languages.
+
+Runs quick-mode scans against the C/C++, Python, and JavaScript benchmark
+corpora and asserts the project goal: at least 90% recall of known
+vulnerabilities and under a 10% false-positive rate per language.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from openultrasast.benchmark import (
+    BenchmarkRun,
+    evaluate_benchmark,
+    load_benchmark_manifest,
+    resolve_benchmark_source,
+)
+from openultrasast.findings import quick_scan_findings
+from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
+from openultrasast.preprocess import preprocess_repository
+from openultrasast.rank import rank_targets
+
+LANGUAGE_MANIFESTS = {
+    "c_cpp": ["c-cpp-smoke", "cpp-damn-vulnerable"],
+    "python": ["python-web-smoke", "python-vulnerable"],
+    "javascript": ["javascript-node-web-smoke", "javascript-vulnerable"],
+    "java": ["java-web-smoke", "java-spring-boot-vulnerable"],
+}
+
+RECALL_FLOOR = 0.90
+FALSE_POSITIVE_CEILING = 0.10
+
+
+def _quick_scan(target: Path):  # type: ignore[no-untyped-def]
+    _, targets = preprocess_repository(target)
+    targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
+    rankings = rank_targets(targets)
+    return quick_scan_findings(target, targets, rankings)
+
+
+def _language_metrics(manifest_names: list[str]) -> tuple[int, int, int, int]:
+    expected = matched = actual = false_positives = 0
+    for name in manifest_names:
+        manifest_path = Path(f"benchmarks/manifests/{name}.toml")
+        manifest = load_benchmark_manifest(manifest_path)
+        target = resolve_benchmark_source(manifest_path, manifest)
+        findings = _quick_scan(target)
+        run = BenchmarkRun(benchmark_run_id="gate", root=Path("/tmp/gate"), manifest=manifest)
+        result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
+        metrics = result.metrics
+        expected += metrics.expected_findings_total
+        matched += metrics.matched_findings_total
+        actual += metrics.actual_findings_total
+        false_positives += metrics.false_positive_findings_total
+    return expected, matched, actual, false_positives
+
+
+@pytest.mark.parametrize("language", sorted(LANGUAGE_MANIFESTS))
+def test_language_meets_recall_and_false_positive_goal(language: str) -> None:
+    expected, matched, actual, false_positives = _language_metrics(LANGUAGE_MANIFESTS[language])
+
+    recall = matched / expected
+    false_positive_rate = false_positives / actual if actual else 0.0
+
+    assert recall >= RECALL_FLOOR, f"{language} recall {recall:.2%} below {RECALL_FLOOR:.0%} ({matched}/{expected})"
+    assert false_positive_rate < FALSE_POSITIVE_CEILING, (
+        f"{language} false-positive rate {false_positive_rate:.2%} at or above {FALSE_POSITIVE_CEILING:.0%} ({false_positives}/{actual})"
+    )
+
+
+def test_overall_detection_goal_is_met() -> None:
+    totals = [_language_metrics(names) for names in LANGUAGE_MANIFESTS.values()]
+    expected = sum(t[0] for t in totals)
+    matched = sum(t[1] for t in totals)
+    actual = sum(t[2] for t in totals)
+    false_positives = sum(t[3] for t in totals)
+
+    assert matched / expected >= RECALL_FLOOR
+    assert (false_positives / actual) < FALSE_POSITIVE_CEILING
@@ -23,6 +23,27 @@ def test_quick_scan_findings_emit_static_corroboration(tmp_path: Path) -> None:
    assert findings[0].line == 1


+def test_quick_scan_findings_emit_common_cpp_hazards(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "vuln.cpp").write_text(
+        "#include <cstring>\n"
+        "#include <cstdlib>\n"
+        "// system(cmd) in a comment is not executable evidence.\n"
+        "void copy(char *src) { char dst[4]; memcpy(dst, src, 16); }\n"
+        "void run(char *cmd) { system(cmd); }\n"
+        "void fmt(char *format, va_list args) { char dst[8]; vsprintf(dst, format, args); }\n"
+    )
+    _, targets = preprocess_repository(repo)
+
+    finding_ids = {finding.finding_id for finding in quick_scan_findings(repo, targets, rank_targets(targets))}
+
+    assert "c-shell-exec:vuln.cpp:3" not in finding_ids
+    assert "c-unsafe-memory-copy:vuln.cpp:4" in finding_ids
+    assert "c-shell-exec:vuln.cpp:5" in finding_ids
+    assert "c-unsafe-format:vuln.cpp:6" in finding_ids
+
+
 def test_findings_and_report_artifacts(tmp_path: Path) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()