mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 4.8
cf9f945992
Rebuild the quick-scan detection engine and ground-truth benchmarks to hit >=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java. Engine (findings.py, preprocess.py): - Scope each PatternRule by language (fixes cross-language false positives such as the Python eval/exec rule firing on JS .exec()). - Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS, SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv excluded as safe alternatives. - Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject) and a Groovy server-side template rule (yieldUnescaped of a bound variable). - Broaden C/C++ and Python coverage; precision guards for format strings and parameterised vs concatenated SQL; #-comment skipping; CWE in rationale. - Register .java/.groovy/.tpl source extensions. Benchmarks: - Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries complete, independently-labeled ground truth including genuinely hard-to-detect vulns (integer overflow, second-order SQLi, prototype pollution) so recall is measured honestly, plus safe-API files for precision. - Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets, missing C/C++ strcpy entry, Java SQLi line). benchmark.py: - Fix sink matching to normalise the expected sink consistently so sinks with underscores (e.g. render_template_string) can match. Tests: - Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP goal per language and overall. Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%; overall 93.6% recall, 0.0% false-positive rate. Co-Authored-By: Claude Opus 4.8 <[email protected]>
358 lines
11 KiB
Python
358 lines
11 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
from openultrasast.benchmark import (
|
|
create_benchmark_run,
|
|
evaluate_benchmark,
|
|
load_baseline_findings,
|
|
load_benchmark_manifest,
|
|
write_benchmark_artifacts,
|
|
)
|
|
from openultrasast.cli import main
|
|
from openultrasast.findings import StaticFinding
|
|
|
|
|
|
def test_benchmark_manifest_loads_expected_findings(tmp_path: Path) -> None:
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
frameworks = ["native"]
|
|
setup = ["make"]
|
|
modes = ["quick"]
|
|
known_noise = ["sample-only fixture warning"]
|
|
baselines = ["clang-tidy"]
|
|
|
|
[source]
|
|
type = "local"
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 3
|
|
rule_id = "c-unsafe-memory-copy"
|
|
evidence = "memcpy copies attacker-controlled input"
|
|
function = "copy_user"
|
|
sink = "memcpy"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
|
|
assert manifest.name == "cpp-smoke"
|
|
assert manifest.language == "c_cpp"
|
|
assert manifest.setup == ["make"]
|
|
assert manifest.source.path == "repo"
|
|
assert manifest.known_noise == ["sample-only fixture warning"]
|
|
assert manifest.expected[0].rule_id == "c-unsafe-memory-copy"
|
|
assert manifest.expected[0].function == "copy_user"
|
|
assert manifest.expected[0].sink == "memcpy"
|
|
|
|
|
|
def test_repository_benchmark_manifests_load_and_point_to_fixtures() -> None:
|
|
manifest_paths = sorted(Path("benchmarks/manifests").glob("*.toml"))
|
|
|
|
assert {path.name for path in manifest_paths} == {
|
|
"c-cpp-smoke.toml",
|
|
"cpp-damn-vulnerable.toml",
|
|
"java-spring-boot-vulnerable.toml",
|
|
"java-web-smoke.toml",
|
|
"javascript-node-web-smoke.toml",
|
|
"javascript-vulnerable.toml",
|
|
"python-vulnerable.toml",
|
|
"python-web-smoke.toml",
|
|
}
|
|
for manifest_path in manifest_paths:
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
assert manifest.expected
|
|
assert manifest.language in {"c_cpp", "java_web", "javascript_node_web", "python_web"}
|
|
assert (manifest_path.parent / manifest.source.path).resolve().exists()
|
|
|
|
|
|
def test_benchmark_result_records_missed_expected_findings(tmp_path: Path) -> None:
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
|
|
[source]
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 3
|
|
rule_id = "c-unsafe-memory-copy"
|
|
evidence = "memcpy copies attacker-controlled input"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
run = create_benchmark_run(repo, manifest)
|
|
|
|
result = evaluate_benchmark(run=run, mode="quick", findings=[], scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
|
|
write_benchmark_artifacts(run, manifest_path, result)
|
|
|
|
payload = json.loads((run.root / "benchmark_result.json").read_text())
|
|
assert payload["metrics"]["expected_findings_total"] == 1
|
|
assert payload["metrics"]["matched_findings_total"] == 0
|
|
assert payload["metrics"]["missed_findings_total"] == 1
|
|
assert payload["metrics"]["false_positive_findings_total"] == 0
|
|
assert payload["metrics"]["recall"] == 0.0
|
|
assert payload["metrics"]["precision"] is None
|
|
assert payload["misses"][0]["reason"] == "no OpenUltraSAST finding matched the expected benchmark vulnerability"
|
|
|
|
|
|
def test_benchmark_cli_runs_scan_and_writes_scoreboard(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
|
repo = tmp_path / "repo"
|
|
source = repo / "src"
|
|
source.mkdir(parents=True)
|
|
(source / "vuln.cpp").write_text("#include <string.h>\nvoid f(char *a, char *b) {\n memcpy(a, b, 10);\n}\n")
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
|
|
[source]
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 3
|
|
rule_id = "c-unsafe-memory-copy"
|
|
evidence = "memcpy copies attacker-controlled input"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
|
|
|
assert main(["benchmark", str(manifest_path), "--mode", "quick"]) == 0
|
|
|
|
benchmark_dir = sorted((repo / ".openultrasast" / "benchmarks").iterdir())[-1]
|
|
payload = json.loads((benchmark_dir / "benchmark_result.json").read_text())
|
|
|
|
assert payload["benchmark_name"] == "cpp-smoke"
|
|
assert payload["metrics"]["expected_findings_total"] == 1
|
|
assert payload["metrics"]["matched_findings_total"] == 1
|
|
assert payload["metrics"]["recall"] == 1.0
|
|
assert payload["metrics"]["precision"] == 1.0
|
|
assert payload["metrics"]["artifact_links"]["findings"].endswith("findings.json")
|
|
|
|
|
|
def test_benchmark_matching_uses_function_sink_and_evidence_text(tmp_path: Path) -> None:
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
|
|
[source]
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 9
|
|
rule_id = "c-unsafe-memory-copy"
|
|
function = "copy_user"
|
|
sink = "memcpy"
|
|
evidence = "attacker-controlled memcpy"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
run = create_benchmark_run(repo, manifest)
|
|
findings = [
|
|
_finding(
|
|
finding_id="c-unsafe-memory-copy:src/vuln.cpp:9",
|
|
path="src/vuln.cpp",
|
|
line=9,
|
|
function_name="copy_user",
|
|
rationale="Static pattern c-unsafe-memory-copy matched memcpy with attacker-controlled input.",
|
|
)
|
|
]
|
|
|
|
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
|
|
|
|
assert result.metrics.matched_findings_total == 1
|
|
assert result.metrics.missed_findings_total == 0
|
|
assert result.metrics.false_positive_findings_total == 0
|
|
assert result.metrics.recall == 1.0
|
|
assert result.metrics.precision == 1.0
|
|
|
|
|
|
def test_benchmark_records_unmatched_findings_as_false_positives(tmp_path: Path) -> None:
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
|
|
[source]
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 9
|
|
rule_id = "c-unsafe-memory-copy"
|
|
sink = "memcpy"
|
|
evidence = "memcpy"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
run = create_benchmark_run(repo, manifest)
|
|
findings = [
|
|
_finding(
|
|
finding_id="c-shell-exec:src/other.cpp:12",
|
|
path="src/other.cpp",
|
|
line=12,
|
|
rationale="Static pattern c-shell-exec matched system.",
|
|
)
|
|
]
|
|
|
|
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
|
|
|
|
assert result.metrics.matched_findings_total == 0
|
|
assert result.metrics.missed_findings_total == 1
|
|
assert result.metrics.false_positive_findings_total == 1
|
|
assert result.metrics.precision == 0.0
|
|
assert result.false_positives[0].finding_id == "c-shell-exec:src/other.cpp:12"
|
|
assert result.false_positives[0].reason == "finding did not match any expected benchmark vulnerability"
|
|
|
|
|
|
def test_benchmark_ingests_baselines_and_writes_calibration_records(tmp_path: Path) -> None:
|
|
manifest_path = tmp_path / "benchmark.toml"
|
|
baseline_path = tmp_path / "clearwing-normalized.json"
|
|
repo = tmp_path / "repo"
|
|
repo.mkdir()
|
|
baseline_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"findings": [
|
|
{
|
|
"tool": "clearwing",
|
|
"finding_id": "cw-1",
|
|
"path": "src/vuln.cpp",
|
|
"line": 9,
|
|
"rule_id": "c-unsafe-memory-copy",
|
|
"cwe": "CWE-120",
|
|
"evidence": "memcpy copies attacker-controlled input",
|
|
},
|
|
{
|
|
"tool": "clearwing",
|
|
"finding_id": "cw-extra",
|
|
"path": "src/other.cpp",
|
|
"line": 12,
|
|
"rule_id": "c-shell-exec",
|
|
"cwe": "CWE-78",
|
|
"evidence": "system executes shell command",
|
|
},
|
|
]
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
manifest_path.write_text(
|
|
"""
|
|
name = "cpp-smoke"
|
|
language = "c_cpp"
|
|
baselines = ["clearwing-normalized.json"]
|
|
|
|
[source]
|
|
path = "repo"
|
|
|
|
[[expected]]
|
|
cwe = "CWE-120"
|
|
class = "unsafe memory copy"
|
|
path = "src/vuln.cpp"
|
|
line = 9
|
|
rule_id = "c-unsafe-memory-copy"
|
|
sink = "memcpy"
|
|
evidence = "memcpy copies attacker-controlled input"
|
|
""".strip()
|
|
+ "\n"
|
|
)
|
|
manifest = load_benchmark_manifest(manifest_path)
|
|
run = create_benchmark_run(repo, manifest)
|
|
baseline_findings = load_baseline_findings(manifest_path, manifest)
|
|
unrelated_finding = _finding(
|
|
finding_id="c-shell-exec:src/other.cpp:12",
|
|
path="src/other.cpp",
|
|
line=12,
|
|
rationale="Static pattern c-shell-exec matched system.",
|
|
)
|
|
|
|
result = evaluate_benchmark(
|
|
run=run,
|
|
mode="quick",
|
|
findings=[unrelated_finding],
|
|
scan_id="scan-1",
|
|
scan_run_dir=repo / ".runs" / "scan-1",
|
|
baseline_findings=baseline_findings,
|
|
)
|
|
write_benchmark_artifacts(run, manifest_path, result)
|
|
|
|
payload = json.loads((run.root / "benchmark_result.json").read_text())
|
|
calibration = json.loads((run.root / "calibration_records.json").read_text())
|
|
deltas = json.loads((run.root / "external_baseline_deltas.json").read_text())
|
|
assert payload["metrics"]["missed_findings_total"] == 1
|
|
assert payload["metrics"]["false_positive_findings_total"] == 1
|
|
assert payload["false_positives"][0]["finding_id"] == "c-shell-exec:src/other.cpp:12"
|
|
assert calibration == [
|
|
{
|
|
"cwe": "CWE-120",
|
|
"evidence": "memcpy copies attacker-controlled input",
|
|
"failed_stage": "benchmark_ground_truth_matching",
|
|
"next_improvement_candidate": "rules_or_language_hunter",
|
|
"path": "src/vuln.cpp",
|
|
"reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
|
|
"vulnerability_class": "unsafe memory copy",
|
|
}
|
|
]
|
|
assert deltas == [{"extra_findings_total": 1, "matched_expected_total": 1, "missed_expected_total": 0, "tool": "clearwing"}]
|
|
|
|
|
|
def _finding(
|
|
*,
|
|
finding_id: str,
|
|
path: str,
|
|
line: int,
|
|
rationale: str,
|
|
function_name: str | None = None,
|
|
) -> StaticFinding:
|
|
return StaticFinding(
|
|
finding_id=finding_id,
|
|
path=path,
|
|
title="Test finding",
|
|
severity="high",
|
|
confidence="medium",
|
|
evidence_level="static_corroboration",
|
|
rationale=rationale,
|
|
line=line,
|
|
function_name=function_name,
|
|
reachability_status="unknown",
|
|
reachability_evidence=[],
|
|
reachability_conditions=[],
|
|
tags=[],
|
|
ranking_priority=1.0,
|
|
)
|