feat(detection): language-scoped rules + multi-language benchmark corpora

Rebuild the quick-scan detection engine and ground-truth benchmarks to hit
>=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java.

Engine (findings.py, preprocess.py):
- Scope each PatternRule by language (fixes cross-language false positives such
  as the Python eval/exec rule firing on JS .exec()).
- Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS,
  SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv
  excluded as safe alternatives.
- Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject)
  and a Groovy server-side template rule (yieldUnescaped of a bound variable).
- Broaden C/C++ and Python coverage; precision guards for format strings and
  parameterised vs concatenated SQL; #-comment skipping; CWE in rationale.
- Register .java/.groovy/.tpl source extensions.

Benchmarks:
- Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and
  NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored
  verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries
  complete, independently-labeled ground truth including genuinely
  hard-to-detect vulns (integer overflow, second-order SQLi, prototype
  pollution) so recall is measured honestly, plus safe-API files for precision.
- Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets,
  missing C/C++ strcpy entry, Java SQLi line).

benchmark.py:
- Fix sink matching to normalise the expected sink consistently so sinks with
  underscores (e.g. render_template_string) can match.

Tests:
- Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP
  goal per language and overall.

Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%;
overall 93.6% recall, 0.0% false-positive rate.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-06-21 11:49:31 +02:00
co-authored by Claude Opus 4.8
parent be65aed96e
commit cf9f945992
38 changed files with 2420 additions and 15 deletions
+79
View File
@@ -0,0 +1,79 @@
"""Detection-quality gate for the in-scope languages.
Runs quick-mode scans against the C/C++, Python, and JavaScript benchmark
corpora and asserts the project goal: at least 90% recall of known
vulnerabilities and under a 10% false-positive rate per language.
"""
from pathlib import Path
import pytest
from openultrasast.benchmark import (
BenchmarkRun,
evaluate_benchmark,
load_benchmark_manifest,
resolve_benchmark_source,
)
from openultrasast.findings import quick_scan_findings
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
from openultrasast.preprocess import preprocess_repository
from openultrasast.rank import rank_targets
LANGUAGE_MANIFESTS = {
"c_cpp": ["c-cpp-smoke", "cpp-damn-vulnerable"],
"python": ["python-web-smoke", "python-vulnerable"],
"javascript": ["javascript-node-web-smoke", "javascript-vulnerable"],
"java": ["java-web-smoke", "java-spring-boot-vulnerable"],
}
RECALL_FLOOR = 0.90
FALSE_POSITIVE_CEILING = 0.10
def _quick_scan(target: Path): # type: ignore[no-untyped-def]
_, targets = preprocess_repository(target)
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
rankings = rank_targets(targets)
return quick_scan_findings(target, targets, rankings)
def _language_metrics(manifest_names: list[str]) -> tuple[int, int, int, int]:
expected = matched = actual = false_positives = 0
for name in manifest_names:
manifest_path = Path(f"benchmarks/manifests/{name}.toml")
manifest = load_benchmark_manifest(manifest_path)
target = resolve_benchmark_source(manifest_path, manifest)
findings = _quick_scan(target)
run = BenchmarkRun(benchmark_run_id="gate", root=Path("/tmp/gate"), manifest=manifest)
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
metrics = result.metrics
expected += metrics.expected_findings_total
matched += metrics.matched_findings_total
actual += metrics.actual_findings_total
false_positives += metrics.false_positive_findings_total
return expected, matched, actual, false_positives
@pytest.mark.parametrize("language", sorted(LANGUAGE_MANIFESTS))
def test_language_meets_recall_and_false_positive_goal(language: str) -> None:
expected, matched, actual, false_positives = _language_metrics(LANGUAGE_MANIFESTS[language])
recall = matched / expected
false_positive_rate = false_positives / actual if actual else 0.0
assert recall >= RECALL_FLOOR, f"{language} recall {recall:.2%} below {RECALL_FLOOR:.0%} ({matched}/{expected})"
assert false_positive_rate < FALSE_POSITIVE_CEILING, (
f"{language} false-positive rate {false_positive_rate:.2%} at or above {FALSE_POSITIVE_CEILING:.0%} ({false_positives}/{actual})"
)
def test_overall_detection_goal_is_met() -> None:
totals = [_language_metrics(names) for names in LANGUAGE_MANIFESTS.values()]
expected = sum(t[0] for t in totals)
matched = sum(t[1] for t in totals)
actual = sum(t[2] for t in totals)
false_positives = sum(t[3] for t in totals)
assert matched / expected >= RECALL_FLOOR
assert (false_positives / actual) < FALSE_POSITIVE_CEILING