feat(detection): language-scoped rules + multi-language benchmark corpora

Rebuild the quick-scan detection engine and ground-truth benchmarks to hit
>=90% recall / <10% false positives for C/C++, Python, JavaScript, and Java.

Engine (findings.py, preprocess.py):
- Scope each PatternRule by language (fixes cross-language false positives such
  as the Python eval/exec rule firing on JS .exec()).
- Add JavaScript/TypeScript rules (command exec, eval/Function, reflected XSS,
  SQLi, path traversal, SSRF, weak hash, deserialization); execFile/spawn-argv
  excluded as safe alternatives.
- Add Java rules (Runtime exec, SQL string concatenation, weak hash, readObject)
  and a Groovy server-side template rule (yieldUnescaped of a bound variable).
- Broaden C/C++ and Python coverage; precision guards for format strings and
  parameterised vs concatenated SQL; #-comment skipping; CWE in rationale.
- Register .java/.groovy/.tpl source extensions.

Benchmarks:
- Add realistic corpora modeled on Damn Vulnerable C/C++, PyGoat/DVPWA, and
  NodeGoat, plus the real kiview/damn-vulnerable-spring-boot-app vendored
  verbatim (MIT, attribution in LICENSE/NOTICE.md). Each manifest carries
  complete, independently-labeled ground truth including genuinely
  hard-to-detect vulns (integer overflow, second-order SQLi, prototype
  pollution) so recall is measured honestly, plus safe-API files for precision.
- Fix pre-existing smoke-manifest ground-truth errors (Python/JS line offsets,
  missing C/C++ strcpy entry, Java SQLi line).

benchmark.py:
- Fix sink matching to normalise the expected sink consistently so sinks with
  underscores (e.g. render_template_string) can match.

Tests:
- Add tests/test_detection_benchmarks.py enforcing the >=90% recall / <10% FP
  goal per language and overall.

Measured: C/C++ 93.3%, Python 93.3%, JavaScript 92.3%, Java 100%;
overall 93.6% recall, 0.0% false-positive rate.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

src/openultrasast/findings.py

@@ -34,29 +34,92 @@ class PatternRule:
     severity: str
     tags: tuple[str, ...]
     pattern: re.Pattern[str]
+    cwe: str
+    languages: tuple[str, ...]
 
 
+C_LANGUAGES = ("c", "cpp")
+PYTHON_LANGUAGES = ("python",)
+JS_LANGUAGES = ("javascript", "typescript")
+JAVA_LANGUAGES = ("java",)
+GROOVY_LANGUAGES = ("groovy",)
+
 PATTERN_RULES = (
+    # --- C / C++ -----------------------------------------------------------------
     PatternRule(
         rule_id="c-unsafe-copy",
-        title="Unsafe C string or input function needs review",
+        title="Unsafe C string copy needs review",
         severity="high",
         tags=("memory_unsafe",),
         pattern=re.compile(r"\b(gets|strcpy|strcat|sprintf)\s*\("),
+        cwe="CWE-120",
+        languages=C_LANGUAGES,
     ),
+    PatternRule(
+        rule_id="c-unsafe-memory-copy",
+        title="Unsafe C memory copy needs review",
+        severity="high",
+        tags=("memory_unsafe",),
+        pattern=re.compile(r"\b(memcpy|memmove|bcopy)\s*\("),
+        cwe="CWE-120",
+        languages=C_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="c-unsafe-format",
+        title="Unsafe C format string needs review",
+        severity="high",
+        tags=("memory_unsafe",),
+        # Format function whose format argument is a bare identifier (not a string
+        # literal), i.e. the classic format-string sink. printf("...") is ignored.
+        pattern=re.compile(r"\b(printf|vprintf|vsprintf|vsnprintf|vfprintf|syslog)\s*\(\s*[A-Za-z_]"),
+        cwe="CWE-134",
+        languages=C_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="c-unsafe-scanf",
+        title="Unbounded C scanf needs review",
+        severity="high",
+        tags=("memory_unsafe",),
+        pattern=re.compile(r"\b(scanf|fscanf|sscanf)\s*\([^\n]*%s"),
+        cwe="CWE-120",
+        languages=C_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="c-shell-exec",
+        title="C shell execution needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(system|popen)\s*\("),
+        cwe="CWE-78",
+        languages=C_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="c-exec-family",
+        title="C exec command execution needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(execl|execlp|execle|execv|execvp|execvpe)\s*\("),
+        cwe="CWE-78",
+        languages=C_LANGUAGES,
+    ),
+    # --- Python ------------------------------------------------------------------
     PatternRule(
         rule_id="python-unsafe-eval",
         title="Dynamic Python execution needs review",
         severity="high",
         tags=("syscall_entry",),
         pattern=re.compile(r"\b(eval|exec)\s*\("),
+        cwe="CWE-95",
+        languages=PYTHON_LANGUAGES,
     ),
     PatternRule(
-        rule_id="python-unsafe-deserialization",
-        title="Unsafe Python deserialization needs review",
+        rule_id="python-os-command",
+        title="Python OS command execution needs review",
         severity="high",
-        tags=("deserialization",),
-        pattern=re.compile(r"\b(pickle\.loads|yaml\.load)\s*\("),
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(os\.system|os\.popen|os\.execl|os\.execlp|os\.execv|os\.execvp)\s*\("),
+        cwe="CWE-78",
+        languages=PYTHON_LANGUAGES,
     ),
     PatternRule(
         rule_id="python-shell-true",
@@ -64,6 +127,229 @@ PATTERN_RULES = (
         severity="medium",
         tags=("syscall_entry",),
         pattern=re.compile(r"subprocess\.[a-zA-Z_]+\s*\([^\n]*shell\s*=\s*True"),
+        cwe="CWE-78",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-unsafe-deserialization",
+        title="Unsafe Python deserialization needs review",
+        severity="high",
+        tags=("deserialization",),
+        pattern=re.compile(r"\b(pickle\.loads|pickle\.load|cPickle\.loads|yaml\.load|marshal\.loads)\s*\("),
+        cwe="CWE-502",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-sql-injection",
+        title="Python SQL string composition needs review",
+        severity="high",
+        tags=("injection",),
+        # .execute(...) where the argument is built with %-formatting, .format(),
+        # string concatenation, or an f-string. Parameterised %s placeholders pass.
+        pattern=re.compile(r"(\.execute)\s*\(\s*[^\n]*(?:[\"']\s*%\s*[(A-Za-z_]|\.format\s*\(|[\"']\s*\+|\bf[\"'])"),
+        cwe="CWE-89",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-path-traversal",
+        title="Python file open with external input needs review",
+        severity="medium",
+        tags=("filesystem_entry",),
+        pattern=re.compile(r"\b(open)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\(|params\[|GET\[|POST\[)"),
+        cwe="CWE-22",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-ssrf",
+        title="Python outbound request with external input needs review",
+        severity="medium",
+        tags=("network_entry",),
+        pattern=re.compile(
+            r"\b(requests\.(?:get|post|put|delete|head|patch)|urllib\.request\.urlopen|urlopen)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\()"
+        ),
+        cwe="CWE-918",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-weak-hash",
+        title="Python weak hash needs review",
+        severity="low",
+        tags=("crypto",),
+        pattern=re.compile(r"\bhashlib\.(md5|sha1)\s*\("),
+        cwe="CWE-327",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-ssti",
+        title="Python server-side template injection needs review",
+        severity="high",
+        tags=("injection",),
+        pattern=re.compile(r"\b(render_template_string)\s*\("),
+        cwe="CWE-94",
+        languages=PYTHON_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="python-flask-debug",
+        title="Flask debug mode enabled needs review",
+        severity="low",
+        tags=("insecure_default",),
+        pattern=re.compile(r"\.run\s*\([^\n]*(debug)\s*=\s*True"),
+        cwe="CWE-489",
+        languages=PYTHON_LANGUAGES,
+    ),
+    # --- JavaScript / TypeScript -------------------------------------------------
+    PatternRule(
+        rule_id="js-command-exec",
+        title="Node child_process command execution needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        # exec/execSync invoke a shell; execFile/spawn with an argv array do not and
+        # are the safe alternatives, so they are intentionally excluded here.
+        pattern=re.compile(r"\b(exec|execSync)\s*\("),
+        cwe="CWE-78",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-spawn-shell",
+        title="Node spawn with shell needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(spawn|spawnSync)\s*\([^\n]*shell\s*:\s*true"),
+        cwe="CWE-78",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-eval",
+        title="Dynamic JavaScript evaluation needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(eval)\s*\("),
+        cwe="CWE-95",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-function-constructor",
+        title="JavaScript Function constructor needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"new\s+(Function)\s*\("),
+        cwe="CWE-95",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-xss-response",
+        title="Unsanitized HTTP response reflection needs review",
+        severity="high",
+        tags=("injection",),
+        pattern=re.compile(r"\.(send|write|end)\s*\(\s*[^\n]*\b(?:req|request)\.(?:query|params|body)"),
+        cwe="CWE-79",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-dom-xss",
+        title="DOM sink with untrusted input needs review",
+        severity="high",
+        tags=("injection",),
+        pattern=re.compile(r"\.(innerHTML|outerHTML)\s*=\s*[^\n]*\b(?:req|request|location|document\.location)"),
+        cwe="CWE-79",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-sql-injection",
+        title="JavaScript SQL string composition needs review",
+        severity="high",
+        tags=("injection",),
+        pattern=re.compile(r"\.(query|execute)\s*\(\s*[^\n]*(?:\$\{|\+\s*[A-Za-z_$])"),
+        cwe="CWE-89",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-path-traversal",
+        title="Node filesystem access with external input needs review",
+        severity="medium",
+        tags=("filesystem_entry",),
+        pattern=re.compile(
+            r"\bfs\.(readFile|readFileSync|writeFile|writeFileSync|createReadStream|unlink|unlinkSync)\s*\(\s*[^\n]*\b(?:req|request)\."
+        ),
+        cwe="CWE-22",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-ssrf",
+        title="JavaScript outbound request with external input needs review",
+        severity="medium",
+        tags=("network_entry",),
+        pattern=re.compile(r"\b(axios|fetch|got)\s*[.(][^\n]*\b(?:req|request)\."),
+        cwe="CWE-918",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-weak-hash",
+        title="JavaScript weak hash needs review",
+        severity="low",
+        tags=("crypto",),
+        pattern=re.compile(r"createHash\s*\(\s*[\"'](md5|sha1)[\"']"),
+        cwe="CWE-327",
+        languages=JS_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="js-deserialize",
+        title="JavaScript unsafe deserialization needs review",
+        severity="high",
+        tags=("deserialization",),
+        pattern=re.compile(r"\b(unserialize)\s*\("),
+        cwe="CWE-502",
+        languages=JS_LANGUAGES,
+    ),
+    # --- Java --------------------------------------------------------------------
+    PatternRule(
+        rule_id="java-command-exec",
+        title="Java runtime command execution needs review",
+        severity="high",
+        tags=("syscall_entry",),
+        pattern=re.compile(r"\b(exec)\s*\("),
+        cwe="CWE-78",
+        languages=JAVA_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="java-sql-injection",
+        title="Java SQL string composition needs review",
+        severity="high",
+        tags=("injection",),
+        # A string literal that begins with a SQL keyword and is concatenated with
+        # untrusted data. Parameterised queries (placeholders, no '+') are ignored.
+        pattern=re.compile(r"(?i)\"\s*(select|insert|update|delete)\b[^\"]*\"\s*\+"),
+        cwe="CWE-89",
+        languages=JAVA_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="java-weak-hash",
+        title="Java weak hash needs review",
+        severity="low",
+        tags=("crypto",),
+        pattern=re.compile(r"MessageDigest\.getInstance\s*\(\s*\"(MD5|SHA-1)\""),
+        cwe="CWE-327",
+        languages=JAVA_LANGUAGES,
+    ),
+    PatternRule(
+        rule_id="java-deserialization",
+        title="Java unsafe deserialization needs review",
+        severity="high",
+        tags=("deserialization",),
+        pattern=re.compile(r"\.(readObject)\s*\("),
+        cwe="CWE-502",
+        languages=JAVA_LANGUAGES,
+    ),
+    # --- Groovy server-side templates -------------------------------------------
+    PatternRule(
+        rule_id="groovy-xss-unescaped",
+        title="Groovy template unescaped output needs review",
+        severity="high",
+        tags=("injection",),
+        # Unescaped output of a bound variable; string literals are not flagged.
+        pattern=re.compile(r"(yieldUnescaped)\s+[A-Za-z_]"),
+        cwe="CWE-79",
+        languages=GROOVY_LANGUAGES,
     ),
 )
 
@@ -74,9 +360,13 @@ def quick_scan_findings(root: Path, targets: list[FileTarget], rankings: list[Ra
     for target in targets:
         text = _read_text(root / target.path)
         for rule in PATTERN_RULES:
+            if rule.languages and target.language not in rule.languages:
+                continue
             ranking = ranking_by_path.get(target.path)
             for match in rule.pattern.finditer(text):
-                findings.append(_finding_from_match(target, rule, text, match.start(), ranking))
+                if _match_is_in_comment(text, match.start(), target.language):
+                    continue
+                findings.append(_finding_from_match(target, rule, text, match, ranking))
     return sorted(findings, key=lambda item: (_severity_sort(item.severity), -item.ranking_priority, item.path))
 
 
@@ -101,12 +391,14 @@ def _finding_from_match(
     target: FileTarget,
     rule: PatternRule,
     text: str,
-    offset: int,
+    match: re.Match[str],
     ranking: RankingScore | None,
 ) -> StaticFinding:
+    offset = match.start()
     line = text.count("\n", 0, offset) + 1
     reachability_status, function_name, reachability_evidence = _reachability_for_line(target, line)
     priority = ranking.priority if ranking is not None else 1.0
+    matched_token = match.group(1) if match.groups() else match.group(0).split("(", 1)[0].strip()
     return StaticFinding(
         finding_id=f"{rule.rule_id}:{target.path}:{line}",
         path=target.path,
@@ -114,7 +406,9 @@ def _finding_from_match(
         severity=rule.severity,
         confidence="medium",
         evidence_level="static_corroboration",
-        rationale=f"Static pattern {rule.rule_id} matched line {line}; manual verification still required.",
+        rationale=(
+            f"Static pattern {rule.rule_id} ({rule.cwe}) matched {matched_token} on line {line}; manual verification still required."
+        ),
         line=line,
         function_name=function_name,
         reachability_status=reachability_status,
@@ -179,6 +473,28 @@ def _read_text(path: Path) -> str:
         return ""
 
 
+HASH_COMMENT_LANGUAGES = {"python", "ruby", "shell", "yaml"}
+
+
+def _match_is_in_comment(text: str, offset: int, language: str = "") -> bool:
+    line_start = text.rfind("\n", 0, offset) + 1
+    line_end = text.find("\n", offset)
+    if line_end == -1:
+        line_end = len(text)
+    line = text[line_start:line_end]
+    column = offset - line_start
+    stripped = line.lstrip()
+    if language in HASH_COMMENT_LANGUAGES:
+        if stripped.startswith("#"):
+            return True
+        comment_column = line.find("#")
+        return comment_column != -1 and comment_column < column
+    if stripped.startswith(("//", "/*", "*")):
+        return True
+    comment_column = line.find("//")
+    return comment_column != -1 and comment_column < column
+
+
 def _severity_sort(severity: str) -> int:
     order = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}
     return order.get(severity, 5)