feat(tree-sitter-overlay-extra): parse JS/C/Java overlay via optional extra

Add openultrasast[semantic] with official grammar wheels and one CST walker
into FileIR. quick stays extra-free. Host CLI is not overlay-ready. Default CI
stays extra-free; a semantic job runs pytest -m semantic.

tests/test_semantic_cst.py

@@ -0,0 +1,70 @@
+"""Extra-on CST walker and overlay. Skips when wheels are absent."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openultrasast.findings import quick_scan_findings
+from openultrasast.preprocess import preprocess_repository
+from openultrasast.rank import rank_targets
+from openultrasast.semantic import adjudicate, has_semantic_extra
+from openultrasast.semantic.ir import parse_file
+
+pytestmark = pytest.mark.semantic
+
+
+def _require_extra() -> None:
+    if not has_semantic_extra():
+        pytest.skip("requires openultrasast[semantic]")
+
+
+def test_javascript_eval_query_is_tree_sitter_call() -> None:
+    _require_extra()
+    ir = parse_file("app.js", "eval(req.query.x)\n", "javascript")
+    assert ir.parse_ok is True
+    assert ir.engine == "tree-sitter"
+    calls = [call for function in ir.functions for call in function.calls]
+    eval_calls = [call for call in calls if call.name == "eval"]
+    assert eval_calls
+    assert any("req.query" in arg for arg in eval_calls[0].arg_texts)
+
+
+def test_broken_javascript_is_parse_failed() -> None:
+    _require_extra()
+    ir = parse_file("app.js", "eval(req.query.x\n", "javascript")
+    assert ir.parse_ok is False
+    assert ir.reason == "parse_failed"
+
+
+def test_javascript_eval_promotes(tmp_path: Path) -> None:
+    _require_extra()
+    root = tmp_path / "repo"
+    root.mkdir()
+    (root / "app.js").write_text("eval(req.query.x)\n")
+    _, targets = preprocess_repository(root)
+    findings = quick_scan_findings(root, targets, rank_targets(targets))
+    records = adjudicate(root=root, targets=targets, findings=findings)
+    eval_records = [record for record in records if "js-eval" in record.proposal_id]
+    assert eval_records
+    assert eval_records[0].disposition == "promote"
+    assert eval_records[0].evidence_level == "static_corroboration"
+    assert eval_records[0].engine == "tree-sitter"
+
+
+def test_c_constant_system_demotes(tmp_path: Path) -> None:
+    _require_extra()
+    root = tmp_path / "repo"
+    root.mkdir()
+    (root / "run.c").write_text('void f(void) { system("ls"); }\n')
+    _, targets = preprocess_repository(root)
+    findings = quick_scan_findings(root, targets, rank_targets(targets))
+    records = adjudicate(root=root, targets=targets, findings=findings)
+    hits = [record for record in records if record.path == "run.c"]
+    assert hits
+    assert all(record.disposition != "unadjudicated" or record.reason != "language_unsupported" for record in hits)
+    demoted = [record for record in hits if record.disposition == "demote"]
+    assert demoted
+    assert demoted[0].dominating_fact == "constant"
+    assert demoted[0].evidence_level == "static_corroboration"

tests/test_semantic_extra.py

@@ -0,0 +1,71 @@
+"""Semantic extra packaging and extra-free overlay parse."""
+
+from __future__ import annotations
+
+import json
+import tomllib
+from pathlib import Path
+
+import pytest
+
+from openultrasast.cli import main
+from openultrasast.semantic.extra import grammar_for, has_semantic_extra
+from openultrasast.semantic.ir import parse_file
+
+
+def test_core_dependencies_stay_empty_and_semantic_is_optional() -> None:
+    payload = tomllib.loads(Path("pyproject.toml").read_text())
+    assert payload["project"]["dependencies"] == []
+    extra = payload["project"]["optional-dependencies"]["semantic"]
+    joined = " ".join(extra)
+    assert "tree-sitter" in joined
+    assert "tree-sitter-javascript" in joined
+    assert "tree-sitter-c" in joined
+    assert "tree-sitter-java" in joined
+    assert "semantic:" in " ".join(payload["tool"]["pytest"]["ini_options"]["markers"])
+
+
+def test_cli_import_does_not_load_tree_sitter(assert_cold_of_harnessx) -> None:  # type: ignore[no-untyped-def]
+    proc_code = (
+        "import sys\n"
+        "import openultrasast.cli\n"
+        "leaked = sorted(m for m in sys.modules if m == 'tree_sitter' or m.startswith('tree_sitter_'))\n"
+        "assert not leaked, leaked\n"
+    )
+    assert_cold_of_harnessx(proc_code)
+
+
+def test_probe_off_yields_no_javascript_grammar(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
+    assert has_semantic_extra() is False
+    assert grammar_for("javascript") is None
+    ir = parse_file("app.js", "eval(req.query.x)\n", "javascript")
+    assert ir.parse_ok is False
+    assert ir.reason == "language_unsupported"
+
+
+def test_standard_javascript_is_unadjudicated_without_extra(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.js").write_text("eval(req.query.x)\n")
+    assert main(["scan", str(repo), "--mode", "standard"]) == 0
+    run_dir = sorted((repo / ".runs").iterdir())[-1]
+    overlay = json.loads((run_dir / "overlay.json").read_text())["records"]
+    assert overlay
+    assert all(item["disposition"] == "unadjudicated" for item in overlay)
+    assert all(item["reason"] in {"language_unsupported", "parse_failed"} for item in overlay)
+
+
+def test_quick_scan_without_extra_still_inventories(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def run():\n    return eval(request.args)\n")
+    assert main(["scan", str(repo), "--mode", "quick"]) == 0
+    run_dir = sorted((repo / ".runs").iterdir())[-1]
+    findings = json.loads((run_dir / "findings.json").read_text())["findings"]
+    assert findings
+    assert not (run_dir / "overlay.json").exists()

tests/test_semantic_overlay.py

@@ -13,7 +13,7 @@ from openultrasast.rank import rank_targets
 from openultrasast.semantic import OverlayError, OverlayRecord, adjudicate, load_facts
 from openultrasast.semantic.engines import joern_available, tree_sitter_available
 from openultrasast.semantic.facts import FactLoadError
-from openultrasast.semantic.ir import FileIR, FunctionIR, parse_file
+from openultrasast.semantic.ir import parse_file
 from openultrasast.semantic.taint import TaintPath, analyze_file
 
 
@@ -186,29 +186,13 @@ def test_overlay_length_matches_proposals_plus_coverage_only(tmp_path: Path) ->
     assert all(record.evidence_level == "static_corroboration" for record in records if record.disposition == "coverage")
 
 
-def test_js_language_unsupported_without_tree_sitter(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr("openultrasast.semantic.ir.tree_sitter_available", lambda: False)
-    monkeypatch.setattr("openultrasast.semantic.engines.tree_sitter_available", lambda: False)
+def test_js_language_unsupported_without_semantic_extra(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0")
     ir = parse_file("app.js", "eval(req.query.x)\n", "javascript")
     assert ir.parse_ok is False
     assert ir.reason == "language_unsupported"
 
 
-def test_js_yields_ir_when_tree_sitter_available(monkeypatch: pytest.MonkeyPatch) -> None:
-    fake = FileIR(
-        path="app.js",
-        language="javascript",
-        engine="tree-sitter",
-        functions=(FunctionIR(name="run", params=("x",), start_line=1, end_line=1, binds=(), calls=()),),
-        parse_ok=True,
-    )
-    monkeypatch.setattr("openultrasast.semantic.ir.tree_sitter_available", lambda: True)
-    monkeypatch.setattr("openultrasast.semantic.ir.parse_tree_sitter_cli", lambda *args, **kwargs: fake)
-    ir = parse_file("app.js", "eval(req.query.x)\n", "javascript")
-    assert ir.parse_ok is True
-    assert ir.engine == "tree-sitter"
-
-
 def test_joern_absence_does_not_block_overlay(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr("openultrasast.semantic.overlay.joern_available", lambda: False)
     root = tmp_path / "repo"