feat(model-grounded-detection): tasks 2.1-2.2, the Joern seam and the taint verdict

The whole engine integration is one subprocess boundary: joern-parse builds a
cpg.bin, joern --script runs a shipped CPGQL query that prints its JSON between
fence markers, because Joern's stdout carries a banner and a REPL prompt around
whatever a script emits. Nothing is imported in-process, so the core install
keeps dependencies = [] -- a test walks the seam's AST to assert it imports
only the standard library and names no JVM binding.

Fail closed everywhere. A build or query that errors, times out, or prints
something unparseable yields no verdict and the caller reports suspicion. The
taint verdict distinguishes this from absence deliberately: a query returning
None means "the engine could not decide", which is not "there is no flow", and
conflating them is how a missing engine silently becomes a clean bill of health.

The verdict is where the ladder earns its keep. An unsanitized source->sink
path is model_entailed -- the graph establishes it and the LLM's claim is no
longer load-bearing. A path a sanitizer may break is model_corroborated: the
flow is real, but whether the sanitizer suffices is the semantic residual the
LLM answers, checked against this verdict rather than averaged over runs. The
chosen flow is picked by a total order (shortest, then lexicographic), so two
runs over one CPG cannot disagree -- Req 6.4 asserted directly.

The query itself is the part our flat IR could not do: reachableByFlows walks
the PDG, so `q = "..." + request.args["x"]; execute(q)` is found, which is
exactly the one-hop limit that held measured entailment at 2.2%.

Capability detection reuses semantic/engines.joern_available rather than
growing a second notion of "installed". The audit manifest is regenerated; the
new modules are named by Req 4/5/6 rather than counted as orphans.

Task 2.3, the go/no-go measurement, needs Joern installed and has not run.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_cpg_backend.py

@@ -0,0 +1,94 @@
+"""model-grounded-detection task 2.1: the Joern seam, out of process and capability-detected.
+
+Joern is a JVM tool. The whole integration is one subprocess boundary so that the core install stays
+zero-dependency and an absent engine degrades to `suspicion` with a recorded reason rather than a traceback.
+Nothing here imports a JVM binding, and every test runs with Joern absent.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+
+def test_without_joern_the_resolved_backend_is_the_null_one(monkeypatch: pytest.MonkeyPatch) -> None:
+    from openultrasast.cpg.backend import NullBackend, resolve_cpg_backend
+
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "off")
+    backend = resolve_cpg_backend()
+    assert isinstance(backend, NullBackend)
+    assert backend.available() is False
+    assert backend.build(Path(".")) is None, "an absent engine yields no CPG, and the caller degrades"
+
+
+def test_the_capability_probe_reuses_the_existing_joern_probe(monkeypatch: pytest.MonkeyPatch) -> None:
+    """`semantic/engines.joern_available` already exists; the model layer must not grow a second probe."""
+    from openultrasast.cpg.capability import has_cpg
+
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "off")
+    assert has_cpg() is False
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
+    assert has_cpg() is True
+
+
+def test_the_seam_imports_nothing_outside_the_standard_library() -> None:
+    """Req 4.2: the core install has no dependencies, so the seam may not import one at module load."""
+    import ast
+
+    source = Path("src/openultrasast/cpg/backend.py").read_text()
+    imported = set()
+    for node in ast.walk(ast.parse(source)):
+        if isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
+            imported.add(node.module.split(".")[0])
+        elif isinstance(node, ast.Import):
+            imported.update(alias.name.split(".")[0] for alias in node.names)
+    stdlib = {"__future__", "json", "os", "re", "shutil", "subprocess", "tempfile", "dataclasses", "pathlib", "typing",
+              "collections", "logging", "hashlib"}
+    assert imported <= stdlib, f"the seam imports outside the standard library: {sorted(imported - stdlib)}"
+    assert "import jpype" not in source and "py4j" not in source, "Joern is reached by subprocess, never in-process"
+
+
+def test_a_scripted_backend_parses_a_query_result_without_joern(tmp_path: Path) -> None:
+    """The parsing is exercised by canned JSON, so the query contract is testable with no engine installed."""
+    from openultrasast.cpg.backend import CpgResult
+
+    payload = [{"sink": "os.system", "source": "request.args", "path": ["a", "b"], "sanitized": False}]
+
+    def run(query: str, params: dict[str, object]) -> object:
+        assert query == "taint"
+        return payload
+
+    result = CpgResult(cpg_path=tmp_path / "cpg.bin", run=run)
+    assert result.run("taint", {}) == payload
+
+
+def test_joern_output_is_read_from_a_delimited_block_not_from_noisy_stdout() -> None:
+    """Joern prints a banner and a REPL prompt; the payload is fenced so the parse cannot swallow either."""
+    from openultrasast.cpg.backend import BEGIN, END, extract_payload
+
+    noisy = f"Compiling...\nJoern v4.0\n{BEGIN}\n{json.dumps([{'sink': 'eval'}])}\n{END}\njoern> \n"
+    assert extract_payload(noisy) == [{"sink": "eval"}]
+    assert extract_payload("no markers at all") is None
+    assert extract_payload(f"{BEGIN}\nnot json\n{END}") is None
+
+
+def test_a_failing_or_slow_engine_fails_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Fail closed: a build or query that errors yields no verdict, never a wrong one."""
+    import subprocess
+
+    from openultrasast.cpg.backend import JoernBackend
+
+    def boom(*args: object, **kwargs: object) -> object:
+        raise subprocess.TimeoutExpired(cmd="joern-parse", timeout=1)
+
+    backend = JoernBackend(runner=boom)
+    monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
+    assert backend.build(tmp_path) is None
+
+
+def test_the_query_scripts_are_shipped_beside_the_seam() -> None:
+    """The CPGQL is ours and is data: it lives with the package, not inlined in a Python string."""
+    queries = Path("src/openultrasast/cpg/queries")
+    assert (queries / "taint.sc").is_file(), "the injection taint query must ship with the package"

tests/test_model_taint.py

@@ -0,0 +1,113 @@
+"""model-grounded-detection task 2.2: the taint verdict, and what separates entailed from corroborated.
+
+An `ENTAILED` verdict is the static analogue of a reproduced crash: the CPG establishes the flow on its own and
+the LLM's assertion is no longer load-bearing. `CORROBORATED` means a path exists but a sanitizer may break it,
+so the judgement of sufficiency is the LLM's residual — checked against the graph, never averaged over runs.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def _cpg(rows: list[dict[str, object]]):  # type: ignore[no-untyped-def]
+    from openultrasast.cpg.backend import CpgResult
+
+    return CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: rows)
+
+
+def _spec():  # type: ignore[no-untyped-def]
+    from openultrasast.model.specs import TaintSpec
+
+    return TaintSpec(
+        family="injection",
+        language="python",
+        sources=("request.args",),
+        sinks=("os.system", "execute"),
+        sanitizers=("escape", "quote"),
+    )
+
+
+def test_an_unsanitized_source_to_sink_path_is_entailed() -> None:
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.taint import verdict
+
+    rows = [
+        {"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['x']", "sanitized": False, "length": 3}
+    ]
+    answer = verdict(_cpg(rows), _spec(), function="run")
+    assert answer is not None and answer.rung is Rung.ENTAILED
+    assert answer.family == "injection"
+    assert "request.args" in answer.witness and "os.system" in answer.witness, "the witness names the flow it found"
+
+
+def test_a_path_through_a_sanitizer_is_corroborated_not_entailed() -> None:
+    """The model saw the flow but cannot judge whether the sanitizer is sufficient. That residual is the LLM's."""
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.taint import verdict
+
+    rows = [{"sink": "execute(q)", "sinkLine": "9", "sinkMethod": "run", "source": "request.args['q']", "sanitized": True, "length": 4}]
+    answer = verdict(_cpg(rows), _spec(), function="run")
+    assert answer is not None and answer.rung is Rung.CORROBORATED
+
+
+def test_no_path_yields_no_verdict_so_the_caller_reports_suspicion() -> None:
+    from openultrasast.model.taint import verdict
+
+    assert verdict(_cpg([]), _spec(), function="run") is None
+
+
+def test_a_query_that_failed_is_not_read_as_an_absence_of_flow() -> None:
+    """`None` from the engine means "could not decide", which is not the same as "no path" — Req 4.3."""
+    from openultrasast.cpg.backend import CpgResult
+    from openultrasast.model.taint import verdict
+
+    broken = CpgResult(cpg_path=Path("cpg.bin"), run=lambda query, params: None)
+    assert verdict(broken, _spec(), function="run") is None
+
+
+def test_the_verdict_is_restricted_to_the_labeled_function() -> None:
+    from openultrasast.model.taint import verdict
+
+    rows = [
+        {
+            "sink": "os.system(x)", "sinkLine": "40", "sinkMethod": "elsewhere",
+            "source": "request.args['x']", "sanitized": False, "length": 2,
+        }
+    ]
+    assert verdict(_cpg(rows), _spec(), function="run") is None, "a flow in another function does not judge this one"
+
+
+def test_the_shortest_unsanitized_flow_wins_and_the_sequence_is_deterministic() -> None:
+    """Req 6.4: two runs over one CPG give an identical verdict sequence, so there is nothing to average."""
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.taint import verdict
+
+    rows = [
+        {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['b']", "sanitized": True, "length": 9},
+        {"sink": "os.system(a)", "sinkLine": "7", "sinkMethod": "run", "source": "request.args['a']", "sanitized": False, "length": 2},
+    ]
+    first = verdict(_cpg(rows), _spec(), function="run")
+    second = verdict(_cpg(rows), _spec(), function="run")
+    assert first == second
+    assert first is not None and first.rung is Rung.ENTAILED and "request.args['a']" in first.witness
+
+
+def test_the_spec_drives_the_query_parameters() -> None:
+    """The security modelling is ours; the traversal is the engine's. The spec is what crosses the seam."""
+    from openultrasast.cpg.backend import CpgResult
+    from openultrasast.model.taint import verdict
+
+    seen: dict[str, object] = {}
+
+    def run(query: str, params: dict[str, object]) -> object:
+        seen["query"] = query
+        seen.update(params)
+        return []
+
+    verdict(CpgResult(cpg_path=Path("cpg.bin"), run=run), _spec(), function="run")
+    assert seen["query"] == "taint"
+    assert seen["sources"] == ("request.args",)
+    assert seen["sinks"] == ("os.system", "execute")
+    assert seen["sanitizers"] == ("escape", "quote")
+    assert seen["function"] == "run"