feat(model-grounded-detection): wire the detector -- the enumerator proposes, the model disposes

Everything before this built an ARBITER: something that judges a flow taint
reachability had already found unaided. Nothing proposed to it. That is why its
recall was never comparable to a detector's, and why comparing 41.3% entailment
against the LLM hunter's 76.7% pair-correct was comparing two different kinds
of thing. model/judge.py and enumerate_candidates were called by no production
code at all; every number so far came from a measurement harness.

The wiring turns on a distinction judge() alone conflated:

  the model CONTRADICTS    it has coverage at this site and says no -- a
                           sanitizer on the path, or a declared-safe sink shape.
                           The claim is dropped (Req 8.2).
  the model CANNOT DECIDE  no modelled sink or source reaches the site at all.
                           The model is SILENT, not negative, so the claim is
                           reported at `suspicion` (Req 8.4).

Treating silence as refutation is the tempting mistake and would be wrong here:
23 of the 39 misses in the entailment ceiling were sinks simply absent from the
fact tables, so "no flow" usually means the model cannot see it rather than
that nothing is there. The rung carries that uncertainty on the finding's face
instead of a confidence score nobody can act on.

An entailed site still costs zero model calls. The candidate budget is bounded
so a dense region cannot become an unbounded spend.

Smoke-tested end to end against a live endpoint: 2 pairs, detection 100%,
$0.0024. The full vibe-py measurement -- the first like-for-like comparison
with the hunter's 76.7% pair-correct and 12.3% leak -- is running.

727 passing, mypy/ruff clean, zero orphaned modules.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

benchmarks/measurements/2026-09-08-module-audit.json

@@ -1,13 +1,13 @@
 {
   "counts": {
-    "load_bearing": 85,
+    "load_bearing": 86,
     "standalone_capability": 7
   },
-  "generated": "2026-09-08 (regenerated after group 6: the deferred execution tier)",
+  "generated": "2026-09-08 (regenerated after the detector pipeline was wired)",
   "kept_with_a_lever_stripped": {
     "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them."
   },
-  "modules": 92,
+  "modules": 93,
   "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.",
   "orphaned": [],
   "removed_in_this_pass": {
@@ -131,6 +131,7 @@
         "model.config_value",
         "model.dominance",
         "model.judge",
+        "model.pipeline",
         "model.taint"
       ],
       "module": "cpg",
@@ -142,6 +143,7 @@
         "model.config_value",
         "model.dominance",
         "model.judge",
+        "model.pipeline",
         "model.taint"
       ],
       "module": "cpg.backend",
@@ -333,6 +335,7 @@
         "model.dominance",
         "model.execution",
         "model.judge",
+        "model.pipeline",
         "model.specs",
         "model.taint"
       ],
@@ -396,11 +399,18 @@
         "model.dominance",
         "model.execution",
         "model.judge",
+        "model.pipeline",
         "model.taint"
       ],
       "module": "model.ladder",
       "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries"
     },
+    {
+      "classification": "load_bearing",
+      "importers": [],
+      "module": "model.pipeline",
+      "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes"
+    },
     {
       "classification": "load_bearing",
       "importers": [],
@@ -413,6 +423,7 @@
         "model.config_value",
         "model.dominance",
         "model.judge",
+        "model.pipeline",
         "model.taint"
       ],
       "module": "model.specs",
@@ -421,7 +432,8 @@
     {
       "classification": "load_bearing",
       "importers": [
-        "model.judge"
+        "model.judge",
+        "model.pipeline"
       ],
       "module": "model.taint",
       "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter"
@@ -556,6 +568,7 @@
       "importers": [
         "harness",
         "model.judge",
+        "model.pipeline",
         "reports",
         "semantic.obligations.intent"
       ],

src/openultrasast/model/audit.py

@@ -39,6 +39,7 @@ SPEC_OWNED = {
     "model.calibrate": "model-grounded-detection Req 9: the corpus as the model's calibration set",
     "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap",
     "model.execution": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built",
+    "model.pipeline": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes",
     "cpg": "model-grounded-detection Req 4: the CPG seam package",
     "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary",
     "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion",

src/openultrasast/model/pipeline.py

@@ -0,0 +1,158 @@
+"""The detector: the enumerator proposes, the LLM answers, the model disposes (Req 8).
+
+Everything before this module built an *arbiter* — something that judges a flow taint reachability had already
+found on its own. Nothing proposed candidates to it, which is why the arbiter's recall was never comparable to
+a detector's: it could only report what it happened to find unaided.
+
+This is the wiring the architecture was designed around, and it turns on a distinction that `judge` alone
+conflated:
+
+    the model CONTRADICTS    it has coverage at this site and says no — a sanitizer lies on the path, or the
+                             sink is written in its declared-safe shape. The claim is dropped (Req 8.2).
+    the model CANNOT DECIDE  no modelled sink or source reaches the site at all. The model is *silent*, not
+                             negative, so the LLM's claim is reported at `suspicion` (Req 8.4).
+
+Treating silence as refutation is the tempting mistake, and it would be wrong here: 23 of the 39 misses in the
+entailment ceiling were sinks simply absent from the fact tables. "No flow" usually means the model cannot see
+it, not that nothing is there. The rung is what carries that uncertainty honestly, rather than a confidence
+score nobody can act on.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from ..cpg.backend import CpgResult
+from ..redaction import redact_secrets
+from .ladder import Rung
+from .specs import TaintSpec
+from .taint import verdict as taint_verdict
+
+logger = logging.getLogger(__name__)
+
+# A region with many candidate sites must not become an unbounded spend. The enumerator already orders them,
+# so this takes the strongest few rather than a random slice.
+MAX_JUDGED_CANDIDATES = 8
+
+CANDIDATE_QUESTION = """You are judging ONE site a static enumerator selected. Do not look for other issues.
+
+A dataflow model examined this {language} code and could NOT resolve a source-to-sink path here — which means
+its sink table may simply not cover this API, not that the site is safe.
+
+  site: {site}
+  code: {text}
+
+Is this site a {family} vulnerability?
+
+Answer ONLY with JSON: {{"vulnerable": true|false, "why": "<one sentence>"}}"""
+
+
+@dataclass(frozen=True)
+class ModelFinding:
+    """One reported site, carrying what established it."""
+
+    site: str
+    family: str
+    rung: Rung
+    witness: str = ""
+    contradiction: str = ""
+
+
+def scan_region(
+    cpg: CpgResult,
+    spec: TaintSpec,
+    *,
+    function: str = "",
+    client: Any | None = None,
+    model: str = "",
+    candidates: Sequence[Mapping[str, object]] = (),
+    parameter_sources: bool = False,
+) -> list[ModelFinding]:
+    """Findings for one region and family, ordered strongest first."""
+    answer = taint_verdict(cpg, spec, function=function, parameter_sources=parameter_sources)
+
+    if answer is not None and answer.rung is Rung.ENTAILED:
+        # The graph decided. No model call, and no candidate needs asking about: this region has its finding.
+        return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.ENTAILED,
+                             witness=answer.witness)]
+
+    if answer is not None:
+        # A real flow through a sanitizer. Whether it suffices is the residual, and the model HAS coverage
+        # here — so a "no" from the judge is a genuine contradiction and the claim is dropped.
+        if client is None:
+            return []
+        said = _ask(client, model, _sanitizer_question(spec, answer.witness, function))
+        if said:
+            return [ModelFinding(site=_site(function, answer.witness), family=spec.family,
+                                 rung=Rung.CORROBORATED, witness=answer.witness)]
+        return []
+
+    # The model resolved nothing. It is SILENT, not negative — so the enumerator's candidates are put to the
+    # judge, and anything it affirms is reported at `suspicion` with that uncertainty on its face.
+    if client is None:
+        return []
+    findings: list[ModelFinding] = []
+    for candidate in list(candidates)[:MAX_JUDGED_CANDIDATES]:
+        question = CANDIDATE_QUESTION.format(
+            language=spec.language,
+            site=str(candidate.get("id", "")),
+            text=str(candidate.get("text", ""))[:400],
+            family=spec.family,
+        )
+        if _ask(client, model, question):
+            findings.append(
+                ModelFinding(
+                    site=str(candidate.get("id", "")),
+                    family=spec.family,
+                    rung=Rung.SUSPICION,
+                    contradiction="the model resolved no flow here; its sink table may not cover this API",
+                )
+            )
+    return findings
+
+
+def _sanitizer_question(spec: TaintSpec, witness: str, function: str) -> str:
+    source, _, sink = witness.partition(" -> ")
+    return (
+        f"You are judging ONE candidate a static model already found. Do not look for other issues.\n\n"
+        f"A dataflow analysis resolved this path in {spec.language} code:\n\n"
+        f"  source: {source.strip()}\n  sink:   {sink.strip()}\n  function: {function or '?'}\n"
+        f"  a sanitizer appears on the path: yes\n\n"
+        f"The graph has established that the value reaches the sink. The only open question is whether the\n"
+        f"sanitizer on that path actually makes it safe for the {spec.family} weakness class.\n\n"
+        'Answer ONLY with JSON: {"vulnerable": true|false, "why": "<one sentence>"}'
+    )
+
+
+def _ask(client: Any, model: str, question: str) -> bool | None:
+    """One bounded question, no tools, no retry for agreement."""
+    try:
+        response = client.complete(
+            model=model,
+            messages=[{"role": "user", "content": redact_secrets(question)}],
+            tools=[],
+            json_object=True,
+        )
+    except Exception as exc:  # noqa: BLE001 — an endpoint failure is a degradation, never a verdict
+        logger.warning("judge call failed: %s", exc)
+        return None
+    content = getattr(response, "content", None)
+    if not isinstance(content, str) or not content.strip():
+        return None
+    try:
+        payload = json.loads(content)
+    except json.JSONDecodeError:
+        return None
+    return bool(payload["vulnerable"]) if isinstance(payload, Mapping) and isinstance(payload.get("vulnerable"), bool) else None
+
+
+def _site(function: str, witness: str) -> str:
+    line = witness.partition("(line ")[2].split(",")[0].strip(") ") if "(line " in witness else "?"
+    return f"{function or '?'}:{line}"
+
+
+__all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "scan_region"]

tests/test_model_pipeline.py

@@ -0,0 +1,121 @@
+"""Wiring the model layer into a detector (model-grounded-detection, Req 8).
+
+Everything before this built an *arbiter* — something that judges a flow taint reachability already found.
+Nothing proposed. This is the piece that makes it a detector: the enumerator offers candidates, the LLM
+answers one bounded question about each, and the model checks the answer.
+
+The distinction that matters, and that `judge` alone conflated:
+
+    the model CONTRADICTS   it has coverage here and says no -- a sanitizer on the path, a safe sink shape.
+                            The claim is dropped (Req 8.2).
+    the model CANNOT DECIDE no modelled sink or source reaches this site at all, so the model is silent
+                            rather than negative. The claim is reported at `suspicion` (Req 8.4).
+
+Conflating them is the difference between a precision-first tool that misses most real bugs and an honest one
+that says how sure it is. 23 of 39 misses in the ceiling were absent sinks -- "no flow" usually means the
+model cannot see it, not that nothing is there.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def _spec():  # type: ignore[no-untyped-def]
+    from openultrasast.model.specs import TaintSpec
+
+    return TaintSpec(family="injection", language="python", sources=("request.args",),
+                     sinks=("os.system",), sanitizers=("escape",))
+
+
+def _cpg(rows):  # type: ignore[no-untyped-def]
+    from openultrasast.cpg.backend import CpgResult
+
+    return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: rows)
+
+
+class _Client:
+    def __init__(self, answer: str = '{"vulnerable": true, "why": "unsanitised"}') -> None:
+        self.calls = 0
+        self._answer = answer
+
+    def complete(self, **kw: object):  # type: ignore[no-untyped-def]
+        from openultrasast.tool_hunter import ChatResponse
+
+        self.calls += 1
+        return ChatResponse(content=self._answer)
+
+
+ENTAILING = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']",
+              "sanitized": False, "length": 3, "inLabeledScope": True}]
+SANITIZED = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']",
+              "sanitized": True, "length": 3, "inLabeledScope": True}]
+
+
+def test_an_entailed_site_becomes_a_finding_with_no_model_call() -> None:
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.pipeline import scan_region
+
+    client = _Client()
+    findings = scan_region(_cpg(ENTAILING), _spec(), function="run", client=client, model="m", candidates=())
+    assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED
+    assert client.calls == 0
+
+
+def test_a_sanitized_flow_the_judge_confirms_becomes_corroborated() -> None:
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.pipeline import scan_region
+
+    findings = scan_region(_cpg(SANITIZED), _spec(), function="run", client=_Client(), model="m", candidates=())
+    assert len(findings) == 1 and findings[0].rung is Rung.CORROBORATED
+
+
+def test_a_sanitized_flow_the_judge_rejects_yields_nothing() -> None:
+    """The model found a flow AND has coverage; the judge says the sanitizer suffices. That is a contradiction."""
+    from openultrasast.model.pipeline import scan_region
+
+    client = _Client('{"vulnerable": false, "why": "escaped"}')
+    assert scan_region(_cpg(SANITIZED), _spec(), function="run", client=client, model="m", candidates=()) == []
+
+
+def test_a_candidate_the_model_cannot_see_is_reported_at_suspicion() -> None:
+    """Req 8.4, and the reason recall exists at all: no modelled sink reaches this site, so the model is
+    SILENT, not negative. The LLM's claim is reported, honestly labelled."""
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.pipeline import scan_region
+
+    client = _Client()
+    findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m",
+                           candidates=({"id": "app.py:9:raw_query", "text": "db.raw(q)", "line": 9},))
+    assert client.calls == 1, "one bounded question per candidate"
+    assert len(findings) == 1 and findings[0].rung is Rung.SUSPICION
+    assert findings[0].site == "app.py:9:raw_query"
+
+
+def test_a_candidate_the_judge_declines_is_not_reported() -> None:
+    from openultrasast.model.pipeline import scan_region
+
+    client = _Client('{"vulnerable": false, "why": "constant"}')
+    findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m",
+                           candidates=({"id": "app.py:9:x", "text": "db.raw('literal')", "line": 9},))
+    assert findings == []
+
+
+def test_without_a_client_only_what_the_model_establishes_is_reported() -> None:
+    """Req 11.3: the endpoint is optional. Entailment still works; the suspicion band simply goes unasked."""
+    from openultrasast.model.ladder import Rung
+    from openultrasast.model.pipeline import scan_region
+
+    assert scan_region(_cpg(ENTAILING), _spec(), function="run", client=None, model="", candidates=())[0].rung is Rung.ENTAILED
+    assert scan_region(_cpg([]), _spec(), function="run", client=None, model="",
+                       candidates=({"id": "a:1:x", "text": "t", "line": 1},)) == []
+
+
+def test_the_candidate_budget_is_bounded() -> None:
+    """A region with many sites must not turn into an unbounded spend."""
+    from openultrasast.model.pipeline import MAX_JUDGED_CANDIDATES, scan_region
+
+    client = _Client()
+    many = tuple({"id": f"a:{i}:x", "text": "db.raw(q)", "line": i} for i in range(50))
+    scan_region(_cpg([]), _spec(), function="run", client=client, model="m", candidates=many)
+    assert client.calls == MAX_JUDGED_CANDIDATES