diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index e17025f..f9a24f2 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,13 +1,13 @@ { "counts": { - "load_bearing": 85, + "load_bearing": 86, "standalone_capability": 7 }, - "generated": "2026-09-08 (regenerated after group 6: the deferred execution tier)", + "generated": "2026-09-08 (regenerated after the detector pipeline was wired)", "kept_with_a_lever_stripped": { "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." }, - "modules": 92, + "modules": 93, "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], "removed_in_this_pass": { @@ -131,6 +131,7 @@ "model.config_value", "model.dominance", "model.judge", + "model.pipeline", "model.taint" ], "module": "cpg", @@ -142,6 +143,7 @@ "model.config_value", "model.dominance", "model.judge", + "model.pipeline", "model.taint" ], "module": "cpg.backend", @@ -333,6 +335,7 @@ "model.dominance", "model.execution", "model.judge", + "model.pipeline", "model.specs", "model.taint" ], @@ -396,11 +399,18 @@ "model.dominance", "model.execution", "model.judge", + "model.pipeline", "model.taint" ], "module": "model.ladder", "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries" }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.pipeline", + "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes" + }, { "classification": "load_bearing", "importers": [], @@ -413,6 +423,7 @@ "model.config_value", "model.dominance", "model.judge", + "model.pipeline", "model.taint" ], "module": "model.specs", @@ -421,7 +432,8 @@ { "classification": "load_bearing", "importers": [ - "model.judge" + "model.judge", + "model.pipeline" ], "module": "model.taint", "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter" @@ -556,6 +568,7 @@ "importers": [ "harness", "model.judge", + "model.pipeline", "reports", "semantic.obligations.intent" ], diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py index 2a746fc..06c92b5 100644 --- a/src/openultrasast/model/audit.py +++ b/src/openultrasast/model/audit.py @@ -39,6 +39,7 @@ SPEC_OWNED = { "model.calibrate": "model-grounded-detection Req 9: the corpus as the model's calibration set", "model.report": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", "model.execution": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built", + "model.pipeline": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes", "cpg": "model-grounded-detection Req 4: the CPG seam package", "cpg.backend": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "cpg.capability": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", diff --git a/src/openultrasast/model/pipeline.py b/src/openultrasast/model/pipeline.py new file mode 100644 index 0000000..993c29e --- /dev/null +++ b/src/openultrasast/model/pipeline.py @@ -0,0 +1,158 @@ +"""The detector: the enumerator proposes, the LLM answers, the model disposes (Req 8). + +Everything before this module built an *arbiter* — something that judges a flow taint reachability had already +found on its own. Nothing proposed candidates to it, which is why the arbiter's recall was never comparable to +a detector's: it could only report what it happened to find unaided. + +This is the wiring the architecture was designed around, and it turns on a distinction that `judge` alone +conflated: + + the model CONTRADICTS it has coverage at this site and says no — a sanitizer lies on the path, or the + sink is written in its declared-safe shape. The claim is dropped (Req 8.2). + the model CANNOT DECIDE no modelled sink or source reaches the site at all. The model is *silent*, not + negative, so the LLM's claim is reported at `suspicion` (Req 8.4). + +Treating silence as refutation is the tempting mistake, and it would be wrong here: 23 of the 39 misses in the +entailment ceiling were sinks simply absent from the fact tables. "No flow" usually means the model cannot see +it, not that nothing is there. The rung is what carries that uncertainty honestly, rather than a confidence +score nobody can act on. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from ..cpg.backend import CpgResult +from ..redaction import redact_secrets +from .ladder import Rung +from .specs import TaintSpec +from .taint import verdict as taint_verdict + +logger = logging.getLogger(__name__) + +# A region with many candidate sites must not become an unbounded spend. The enumerator already orders them, +# so this takes the strongest few rather than a random slice. +MAX_JUDGED_CANDIDATES = 8 + +CANDIDATE_QUESTION = """You are judging ONE site a static enumerator selected. Do not look for other issues. + +A dataflow model examined this {language} code and could NOT resolve a source-to-sink path here — which means +its sink table may simply not cover this API, not that the site is safe. + + site: {site} + code: {text} + +Is this site a {family} vulnerability? + +Answer ONLY with JSON: {{"vulnerable": true|false, "why": ""}}""" + + +@dataclass(frozen=True) +class ModelFinding: + """One reported site, carrying what established it.""" + + site: str + family: str + rung: Rung + witness: str = "" + contradiction: str = "" + + +def scan_region( + cpg: CpgResult, + spec: TaintSpec, + *, + function: str = "", + client: Any | None = None, + model: str = "", + candidates: Sequence[Mapping[str, object]] = (), + parameter_sources: bool = False, +) -> list[ModelFinding]: + """Findings for one region and family, ordered strongest first.""" + answer = taint_verdict(cpg, spec, function=function, parameter_sources=parameter_sources) + + if answer is not None and answer.rung is Rung.ENTAILED: + # The graph decided. No model call, and no candidate needs asking about: this region has its finding. + return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.ENTAILED, + witness=answer.witness)] + + if answer is not None: + # A real flow through a sanitizer. Whether it suffices is the residual, and the model HAS coverage + # here — so a "no" from the judge is a genuine contradiction and the claim is dropped. + if client is None: + return [] + said = _ask(client, model, _sanitizer_question(spec, answer.witness, function)) + if said: + return [ModelFinding(site=_site(function, answer.witness), family=spec.family, + rung=Rung.CORROBORATED, witness=answer.witness)] + return [] + + # The model resolved nothing. It is SILENT, not negative — so the enumerator's candidates are put to the + # judge, and anything it affirms is reported at `suspicion` with that uncertainty on its face. + if client is None: + return [] + findings: list[ModelFinding] = [] + for candidate in list(candidates)[:MAX_JUDGED_CANDIDATES]: + question = CANDIDATE_QUESTION.format( + language=spec.language, + site=str(candidate.get("id", "")), + text=str(candidate.get("text", ""))[:400], + family=spec.family, + ) + if _ask(client, model, question): + findings.append( + ModelFinding( + site=str(candidate.get("id", "")), + family=spec.family, + rung=Rung.SUSPICION, + contradiction="the model resolved no flow here; its sink table may not cover this API", + ) + ) + return findings + + +def _sanitizer_question(spec: TaintSpec, witness: str, function: str) -> str: + source, _, sink = witness.partition(" -> ") + return ( + f"You are judging ONE candidate a static model already found. Do not look for other issues.\n\n" + f"A dataflow analysis resolved this path in {spec.language} code:\n\n" + f" source: {source.strip()}\n sink: {sink.strip()}\n function: {function or '?'}\n" + f" a sanitizer appears on the path: yes\n\n" + f"The graph has established that the value reaches the sink. The only open question is whether the\n" + f"sanitizer on that path actually makes it safe for the {spec.family} weakness class.\n\n" + 'Answer ONLY with JSON: {"vulnerable": true|false, "why": ""}' + ) + + +def _ask(client: Any, model: str, question: str) -> bool | None: + """One bounded question, no tools, no retry for agreement.""" + try: + response = client.complete( + model=model, + messages=[{"role": "user", "content": redact_secrets(question)}], + tools=[], + json_object=True, + ) + except Exception as exc: # noqa: BLE001 — an endpoint failure is a degradation, never a verdict + logger.warning("judge call failed: %s", exc) + return None + content = getattr(response, "content", None) + if not isinstance(content, str) or not content.strip(): + return None + try: + payload = json.loads(content) + except json.JSONDecodeError: + return None + return bool(payload["vulnerable"]) if isinstance(payload, Mapping) and isinstance(payload.get("vulnerable"), bool) else None + + +def _site(function: str, witness: str) -> str: + line = witness.partition("(line ")[2].split(",")[0].strip(") ") if "(line " in witness else "?" + return f"{function or '?'}:{line}" + + +__all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "scan_region"] diff --git a/tests/test_model_pipeline.py b/tests/test_model_pipeline.py new file mode 100644 index 0000000..00d4e7b --- /dev/null +++ b/tests/test_model_pipeline.py @@ -0,0 +1,121 @@ +"""Wiring the model layer into a detector (model-grounded-detection, Req 8). + +Everything before this built an *arbiter* — something that judges a flow taint reachability already found. +Nothing proposed. This is the piece that makes it a detector: the enumerator offers candidates, the LLM +answers one bounded question about each, and the model checks the answer. + +The distinction that matters, and that `judge` alone conflated: + + the model CONTRADICTS it has coverage here and says no -- a sanitizer on the path, a safe sink shape. + The claim is dropped (Req 8.2). + the model CANNOT DECIDE no modelled sink or source reaches this site at all, so the model is silent + rather than negative. The claim is reported at `suspicion` (Req 8.4). + +Conflating them is the difference between a precision-first tool that misses most real bugs and an honest one +that says how sure it is. 23 of 39 misses in the ceiling were absent sinks -- "no flow" usually means the +model cannot see it, not that nothing is there. +""" + +from __future__ import annotations + +from pathlib import Path + + +def _spec(): # type: ignore[no-untyped-def] + from openultrasast.model.specs import TaintSpec + + return TaintSpec(family="injection", language="python", sources=("request.args",), + sinks=("os.system",), sanitizers=("escape",)) + + +def _cpg(rows): # type: ignore[no-untyped-def] + from openultrasast.cpg.backend import CpgResult + + return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: rows) + + +class _Client: + def __init__(self, answer: str = '{"vulnerable": true, "why": "unsanitised"}') -> None: + self.calls = 0 + self._answer = answer + + def complete(self, **kw: object): # type: ignore[no-untyped-def] + from openultrasast.tool_hunter import ChatResponse + + self.calls += 1 + return ChatResponse(content=self._answer) + + +ENTAILING = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']", + "sanitized": False, "length": 3, "inLabeledScope": True}] +SANITIZED = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run", "source": "request.args['c']", + "sanitized": True, "length": 3, "inLabeledScope": True}] + + +def test_an_entailed_site_becomes_a_finding_with_no_model_call() -> None: + from openultrasast.model.ladder import Rung + from openultrasast.model.pipeline import scan_region + + client = _Client() + findings = scan_region(_cpg(ENTAILING), _spec(), function="run", client=client, model="m", candidates=()) + assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED + assert client.calls == 0 + + +def test_a_sanitized_flow_the_judge_confirms_becomes_corroborated() -> None: + from openultrasast.model.ladder import Rung + from openultrasast.model.pipeline import scan_region + + findings = scan_region(_cpg(SANITIZED), _spec(), function="run", client=_Client(), model="m", candidates=()) + assert len(findings) == 1 and findings[0].rung is Rung.CORROBORATED + + +def test_a_sanitized_flow_the_judge_rejects_yields_nothing() -> None: + """The model found a flow AND has coverage; the judge says the sanitizer suffices. That is a contradiction.""" + from openultrasast.model.pipeline import scan_region + + client = _Client('{"vulnerable": false, "why": "escaped"}') + assert scan_region(_cpg(SANITIZED), _spec(), function="run", client=client, model="m", candidates=()) == [] + + +def test_a_candidate_the_model_cannot_see_is_reported_at_suspicion() -> None: + """Req 8.4, and the reason recall exists at all: no modelled sink reaches this site, so the model is + SILENT, not negative. The LLM's claim is reported, honestly labelled.""" + from openultrasast.model.ladder import Rung + from openultrasast.model.pipeline import scan_region + + client = _Client() + findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m", + candidates=({"id": "app.py:9:raw_query", "text": "db.raw(q)", "line": 9},)) + assert client.calls == 1, "one bounded question per candidate" + assert len(findings) == 1 and findings[0].rung is Rung.SUSPICION + assert findings[0].site == "app.py:9:raw_query" + + +def test_a_candidate_the_judge_declines_is_not_reported() -> None: + from openultrasast.model.pipeline import scan_region + + client = _Client('{"vulnerable": false, "why": "constant"}') + findings = scan_region(_cpg([]), _spec(), function="run", client=client, model="m", + candidates=({"id": "app.py:9:x", "text": "db.raw('literal')", "line": 9},)) + assert findings == [] + + +def test_without_a_client_only_what_the_model_establishes_is_reported() -> None: + """Req 11.3: the endpoint is optional. Entailment still works; the suspicion band simply goes unasked.""" + from openultrasast.model.ladder import Rung + from openultrasast.model.pipeline import scan_region + + assert scan_region(_cpg(ENTAILING), _spec(), function="run", client=None, model="", candidates=())[0].rung is Rung.ENTAILED + assert scan_region(_cpg([]), _spec(), function="run", client=None, model="", + candidates=({"id": "a:1:x", "text": "t", "line": 1},)) == [] + + +def test_the_candidate_budget_is_bounded() -> None: + """A region with many sites must not turn into an unbounded spend.""" + from openultrasast.model.pipeline import MAX_JUDGED_CANDIDATES, scan_region + + client = _Client() + many = tuple({"id": f"a:{i}:x", "text": "db.raw(q)", "line": i} for i in range(50)) + scan_region(_cpg([]), _spec(), function="run", client=client, model="m", candidates=many) + assert client.calls == MAX_JUDGED_CANDIDATES