From 52f2ab5b4dd5ad9f0647f505cec030c8a400d0ce Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:19:18 +0200 Subject: [PATCH] fix(model-grounded-detection): ask the residual question the arbiter actually raised The corrected re-measure returned numbers IDENTICAL to the buggy one, which was the clue. Routing was fixed, the dominance arbiter was reached, and the finding was discarded anyway -- because every corroboration was put to the judge as a TAINT question: "a sanitizer appears on the path: yes, does it suffice?". For a missing authorization guard that question is meaningless. The judge answers no to something that does not apply and the claim is dropped. All three vibe-py access_control pairs scored both_silent this way even though the dominance arbiter had returned model_corroborated for each of them, which I had verified by hand earlier in the session. Fourth instance in this feature of one shape: taint-specific behaviour applied to every family. Sanitizer matching, config routing, family dispatch, now the residual question. Each was exposed by a measurement rather than by a check. The question is now phrased for the arbiter that raised it -- a guard asymmetry for dominance, a permissive setting for constant abstraction, sanitizer sufficiency only for taint. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- src/openultrasast/model/pipeline.py | 30 ++++++++++++++++++++++--- tests/test_model_pipeline.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/openultrasast/model/pipeline.py b/src/openultrasast/model/pipeline.py index 7dcb292..81baeff 100644 --- a/src/openultrasast/model/pipeline.py +++ b/src/openultrasast/model/pipeline.py @@ -104,7 +104,7 @@ def scan_region( # here — so a "no" from the judge is a genuine contradiction and the claim is dropped. if client is None: return [] - said = _ask(client, model, _sanitizer_question(spec, answer.witness, function)) + said = _ask(client, model, residual_question(spec, answer.witness, function)) if said: return [ModelFinding(site=_site(function, answer.witness), family=spec.family, rung=Rung.CORROBORATED, witness=answer.witness)] @@ -134,7 +134,31 @@ def scan_region( return findings -def _sanitizer_question(spec: TaintSpec | DominanceSpec | ConfigSpec, witness: str, function: str) -> str: +def residual_question(spec: TaintSpec | DominanceSpec | ConfigSpec, witness: str, function: str) -> str: + """The one question the arbiter could not settle -- phrased for the arbiter that raised it. + + Asking a sanitizer question about a missing guard is not merely odd, it silently loses findings: the judge + answers "no" to a question that does not apply and the claim is dropped. Every vibe-py access_control pair + was lost this way. + """ + if isinstance(spec, DominanceSpec): + return ( + f"You are judging ONE finding a static model already made. Do not look for other issues.\n\n" + f"A dominance analysis of {spec.language} code found an obligated operation that NO discharging\n" + f"guard governs, in a file where sibling handlers are guarded:\n\n" + f" {witness}\n function: {function or '?'}\n\n" + f"The model has established the asymmetry. The open question is whether this operation genuinely\n" + f"needs the guard its siblings have -- or is legitimately public.\n\n" + 'Answer ONLY with JSON: {"vulnerable": true|false, "why": ""}' + ) + if isinstance(spec, ConfigSpec): + return ( + f"You are judging ONE finding a static model already made. Do not look for other issues.\n\n" + f"Constant evaluation of {spec.language} code found a security setting given a value the model\n" + f"could not read as safe:\n\n {witness}\n function: {function or '?'}\n\n" + f"The open question is whether this setting is genuinely permissive in a deployed configuration.\n\n" + 'Answer ONLY with JSON: {"vulnerable": true|false, "why": ""}' + ) source, _, sink = witness.partition(" -> ") return ( f"You are judging ONE candidate a static model already found. Do not look for other issues.\n\n" @@ -174,4 +198,4 @@ def _site(function: str, witness: str) -> str: return f"{function or '?'}:{line}" -__all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "scan_region"] +__all__ = ["CANDIDATE_QUESTION", "MAX_JUDGED_CANDIDATES", "ModelFinding", "residual_question", "scan_region"] diff --git a/tests/test_model_pipeline.py b/tests/test_model_pipeline.py index f32f8a2..8d851e2 100644 --- a/tests/test_model_pipeline.py +++ b/tests/test_model_pipeline.py @@ -163,3 +163,37 @@ def test_a_configuration_family_is_arbitrated_by_constant_abstraction() -> None: findings = scan_region(_cpg(rows), _config_spec(), function="create_app", client=None, model="", candidates=()) assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED assert findings[0].family == "config_secrets" + + +def test_the_residual_question_matches_the_arbiter_that_raised_it() -> None: + """A corroborated verdict from dominance is not a question about a sanitizer. + + The pipeline asked `_sanitizer_question` for every corroboration, so an absence finding was put to the + judge as "a sanitizer appears on the path: yes -- does it suffice?", which is meaningless for a missing + guard. The judge answered no and the finding was discarded: all three vibe-py access_control pairs scored + both_silent even though the dominance arbiter had returned model_corroborated for each. Fourth instance in + this feature of taint-specific behaviour applied to every family. + """ + from openultrasast.model.pipeline import residual_question + + rows = [{"operation": "q.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []}, + {"operation": "q.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user"]}] + dom = residual_question(_dominance_spec(), "leaky line 9: no discharging guard", "leaky") + assert "sanitizer" not in dom.lower(), "an absence bug has no sanitizer to judge" + assert "guard" in dom.lower() + + cfg = residual_question(_config_spec(), 'CORS(...): permissive literal "*"', "create_app") + assert "sanitizer" not in cfg.lower() and "setting" in cfg.lower() + + taint = residual_question(_spec(), "request.args['c'] -> os.system(cmd)", "run") + assert "sanitizer" in taint.lower() + + +def test_a_corroborated_absence_finding_survives_a_judge_that_agrees() -> None: + from openultrasast.model.ladder import Rung + from openultrasast.model.pipeline import scan_region + + rows = [{"operation": "q.filter_by(id=n)", "opLine": "9", "opMethod": "leaky", "dominatingGuards": []}, + {"operation": "q.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user"]}] + findings = scan_region(_cpg(rows), _dominance_spec(), function="leaky", client=_Client(), model="m", candidates=()) + assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED