From 3fe7b9ded79c0a86e6aa26c8904079bda09cf009 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:22:55 +0200 Subject: [PATCH] fix(model-grounded-detection): the dominance question contradicted its own evidence vampi-users-update-password scored both_silent while the arbiter returned model_corroborated on the vulnerable side and nothing on the fixed one -- so the finding existed and the pipeline threw it away. The residual question was written for the ENTAILED case, but entailed returns before the judge is ever consulted, so the question is only ever asked for CORROBORATED. It asserted "in a file where sibling handlers are guarded" and asked about "the guard its siblings have", printed directly above a witness reading "and no guarded sibling" -- because corroborated means precisely that no sibling is guarded. The judge was handed a prompt contradicting its own evidence line and reasonably answered no. The question now describes what corroborated actually is: an obligated operation no guard governs, with no sibling evidence either way, so the model cannot tell from consistency alone whether the guard is missing or the module is deliberately public. That is the real open question and the one worth asking. Verified end to end against a live endpoint: vuln model_corroborated, fixed silent, the pair splits. The taint and config questions were checked for the same defect and are correct -- each describes the corroborated case it is actually asked about. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- src/openultrasast/model/pipeline.py | 13 +++++++++---- tests/test_model_pipeline.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/openultrasast/model/pipeline.py b/src/openultrasast/model/pipeline.py index 81baeff..3bb892a 100644 --- a/src/openultrasast/model/pipeline.py +++ b/src/openultrasast/model/pipeline.py @@ -142,13 +142,18 @@ def residual_question(spec: TaintSpec | DominanceSpec | ConfigSpec, witness: str was lost this way. """ if isinstance(spec, DominanceSpec): + # Only the CORROBORATED case reaches this question -- an entailed verdict returns before the judge is + # consulted. Corroborated means precisely that no sibling is guarded, so there is no asymmetry to + # appeal to and claiming one contradicts the witness printed directly beneath it. return ( f"You are judging ONE finding a static model already made. Do not look for other issues.\n\n" - f"A dominance analysis of {spec.language} code found an obligated operation that NO discharging\n" - f"guard governs, in a file where sibling handlers are guarded:\n\n" + f"A dominance analysis of {spec.language} code found an obligated operation on a protected\n" + f"resource that NO discharging guard governs on any path:\n\n" f" {witness}\n function: {function or '?'}\n\n" - f"The model has established the asymmetry. The open question is whether this operation genuinely\n" - f"needs the guard its siblings have -- or is legitimately public.\n\n" + f"No sibling handler in this file is guarded either, so the model cannot tell from consistency\n" + f"alone whether the guard is missing or the whole module is deliberately public. That is the\n" + f"open question: does this operation require an authorization check, or is it legitimately\n" + f"public?\n\n" 'Answer ONLY with JSON: {"vulnerable": true|false, "why": ""}' ) if isinstance(spec, ConfigSpec): diff --git a/tests/test_model_pipeline.py b/tests/test_model_pipeline.py index 6b6a480..225003c 100644 --- a/tests/test_model_pipeline.py +++ b/tests/test_model_pipeline.py @@ -195,3 +195,21 @@ def test_a_corroborated_absence_finding_survives_a_judge_that_agrees() -> None: {"operation": "q.filter_by(id=n)", "opLine": "3", "opMethod": "safe", "dominatingGuards": ["current_user"]}] findings = scan_region(_cpg(rows), _dominance_spec(), function="leaky", client=_Client(), model="m", candidates=()) assert len(findings) == 1 and findings[0].rung is Rung.ENTAILED + + +def test_the_dominance_question_describes_the_evidence_the_model_actually_has() -> None: + """The residual question is only ever asked for CORROBORATED, never ENTAILED -- entailed returns early. + + It was written for the entailed case, so it asserted "in a file where sibling handlers are guarded" and + asked about "the guard its siblings have", directly above a witness reading "and no guarded sibling". + The judge, handed a prompt contradicting its own evidence line, answered no and the finding was dropped: + vampi-users-update-password scored both_silent while the arbiter returned corroborated on the vulnerable + side and nothing on the fixed one. + """ + from openultrasast.model.pipeline import residual_question + + witness = "update_password line 16: obligated operation with no discharging guard, and no guarded sibling" + q = residual_question(_dominance_spec(), witness, "update_password") + assert "siblings have" not in q, "the corroborated case has no guarded siblings to appeal to" + assert "sibling handlers are guarded" not in q + assert "guard" in q.lower() and "public" in q.lower(), "it must still ask whether the operation needs a guard"