Open_Ultra_SAST/tests/test_model_judge.py

"""model-grounded-detection task 3.2: the LLM proposes, the model disposes (Req 8).

The judge is where the prior architecture's whole premise is inverted. There is no K-run averaging, no noise
floor and no acceptance rule; there is one bounded, typed question per candidate, and the answer is CHECKED
against the CPG rather than voted on. The three outcomes are exhaustive:

    the model entails it        -> report it, and never call the model at all
    the model contradicts it    -> drop the claim, and record the contradiction
    the model cannot decide     -> report `suspicion`, honestly

Every test here runs with no network and a scripted client.
"""

from __future__ import annotations

from pathlib import Path


def _spec():  # type: ignore[no-untyped-def]
    from openultrasast.model.specs import TaintSpec

    return TaintSpec(family="injection", language="python", sources=("request.args",),
                     sinks=("os.system",), sanitizers=("escape",))


def _cpg(rows):  # type: ignore[no-untyped-def]
    from openultrasast.cpg.backend import CpgResult

    return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: rows)


ENTAILING_ROW = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run",
                  "source": "request.args['c']", "sanitized": False, "length": 3}]


class _Client:
    """A scripted chat client that records whether it was called at all."""

    def __init__(self, answer: str = '{"vulnerable": true, "family": "injection"}') -> None:
        self.calls: list[object] = []
        self._answer = answer

    def complete(self, **kwargs: object):  # type: ignore[no-untyped-def]
        from openultrasast.tool_hunter import ChatResponse

        self.calls.append(kwargs)
        return ChatResponse(content=self._answer, tool_calls=[], reasoning=None)


def test_an_entailed_candidate_costs_no_model_call() -> None:
    """The point of the arbiter: where the graph decides, the LLM is not consulted at all."""
    from openultrasast.model.judge import judge
    from openultrasast.model.ladder import Rung

    client = _Client()
    answer = judge(_cpg(ENTAILING_ROW), _spec(), function="run", client=client, model="m")
    assert answer.rung is Rung.ENTAILED
    assert client.calls == [], "an entailed candidate must not reach the model"


def test_a_claim_the_model_contradicts_is_dropped_with_its_reason() -> None:
    """Req 8.2. The LLM says vulnerable; the CPG shows no flow reaches the sink. The graph wins."""
    from openultrasast.model.judge import judge
    from openultrasast.model.ladder import Rung

    client = _Client('{"vulnerable": true, "family": "injection"}')
    answer = judge(_cpg([]), _spec(), function="run", client=client, model="m")
    assert answer.rung is Rung.SUSPICION
    assert answer.contradiction, "a dropped claim must say why it was dropped"
    assert "no flow" in answer.contradiction.lower() or "not reach" in answer.contradiction.lower()


def test_a_claim_the_model_supports_advances_to_corroborated() -> None:
    """Req 8.3: a sanitized path is a real flow whose sufficiency only the LLM can judge."""
    from openultrasast.model.judge import judge
    from openultrasast.model.ladder import Rung

    rows = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run",
             "source": "request.args['c']", "sanitized": True, "length": 3}]
    client = _Client('{"vulnerable": true, "family": "injection"}')
    answer = judge(_cpg(rows), _spec(), function="run", client=client, model="m")
    assert answer.rung is Rung.CORROBORATED
    assert len(client.calls) == 1, "exactly one bounded question, never a loop"


def test_the_llm_declining_leaves_a_corroborated_flow_at_suspicion() -> None:
    """The model saw a flow but the LLM says the sanitizer suffices; the claim does not advance."""
    from openultrasast.model.judge import judge
    from openultrasast.model.ladder import Rung

    rows = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run",
             "source": "request.args['c']", "sanitized": True, "length": 3}]
    client = _Client('{"vulnerable": false, "family": "injection"}')
    answer = judge(_cpg(rows), _spec(), function="run", client=client, model="m")
    assert answer.rung is Rung.SUSPICION


def test_without_a_client_the_verdict_still_stands_on_its_own() -> None:
    """Req 11.3: the endpoint is optional. The model layer does not need it to entail."""
    from openultrasast.model.judge import judge
    from openultrasast.model.ladder import Rung

    assert judge(_cpg(ENTAILING_ROW), _spec(), function="run", client=None, model="").rung is Rung.ENTAILED
    sanitized = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run",
                  "source": "request.args['c']", "sanitized": True, "length": 3}]
    answer = judge(_cpg(sanitized), _spec(), function="run", client=None, model="")
    assert answer.rung is Rung.SUSPICION and "endpoint" in answer.contradiction.lower()


def test_the_question_is_bounded_typed_and_offers_no_tools() -> None:
    """Req 8.1: the LLM answers one question about a site it was handed; it never searches for the site."""
    from openultrasast.model.judge import judge

    rows = [{"sink": "os.system(cmd)", "sinkLine": "4", "sinkMethod": "run",
             "source": "request.args['c']", "sanitized": True, "length": 3}]
    client = _Client()
    judge(_cpg(rows), _spec(), function="run", client=client, model="m")
    call = client.calls[0]
    assert call["tools"] == [], "no tools: the candidate is given, not searched for"
    assert call["json_object"] is True, "a typed answer, not prose"
    text = " ".join(str(m.get("content", "")) for m in call["messages"])  # type: ignore[union-attr]
    assert "os.system(cmd)" in text and "request.args['c']" in text, "the question carries the model's own evidence"


def test_the_prompt_is_redacted() -> None:
    """Req 11.4: every prompt passes through redaction."""
    from openultrasast.model.judge import judge

    rows = [{"sink": "connect(pw)", "sinkLine": "4", "sinkMethod": "run",
             "source": "AKIA" + "IOSFODNN7" + "EXAMPLE", "sanitized": True, "length": 2}]
    client = _Client()
    judge(_cpg(rows), _spec(), function="run", client=client, model="m")
    text = " ".join(str(m.get("content", "")) for m in client.calls[0]["messages"])  # type: ignore[index,union-attr]
    assert "AKIAIOSFODNN7EXAMPLE" not in text


def test_there_is_no_k_run_averaging_anywhere_in_the_judge() -> None:
    """Req 8.5, asserted structurally: the noise architecture must not grow back."""
    import ast

    source = Path("src/openultrasast/model/judge.py").read_text()
    tree = ast.parse(source)

    # Check CODE, not prose: the docstring is allowed to name the machinery it replaced.
    names = {
        node.id for node in ast.walk(tree) if isinstance(node, ast.Name)
    } | {
        node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute)
    } | {
        node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)
    }
    for banned in ("k_runs", "majority", "vote", "votes", "average", "mean", "acceptance", "floor", "budget"):
        assert banned not in names, f"{banned} has no place in the judge"

    # No loop over runs anywhere in the module.
    assert not [n for n in ast.walk(tree) if isinstance(n, ast.For | ast.While)], "the judge asks once"

    imported = {n.module for n in ast.walk(tree) if isinstance(n, ast.ImportFrom) and n.module}
    assert not any("rounds" in m or "acceptance" in m for m in imported)