Open_Ultra_SAST/tests/test_learning_verifiers.py

"""learning-harness task 2.6: the verifier registry and the oracle callback (offline)."""

from __future__ import annotations

from pathlib import Path

import pytest

from openultrasast.findings import StaticFinding
from openultrasast.learning.families import load_families
from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result
from openultrasast.sandbox import SandboxResult

APP = (
    "from flask import request\n\n\n"
    "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n"
    "    return Book.query.filter_by(book_title=title).first()\n\n\n"
    "@app.route('/books/any/<title>')\ndef leaky(title):\n"
    "    owner = request.args.get('owner')\n"
    "    return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
    "@app.route('/health')\ndef health():\n    return {'ok': True}\n"
)


def _claim(function: str, line: int, family: str) -> StaticFinding:
    return StaticFinding(
        finding_id=f"detector:app.py:{line}",
        path="app.py",
        title="t",
        severity="medium",
        confidence="low",
        evidence_level="suspicion",
        rationale="r",
        line=line,
        function_name=function,
        reachability_status="unknown",
        reachability_evidence=[],
        reachability_conditions=[],
        tags=[f"family:{family}"],
        ranking_priority=0.0,
    )


@pytest.fixture
def repo(tmp_path: Path) -> Path:
    (tmp_path / "app.py").write_text(APP)
    return tmp_path


def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None:
    from openultrasast.learning.verifiers import token_oracle

    oracle = token_oracle("CANARY:9f2a")
    fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False)
    quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False)
    assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE
    assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE  # a crash is not this family's oracle
    assert verdict_from_result(quiet).verdict == TRIGGERABLE  # without an oracle the exit code still decides
    assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired"


def test_an_oracle_never_overrides_an_inconclusive_run() -> None:
    from openultrasast.learning.verifiers import token_oracle

    oracle = token_oracle("CANARY:9f2a")
    timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True)
    assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive"
    assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive"
    assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable"


def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None:
    from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for

    taxonomy = load_families()
    assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier)
    assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier)
    assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier)
    # a canary family with no canary implementation registered yet reports at suspicion rather than pretending
    injection = verifier_for(taxonomy.by_id("injection"))
    assert isinstance(injection, NoVerifier) and injection.kind == "canary"


def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None:
    from openultrasast.learning.verifiers import verifier_for

    taxonomy = load_families()
    verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python")
    assert verification.rung == "suspicion" and verification.reason == "no_verifier"


def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None:
    from openultrasast.learning.verifiers import verifier_for

    taxonomy = load_families()
    verifier = verifier_for(taxonomy.by_id("access_control"))
    agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python")
    assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output
    # the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate
    healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python")
    assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees"
    elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python")
    assert elsewhere.rung == "suspicion"


def test_a_verification_can_be_written_onto_the_claim_as_a_tag(repo: Path) -> None:
    from openultrasast.learning.verifiers import apply_verification, verifier_for

    taxonomy = load_families()
    claim = _claim("leaky", 12, "access_control")
    verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python"))
    assert "verifier:static_corroboration" in verified.tags
    assert verified.evidence_level == "static_corroboration" and claim.evidence_level == "suspicion"
    plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python"))
    assert "verifier:suspicion" in plain.tags and plain.evidence_level == "suspicion"