"""learning-harness task 2.6: the verifier registry and the oracle callback (offline)."""
from __future__ import annotations
from pathlib import Path
import pytest
from openultrasast.findings import StaticFinding
from openultrasast.learning.families import load_families
from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result
from openultrasast.sandbox import SandboxResult
APP = (
"from flask import request\n\n\n"
"@app.route('/books/
')\n@login_required\ndef guarded(title):\n"
" return Book.query.filter_by(book_title=title).first()\n\n\n"
"@app.route('/books/any/')\ndef leaky(title):\n"
" owner = request.args.get('owner')\n"
" return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n"
"@app.route('/health')\ndef health():\n return {'ok': True}\n"
)
def _claim(function: str, line: int, family: str) -> StaticFinding:
return StaticFinding(
finding_id=f"detector:app.py:{line}",
path="app.py",
title="t",
severity="medium",
confidence="low",
evidence_level="suspicion",
rationale="r",
line=line,
function_name=function,
reachability_status="unknown",
reachability_evidence=[],
reachability_conditions=[],
tags=[f"family:{family}"],
ranking_priority=0.0,
)
@pytest.fixture
def repo(tmp_path: Path) -> Path:
(tmp_path / "app.py").write_text(APP)
return tmp_path
def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None:
from openultrasast.learning.verifiers import token_oracle
oracle = token_oracle("CANARY:9f2a")
fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False)
quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False)
assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE
assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE # a crash is not this family's oracle
assert verdict_from_result(quiet).verdict == TRIGGERABLE # without an oracle the exit code still decides
assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired"
def test_an_oracle_never_overrides_an_inconclusive_run() -> None:
from openultrasast.learning.verifiers import token_oracle
oracle = token_oracle("CANARY:9f2a")
timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True)
assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive"
assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive"
assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable"
def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None:
from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for
taxonomy = load_families()
assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier)
assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier)
assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier)
# a canary family with no canary implementation registered yet reports at suspicion rather than pretending
injection = verifier_for(taxonomy.by_id("injection"))
assert isinstance(injection, NoVerifier) and injection.kind == "canary"
def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None:
from openultrasast.learning.verifiers import verifier_for
taxonomy = load_families()
verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python")
assert verification.rung == "suspicion" and verification.reason == "no_verifier"
def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None:
from openultrasast.learning.verifiers import verifier_for
taxonomy = load_families()
verifier = verifier_for(taxonomy.by_id("access_control"))
agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python")
assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output
# the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate
healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python")
assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees"
elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python")
assert elsewhere.rung == "suspicion"
def test_a_verification_is_written_onto_the_claim_as_a_tag_and_nothing_else(repo: Path) -> None:
"""The evidence ladder belongs to another boundary; a verifier says what it saw with a tag."""
from openultrasast.learning.verifiers import apply_verification, verifier_for
taxonomy = load_families()
claim = _claim("leaky", 12, "access_control")
verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python"))
assert "verifier:static_corroboration" in verified.tags
assert verified.evidence_level == "suspicion" and claim.evidence_level == "suspicion" # the ladder is untouched
plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python"))
assert "verifier:suspicion" in plain.tags and plain.tags.count("verifier:suspicion") == 1
def test_a_rung_only_reaches_the_ladder_through_an_explicit_mapping() -> None:
"""Every rung must land on a level the project's ladder actually defines, or the report raises later."""
from openultrasast.learning.verifiers import RUNGS, evidence_level_for
from openultrasast.verification import EvidenceLevel
assert {rung: evidence_level_for(rung) for rung in RUNGS} == {
"suspicion": EvidenceLevel.SUSPICION,
"static_corroboration": EvidenceLevel.STATIC_CORROBORATION,
"proven": EvidenceLevel.EXPLOIT_DEMONSTRATED,
}
assert all(EvidenceLevel(evidence_level_for(rung).value) for rung in RUNGS) # never a value outside the ladder
def test_a_verified_claim_still_passes_the_projects_own_verification_helpers(repo: Path) -> None:
"""The bug this pins: stamping a learning rung onto evidence_level raised ValueError in three call sites."""
from openultrasast.learning.verifiers import Verification, apply_verification, raise_to
from openultrasast.verification import VerificationStatus, is_report_verified, verify_finding
claim = _claim("lookup", 3, "injection")
proven = apply_verification(claim, Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY x"))
assert "verifier:proven" in proven.tags
assert verify_finding(proven).evidence_level == "suspicion" # would raise ValueError if the rung were stamped on
assert is_report_verified(proven.evidence_level, VerificationStatus.ACCEPTED) is False
raised = raise_to(proven, "proven")
assert raised.evidence_level == "exploit_demonstrated"
assert verify_finding(raised).evidence_level == "exploit_demonstrated"
assert is_report_verified(raised.evidence_level, VerificationStatus.ACCEPTED) is True