"""learning-harness task 2.6: the verifier registry and the oracle callback (offline).""" from __future__ import annotations from pathlib import Path import pytest from openultrasast.findings import StaticFinding from openultrasast.learning.families import load_families from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result from openultrasast.sandbox import SandboxResult APP = ( "from flask import request\n\n\n" "@app.route('/books/')\n@login_required\ndef guarded(title):\n" " return Book.query.filter_by(book_title=title).first()\n\n\n" "@app.route('/books/any/<title>')\ndef leaky(title):\n" " owner = request.args.get('owner')\n" " return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n" "@app.route('/health')\ndef health():\n return {'ok': True}\n" ) def _claim(function: str, line: int, family: str) -> StaticFinding: return StaticFinding( finding_id=f"detector:app.py:{line}", path="app.py", title="t", severity="medium", confidence="low", evidence_level="suspicion", rationale="r", line=line, function_name=function, reachability_status="unknown", reachability_evidence=[], reachability_conditions=[], tags=[f"family:{family}"], ranking_priority=0.0, ) @pytest.fixture def repo(tmp_path: Path) -> Path: (tmp_path / "app.py").write_text(APP) return tmp_path def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None: from openultrasast.learning.verifiers import token_oracle oracle = token_oracle("CANARY:9f2a") fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False) quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False) assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE # a crash is not this family's oracle assert verdict_from_result(quiet).verdict == TRIGGERABLE # without an oracle the exit code still decides assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired" def test_an_oracle_never_overrides_an_inconclusive_run() -> None: from openultrasast.learning.verifiers import token_oracle oracle = token_oracle("CANARY:9f2a") timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True) assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive" assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive" assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable" def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None: from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for taxonomy = load_families() assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier) assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier) assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier) # a canary family with no canary implementation registered yet reports at suspicion rather than pretending injection = verifier_for(taxonomy.by_id("injection")) assert isinstance(injection, NoVerifier) and injection.kind == "canary" def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None: from openultrasast.learning.verifiers import verifier_for taxonomy = load_families() verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python") assert verification.rung == "suspicion" and verification.reason == "no_verifier" def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None: from openultrasast.learning.verifiers import verifier_for taxonomy = load_families() verifier = verifier_for(taxonomy.by_id("access_control")) agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python") assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output # the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python") assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees" elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python") assert elsewhere.rung == "suspicion" def test_a_verification_can_be_written_onto_the_claim_as_a_tag(repo: Path) -> None: from openultrasast.learning.verifiers import apply_verification, verifier_for taxonomy = load_families() claim = _claim("leaky", 12, "access_control") verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python")) assert "verifier:static_corroboration" in verified.tags assert verified.evidence_level == "static_corroboration" and claim.evidence_level == "suspicion" plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python")) assert "verifier:suspicion" in plain.tags and plain.evidence_level == "suspicion"