feat(learning-harness): verifier registry and the sandbox oracle callback

Task 2.6. A detector's claim is a suspicion until something other than the model
agrees with it, and what counts as agreement depends on the family. The regress
verdict now takes an optional oracle over the sandbox result: when a family
supplies one it decides triggerability instead of the exit code, because a crash
and a demonstrated injection are different evidence and one family's oracle must
not answer for another. Without an oracle the exit code decides exactly as
before, and an inconclusive run stays inconclusive either way.

The registry answers by the verifier kind the taxonomy declares. Access control
gets static corroboration when the obligations checker looks at the same
function and agrees the obligation is undischarged, and refuses to corroborate a
handler that owes nothing. A family whose verifier does not exist yet reports at
suspicion rather than pretending, which is the honest default rather than a gap.
A verification is written onto a copy of the claim as a tag, never onto the
original.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/learning/verifiers.py

@@ -0,0 +1,111 @@
+"""What may raise a claim above suspicion, keyed by family (learning-harness, Req 7.1, 7.3, 7.4).
+
+A detector's claim is a suspicion until something other than the model agrees with it. What counts as
+agreement depends on the family: a canary oracle observed in the sandbox, corroboration from a static
+checker, or nothing at all. A family whose verifier does not exist yet reports at suspicion rather than
+pretending, and that is the honest default rather than a gap.
+
+The canary oracle is a function over a sandbox result, so the sandbox keeps deciding what happened and
+this module only says what would count. Exit codes are not oracles: a crash and a demonstrated
+injection are different evidence, and one family's oracle must not silently answer for another.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Literal, Protocol
+
+from ..findings import StaticFinding
+from ..sandbox import SandboxResult
+from .families import Family
+
+Rung = Literal["suspicion", "static_corroboration", "proven"]
+VerifierKind = Literal["canary", "static", "none"]
+
+
+@dataclass(frozen=True)
+class Verification:
+    """What a verifier saw, and how high it lets the claim go."""
+
+    rung: Rung
+    reason: str
+    oracle_output: str = ""
+
+
+class Verifier(Protocol):
+    @property
+    def kind(self) -> VerifierKind: ...
+
+    def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: ...
+
+
+def token_oracle(token: str) -> Callable[[SandboxResult], bool]:
+    """The oracle a canary snippet is written against: the token appears only if the vulnerability fired."""
+
+    def fired(result: SandboxResult) -> bool:
+        return token in result.stdout or token in result.stderr
+
+    return fired
+
+
+@dataclass(frozen=True)
+class NoVerifier:
+    """Nothing can confirm this family yet, so the claim stays where the detector left it."""
+
+    kind: VerifierKind = "none"
+
+    def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification:
+        del claim, root, language
+        return Verification(rung="suspicion", reason="no_verifier")
+
+
+@dataclass(frozen=True)
+class StaticVerifier:
+    """Another checker looked at the same function and agrees the obligation is undischarged."""
+
+    kind: VerifierKind = "static"
+
+    def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification:
+        del language
+        from ..hunter_tools import PathEscapesRepo, obligations
+
+        if not claim.function_name:
+            return Verification(rung="suspicion", reason="no_function")
+        try:
+            rows = obligations(root, path=claim.path, function=claim.function_name)
+        except (PathEscapesRepo, OSError):
+            return Verification(rung="suspicion", reason="checker_unavailable")
+        if not rows:
+            return Verification(rung="suspicion", reason="checker_disagrees")
+        summary = "; ".join(f"{row.get('operation')} missing {row.get('missing')}" for row in rows)
+        return Verification(rung="static_corroboration", reason="checker_agrees", oracle_output=summary)
+
+
+def verifier_for(family: Family, *, canary: Verifier | None = None) -> Verifier:
+    """The verifier a family declares. A canary family with no implementation registered stays at suspicion."""
+    if family.verifier == "static":
+        return StaticVerifier()
+    if family.verifier == "canary":
+        return canary if canary is not None else NoVerifier(kind="canary")
+    return NoVerifier()
+
+
+def apply_verification(claim: StaticFinding, verification: Verification) -> StaticFinding:
+    """A copy of the claim carrying what the verifier said. The original is never mutated."""
+    tags = [tag for tag in claim.tags if not tag.startswith("verifier:")] + [f"verifier:{verification.rung}"]
+    return replace(claim, tags=tags, evidence_level=verification.rung)
+
+
+__all__ = [
+    "NoVerifier",
+    "Rung",
+    "StaticVerifier",
+    "Verification",
+    "Verifier",
+    "VerifierKind",
+    "apply_verification",
+    "token_oracle",
+    "verifier_for",
+]

src/openultrasast/regress/verdict.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Callable
 from dataclasses import dataclass
 
 from ..sandbox import SandboxResult
@@ -21,6 +22,8 @@ SANITIZER_ABORT = "sanitizer_abort"
 CRASH = "crash"
 NONZERO_EXIT = "nonzero_exit"
 EXIT_ZERO = "exit_zero"
+ORACLE_FIRED = "oracle_fired"
+ORACLE_QUIET = "oracle_quiet"
 
 WORTH_FIXING_REACHABILITY = frozenset({"reachable", "inferred-file-surface"})
 
@@ -65,8 +68,14 @@ def verdict_from_result(
     sandbox_missing: bool = False,
     safety_rejected: bool = False,
     covering_test: bool = False,
+    oracle: Callable[[SandboxResult], bool] | None = None,
 ) -> RegressionVerdict:
-    """Map a sandbox observation onto one of the four regression verdicts."""
+    """Map a sandbox observation onto one of the four regression verdicts.
+
+    ``oracle`` is how a family says what "the vulnerability fired" means for it: when one is given it
+    decides triggerability instead of the exit code, because a crash and a demonstrated injection are
+    not the same evidence. An inconclusive run stays inconclusive either way.
+    """
     if recipe_missing:
         return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
     if sandbox_missing:
@@ -79,6 +88,12 @@ def verdict_from_result(
         return RegressionVerdict(verdict=INCONCLUSIVE, reason=TIMEOUT)
     if covering_test and result.exit_code == 0:
         return RegressionVerdict(verdict=ALREADY_COVERED, reason=COVERED_TEST_PASSED)
+    if oracle is not None:
+        return (
+            RegressionVerdict(verdict=TRIGGERABLE, reason=ORACLE_FIRED)
+            if oracle(result)
+            else RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=ORACLE_QUIET)
+        )
     if result.exit_code != 0:
         return RegressionVerdict(verdict=TRIGGERABLE, reason=_nonzero_reason(result))
     return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=EXIT_ZERO)