From 43fd59f960bcb5b3f2cbcd5c86c285420fa0906f Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:27:27 +0100 Subject: [PATCH] feat(detection-core): anti-shortcut probes and fitted-component admission gate --- .kiro/specs/detection-core/tasks.md | 4 +- src/openultrasast/core/__init__.py | 24 +++++++++++ src/openultrasast/core/probes.py | 67 +++++++++++++++++++++++++++++ tests/test_core_probes.py | 52 ++++++++++++++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 src/openultrasast/core/probes.py create mode 100644 tests/test_core_probes.py diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index e51ff86..7dd9cf5 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -87,13 +87,13 @@ _Requirement IDs use N.M where N is the requirement number in requirements.md an - _Depends: 4.2, 3.2_ - [ ] 5. Anti-shortcut probes for any fitted component -- [ ] 5.1 (P) Reversed-pair probe harness +- [x] 5.1 (P) Reversed-pair probe harness - Build the fix/reversed-fix probe; the deletion-count shape feature must fail it (control test). - Done when the control probe fails a shape-only component and passes a relation-based one. - _Requirements: 8.1, 8.2_ - _Boundary: core probes_ - _Depends: 1.2_ -- [ ] 5.2 Fitted-component gate +- [x] 5.2 Fitted-component gate - Require per-repository holdout and untouched-population admission before any fitted component makes a claim; BLOCK reserved for demonstrated proofs. - Done when a fitted component without admission evidence is refused by the gate in a test. - _Requirements: 8.2, 9.2_ diff --git a/src/openultrasast/core/__init__.py b/src/openultrasast/core/__init__.py index ddc88ce..11e72d3 100644 --- a/src/openultrasast/core/__init__.py +++ b/src/openultrasast/core/__init__.py @@ -4,31 +4,55 @@ Owns contracts only -- no analysis logic. Arms implement the trace contract; runners and consumers adopt coverage, budget, provenance, and admission contracts. """ +from openultrasast.core.admission import ( + REPORT_SCHEMA_KEYS, + AdmissionManifest, + AdmissionManifestError, + validate_admission_manifest, +) from openultrasast.core.budget import Budget, BudgetGateFailure, Complexity, check_budget, derive_budget from openultrasast.core.coverage import Coverage, InstrumentFailure, ResolutionOutcome, check_reads +from openultrasast.core.probes import ( + FittedComponentBlocked, + ProbeResult, + ShortcutDetected, + assert_no_shortcut, + require_admission_evidence, + reversed_pair_probe, +) from openultrasast.core.provenance import ProvenanceMismatch, stamp, verify_stamp from openultrasast.core.trace import Guard, Hop, TraceRecord, Witness from openultrasast.core.verdict import BoundedSilence, Judgment, ProvenClaim, as_clean __all__ = [ + "REPORT_SCHEMA_KEYS", + "AdmissionManifest", + "AdmissionManifestError", "BoundedSilence", "Budget", "BudgetGateFailure", "Complexity", "Coverage", + "FittedComponentBlocked", "Guard", "Hop", "InstrumentFailure", "Judgment", + "ProbeResult", "ProvenClaim", "ProvenanceMismatch", "ResolutionOutcome", + "ShortcutDetected", "TraceRecord", "Witness", "as_clean", + "assert_no_shortcut", "check_budget", "check_reads", "derive_budget", + "require_admission_evidence", + "reversed_pair_probe", "stamp", + "validate_admission_manifest", "verify_stamp", ] diff --git a/src/openultrasast/core/probes.py b/src/openultrasast/core/probes.py new file mode 100644 index 0000000..5732dff --- /dev/null +++ b/src/openultrasast/core/probes.py @@ -0,0 +1,67 @@ +"""Anti-shortcut probes for fitted components (Req 8, 9). + +A fitted prompt, threshold, or model must not separate a fix from its reversed fix +using shape features (diff size, deletion count, line counts): that is pool-specific +shortcut learning, the 2026-10-14 deletion-count 0.90 finding. A component making any +detection claim must also carry per-repository holdout evidence and a GO admission on +an untouched population. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + + +class ShortcutDetected(ValueError): + """A component separates a fix from its reversed fix by its decision features.""" + + +class FittedComponentBlocked(ValueError): + """A fitted component claims detection without holdout + admission evidence.""" + + +@dataclass(frozen=True) +class ProbeResult: + """Outcome of one reversed-pair probe.""" + + separated: bool + fix_score: float + reversed_score: float + + +def reversed_pair_probe( + score_fn: Callable[[Mapping[str, Any]], float], fix_change: Mapping[str, Any], reversed_change: Mapping[str, Any] +) -> ProbeResult: + """Score a fix and its reversed fix; separated=True means the features shortcut.""" + fix_score = float(score_fn(fix_change)) + reversed_score = float(score_fn(reversed_change)) + return ProbeResult(separated=fix_score != reversed_score, fix_score=fix_score, reversed_score=reversed_score) + + +def assert_no_shortcut( + score_fn: Callable[[Mapping[str, Any]], float], fix_change: Mapping[str, Any], reversed_change: Mapping[str, Any] +) -> None: + """Raise ShortcutDetected when the component separates fix from reversed fix.""" + result = reversed_pair_probe(score_fn, fix_change, reversed_change) + if result.separated: + raise ShortcutDetected(f"shape shortcut: fix={result.fix_score} vs reversed={result.reversed_score}") + + +def require_admission_evidence(*, holdout_passed: bool, admission_decision: str) -> None: + """Block any fitted-component claim without holdout and an admission GO (Req 8.2).""" + if not holdout_passed: + raise FittedComponentBlocked("fitted component lacks per-repository holdout evidence") + if admission_decision != "GO": + raise FittedComponentBlocked(f"fitted component admission decision is {admission_decision!r}, not GO") + + +__all__ = [ + "FittedComponentBlocked", + "ProbeResult", + "ShortcutDetected", + "assert_no_shortcut", + "require_admission_evidence", + "reversed_pair_probe", +] diff --git a/tests/test_core_probes.py b/tests/test_core_probes.py new file mode 100644 index 0000000..a1ed25f --- /dev/null +++ b/tests/test_core_probes.py @@ -0,0 +1,52 @@ +"""Anti-shortcut probe tests for detection-core tasks 5.1-5.2 (Req 8, 9).""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from collections.abc import Mapping +from typing import Any + +from openultrasast.core.probes import ( # noqa: E402 + FittedComponentBlocked, + ShortcutDetected, + assert_no_shortcut, + require_admission_evidence, + reversed_pair_probe, +) + +FIX: dict[str, Any] = {"id": "fix", "files": [{"path": "a.py", "head_lines": [[1, 10]]}], "seconds": 1.0} +REVERSED: dict[str, Any] = {"id": "reversed", "files": [{"path": "a.py", "head_lines": [[1, 50]]}], "seconds": 1.0} + + +def test_reversed_pair_probe_detects_shape_shortcut() -> None: + # deletion-count style component: score = lines removed -> separates fix from reversed + def shape_score(change: Mapping[str, Any]) -> float: + return float(50.0 - change["files"][0]["head_lines"][0][1]) + + result = reversed_pair_probe(shape_score, FIX, REVERSED) + assert result.separated is True + try: + assert_no_shortcut(shape_score, FIX, REVERSED) + raise AssertionError("shape shortcut must be detected") + except ShortcutDetected: + pass + + +def test_reversed_pair_probe_passes_relation_based_component() -> None: + def relation_score(change: Mapping[str, Any]) -> float: + return float(bool(change.get("has_guard"))) + + assert reversed_pair_probe(relation_score, {**FIX, "has_guard": True}, {**REVERSED, "has_guard": True}).separated is False + assert_no_shortcut(relation_score, {**FIX, "has_guard": True}, {**REVERSED, "has_guard": True}) + + +def test_fitted_component_requires_holdout_and_admission_go() -> None: + require_admission_evidence(holdout_passed=True, admission_decision="GO") + for holdout, decision in ((False, "GO"), (True, "NO-GO"), (False, "NO-GO")): + try: + require_admission_evidence(holdout_passed=holdout, admission_decision=decision) + raise AssertionError(f"holdout={holdout} decision={decision} must block the claim") + except FittedComponentBlocked: + pass