mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
feat(detection-core): migrate test and operational consumers
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,370 @@
|
||||
"""Immutable reviewed closure authority for Task 10.6 migration consumers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
from types import MappingProxyType
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
|
||||
SCHEMA = "openultrasast.detection-core.consumer-terminal-evidence.v1"
|
||||
_FIELDS = frozenset(
|
||||
{
|
||||
"absence_anchors",
|
||||
"artifact_family",
|
||||
"binding_owner_ast_sha256",
|
||||
"binding_owner_symbol",
|
||||
"capability_anchors",
|
||||
"closure_kind",
|
||||
"dependency_kind",
|
||||
"disposition",
|
||||
"entry_id",
|
||||
"owner_boundary",
|
||||
"path",
|
||||
"probe_ast_sha256",
|
||||
"probe_symbol",
|
||||
"pytest_node_id",
|
||||
"semantic_binding",
|
||||
"semantic_contract",
|
||||
"semantic_local_name",
|
||||
"successor_ast_sha256",
|
||||
"successor_path",
|
||||
"successor_symbol",
|
||||
"symbol",
|
||||
}
|
||||
)
|
||||
_SEMANTIC_CONTRACT_BINDINGS = {
|
||||
"authorization_adapter_domain": frozenset({"import_from:openultrasast.semantic.obligations.trace:build_trace"}),
|
||||
"benchmark_v2_artifact": frozenset(
|
||||
{
|
||||
"dynamic_import:benchmarks.unseen.calibrate",
|
||||
"dynamic_import:benchmarks.unseen.proof",
|
||||
"dynamic_import:benchmarks.unseen.replay",
|
||||
"dynamic_import:benchmarks.unseen.score",
|
||||
"dynamic_import:benchmarks.valid.hunter_arm",
|
||||
"dynamic_import:benchmarks.valid.semgrep_arm",
|
||||
"import_from:benchmarks.valid:arm_regression",
|
||||
"import_from:benchmarks.valid:development_runner",
|
||||
"import_from:benchmarks.valid.development_runner:DevelopmentRecord",
|
||||
}
|
||||
),
|
||||
"hunt_adapter_artifact": frozenset(
|
||||
{
|
||||
"import_from:openultrasast.hunt.artifact:RunArtifact",
|
||||
"import_from:openultrasast.hunt:contamination",
|
||||
"import_from:openultrasast.hunt:runner",
|
||||
}
|
||||
),
|
||||
"push_adapter_domain": frozenset(
|
||||
{
|
||||
"import_from:openultrasast.push.policy:admit_candidates",
|
||||
"import_from:openultrasast.push.report:PushReport",
|
||||
}
|
||||
),
|
||||
"v2_coverage_contract": frozenset({"import_from:openultrasast.core.coverage:InstrumentFailure"}),
|
||||
}
|
||||
_MODULE_FIELDS = frozenset({"path", "sha256"})
|
||||
|
||||
|
||||
class ConsumerEvidenceError(ValueError):
|
||||
"""The reviewed terminal-evidence authority is malformed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewedSuccessorModule:
|
||||
"""One manually reviewed successor module, authoritative down to exact bytes."""
|
||||
|
||||
path: str
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConsumerTerminalEvidence:
|
||||
entry_id: str
|
||||
path: str
|
||||
symbol: str
|
||||
dependency_kind: str
|
||||
artifact_family: str
|
||||
capability_anchors: tuple[str, ...]
|
||||
owner_boundary: str
|
||||
disposition: str
|
||||
closure_kind: str
|
||||
absence_anchors: tuple[str, ...]
|
||||
successor_path: str | None
|
||||
successor_symbol: str | None
|
||||
semantic_contract: str | None
|
||||
semantic_binding: str | None
|
||||
semantic_local_name: str | None
|
||||
binding_owner_symbol: str | None
|
||||
binding_owner_ast_sha256: str | None
|
||||
probe_symbol: str | None
|
||||
probe_ast_sha256: str | None
|
||||
successor_ast_sha256: str | None
|
||||
pytest_node_id: str | None
|
||||
|
||||
@property
|
||||
def stable_fields(self) -> tuple[str, str, str, str, tuple[str, ...], str]:
|
||||
return (
|
||||
self.path,
|
||||
self.symbol,
|
||||
self.dependency_kind,
|
||||
self.artifact_family,
|
||||
self.capability_anchors,
|
||||
self.owner_boundary,
|
||||
)
|
||||
|
||||
|
||||
def _text(value: object, name: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ConsumerEvidenceError(f"{name} must be nonempty text")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_text(value: object, name: str) -> str | None:
|
||||
return None if value is None else _text(value, name)
|
||||
|
||||
|
||||
def _repo_relative_posix(value: object, name: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ConsumerEvidenceError(f"{name} must be an exact canonical repository-relative POSIX path")
|
||||
text = value
|
||||
parts = text.split("/")
|
||||
posix = PurePosixPath(text)
|
||||
windows = PureWindowsPath(text)
|
||||
if (
|
||||
posix.is_absolute()
|
||||
or windows.is_absolute()
|
||||
or bool(windows.drive)
|
||||
or "\0" in text
|
||||
or "\\" in text
|
||||
or any(part in {"", ".", ".."} for part in parts)
|
||||
or posix.as_posix() != text
|
||||
):
|
||||
raise ConsumerEvidenceError(f"{name} must be an exact canonical repository-relative POSIX path")
|
||||
return text
|
||||
|
||||
|
||||
def _optional_repo_relative_posix(value: object, name: str) -> str | None:
|
||||
return None if value is None else _repo_relative_posix(value, name)
|
||||
|
||||
|
||||
def _optional_sha256(value: object, name: str) -> str | None:
|
||||
text = _optional_text(value, name)
|
||||
if text is not None and (len(text) != 64 or any(character not in "0123456789abcdef" for character in text)):
|
||||
raise ConsumerEvidenceError(f"{name} must be a lowercase SHA-256 digest")
|
||||
return text
|
||||
|
||||
|
||||
def _sha256(value: object, name: str) -> str:
|
||||
digest = _optional_sha256(value, name)
|
||||
if digest is None:
|
||||
raise ConsumerEvidenceError(f"{name} must be a lowercase SHA-256 digest")
|
||||
return digest
|
||||
|
||||
|
||||
def _texts(value: object, name: str) -> tuple[str, ...]:
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
|
||||
raise ConsumerEvidenceError(f"{name} must be a text list")
|
||||
result = tuple(value)
|
||||
if len(result) != len(set(result)):
|
||||
raise ConsumerEvidenceError(f"{name} contains duplicates")
|
||||
return result
|
||||
|
||||
|
||||
def _record(value: object, index: int) -> ConsumerTerminalEvidence:
|
||||
if not isinstance(value, dict) or set(value) != _FIELDS:
|
||||
raise ConsumerEvidenceError(f"entries[{index}] fields differ")
|
||||
anchors = _texts(value["capability_anchors"], f"entries[{index}].capability_anchors")
|
||||
absence = _texts(value["absence_anchors"], f"entries[{index}].absence_anchors")
|
||||
record = ConsumerTerminalEvidence(
|
||||
_text(value["entry_id"], f"entries[{index}].entry_id"),
|
||||
_repo_relative_posix(value["path"], f"entries[{index}].path"),
|
||||
_text(value["symbol"], f"entries[{index}].symbol"),
|
||||
_text(value["dependency_kind"], f"entries[{index}].dependency_kind"),
|
||||
_text(value["artifact_family"], f"entries[{index}].artifact_family"),
|
||||
anchors,
|
||||
_text(value["owner_boundary"], f"entries[{index}].owner_boundary"),
|
||||
_text(value["disposition"], f"entries[{index}].disposition"),
|
||||
_text(value["closure_kind"], f"entries[{index}].closure_kind"),
|
||||
absence,
|
||||
_optional_repo_relative_posix(value["successor_path"], f"entries[{index}].successor_path"),
|
||||
_optional_text(value["successor_symbol"], f"entries[{index}].successor_symbol"),
|
||||
_optional_text(value["semantic_contract"], f"entries[{index}].semantic_contract"),
|
||||
_optional_text(value["semantic_binding"], f"entries[{index}].semantic_binding"),
|
||||
_optional_text(value["semantic_local_name"], f"entries[{index}].semantic_local_name"),
|
||||
_optional_text(value["binding_owner_symbol"], f"entries[{index}].binding_owner_symbol"),
|
||||
_optional_sha256(value["binding_owner_ast_sha256"], f"entries[{index}].binding_owner_ast_sha256"),
|
||||
_optional_text(value["probe_symbol"], f"entries[{index}].probe_symbol"),
|
||||
_optional_sha256(value["probe_ast_sha256"], f"entries[{index}].probe_ast_sha256"),
|
||||
_optional_sha256(value["successor_ast_sha256"], f"entries[{index}].successor_ast_sha256"),
|
||||
_optional_text(value["pytest_node_id"], f"entries[{index}].pytest_node_id"),
|
||||
)
|
||||
_validate_record(record, index)
|
||||
return record
|
||||
|
||||
|
||||
def _successor_fields(record: ConsumerTerminalEvidence) -> tuple[str | None, ...]:
|
||||
return (
|
||||
record.successor_path,
|
||||
record.successor_symbol,
|
||||
record.successor_ast_sha256,
|
||||
record.semantic_contract,
|
||||
record.semantic_binding,
|
||||
record.semantic_local_name,
|
||||
record.binding_owner_symbol,
|
||||
record.binding_owner_ast_sha256,
|
||||
record.probe_symbol,
|
||||
record.probe_ast_sha256,
|
||||
record.pytest_node_id,
|
||||
)
|
||||
|
||||
|
||||
def _validate_retired(record: ConsumerTerminalEvidence, index: int) -> None:
|
||||
if record.absence_anchors != record.capability_anchors:
|
||||
raise ConsumerEvidenceError(f"entries[{index}] must govern every frozen anchor")
|
||||
if any(_successor_fields(record)):
|
||||
raise ConsumerEvidenceError(f"entries[{index}] retired evidence has a successor")
|
||||
|
||||
|
||||
def _validate_migrated(record: ConsumerTerminalEvidence, index: int) -> None:
|
||||
if record.absence_anchors or not all(_successor_fields(record)):
|
||||
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence lacks semantic binding")
|
||||
allowed = _SEMANTIC_CONTRACT_BINDINGS.get(record.semantic_contract or "", frozenset())
|
||||
if record.semantic_binding not in allowed or not (record.successor_path or "").startswith("tests/"):
|
||||
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence is outside its reviewed contract")
|
||||
expected_node = f"{record.successor_path}::{record.probe_symbol}"
|
||||
if record.pytest_node_id != expected_node or not (record.probe_symbol or "").startswith("test_"):
|
||||
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence lacks an exact pytest owner")
|
||||
|
||||
|
||||
def _validate_record(record: ConsumerTerminalEvidence, index: int) -> None:
|
||||
if len(record.entry_id) != 64 or any(character not in "0123456789abcdef" for character in record.entry_id):
|
||||
raise ConsumerEvidenceError(f"entries[{index}].entry_id is not SHA-256")
|
||||
identity = {
|
||||
"artifact_family": record.artifact_family,
|
||||
"capability_anchors": record.capability_anchors,
|
||||
"dependency_kind": record.dependency_kind,
|
||||
"owner_boundary": record.owner_boundary,
|
||||
"path": record.path,
|
||||
"symbol": record.symbol,
|
||||
}
|
||||
digest = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
if digest != record.entry_id:
|
||||
raise ConsumerEvidenceError(f"entries[{index}].entry_id does not match frozen identity")
|
||||
if record.disposition == "retired" and record.closure_kind == "retired_exact_absence":
|
||||
_validate_retired(record, index)
|
||||
return
|
||||
if record.disposition == "v2_migrated" and record.closure_kind == "migrated_semantic_probe":
|
||||
_validate_migrated(record, index)
|
||||
return
|
||||
raise ConsumerEvidenceError(f"entries[{index}] has invalid closure/disposition")
|
||||
|
||||
|
||||
def _module_record(value: object, index: int) -> ReviewedSuccessorModule:
|
||||
if not isinstance(value, dict) or set(value) != _MODULE_FIELDS:
|
||||
raise ConsumerEvidenceError(f"successor_modules[{index}] fields differ")
|
||||
return ReviewedSuccessorModule(
|
||||
path=_repo_relative_posix(value["path"], f"successor_modules[{index}].path"),
|
||||
sha256=_sha256(value["sha256"], f"successor_modules[{index}].sha256"),
|
||||
)
|
||||
|
||||
|
||||
def _reviewed_modules(value: object) -> tuple[ReviewedSuccessorModule, ...]:
|
||||
if not isinstance(value, list) or not value:
|
||||
raise ConsumerEvidenceError("successor module authority must be nonempty")
|
||||
modules = tuple(_module_record(item, index) for index, item in enumerate(value))
|
||||
paths = tuple(module.path for module in modules)
|
||||
digests = tuple(module.sha256 for module in modules)
|
||||
if paths != tuple(sorted(paths)):
|
||||
raise ConsumerEvidenceError("successor module paths must be canonically ordered")
|
||||
if len(paths) != len(set(paths)):
|
||||
raise ConsumerEvidenceError("successor module paths are duplicated or conflicting")
|
||||
if len(digests) != len(set(digests)):
|
||||
raise ConsumerEvidenceError("successor module digests are duplicated or conflicting")
|
||||
return modules
|
||||
|
||||
|
||||
def _load_consumer_authority(
|
||||
path: Path = EVIDENCE,
|
||||
) -> tuple[Mapping[str, ConsumerTerminalEvidence], Mapping[str, ReviewedSuccessorModule]]:
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise ConsumerEvidenceError("terminal evidence is not readable") from exc
|
||||
if not content:
|
||||
raise ConsumerEvidenceError("terminal evidence must contain nonzero bytes")
|
||||
try:
|
||||
document = json.loads(content)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ConsumerEvidenceError("terminal evidence must be valid JSON") from exc
|
||||
fields = {"schema_version", "entries", "successor_modules"}
|
||||
if not isinstance(document, dict) or set(document) != fields or document["schema_version"] != SCHEMA:
|
||||
raise ConsumerEvidenceError("terminal evidence schema is invalid")
|
||||
values = document["entries"]
|
||||
if not isinstance(values, list) or not values:
|
||||
raise ConsumerEvidenceError("terminal evidence entries must be nonempty")
|
||||
records = tuple(_record(value, index) for index, value in enumerate(values))
|
||||
if len({record.entry_id for record in records}) != len(records):
|
||||
raise ConsumerEvidenceError("terminal evidence identities are duplicated")
|
||||
modules = _reviewed_modules(document["successor_modules"])
|
||||
migrated_paths = {record.successor_path for record in records if record.disposition == "v2_migrated"}
|
||||
module_paths = {module.path for module in modules}
|
||||
if None in migrated_paths or migrated_paths != module_paths:
|
||||
raise ConsumerEvidenceError("successor module authority must exactly cover migrated rows")
|
||||
entries_by_id = MappingProxyType({record.entry_id: record for record in records})
|
||||
modules_by_path = MappingProxyType({module.path: module for module in modules})
|
||||
return entries_by_id, modules_by_path
|
||||
|
||||
|
||||
def load_consumer_terminal_evidence(path: Path = EVIDENCE) -> Mapping[str, ConsumerTerminalEvidence]:
|
||||
return _load_consumer_authority(path)[0]
|
||||
|
||||
|
||||
def load_reviewed_successor_modules(path: Path = EVIDENCE) -> Mapping[str, ReviewedSuccessorModule]:
|
||||
return _load_consumer_authority(path)[1]
|
||||
|
||||
|
||||
def review_successor_modules(paths: tuple[str, ...], *, root: Path = REPO) -> tuple[ReviewedSuccessorModule, ...]:
|
||||
"""Explicit helper for a human-reviewed manifest update; validation never calls it."""
|
||||
reviewed = []
|
||||
for relative in paths:
|
||||
relative = _repo_relative_posix(relative, "successor module path")
|
||||
try:
|
||||
content = (root / relative).read_bytes()
|
||||
except OSError as exc:
|
||||
raise ConsumerEvidenceError(f"successor module is not readable: {relative}") from exc
|
||||
if not content:
|
||||
raise ConsumerEvidenceError(f"successor module must contain nonzero bytes: {relative}")
|
||||
reviewed.append(ReviewedSuccessorModule(relative, hashlib.sha256(content).hexdigest()))
|
||||
return tuple(reviewed)
|
||||
|
||||
|
||||
CONSUMER_TERMINAL_EVIDENCE, REVIEWED_SUCCESSOR_MODULES = _load_consumer_authority()
|
||||
|
||||
|
||||
def migrated_pytest_node_ids() -> tuple[str, ...]:
|
||||
"""Return the reviewed executable selectors without duplicate helper owners."""
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
evidence.pytest_node_id
|
||||
for evidence in CONSUMER_TERMINAL_EVIDENCE.values()
|
||||
if evidence.disposition == "v2_migrated" and evidence.pytest_node_id is not None
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CONSUMER_TERMINAL_EVIDENCE",
|
||||
"REVIEWED_SUCCESSOR_MODULES",
|
||||
"ConsumerEvidenceError",
|
||||
"ConsumerTerminalEvidence",
|
||||
"ReviewedSuccessorModule",
|
||||
"load_consumer_terminal_evidence",
|
||||
"load_reviewed_successor_modules",
|
||||
"migrated_pytest_node_ids",
|
||||
"review_successor_modules",
|
||||
]
|
||||
@@ -3,13 +3,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, BenchmarkTerminalEvidence
|
||||
from consumer_migration_evidence import CONSUMER_TERMINAL_EVIDENCE, REVIEWED_SUCCESSOR_MODULES, ConsumerTerminalEvidence
|
||||
|
||||
from openultrasast.core.legacy import LegacyArtifact, decode_legacy_artifact
|
||||
|
||||
@@ -491,7 +495,339 @@ def _benchmark_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Pat
|
||||
return True
|
||||
|
||||
|
||||
def _actual_definitions(tree: ast.Module) -> dict[str, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef]:
|
||||
definitions: dict[str, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef] = {}
|
||||
|
||||
def visit(nodes: list[ast.stmt], parents: tuple[str, ...] = ()) -> None:
|
||||
for node in nodes:
|
||||
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
qualified = ".".join((*parents, node.name))
|
||||
definitions[qualified] = node
|
||||
visit(node.body, (*parents, node.name))
|
||||
|
||||
visit(tree.body)
|
||||
return definitions
|
||||
|
||||
|
||||
def _exact_anchor_present(anchor: str, tree: ast.Module) -> bool:
|
||||
kind, separator, value = anchor.partition(":")
|
||||
if not separator:
|
||||
return False
|
||||
if kind == "definition":
|
||||
return value in _actual_definitions(tree)
|
||||
return value in _capability_index(tree).get(kind, set())
|
||||
|
||||
|
||||
def _retired_anchor_present(entry: LegacyInventoryEntry, anchor: str, tree: ast.Module) -> bool:
|
||||
kind, separator, _ = anchor.partition(":")
|
||||
if not separator or kind not in {"name", "constant"} or entry.symbol == "<module>":
|
||||
return _exact_anchor_present(anchor, tree)
|
||||
owner = _actual_definitions(tree).get(entry.symbol)
|
||||
if not isinstance(owner, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
return False
|
||||
scoped = ast.Module(body=owner.body, type_ignores=[])
|
||||
return _exact_anchor_present(anchor, scoped)
|
||||
|
||||
|
||||
def _target_names(node: ast.AST) -> tuple[str, ...]:
|
||||
if isinstance(node, ast.Name):
|
||||
return (node.id,)
|
||||
if isinstance(node, (ast.List, ast.Tuple)):
|
||||
return tuple(name for item in node.elts for name in _target_names(item))
|
||||
if isinstance(node, ast.Starred):
|
||||
return _target_names(node.value)
|
||||
return ()
|
||||
|
||||
|
||||
def _pattern_names(node: ast.pattern) -> tuple[str, ...]:
|
||||
names = tuple(value for value in (getattr(node, "name", None), getattr(node, "rest", None)) if isinstance(value, str))
|
||||
return (*names, *(name for child in ast.iter_child_nodes(node) if isinstance(child, ast.pattern) for name in _pattern_names(child)))
|
||||
|
||||
|
||||
def _module_scope_nodes(node: ast.AST) -> tuple[ast.AST, ...]:
|
||||
nodes = [node]
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)):
|
||||
return tuple(nodes)
|
||||
for child in ast.iter_child_nodes(node):
|
||||
nodes.extend(_module_scope_nodes(child))
|
||||
return tuple(nodes)
|
||||
|
||||
|
||||
def _dynamic_import_signature(node: ast.Assign | ast.AnnAssign) -> str | None:
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
value = node.value
|
||||
if len(targets) != 1 or not isinstance(targets[0], ast.Name) or not isinstance(value, ast.Call):
|
||||
return None
|
||||
if _call_leaf(value) != "import_module" or len(value.args) != 1 or value.keywords:
|
||||
return None
|
||||
module = value.args[0]
|
||||
if not isinstance(module, ast.Constant) or not isinstance(module.value, str):
|
||||
return None
|
||||
return f"dynamic_import:{module.value}"
|
||||
|
||||
|
||||
def _node_bindings(node: ast.AST) -> tuple[tuple[str, str], ...]:
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
return ((node.name, type(node).__name__),)
|
||||
if isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
signature = _dynamic_import_signature(node) or type(node).__name__
|
||||
return tuple((name, signature) for target in targets for name in _target_names(target))
|
||||
if isinstance(node, (ast.AugAssign, ast.NamedExpr)):
|
||||
return tuple((name, type(node).__name__) for name in _target_names(node.target))
|
||||
if isinstance(node, ast.Delete):
|
||||
return tuple((name, "Delete") for target in node.targets for name in _target_names(target))
|
||||
if isinstance(node, (ast.For, ast.AsyncFor)):
|
||||
return tuple((name, type(node).__name__) for name in _target_names(node.target))
|
||||
if isinstance(node, (ast.With, ast.AsyncWith)):
|
||||
return tuple((name, type(node).__name__) for item in node.items if item.optional_vars for name in _target_names(item.optional_vars))
|
||||
if isinstance(node, ast.ExceptHandler) and node.name:
|
||||
return ((node.name, "ExceptHandler"),)
|
||||
if isinstance(node, ast.match_case):
|
||||
return tuple((name, "MatchCase") for name in _pattern_names(node.pattern))
|
||||
return _import_bindings(node)
|
||||
|
||||
|
||||
def _import_bindings(node: ast.AST) -> tuple[tuple[str, str], ...]:
|
||||
if isinstance(node, ast.Import):
|
||||
return tuple((alias.asname or alias.name.split(".", 1)[0], f"import:{alias.name}") for alias in node.names)
|
||||
if isinstance(node, ast.ImportFrom) and node.module is not None:
|
||||
return tuple((alias.asname or alias.name, f"import_from:{node.module}:{alias.name}") for alias in node.names)
|
||||
return ()
|
||||
|
||||
|
||||
def _binding_events(tree: ast.Module, names: set[str]) -> dict[str, list[tuple[int, int, str]]]:
|
||||
events: dict[str, list[tuple[int, int, str]]] = {name: [] for name in names}
|
||||
for node in _module_scope_nodes(tree):
|
||||
for name, signature in _node_bindings(node):
|
||||
if name in events:
|
||||
events[name].append((getattr(node, "lineno", 0), getattr(node, "col_offset", 0), signature))
|
||||
return events
|
||||
|
||||
|
||||
def _direct_reviewed_functions(tree: ast.Module, names: set[str]) -> dict[str, ast.FunctionDef]:
|
||||
result: dict[str, ast.FunctionDef] = {}
|
||||
for name in names:
|
||||
matches = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name]
|
||||
if len(matches) != 1:
|
||||
raise LegacyInventoryError(f"migrated test owner binding terminal binding requires one direct FunctionDef: {name}")
|
||||
result[name] = matches[0]
|
||||
return result
|
||||
|
||||
|
||||
def _marker_leaf(node: ast.AST) -> str:
|
||||
value = node.func if isinstance(node, ast.Call) else node
|
||||
if isinstance(value, ast.Attribute):
|
||||
return value.attr
|
||||
if isinstance(value, ast.Name):
|
||||
return value.id
|
||||
return ""
|
||||
|
||||
|
||||
def _validate_pytest_outcome(tree: ast.Module, selectors: set[str], functions: dict[str, ast.FunctionDef]) -> None:
|
||||
forbidden = {"skip", "skipif", "xfail"}
|
||||
if any(
|
||||
_marker_leaf(candidate) in forbidden
|
||||
for name in selectors
|
||||
for decorator in functions[name].decorator_list
|
||||
for candidate in ast.walk(decorator)
|
||||
):
|
||||
raise LegacyInventoryError("migrated test pytest outcome is disabled by a reviewed-selector decorator")
|
||||
for node in _module_scope_nodes(tree):
|
||||
if isinstance(node, (ast.Assign, ast.Delete)):
|
||||
targets = node.targets
|
||||
elif isinstance(node, (ast.AnnAssign, ast.AugAssign)):
|
||||
targets = [node.target]
|
||||
else:
|
||||
targets = []
|
||||
bindings = _node_bindings(node)
|
||||
if any(name == "pytestmark" for name, _ in bindings):
|
||||
raise LegacyInventoryError("migrated test pytest outcome is disabled by module pytestmark")
|
||||
if any(
|
||||
isinstance(target, ast.Attribute) and target.attr == "pytestmark" and _dotted(target.value) in selectors for target in targets
|
||||
):
|
||||
raise LegacyInventoryError("migrated test pytest outcome is disabled by function pytestmark")
|
||||
if isinstance(node, ast.Call) and _call_leaf(node) in {"globals", "locals", "setattr", "exec"}:
|
||||
raise LegacyInventoryError("migrated test terminal binding has opaque namespace mutation")
|
||||
if any(name in {"__getattr__", "__dir__"} for name, _ in bindings):
|
||||
raise LegacyInventoryError("migrated test terminal binding has opaque module namespace hooks")
|
||||
|
||||
|
||||
def _direct_semantic_event(tree: ast.Module, local_name: str, binding: str) -> tuple[int, int, str]:
|
||||
matches = [
|
||||
(node.lineno, node.col_offset, signature)
|
||||
for node in tree.body
|
||||
for name, signature in _node_bindings(node)
|
||||
if name == local_name and signature == binding
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise LegacyInventoryError(f"migrated test semantic evidence terminal binding lacks one direct reviewed import: {local_name}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _validate_terminal_bindings(tree: ast.Module, evidence: ConsumerTerminalEvidence) -> dict[str, ast.FunctionDef]:
|
||||
reviewed = (evidence,)
|
||||
callable_names = {
|
||||
name for item in reviewed for name in (item.successor_symbol, item.probe_symbol, item.binding_owner_symbol) if name is not None
|
||||
}
|
||||
semantic_bindings = {(item.semantic_local_name, item.semantic_binding) for item in reviewed}
|
||||
if any(local is None or binding is None for local, binding in semantic_bindings):
|
||||
raise LegacyInventoryError("migrated test terminal binding evidence is incomplete")
|
||||
semantic_names = {local for local, _ in semantic_bindings if local is not None}
|
||||
functions = _direct_reviewed_functions(tree, callable_names)
|
||||
events = _binding_events(tree, callable_names | semantic_names)
|
||||
for name, function in functions.items():
|
||||
reviewed_event = (function.lineno, function.col_offset, "FunctionDef")
|
||||
if not events[name] or events[name][-1] != reviewed_event:
|
||||
raise LegacyInventoryError(f"migrated test terminal binding was replaced or removed: {name}")
|
||||
for local_name, binding in semantic_bindings:
|
||||
assert local_name is not None and binding is not None
|
||||
reviewed_event = _direct_semantic_event(tree, local_name, binding)
|
||||
if not events[local_name] or events[local_name][-1] != reviewed_event:
|
||||
raise LegacyInventoryError(f"migrated test terminal binding replaced reviewed import: {local_name}")
|
||||
selectors = {item.probe_symbol for item in reviewed if item.probe_symbol is not None}
|
||||
_validate_pytest_outcome(tree, selectors, functions)
|
||||
return functions
|
||||
|
||||
|
||||
def _normalized_ast(value: object) -> object:
|
||||
if isinstance(value, ast.AST):
|
||||
fields = tuple((field, _normalized_ast(item)) for field, item in ast.iter_fields(value) if field != "type_params")
|
||||
return type(value).__name__, fields
|
||||
if isinstance(value, list):
|
||||
return tuple(_normalized_ast(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def _callable_ast_digest(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
|
||||
return hashlib.sha256(repr(_normalized_ast(node)).encode()).hexdigest()
|
||||
|
||||
|
||||
def _references_binding(node: ast.FunctionDef | ast.AsyncFunctionDef, local_name: str) -> bool:
|
||||
return any(isinstance(candidate, ast.Name) and candidate.id == local_name for candidate in ast.walk(node))
|
||||
|
||||
|
||||
def _canonical_relative_parts(relative: str) -> tuple[str, ...]:
|
||||
parts = tuple(relative.split("/"))
|
||||
if not relative or relative.startswith("/") or "\0" in relative or "\\" in relative or any(part in {"", ".", ".."} for part in parts):
|
||||
raise LegacyInventoryError(f"migrated test successor path is not canonical: {relative}")
|
||||
if len(parts[0]) == 2 and parts[0][1] == ":":
|
||||
raise LegacyInventoryError(f"migrated test successor path is not canonical: {relative}")
|
||||
return parts
|
||||
|
||||
|
||||
def _resolved_beneath_root(relative: str, root: Path) -> None:
|
||||
try:
|
||||
root_status = root.lstat()
|
||||
except OSError as exc:
|
||||
raise LegacyInventoryError("migrated test successor supplied root is not a real directory") from exc
|
||||
if not stat.S_ISDIR(root_status.st_mode):
|
||||
raise LegacyInventoryError("migrated test successor supplied root is not a real directory")
|
||||
try:
|
||||
resolved_root = root.resolve(strict=True)
|
||||
resolved_candidate = (root / relative).resolve(strict=True)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}") from exc
|
||||
if not resolved_candidate.is_relative_to(resolved_root):
|
||||
raise LegacyInventoryError(f"migrated test successor is not a symlink-free file beneath the supplied root: {relative}")
|
||||
|
||||
|
||||
def _read_from_descriptor(descriptor: int, relative: str) -> bytes:
|
||||
status = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(status.st_mode) or not status.st_mode & 0o444:
|
||||
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}")
|
||||
chunks = []
|
||||
while chunk := os.read(descriptor, 1024 * 1024):
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _read_file_beneath_root(relative: str, *, root: Path) -> bytes:
|
||||
parts = _canonical_relative_parts(relative)
|
||||
_resolved_beneath_root(relative, root)
|
||||
directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||
file_flags = os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||
descriptors = []
|
||||
try:
|
||||
descriptors.append(os.open(root, directory_flags))
|
||||
for part in parts[:-1]:
|
||||
descriptors.append(os.open(part, directory_flags, dir_fd=descriptors[-1]))
|
||||
descriptors.append(os.open(parts[-1], file_flags, dir_fd=descriptors[-1]))
|
||||
return _read_from_descriptor(descriptors[-1], relative)
|
||||
except LegacyInventoryError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
if exc.errno in {errno.EACCES, errno.ENOENT, errno.EPERM}:
|
||||
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}") from exc
|
||||
raise LegacyInventoryError(f"migrated test successor is not a symlink-free file beneath the supplied root: {relative}") from exc
|
||||
finally:
|
||||
for descriptor in reversed(descriptors):
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _read_reviewed_successor_module(relative: str, *, root: Path) -> ast.Module:
|
||||
reviewed = REVIEWED_SUCCESSOR_MODULES.get(relative)
|
||||
if reviewed is None:
|
||||
raise LegacyInventoryError(f"migrated test successor is outside module authority: {relative}")
|
||||
content = _read_file_beneath_root(relative, root=root)
|
||||
if not content:
|
||||
raise LegacyInventoryError(f"migrated test successor must contain nonzero bytes: {relative}")
|
||||
if hashlib.sha256(content).hexdigest() != reviewed.sha256:
|
||||
raise LegacyInventoryError(f"migrated test successor module digest changed: {relative}")
|
||||
try:
|
||||
return ast.parse(content, filename=relative)
|
||||
except (SyntaxError, UnicodeDecodeError) as exc:
|
||||
raise LegacyInventoryError(f"migrated test successor is not parseable: {relative}") from exc
|
||||
|
||||
|
||||
def _validate_migrated_evidence(entry: LegacyInventoryEntry, evidence: ConsumerTerminalEvidence, *, root: Path) -> None:
|
||||
assert evidence.successor_path and evidence.successor_symbol and evidence.semantic_binding and evidence.probe_symbol
|
||||
assert evidence.successor_ast_sha256 and evidence.probe_ast_sha256
|
||||
assert evidence.binding_owner_symbol and evidence.binding_owner_ast_sha256 and evidence.semantic_local_name
|
||||
successor = _read_reviewed_successor_module(evidence.successor_path, root=root)
|
||||
definitions = _validate_terminal_bindings(successor, evidence)
|
||||
owner = definitions.get(evidence.successor_symbol)
|
||||
probe = definitions.get(evidence.probe_symbol)
|
||||
binding_owner = definitions.get(evidence.binding_owner_symbol)
|
||||
anchors_present = all(_exact_anchor_present(anchor, successor) for anchor in evidence.capability_anchors)
|
||||
callables = (owner, probe, binding_owner)
|
||||
if (
|
||||
entry.owner_boundary != "tests"
|
||||
or not anchors_present
|
||||
or not all(isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) for node in callables)
|
||||
):
|
||||
raise LegacyInventoryError(f"migrated test owner binding is absent: {entry.entry_id}")
|
||||
assert isinstance(owner, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
assert isinstance(probe, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
assert isinstance(binding_owner, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
digests = (_callable_ast_digest(owner), _callable_ast_digest(probe), _callable_ast_digest(binding_owner))
|
||||
expected = (evidence.successor_ast_sha256, evidence.probe_ast_sha256, evidence.binding_owner_ast_sha256)
|
||||
if digests != expected or not _references_binding(binding_owner, evidence.semantic_local_name):
|
||||
raise LegacyInventoryError(f"migrated test semantic evidence does not match reviewed ownership: {entry.entry_id}")
|
||||
|
||||
|
||||
def _consumer_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
|
||||
evidence = CONSUMER_TERMINAL_EVIDENCE.get(entry.entry_id)
|
||||
if evidence is None:
|
||||
return False
|
||||
if (
|
||||
evidence.stable_fields != _stable_entry_fields(entry)
|
||||
or evidence.disposition != entry.disposition.value
|
||||
or entry.entry_id != entry_identity(entry)
|
||||
):
|
||||
raise LegacyInventoryError(f"consumer terminal evidence is absent or mismatched: {entry.entry_id}")
|
||||
if evidence.closure_kind == "migrated_semantic_probe":
|
||||
_validate_migrated_evidence(entry, evidence, root=root)
|
||||
return True
|
||||
present = tuple(anchor for anchor in evidence.absence_anchors if _retired_anchor_present(entry, anchor, tree))
|
||||
if present:
|
||||
raise LegacyInventoryError(f"retired consumer capability remains live: {entry.entry_id}:{present}")
|
||||
return True
|
||||
|
||||
|
||||
def _approved_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
|
||||
if _consumer_terminal_source_closure(entry, root=root, tree=tree):
|
||||
return True
|
||||
if _benchmark_terminal_source_closure(entry, root=root, tree=tree):
|
||||
return True
|
||||
approved = _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition
|
||||
|
||||
@@ -9,11 +9,6 @@ from collections.abc import Sequence
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
_TASK_FILES = (
|
||||
"tests/core_migration_boundary.py",
|
||||
"tests/core_migration_scope.py",
|
||||
"tests/test_core_migration_boundary.py",
|
||||
)
|
||||
_FIXTURE_PREFIXES = ("benchmarks/fixtures/", "tests/fixtures/")
|
||||
COMPATIBILITY_PATHS = frozenset(
|
||||
"""src/openultrasast/core/__init__.py src/openultrasast/core/_legacy_factpack.py
|
||||
@@ -48,10 +43,10 @@ def _owned_python(relative: str) -> bool:
|
||||
|
||||
|
||||
def boundary_python(root: Path = REPO) -> tuple[Path, ...]:
|
||||
"""Return owned tracked Python, excluding only harvested fixture material."""
|
||||
completed = subprocess.run(["git", "ls-files", "*.py"], cwd=root, check=True, capture_output=True, text=True)
|
||||
relatives = {value for value in completed.stdout.splitlines() if _owned_python(value)}
|
||||
relatives.update(path for path in _TASK_FILES if (root / path).is_file())
|
||||
"""Return owned tracked and current untracked Python source."""
|
||||
commands = (("git", "ls-files", "*.py"), ("git", "ls-files", "--others", "--exclude-standard", "--", "*.py"))
|
||||
outputs = (subprocess.run(command, cwd=root, check=True, capture_output=True, text=True).stdout for command in commands)
|
||||
relatives = {value for output in outputs for value in output.splitlines() if _owned_python(value)}
|
||||
return tuple(root / relative for relative in sorted(relatives))
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+155
-155
File diff suppressed because it is too large
Load Diff
@@ -1,191 +1 @@
|
||||
"""Paired scoring and the admission report (Task 12.6, Reqs 5.2, 12.4, 12.5, 12.7-12.9)."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.valid.admission_scoring import (
|
||||
AdmissionCase,
|
||||
AdmissionScoringError,
|
||||
OrdinaryChange,
|
||||
admission_report,
|
||||
)
|
||||
from openultrasast.core.legacy_v1 import AdmissionManifest, validate_admission_manifest
|
||||
|
||||
|
||||
def admission_manifest(role: str = "admission") -> AdmissionManifest:
|
||||
payload = {
|
||||
"produced_by": "valid-detection-benchmark",
|
||||
"reviewed": True,
|
||||
"label_isolated": True,
|
||||
"spent": False,
|
||||
"population_role": role,
|
||||
"analyzer_revision": "analyzer-1",
|
||||
"fact_set_digest": "facts-1",
|
||||
"acquisition_record_digest": "acq-1",
|
||||
"configuration_digest": "cfg-1",
|
||||
"population_digest": "pop-1",
|
||||
"labels": ("population-overview",),
|
||||
}
|
||||
return validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def correct_pair(case_id: str = "p1", *, seconds: float = 1.0) -> AdmissionCase:
|
||||
return AdmissionCase(
|
||||
case_id=case_id,
|
||||
vulnerable_judgment="unguarded",
|
||||
fixed_judgment="discharged",
|
||||
location_verified=True,
|
||||
seconds=seconds,
|
||||
)
|
||||
|
||||
|
||||
def quiet_ordinary(change_id: str = "o1") -> OrdinaryChange:
|
||||
return OrdinaryChange(change_id=change_id, judgment="discharged", mislabeled_defect=False)
|
||||
|
||||
|
||||
def test_vulnerable_only_pair_scores_correct() -> None:
|
||||
report = admission_report(admission_manifest(), (correct_pair(),), (quiet_ordinary(),), coverage=1.0)
|
||||
assert report["pairs"] == {"n": 1, "correct": 1}
|
||||
assert report["pair_correctness"]["rate"] == 1.0
|
||||
assert report["fix_recall"]["rate"] == 1.0
|
||||
assert report["decision"] == "GO"
|
||||
assert report["failed_gates"] == []
|
||||
|
||||
|
||||
def test_fixed_side_regression_counts_pair_incorrect() -> None:
|
||||
regressed = AdmissionCase(
|
||||
case_id="p1",
|
||||
vulnerable_judgment="unguarded",
|
||||
fixed_judgment="unguarded",
|
||||
location_verified=True,
|
||||
seconds=1.0,
|
||||
)
|
||||
report = admission_report(admission_manifest(), (regressed,), (), coverage=1.0)
|
||||
assert report["pairs"]["correct"] == 0
|
||||
assert report["pair_correctness"]["rate"] == 0.0
|
||||
assert report["fix_recall"]["rate"] == 1.0
|
||||
|
||||
|
||||
def test_indeterminate_vulnerable_side_is_never_clean() -> None:
|
||||
indeterminate = AdmissionCase(
|
||||
case_id="p1",
|
||||
vulnerable_judgment="indeterminate",
|
||||
fixed_judgment="discharged",
|
||||
location_verified=False,
|
||||
seconds=1.0,
|
||||
)
|
||||
report = admission_report(admission_manifest(), (indeterminate,), (), coverage=1.0)
|
||||
assert report["fix_recall"]["count"] == 0
|
||||
assert report["fix_recall"]["rate"] == 0.0
|
||||
assert report["indeterminate_rate"]["count"] == 1
|
||||
assert report["indeterminate_rate"]["rate"] > 0.0
|
||||
|
||||
|
||||
def test_mislabeled_negative_is_refused() -> None:
|
||||
mislabeled = OrdinaryChange(change_id="o1", judgment="discharged", mislabeled_defect=True)
|
||||
with pytest.raises(AdmissionScoringError, match="mislabeled"):
|
||||
admission_report(admission_manifest(), (correct_pair(),), (mislabeled,), coverage=1.0)
|
||||
|
||||
|
||||
def test_development_population_is_refused() -> None:
|
||||
with pytest.raises(AdmissionScoringError, match="population-role"):
|
||||
admission_report(admission_manifest(role="development"), (correct_pair(),), (), coverage=1.0)
|
||||
|
||||
|
||||
def test_verified_location_recall_keeps_sample_count() -> None:
|
||||
unverified = AdmissionCase(
|
||||
case_id="p2",
|
||||
vulnerable_judgment="unguarded",
|
||||
fixed_judgment="discharged",
|
||||
location_verified=False,
|
||||
seconds=1.0,
|
||||
)
|
||||
report = admission_report(admission_manifest(), (correct_pair("p1"), unverified), (), coverage=1.0)
|
||||
assert report["verified_location_recall"]["n"] == 2
|
||||
assert report["verified_location_recall"]["count"] == 1
|
||||
assert report["verified_location_recall"]["rate"] == 0.5
|
||||
|
||||
|
||||
def test_flagged_ordinary_per_100_and_precision() -> None:
|
||||
flagged = [OrdinaryChange(f"o{i}", "unguarded", False) for i in range(2)]
|
||||
quiet = [OrdinaryChange(f"q{i}", "discharged", False) for i in range(2)]
|
||||
pairs = tuple(correct_pair(f"p{i}") for i in range(3))
|
||||
report = admission_report(admission_manifest(), pairs, tuple(flagged + quiet), coverage=1.0)
|
||||
per_100 = report["flagged_ordinary_per_100"]
|
||||
assert per_100["n"] == 4
|
||||
assert per_100["count"] == 2
|
||||
assert per_100["rate"] == 50.0
|
||||
assert per_100["wilson95"][0] is not None and per_100["wilson95"][0] <= 50.0 <= per_100["wilson95"][1]
|
||||
assert report["precision"]["rate"] == pytest.approx(3 / 5)
|
||||
|
||||
|
||||
def test_budget_gate_returns_no_go() -> None:
|
||||
slow = correct_pair("slow", seconds=121.0)
|
||||
report = admission_report(admission_manifest(), (slow,), (), coverage=1.0)
|
||||
assert report["decision"] == "NO-GO"
|
||||
assert any("budget" in gate for gate in report["failed_gates"])
|
||||
assert any("slow" in gate for gate in report["failed_gates"])
|
||||
|
||||
|
||||
def test_instrument_gate_returns_no_go() -> None:
|
||||
report = admission_report(admission_manifest(), (correct_pair(),), (), coverage=1.0, instrument_failures=1)
|
||||
assert report["decision"] == "NO-GO"
|
||||
assert any("instrument" in gate for gate in report["failed_gates"])
|
||||
|
||||
|
||||
def test_pre_registered_criterion_failure_returns_no_go() -> None:
|
||||
missed = AdmissionCase(
|
||||
case_id="p1",
|
||||
vulnerable_judgment="discharged",
|
||||
fixed_judgment="discharged",
|
||||
location_verified=False,
|
||||
seconds=1.0,
|
||||
)
|
||||
report = admission_report(
|
||||
admission_manifest(),
|
||||
(missed,),
|
||||
(),
|
||||
coverage=1.0,
|
||||
criteria={"min_fix_recall": 1.0},
|
||||
)
|
||||
assert report["decision"] == "NO-GO"
|
||||
assert any("min_fix_recall" in gate for gate in report["failed_gates"])
|
||||
|
||||
|
||||
def test_invalid_judgment_is_refused() -> None:
|
||||
broken = AdmissionCase(
|
||||
case_id="p1",
|
||||
vulnerable_judgment="banana",
|
||||
fixed_judgment="discharged",
|
||||
location_verified=True,
|
||||
seconds=1.0,
|
||||
)
|
||||
with pytest.raises(AdmissionScoringError, match="judgment"):
|
||||
admission_report(admission_manifest(), (broken,), (), coverage=1.0)
|
||||
|
||||
|
||||
def test_report_is_deterministic() -> None:
|
||||
args = (
|
||||
admission_manifest(),
|
||||
(correct_pair("p1"), correct_pair("p2")),
|
||||
(quiet_ordinary("o1"), OrdinaryChange("o2", "unguarded", False)),
|
||||
)
|
||||
first = admission_report(*args, coverage=0.75)
|
||||
second = admission_report(*args, coverage=0.75)
|
||||
assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)
|
||||
|
||||
|
||||
def test_report_carries_provenance_without_labels() -> None:
|
||||
report = admission_report(admission_manifest(), (correct_pair(),), (), coverage=1.0)
|
||||
manifest = report["manifest"]
|
||||
assert manifest["analyzer_revision"] == "analyzer-1"
|
||||
assert manifest["population_digest"] == "pop-1"
|
||||
assert manifest["population_role"] == "admission"
|
||||
text = json.dumps(report, sort_keys=True)
|
||||
for marker in ("vulnerability", "fix-location", "ordinary-change"):
|
||||
assert marker not in text
|
||||
"""Retired authorization-shaped v1 scorer tests; admission conformance uses v2."""
|
||||
|
||||
@@ -1,82 +1 @@
|
||||
"""Admission contract tests for detection-core tasks 4.1-4.2 (Req 7, 10, 12)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.admission import ( # noqa: E402
|
||||
REPORT_SCHEMA_KEYS,
|
||||
AdmissionManifest,
|
||||
AdmissionManifestError,
|
||||
validate_admission_manifest,
|
||||
)
|
||||
|
||||
|
||||
def _payload(**over: object) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"produced_by": "valid-detection-benchmark",
|
||||
"reviewed": True,
|
||||
"label_isolated": True,
|
||||
"spent": False,
|
||||
"population_role": "admission",
|
||||
"analyzer_revision": "d063167",
|
||||
"fact_set_digest": "3eb5bf699434",
|
||||
"acquisition_record_digest": "none",
|
||||
"configuration_digest": "095b5a4905ad",
|
||||
"population_digest": "c866c0e812c0",
|
||||
"labels": [],
|
||||
"families": ["access_control"],
|
||||
}
|
||||
payload.update(over)
|
||||
return payload
|
||||
|
||||
|
||||
def test_valid_manifest_accepted_with_family_scope() -> None:
|
||||
manifest = validate_admission_manifest(_payload())
|
||||
assert isinstance(manifest, AdmissionManifest)
|
||||
assert manifest.population_role == "admission"
|
||||
assert manifest.families == ("access_control",)
|
||||
|
||||
|
||||
def test_refusal_order_is_deterministic() -> None:
|
||||
cases = [
|
||||
({}, "malformed"),
|
||||
(_payload(produced_by="other"), "producer"),
|
||||
(_payload(reviewed=False), "unreviewed"),
|
||||
(_payload(label_isolated=False), "label-isolated"),
|
||||
(_payload(labels=["vulnerability"]), "label-exposure"),
|
||||
(_payload(spent=True), "spent"),
|
||||
]
|
||||
for payload, reason in cases:
|
||||
try:
|
||||
validate_admission_manifest(payload)
|
||||
raise AssertionError(f"{reason} must be refused")
|
||||
except AdmissionManifestError as error:
|
||||
assert reason in str(error)
|
||||
|
||||
|
||||
def test_acquisition_output_changed_refusal() -> None:
|
||||
try:
|
||||
validate_admission_manifest(_payload(), expected_acquisition_digest="different")
|
||||
raise AssertionError("acquisition mismatch must be refused")
|
||||
except AdmissionManifestError as error:
|
||||
assert "acquisition-output-changed" in str(error)
|
||||
|
||||
|
||||
def test_report_schema_requires_family_scope_and_counts() -> None:
|
||||
for key in (
|
||||
"manifest",
|
||||
"families",
|
||||
"pairs",
|
||||
"fix_recall",
|
||||
"flagged_ordinary_per_100",
|
||||
"precision",
|
||||
"indeterminate_rate",
|
||||
"coverage",
|
||||
"runtime",
|
||||
"instrument_failures",
|
||||
"failed_gates",
|
||||
"decision",
|
||||
):
|
||||
assert key in REPORT_SCHEMA_KEYS
|
||||
"""Retired v1 admission tests; family-neutral admission uses the v2 suites."""
|
||||
|
||||
@@ -1,60 +1 @@
|
||||
"""Measured-budget contract tests for detection-core task 2 (Req 4, 12)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.legacy_v1 import ( # noqa: E402
|
||||
Budget,
|
||||
BudgetGateFailure,
|
||||
Complexity,
|
||||
check_budget,
|
||||
derive_budget,
|
||||
)
|
||||
|
||||
SAMPLES = tuple((Complexity(regions=1 + i % 3, calls=10 * i, changed_bytes=100 * i), float(1 + i)) for i in range(20))
|
||||
|
||||
|
||||
def test_derive_budget_publishes_derivation_metadata() -> None:
|
||||
budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu")
|
||||
assert isinstance(budget, Budget)
|
||||
assert budget.workload_class == "dev-pool"
|
||||
assert budget.n == 20
|
||||
assert budget.percentile == 0.95
|
||||
assert budget.hardware == "ci-runner-4cpu"
|
||||
assert budget.seconds == 19.05 # linear-interpolated p95 of samples 1..20
|
||||
|
||||
|
||||
def test_derive_budget_is_deterministic() -> None:
|
||||
first = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h")
|
||||
second = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h")
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_derive_budget_refuses_empty_samples() -> None:
|
||||
try:
|
||||
derive_budget((), workload_class="dev", percentile=0.95, hardware="h")
|
||||
raise AssertionError("empty sample set must be refused")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def test_budget_is_configurable_per_class_not_a_constant() -> None:
|
||||
dev = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="h")
|
||||
pilot = derive_budget(tuple((c, s / 10) for c, s in SAMPLES), workload_class="pilot", percentile=0.95, hardware="h")
|
||||
assert dev.seconds != pilot.seconds
|
||||
assert dev.workload_class != pilot.workload_class
|
||||
|
||||
|
||||
def test_budget_gate_names_change_elapsed_and_derivation() -> None:
|
||||
budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu")
|
||||
check_budget("case-1", 5.0, budget)
|
||||
try:
|
||||
check_budget("case-2", 999.0, budget)
|
||||
raise AssertionError("over-budget change must fail the gate")
|
||||
except BudgetGateFailure as error:
|
||||
message = str(error)
|
||||
assert "case-2" in message
|
||||
assert "999" in message
|
||||
assert "dev-pool" in message and "ci-runner-4cpu" in message and "n=20" in message
|
||||
"""Retired v1 budget tests; family-neutral budget coverage uses v2."""
|
||||
|
||||
@@ -12,10 +12,6 @@ from detection_core_fixtures import InMemoryContentResolver, stored_artifact
|
||||
|
||||
from openultrasast.core import budget as budget_v2
|
||||
from openultrasast.core import canonical, evidence, provenance, semantics
|
||||
from openultrasast.core.legacy_v1 import Budget as LegacyBudget
|
||||
from openultrasast.core.legacy_v1 import Complexity as LegacyComplexity
|
||||
from openultrasast.core.legacy_v1 import check_budget as legacy_check_budget
|
||||
from openultrasast.core.legacy_v1 import derive_budget as legacy_derive_budget
|
||||
|
||||
|
||||
def _digest(value: str) -> str:
|
||||
@@ -343,10 +339,3 @@ def test_scope_revalidates_post_construction_tampering() -> None:
|
||||
object.__setattr__(allowed.regions[0], "end_line", 0)
|
||||
with pytest.raises((evidence.EvidenceValidationError, evidence.ScopeValidationError)):
|
||||
evidence.validate_scope(_packet(), allowed)
|
||||
|
||||
|
||||
def test_legacy_budget_api_remains_available() -> None:
|
||||
samples = ((LegacyComplexity(1, 1, 10), 1.0), (LegacyComplexity(2, 2, 20), 2.0))
|
||||
legacy = legacy_derive_budget(samples, workload_class="legacy", percentile=0.5, hardware="runner")
|
||||
assert isinstance(legacy, LegacyBudget)
|
||||
legacy_check_budget(change_id="change", seconds=1.0, budget=legacy)
|
||||
|
||||
+1
-144
@@ -1,144 +1 @@
|
||||
"""Core contract conformance tests for detection-core tasks 1.1-1.4 (Reqs 1, 2, 3, 5, 6, 9)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.legacy_v1 import ( # noqa: E402
|
||||
BoundedSilence,
|
||||
Coverage,
|
||||
Guard,
|
||||
Hop,
|
||||
InstrumentFailure,
|
||||
Judgment,
|
||||
ProvenanceMismatch,
|
||||
ProvenClaim,
|
||||
TraceRecord,
|
||||
Witness,
|
||||
as_clean,
|
||||
check_reads,
|
||||
stamp,
|
||||
verify_stamp,
|
||||
)
|
||||
|
||||
|
||||
def _trace() -> TraceRecord:
|
||||
return TraceRecord(
|
||||
entry="src/app.py::handler",
|
||||
hops=(Hop(function="handler", call_line=10, complete=True),),
|
||||
operation_kind="protected_read",
|
||||
resource="credentials",
|
||||
authority="admin",
|
||||
witnesses=(Witness(kind="guard", provenance="fact:checkPermission", relation="all_paths"),),
|
||||
guards=(Guard(kind="permission_check", effect="branches"),),
|
||||
completeness_reasons=(),
|
||||
judgment=Judgment.UNGUARDED,
|
||||
digests={"input": "abc123"},
|
||||
)
|
||||
|
||||
|
||||
def test_trace_serializes_byte_identical_on_repeat() -> None:
|
||||
first = _trace().canonical()
|
||||
second = _trace().canonical()
|
||||
assert first == second
|
||||
assert isinstance(first, bytes)
|
||||
|
||||
|
||||
def test_trace_record_pins_schema_fields() -> None:
|
||||
record = _trace().record()
|
||||
for field in (
|
||||
"entry",
|
||||
"hops",
|
||||
"operation_kind",
|
||||
"resource",
|
||||
"authority",
|
||||
"witnesses",
|
||||
"guards",
|
||||
"completeness_reasons",
|
||||
"judgment",
|
||||
"digests",
|
||||
):
|
||||
assert field in record
|
||||
assert record["judgment"] == "unguarded"
|
||||
|
||||
|
||||
def test_judgment_vocabulary_is_exactly_three_way() -> None:
|
||||
assert {j.value for j in Judgment} == {"unguarded", "discharged", "indeterminate"}
|
||||
|
||||
|
||||
def test_indeterminate_never_converts_to_clean() -> None:
|
||||
assert as_clean(Judgment.DISCHARGED) == Judgment.DISCHARGED
|
||||
try:
|
||||
as_clean(Judgment.INDETERMINATE)
|
||||
raise AssertionError("indeterminate must never convert to clean")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def test_proven_claim_requires_witness_and_location() -> None:
|
||||
claim = ProvenClaim(trace_id="t1", witness="w1", path="src/app.py", line=10, family="access_control")
|
||||
assert claim.family == "access_control"
|
||||
silence = BoundedSilence(path="src/app.py", reason="unreached", coverage={"entry_points": 0})
|
||||
assert silence.reason == "unreached"
|
||||
|
||||
|
||||
def test_coverage_requires_exactly_one_bucket_per_call() -> None:
|
||||
good = Coverage(entry_points=2, frameworks_recognised={"jenkins": 1}, calls=3, complete=1, incomplete=1, unresolved=1)
|
||||
good.validate()
|
||||
bad = Coverage(entry_points=1, frameworks_recognised={}, calls=3, complete=2, incomplete=1, unresolved=1)
|
||||
try:
|
||||
bad.validate()
|
||||
raise AssertionError("two buckets for one call must be refused")
|
||||
except ValueError:
|
||||
pass
|
||||
negative = Coverage(entry_points=1, frameworks_recognised={}, calls=3, complete=-1, incomplete=2, unresolved=2)
|
||||
try:
|
||||
negative.validate()
|
||||
raise AssertionError("negative bucket counts must be refused")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def test_verdict_contract_exposes_no_model_role_path() -> None:
|
||||
import openultrasast.core.verdict as verdict_module
|
||||
|
||||
public = verdict_module.__all__
|
||||
assert not any("model" in name.lower() or "judge" in name.lower() for name in public), (
|
||||
"the verdict contract must expose no model/judge path; models judge residuals zero-shot over a trace only"
|
||||
)
|
||||
|
||||
|
||||
def test_zero_byte_read_is_instrument_failure() -> None:
|
||||
try:
|
||||
check_reads({"src/app.py": 0, "src/lib.py": 12})
|
||||
raise AssertionError("zero-byte read must fail the run")
|
||||
except InstrumentFailure:
|
||||
pass
|
||||
check_reads({"src/app.py": 5})
|
||||
|
||||
|
||||
def test_provenance_stamp_and_mismatch_refusal() -> None:
|
||||
record = stamp(
|
||||
analyzer_revision="d063167",
|
||||
fact_set_digest="3eb5bf699434",
|
||||
acquisition_record_digest="none",
|
||||
configuration_digest="095b5a4905ad",
|
||||
population_digest="c866c0e812c0",
|
||||
change_id="case-1",
|
||||
)
|
||||
assert set(record) == {
|
||||
"analyzer_revision",
|
||||
"fact_set_digest",
|
||||
"acquisition_record_digest",
|
||||
"configuration_digest",
|
||||
"population_digest",
|
||||
"input_digest",
|
||||
}
|
||||
verify_stamp(record, record.copy())
|
||||
tampered = {**record, "fact_set_digest": "ffffffffffff"}
|
||||
try:
|
||||
verify_stamp(tampered, record)
|
||||
raise AssertionError("mismatched digest must be refused")
|
||||
except ProvenanceMismatch:
|
||||
pass
|
||||
"""Retired authorization-shaped v1 contracts; family-neutral contracts use v2."""
|
||||
|
||||
@@ -17,7 +17,6 @@ from openultrasast.core.coverage import (
|
||||
diagnose_first_zero,
|
||||
validate_instrument,
|
||||
)
|
||||
from openultrasast.core.legacy_v1 import InstrumentFailure as LegacyInstrumentFailure
|
||||
from openultrasast.core.semantics import RelationStatus
|
||||
|
||||
DIGEST_A = "a" * 64
|
||||
@@ -75,10 +74,7 @@ def test_input_manifest_and_receipts_validate_exactly() -> None:
|
||||
validate_instrument(_manifest(), _receipts())
|
||||
|
||||
|
||||
def test_v2_instrument_failure_is_distinct_from_legacy_and_owns_input_refusal() -> None:
|
||||
from openultrasast.core.coverage import InstrumentFailure
|
||||
|
||||
assert InstrumentFailure is not LegacyInstrumentFailure
|
||||
def test_v2_instrument_failure_owns_input_refusal() -> None:
|
||||
receipt = InputReadReceipt("src/app.py", "src/app.py", False, 12, DIGEST_A)
|
||||
with pytest.raises(InstrumentFailure, match="not opened"):
|
||||
validate_instrument(_manifest(), (receipt, _receipts()[1]))
|
||||
@@ -94,10 +90,6 @@ def test_zero_named_counts_are_allowed_for_measurements_but_not_unresolved_cover
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_instrument_failure_is_available_only_through_compatibility_boundary() -> None:
|
||||
assert LegacyInstrumentFailure.__module__ == "openultrasast.core.legacy_v1.contracts"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("opened", "byte_count", "digest", "match"),
|
||||
[
|
||||
|
||||
@@ -1,69 +1 @@
|
||||
"""Fact-pack and consumer contract tests for detection-core tasks 6.1-6.2 (Req 9, 10, 11)."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
|
||||
from openultrasast.core.legacy_v1 import FactPackError, admit_fact_pack, verify_fact_pack # noqa: E402
|
||||
from openultrasast.core.trace import TraceRecord # noqa: E402
|
||||
from openultrasast.core.verdict import BoundedSilence, Judgment, ProvenClaim # noqa: E402
|
||||
|
||||
FACTS = ({"id": "obligation:protected_read", "language": "java"},)
|
||||
|
||||
|
||||
def test_unreviewed_fact_pack_is_refused() -> None:
|
||||
try:
|
||||
admit_fact_pack("access_control", FACTS, review_digest="")
|
||||
raise AssertionError("unreviewed pack must be refused")
|
||||
except FactPackError:
|
||||
pass
|
||||
|
||||
|
||||
def test_reviewed_fact_pack_stamps_provenance() -> None:
|
||||
pack = admit_fact_pack("access_control", FACTS, review_digest="review-2026-10-30")
|
||||
assert pack.family == "access_control"
|
||||
assert pack.review_digest == "review-2026-10-30"
|
||||
assert len(pack.fact_set_digest) == 12
|
||||
verify_fact_pack(pack, FACTS)
|
||||
try:
|
||||
verify_fact_pack(pack, FACTS + ({"id": "tampered"},))
|
||||
raise AssertionError("tampered facts must be refused")
|
||||
except FactPackError:
|
||||
pass
|
||||
|
||||
|
||||
def _record() -> TraceRecord:
|
||||
return TraceRecord(
|
||||
entry="src/app.py::handler",
|
||||
hops=(),
|
||||
operation_kind="protected_read",
|
||||
resource="credentials",
|
||||
authority="admin",
|
||||
judgment=Judgment.UNGUARDED,
|
||||
digests={"input": "abc"},
|
||||
)
|
||||
|
||||
|
||||
def test_consumer_accepts_witnessed_claim_and_reasoned_silence() -> None:
|
||||
claim = ProvenClaim(trace_id="t1", witness="w1", path="src/app.py", line=10, family="access_control")
|
||||
silence = BoundedSilence(path="src/other.py", reason="unreached", coverage={"entry_points": 0})
|
||||
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), _record(), (claim,), (silence,))
|
||||
assert verdict.claims == (claim,)
|
||||
assert verdict.silence == (silence,)
|
||||
|
||||
|
||||
def test_consumer_refuses_claim_without_witness_or_silence_without_reason() -> None:
|
||||
bad_claim = ProvenClaim(trace_id="t1", witness="", path="src/app.py", line=10, family="access_control")
|
||||
try:
|
||||
consume_verdict(PushRequest(base="b", head="h", budget_class="c"), _record(), (bad_claim,), ())
|
||||
raise AssertionError("witnessless claim must be refused")
|
||||
except ValueError:
|
||||
pass
|
||||
bad_silence = BoundedSilence(path="src/other.py", reason="", coverage={})
|
||||
try:
|
||||
consume_verdict(PushRequest(base="b", head="h", budget_class="c"), _record(), (), (bad_silence,))
|
||||
raise AssertionError("reasonless silence must be refused")
|
||||
except ValueError:
|
||||
pass
|
||||
"""Retired v1 fact-pack consumer tests; generic fact and consumer seams use v2."""
|
||||
|
||||
@@ -1,65 +1 @@
|
||||
"""Fact-pack file admission tests for detection-core task 8.2 (Req 10.1)."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.legacy_v1 import FactPackError, admit_pack_file, verify_fact_pack # noqa: E402
|
||||
from openultrasast.mapping.installed_fact_authority import ( # noqa: E402
|
||||
InstalledAuthorityWireError,
|
||||
decode_fact_pack_bytes,
|
||||
)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
ENTRIES = REPO / "src/openultrasast/ruleset/entries"
|
||||
LEGACY_DIGEST = "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce"
|
||||
PACK = ENTRIES / "artifacts" / "sha256" / LEGACY_DIGEST
|
||||
PROJECTION = ENTRIES / "express.json"
|
||||
|
||||
|
||||
def _review_digest() -> str:
|
||||
payload = json.loads(PACK.read_text(encoding="utf-8"))
|
||||
return hashlib.sha256(payload["review"]["note"].encode()).hexdigest()
|
||||
|
||||
|
||||
def test_express_pack_admitted_with_verifiable_review() -> None:
|
||||
content = PACK.read_bytes()
|
||||
assert len(content) == 963
|
||||
assert hashlib.sha256(content).hexdigest() == LEGACY_DIGEST
|
||||
pack = admit_pack_file(PACK, review_digest=_review_digest())
|
||||
assert pack.family == "access_control"
|
||||
assert len(pack.facts) == 3
|
||||
assert pack.review_digest == _review_digest()
|
||||
verify_fact_pack(pack, pack.facts)
|
||||
|
||||
|
||||
def test_pack_refused_without_matching_review() -> None:
|
||||
try:
|
||||
admit_pack_file(PACK, review_digest="0" * 64)
|
||||
raise AssertionError("wrong review digest must be refused")
|
||||
except FactPackError:
|
||||
pass
|
||||
|
||||
|
||||
def test_tampered_pack_file_refused(tmp_path: Path) -> None:
|
||||
payload = json.loads(PACK.read_text(encoding="utf-8"))
|
||||
payload["facts"].append({"id": "entry:express:evil", "pattern": "eval("})
|
||||
tampered = tmp_path / "tampered.json"
|
||||
tampered.write_text(json.dumps(payload), encoding="utf-8")
|
||||
pack = admit_pack_file(tampered, review_digest=_review_digest())
|
||||
# review attestation still verifies, but content no longer matches the stamped digest
|
||||
original = admit_pack_file(PACK, review_digest=_review_digest())
|
||||
with pytest.raises(FactPackError):
|
||||
verify_fact_pack(original, pack.facts)
|
||||
|
||||
|
||||
def test_compatibility_projection_is_neither_a_legacy_nor_v2_fact_pack() -> None:
|
||||
with pytest.raises(FactPackError):
|
||||
admit_pack_file(PROJECTION, review_digest=_review_digest())
|
||||
with pytest.raises(InstalledAuthorityWireError):
|
||||
decode_fact_pack_bytes(PROJECTION.read_bytes())
|
||||
"""Retired v1 fact-pack file tests; reviewed family-neutral packs use v2."""
|
||||
|
||||
@@ -138,6 +138,16 @@ def test_legacy_decoder_has_no_compatibility_or_v2_domain_dependency() -> None:
|
||||
|
||||
|
||||
def test_legacy_reader_is_not_reexported_from_the_public_core_api() -> None:
|
||||
import openultrasast.core as core
|
||||
path = REPO / "src/openultrasast/core/__init__.py"
|
||||
content = path.read_bytes()
|
||||
assert content, "the public-boundary instrument must read nonzero package bytes"
|
||||
tree = ast.parse(content, filename=str(path.relative_to(REPO)))
|
||||
exported = {
|
||||
value.value
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for value in ast.walk(node.value)
|
||||
if isinstance(value, ast.Constant) and isinstance(value.value, str)
|
||||
}
|
||||
|
||||
assert not ({"LegacyArtifact", "decode_legacy_artifact", "read_legacy_artifact"} & set(core.__all__))
|
||||
assert not ({"LegacyArtifact", "decode_legacy_artifact", "read_legacy_artifact"} & exported)
|
||||
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE
|
||||
from consumer_migration_evidence import CONSUMER_TERMINAL_EVIDENCE
|
||||
from core_migration_boundary import inspect_legacy_boundary
|
||||
from core_migration_inventory import (
|
||||
REPO,
|
||||
@@ -77,13 +78,22 @@ def test_source_validation_refuses_removed_migrated_owner(tmp_path: Path) -> Non
|
||||
validate_inventory_sources(replace(inventory, entries=(original,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_test_and_operational_entries_remain_open_for_task_10_6() -> None:
|
||||
def test_test_and_operational_entries_have_exact_terminal_evidence() -> None:
|
||||
candidate = load_migration_inventory(INVENTORY)
|
||||
deferred = {
|
||||
entry.disposition for entry in candidate.entries if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"}
|
||||
migrated_ids = {entry_id for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated"}
|
||||
consumers = {
|
||||
entry.entry_id: entry.disposition
|
||||
for entry in candidate.entries
|
||||
if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"}
|
||||
}
|
||||
|
||||
assert deferred == {MigrationDisposition.OPEN}
|
||||
assert set(consumers) == {
|
||||
entry_id
|
||||
for entry_id in CONSUMER_TERMINAL_EVIDENCE
|
||||
if next(entry for entry in candidate.entries if entry.entry_id == entry_id).owner_boundary != "core"
|
||||
}
|
||||
assert all(consumers[entry_id] is MigrationDisposition.V2_MIGRATED for entry_id in migrated_ids)
|
||||
assert all(consumers[entry_id] is MigrationDisposition.RETIRED for entry_id in set(consumers) - migrated_ids)
|
||||
|
||||
|
||||
def test_old_benchmark_admission_contract_and_runner_are_not_live() -> None:
|
||||
|
||||
@@ -151,8 +151,10 @@ def test_migration_phase_separates_compatibility_implementation_from_consumers(t
|
||||
def test_cutover_requires_terminal_inventory_and_structural_zero(tmp_path: Path) -> None:
|
||||
clean = _write(tmp_path, "clean.py", "VALUE = 1\n")
|
||||
frozen = load_migration_inventory(INVENTORY)
|
||||
opened = replace(frozen.entries[0], disposition=MigrationDisposition.OPEN)
|
||||
open_inventory = replace(frozen, entries=(opened, *frozen.entries[1:]))
|
||||
open_report = inspect_legacy_boundary(
|
||||
(clean,), frozen_inventory=frozen, expected_scope=(clean,), root=tmp_path, phase=BoundaryPhase.POST_DELETION
|
||||
(clean,), frozen_inventory=open_inventory, expected_scope=(clean,), root=tmp_path, phase=BoundaryPhase.POST_DELETION
|
||||
)
|
||||
closed = _closed_inventory()
|
||||
migration_report = inspect_legacy_boundary((clean,), frozen_inventory=frozen, candidate_inventory=closed, root=tmp_path)
|
||||
@@ -175,7 +177,7 @@ def test_cutover_requires_terminal_inventory_and_structural_zero(tmp_path: Path)
|
||||
)
|
||||
|
||||
with pytest.raises(LegacyBoundaryError, match="open inventory"):
|
||||
_require_cutover(open_report, frozen=frozen, candidate=frozen, root=tmp_path, isolated_scope=(clean,))
|
||||
_require_cutover(open_report, frozen=open_inventory, candidate=open_inventory, root=tmp_path, isolated_scope=(clean,))
|
||||
with pytest.raises(LegacyBoundaryError, match="post-deletion"):
|
||||
_require_cutover(migration_report, frozen=frozen, candidate=closed, root=tmp_path, isolated_scope=(clean,))
|
||||
_require_cutover(clean_report, frozen=frozen, candidate=closed, root=tmp_path, isolated_scope=(clean,))
|
||||
@@ -286,7 +288,11 @@ def test_repository_scope_proves_nonzero_inputs_before_reporting_sites() -> None
|
||||
assert report.files_read == len(paths) > 0
|
||||
assert report.bytes_read > 0
|
||||
assert report.legacy_import_sites
|
||||
assert report.open_inventory_entries
|
||||
assert report.open_inventory_entries == ()
|
||||
assert report.consumer_legacy_import_sites == ()
|
||||
assert report.consumer_legacy_writer_sites == ()
|
||||
assert report.forbidden_dynamic_sites == ()
|
||||
assert report.compatibility_implementation_sites
|
||||
|
||||
|
||||
def test_cutover_rejects_positive_but_incomplete_repository_scope() -> None:
|
||||
|
||||
@@ -0,0 +1,592 @@
|
||||
"""Final consumer closure for detection-core migration task 10.6."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import core_migration_inventory as migration_inventory
|
||||
import pytest
|
||||
from consumer_migration_evidence import (
|
||||
CONSUMER_TERMINAL_EVIDENCE,
|
||||
REVIEWED_SUCCESSOR_MODULES,
|
||||
ConsumerEvidenceError,
|
||||
ReviewedSuccessorModule,
|
||||
load_consumer_terminal_evidence,
|
||||
load_reviewed_successor_modules,
|
||||
migrated_pytest_node_ids,
|
||||
)
|
||||
from core_migration_boundary import inspect_legacy_boundary
|
||||
from core_migration_inventory import REPO, LegacyInventoryError, MigrationDisposition, load_migration_inventory, validate_inventory_sources
|
||||
from core_migration_scope import boundary_python
|
||||
|
||||
INVENTORY = REPO / "tests/fixtures/detection_core/v1/migration_inventory.json"
|
||||
|
||||
|
||||
def test_complete_original_inventory_is_terminal() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
|
||||
assert inventory.entries
|
||||
assert all(entry.disposition is not MigrationDisposition.OPEN for entry in inventory.entries)
|
||||
|
||||
|
||||
def test_terminal_evidence_exactly_binds_every_task_10_6_entry() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
evidence_ids = tuple(CONSUMER_TERMINAL_EVIDENCE)
|
||||
evidence_id_set = set(evidence_ids)
|
||||
selected = tuple(entry for entry in inventory.entries if entry.entry_id in evidence_id_set)
|
||||
entries = {entry.entry_id: entry for entry in selected}
|
||||
|
||||
assert len(entries) == len(evidence_ids) == 155
|
||||
assert tuple(entry.entry_id for entry in selected) == evidence_ids
|
||||
assert all(entries[entry_id].disposition.value == evidence.disposition for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items())
|
||||
validate_inventory_sources(inventory, root=REPO)
|
||||
|
||||
|
||||
def test_migrated_evidence_owns_exact_bounded_pytest_nodes() -> None:
|
||||
migrated = tuple(evidence for evidence in CONSUMER_TERMINAL_EVIDENCE.values() if evidence.disposition == "v2_migrated")
|
||||
selectors = migrated_pytest_node_ids()
|
||||
|
||||
assert len(migrated) == 40
|
||||
assert len(selectors) == 37
|
||||
assert all(evidence.pytest_node_id == f"{evidence.successor_path}::{evidence.probe_symbol}" for evidence in migrated)
|
||||
assert all(selector in selectors for selector in (evidence.pytest_node_id for evidence in migrated))
|
||||
assert all((REPO / selector.partition("::")[0]).read_bytes() for selector in selectors)
|
||||
|
||||
|
||||
def test_successor_module_authority_is_exact_nonzero_and_digest_matched() -> None:
|
||||
migrated_paths = {evidence.successor_path for evidence in CONSUMER_TERMINAL_EVIDENCE.values() if evidence.disposition == "v2_migrated"}
|
||||
|
||||
assert len(REVIEWED_SUCCESSOR_MODULES) == 18
|
||||
assert set(REVIEWED_SUCCESSOR_MODULES) == migrated_paths
|
||||
for relative, reviewed in REVIEWED_SUCCESSOR_MODULES.items():
|
||||
content = (REPO / relative).read_bytes()
|
||||
assert content, f"module authority instrument must read nonzero bytes from {relative}"
|
||||
assert hashlib.sha256(content).hexdigest() == reviewed.sha256
|
||||
|
||||
|
||||
def _mutated_authority(tmp_path: Path, mutation: str) -> Path:
|
||||
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
|
||||
modules = document["successor_modules"]
|
||||
if mutation == "missing":
|
||||
modules.pop()
|
||||
elif mutation == "extra":
|
||||
modules.append({"path": "tests/forged_successor.py", "sha256": "f" * 64})
|
||||
elif mutation == "duplicate":
|
||||
modules.append(dict(modules[-1]))
|
||||
elif mutation == "conflicting_path":
|
||||
modules.append({**modules[-1], "sha256": "f" * 64})
|
||||
elif mutation == "duplicate_digest":
|
||||
modules[1]["sha256"] = modules[0]["sha256"]
|
||||
else:
|
||||
raise AssertionError(f"unknown mutation: {mutation}")
|
||||
target = tmp_path / f"consumer_terminal_evidence-{mutation}.json"
|
||||
target.write_text(json.dumps(document), encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def _write_authority(tmp_path: Path, document: object, name: str = "consumer_terminal_evidence.json") -> Path:
|
||||
target = tmp_path / name
|
||||
target.write_text(json.dumps(document), encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_path",
|
||||
(
|
||||
"",
|
||||
"/tests/test_arm_regression.py",
|
||||
"C:/tests/test_arm_regression.py",
|
||||
"C:tests/test_arm_regression.py",
|
||||
"C:\\tests\\test_arm_regression.py",
|
||||
"tests\\test_arm_regression.py",
|
||||
"./tests/test_arm_regression.py",
|
||||
"tests/./test_arm_regression.py",
|
||||
"tests/../tests/test_arm_regression.py",
|
||||
"tests/../../outside.py",
|
||||
"tests//test_arm_regression.py",
|
||||
"tests/test_arm_regression.py/.",
|
||||
"tests/test_arm_regression.py\0forged",
|
||||
),
|
||||
)
|
||||
def test_migrated_successor_paths_require_exact_canonical_repo_relative_posix(tmp_path: Path, invalid_path: str) -> None:
|
||||
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
|
||||
migrated = next(entry for entry in document["entries"] if entry["disposition"] == "v2_migrated")
|
||||
migrated["successor_path"] = invalid_path
|
||||
migrated["pytest_node_id"] = f"{invalid_path}::{migrated['probe_symbol']}"
|
||||
|
||||
with pytest.raises(ConsumerEvidenceError, match="canonical repository-relative POSIX path"):
|
||||
load_consumer_terminal_evidence(_write_authority(tmp_path, document))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_path",
|
||||
(
|
||||
"/tests/test_arm_regression.py",
|
||||
"C:/tests/test_arm_regression.py",
|
||||
"C:tests/test_arm_regression.py",
|
||||
"tests\\test_arm_regression.py",
|
||||
"tests/./test_arm_regression.py",
|
||||
"tests/../tests/test_arm_regression.py",
|
||||
"tests/../../outside.py",
|
||||
"tests//test_arm_regression.py",
|
||||
"tests/test_arm_regression.py\0forged",
|
||||
),
|
||||
)
|
||||
def test_reviewed_module_paths_require_exact_canonical_repo_relative_posix(tmp_path: Path, invalid_path: str) -> None:
|
||||
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
|
||||
original = document["successor_modules"][0]["path"]
|
||||
document["successor_modules"][0]["path"] = invalid_path
|
||||
for entry in document["entries"]:
|
||||
if entry["successor_path"] == original:
|
||||
entry["successor_path"] = invalid_path
|
||||
entry["pytest_node_id"] = f"{invalid_path}::{entry['probe_symbol']}"
|
||||
|
||||
with pytest.raises(ConsumerEvidenceError, match="canonical repository-relative POSIX path"):
|
||||
load_reviewed_successor_modules(_write_authority(tmp_path, document))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ("missing", "extra", "duplicate", "conflicting_path", "duplicate_digest"))
|
||||
def test_successor_module_authority_rejects_set_and_identity_drift(tmp_path: Path, mutation: str) -> None:
|
||||
with pytest.raises(ConsumerEvidenceError, match="successor module"):
|
||||
load_reviewed_successor_modules(_mutated_authority(tmp_path, mutation))
|
||||
|
||||
|
||||
def test_migrated_row_outside_module_authority_is_rejected(tmp_path: Path) -> None:
|
||||
evidence_path = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
|
||||
document = json.loads(evidence_path.read_bytes())
|
||||
migrated = next(entry for entry in document["entries"] if entry["disposition"] == "v2_migrated")
|
||||
migrated["successor_path"] = "tests/forged_successor.py"
|
||||
migrated["pytest_node_id"] = f"tests/forged_successor.py::{migrated['probe_symbol']}"
|
||||
target = tmp_path / "consumer_terminal_evidence.json"
|
||||
target.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConsumerEvidenceError, match="exactly cover migrated rows"):
|
||||
load_consumer_terminal_evidence(target)
|
||||
|
||||
|
||||
def test_clean_reviewed_successor_module_is_accepted() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
|
||||
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=REPO)
|
||||
|
||||
|
||||
def _install_reviewed_module(monkeypatch: pytest.MonkeyPatch, relative: str, content: bytes) -> None:
|
||||
monkeypatch.setattr(
|
||||
migration_inventory,
|
||||
"REVIEWED_SUCCESSOR_MODULES",
|
||||
{relative: ReviewedSuccessorModule(relative, hashlib.sha256(content).hexdigest())},
|
||||
)
|
||||
|
||||
|
||||
def test_reviewed_successor_safe_reader_accepts_nested_real_directories(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
relative = "tests/nested/reviewed.py"
|
||||
content = b"VALUE = 1\n"
|
||||
target = tmp_path / relative
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_bytes(content)
|
||||
_install_reviewed_module(monkeypatch, relative, content)
|
||||
|
||||
tree = migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
|
||||
|
||||
assert isinstance(tree, ast.Module)
|
||||
|
||||
|
||||
def test_reviewed_successor_safe_reader_rejects_nul_before_filesystem_access() -> None:
|
||||
with pytest.raises(LegacyInventoryError, match="not canonical"):
|
||||
migration_inventory._canonical_relative_parts("tests/reviewed.py\0forged")
|
||||
|
||||
|
||||
def test_reviewed_successor_safe_reader_refuses_fifo_without_blocking(tmp_path: Path) -> None:
|
||||
relative = "tests/reviewed.py"
|
||||
target = tmp_path / relative
|
||||
target.parent.mkdir()
|
||||
os.mkfifo(target)
|
||||
assert target.is_fifo(), "FIFO refusal instrument must create the special file it exercises"
|
||||
program = (
|
||||
"from pathlib import Path\n"
|
||||
"from core_migration_inventory import LegacyInventoryError, _read_file_beneath_root\n"
|
||||
f"root = Path({str(tmp_path)!r})\n"
|
||||
"try:\n"
|
||||
f" _read_file_beneath_root({relative!r}, root=root)\n"
|
||||
"except LegacyInventoryError:\n"
|
||||
" raise SystemExit(0)\n"
|
||||
"raise SystemExit(2)\n"
|
||||
)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
(sys.executable, "-c", program),
|
||||
cwd=REPO,
|
||||
env={**os.environ, "PYTHONPATH": "src:tests"},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail("reviewed successor FIFO open blocked past the two-second bound")
|
||||
assert completed.returncode == 0, completed.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_location", ("outside", "inside"))
|
||||
def test_reviewed_successor_safe_reader_rejects_final_file_symlinks(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target_location: str
|
||||
) -> None:
|
||||
relative = "tests/reviewed.py"
|
||||
content = b"VALUE = 1\n"
|
||||
tests = tmp_path / "tests"
|
||||
tests.mkdir()
|
||||
target = (tmp_path.parent / f"{tmp_path.name}-outside.py") if target_location == "outside" else (tests / "target.py")
|
||||
target.write_bytes(content)
|
||||
(tmp_path / relative).symlink_to(target)
|
||||
_install_reviewed_module(monkeypatch, relative, content)
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="not a symlink-free file beneath the supplied root"):
|
||||
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_location", ("outside", "inside"))
|
||||
def test_reviewed_successor_safe_reader_rejects_intermediate_directory_symlinks(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target_location: str
|
||||
) -> None:
|
||||
relative = "linked/reviewed.py"
|
||||
content = b"VALUE = 1\n"
|
||||
target = (tmp_path.parent / f"{tmp_path.name}-outside") if target_location == "outside" else (tmp_path / "real")
|
||||
target.mkdir()
|
||||
(target / "reviewed.py").write_bytes(content)
|
||||
(tmp_path / "linked").symlink_to(target, target_is_directory=True)
|
||||
_install_reviewed_module(monkeypatch, relative, content)
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="not a symlink-free file beneath the supplied root"):
|
||||
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root_kind", ("missing", "file", "symlink"))
|
||||
def test_reviewed_successor_safe_reader_rejects_non_directory_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, root_kind: str) -> None:
|
||||
relative = "tests/reviewed.py"
|
||||
content = b"VALUE = 1\n"
|
||||
real_root = tmp_path / "real-root"
|
||||
real_root.mkdir()
|
||||
supplied_root = tmp_path / "supplied-root"
|
||||
if root_kind == "file":
|
||||
supplied_root.write_bytes(b"not a directory")
|
||||
elif root_kind == "symlink":
|
||||
supplied_root.symlink_to(real_root, target_is_directory=True)
|
||||
_install_reviewed_module(monkeypatch, relative, content)
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="supplied root is not a real directory"):
|
||||
migration_inventory._read_reviewed_successor_module(relative, root=supplied_root)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ("missing", "unreadable", "zero"))
|
||||
def test_reviewed_successor_safe_reader_rejects_unusable_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str) -> None:
|
||||
relative = "tests/reviewed.py"
|
||||
expected = b"VALUE = 1\n"
|
||||
target = tmp_path / relative
|
||||
target.parent.mkdir()
|
||||
if failure != "missing":
|
||||
target.write_bytes(b"" if failure == "zero" else expected)
|
||||
if failure == "unreadable":
|
||||
target.chmod(0)
|
||||
_install_reviewed_module(monkeypatch, relative, expected)
|
||||
|
||||
message = "nonzero bytes" if failure == "zero" else "not readable"
|
||||
with pytest.raises(LegacyInventoryError, match=message):
|
||||
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
|
||||
|
||||
|
||||
def test_migrated_owner_binding_cannot_disappear(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
migrated = {entry_id for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated"}
|
||||
entry = next(item for item in inventory.entries if item.entry_id in migrated and item.symbol.startswith("test_"))
|
||||
source = (REPO / entry.path).read_text(encoding="utf-8").replace(f"def {entry.symbol}(", f"def removed_{entry.symbol}(", 1)
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_retired_operational_owner_cannot_be_reintroduced(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.owner_boundary == "operational-tooling")
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text("import json\ndef main():\n return json.dumps({})\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="retired consumer capability remains live"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("entry_id", "source"),
|
||||
(
|
||||
(
|
||||
"887c9abe9a6d827c39655c6835fb0c2471be035914e2bbff888a4cd54635b8e0",
|
||||
"from openultrasast.core._legacy_factpack import LegacyFactPack\n",
|
||||
),
|
||||
(
|
||||
"36e5d60b410bf7f9c676a5da9e95718cab95dadfbf61cda0d0e2f24b56cf3c92",
|
||||
"class TraceRecord:\n def canonical(self):\n return record()\n",
|
||||
),
|
||||
(
|
||||
"a8685de9229906307386c537ff096b385bff727e270744ccc3dd02978a7d5b2a",
|
||||
"def test_budget_gate_returns_no_go():\n return admission_report()\n",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_retired_module_or_dotted_capability_cannot_be_reintroduced(tmp_path: Path, entry_id: str, source: str) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == entry_id)
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="retired consumer capability"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_dead_syntax_cannot_stand_in_for_migrated_test_semantics(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
|
||||
constants = "fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"
|
||||
source = f"def test_summarize_arm_publishes_budget_result_and_masks_repositories():\n if False:\n dumps({constants!r})\n"
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_pass_only_migrated_successor_cannot_claim_semantic_closure(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
|
||||
source = """from benchmarks.valid import arm_regression
|
||||
|
||||
def test_summarize_arm_publishes_budget_result_and_masks_repositories():
|
||||
if False:
|
||||
dumps(("fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"))
|
||||
pass
|
||||
"""
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"replacement_body",
|
||||
(
|
||||
"return True",
|
||||
"assert True",
|
||||
"pytest.skip('not semantic execution')",
|
||||
"len(())",
|
||||
"if False:\n arm_regression.summarize_arm(None)\n return True",
|
||||
),
|
||||
)
|
||||
def test_arbitrary_behavior_cannot_certify_migrated_semantics(tmp_path: Path, replacement_body: str) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
|
||||
source = f"""import pytest
|
||||
from benchmarks.valid import arm_regression
|
||||
|
||||
def test_summarize_arm_publishes_budget_result_and_masks_repositories():
|
||||
if False:
|
||||
dumps(("fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"))
|
||||
{replacement_body}
|
||||
"""
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_rebound_semantic_import_cannot_certify_real_migrated_owner(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
|
||||
source = (
|
||||
(REPO / entry.path)
|
||||
.read_text(encoding="utf-8")
|
||||
.replace(
|
||||
"from benchmarks.valid import arm_regression",
|
||||
"from benchmarks.valid import arm_regression as rebound_arm_regression",
|
||||
)
|
||||
)
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
_TERMINAL_BINDING_ENTRY_ID = "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b"
|
||||
_TERMINAL_BINDING_TEST = "test_summarize_arm_publishes_budget_result_and_masks_repositories"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"later_binding",
|
||||
(
|
||||
f"{_TERMINAL_BINDING_TEST} = lambda: None",
|
||||
f"{_TERMINAL_BINDING_TEST}: object = None",
|
||||
f"{_TERMINAL_BINDING_TEST} += ()",
|
||||
f"del {_TERMINAL_BINDING_TEST}",
|
||||
f"from builtins import len as {_TERMINAL_BINDING_TEST}",
|
||||
f"{_TERMINAL_BINDING_TEST} = wrapper({_TERMINAL_BINDING_TEST})",
|
||||
"arm_regression = object()",
|
||||
"pytestmark = pytest.mark.skip(reason='disabled')",
|
||||
f"{_TERMINAL_BINDING_TEST}.pytestmark = pytest.mark.xfail(reason='disabled')",
|
||||
),
|
||||
)
|
||||
def test_reviewed_migrated_bindings_must_remain_terminal(tmp_path: Path, later_binding: str) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
|
||||
source = (REPO / entry.path).read_text(encoding="utf-8") + f"\n{later_binding}\n"
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"namespace_mutation",
|
||||
(
|
||||
f"vars()[{_TERMINAL_BINDING_TEST!r}] = lambda: None",
|
||||
f"sys.modules[__name__].{_TERMINAL_BINDING_TEST} = lambda: None",
|
||||
f"object.__setattr__(sys.modules[__name__], {_TERMINAL_BINDING_TEST!r}, lambda: None)",
|
||||
f"vars({_TERMINAL_BINDING_TEST})['pytestmark'] = pytest.mark.skip(reason='disabled')",
|
||||
f"{_TERMINAL_BINDING_TEST}.__dict__['pytestmark'] = pytest.mark.xfail(reason='disabled')",
|
||||
),
|
||||
)
|
||||
def test_exact_module_bytes_reject_unbounded_namespace_mutations(tmp_path: Path, namespace_mutation: str) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
|
||||
original = (REPO / entry.path).read_bytes()
|
||||
assert original, "namespace-mutation RED instrument must read nonzero successor bytes"
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_bytes(original + f"\n{namespace_mutation}\n".encode())
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_terminal_ast_diagnostics_remain_defense_in_depth(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
|
||||
original = (REPO / entry.path).read_bytes()
|
||||
assert original, "defense-in-depth instrument must read nonzero successor bytes"
|
||||
changed = original + f"\n{_TERMINAL_BINDING_TEST} = lambda: None\n".encode()
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_bytes(changed)
|
||||
reviewed = dict(REVIEWED_SUCCESSOR_MODULES)
|
||||
reviewed[entry.path] = ReviewedSuccessorModule(entry.path, hashlib.sha256(changed).hexdigest())
|
||||
monkeypatch.setattr(migration_inventory, "REVIEWED_SUCCESSOR_MODULES", reviewed)
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="terminal binding"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_even_safely_overwritten_byte_change_requires_explicit_review(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
|
||||
source = (
|
||||
(REPO / entry.path)
|
||||
.read_text(encoding="utf-8")
|
||||
.replace(
|
||||
f"def {_TERMINAL_BINDING_TEST}(",
|
||||
f"{_TERMINAL_BINDING_TEST} = None\n\ndef {_TERMINAL_BINDING_TEST}(",
|
||||
1,
|
||||
)
|
||||
)
|
||||
target = tmp_path / entry.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text(source, encoding="utf-8")
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="module digest"):
|
||||
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_reviewed_parametrize_decorators_remain_collectable() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
selected_paths = {"tests/test_hunter_arm.py", "tests/test_push_admission.py"}
|
||||
selected = tuple(
|
||||
entry
|
||||
for entry in inventory.entries
|
||||
if entry.entry_id in CONSUMER_TERMINAL_EVIDENCE and CONSUMER_TERMINAL_EVIDENCE[entry.entry_id].successor_path in selected_paths
|
||||
)
|
||||
|
||||
assert selected
|
||||
validate_inventory_sources(replace(inventory, entries=selected), root=REPO)
|
||||
decorators = [
|
||||
decorator
|
||||
for path in selected_paths
|
||||
for node in ast.parse((REPO / path).read_bytes(), filename=path).body
|
||||
if isinstance(node, ast.FunctionDef) and f"{path}::{node.name}" in migrated_pytest_node_ids()
|
||||
for decorator in node.decorator_list
|
||||
if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute) and decorator.func.attr == "parametrize"
|
||||
]
|
||||
assert len(decorators) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "replacement"),
|
||||
(
|
||||
("path", "tests/forged_successor.py"),
|
||||
("capability_anchors", ["definition:forged_owner"]),
|
||||
("disposition", "retired"),
|
||||
),
|
||||
)
|
||||
def test_terminal_evidence_rejects_frozen_identity_or_disposition_drift(tmp_path: Path, field: str, replacement: object) -> None:
|
||||
evidence_path = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
|
||||
document = json.loads(evidence_path.read_bytes())
|
||||
entry = next(item for item in document["entries"] if field != "disposition" or item["disposition"] == "v2_migrated")
|
||||
entry[field] = replacement
|
||||
forged = tmp_path / "consumer_terminal_evidence.json"
|
||||
forged.write_text(json.dumps(document), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ConsumerEvidenceError):
|
||||
load_consumer_terminal_evidence(forged)
|
||||
|
||||
|
||||
def test_authoritative_boundary_includes_task_source_files() -> None:
|
||||
paths = {path.relative_to(REPO).as_posix(): path for path in boundary_python(REPO)}
|
||||
|
||||
for relative in ("tests/consumer_migration_evidence.py", "tests/test_core_migration_consumers.py"):
|
||||
assert relative in paths
|
||||
assert paths[relative].read_bytes()
|
||||
|
||||
|
||||
def test_nonzero_repository_scope_has_no_forbidden_consumers() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
paths = boundary_python(REPO)
|
||||
report = inspect_legacy_boundary(
|
||||
paths,
|
||||
frozen_inventory=inventory,
|
||||
candidate_inventory=inventory,
|
||||
expected_scope=paths,
|
||||
root=REPO,
|
||||
)
|
||||
|
||||
assert report.files_read == len(paths) > 0
|
||||
assert report.bytes_read > 0
|
||||
assert report.scanned_scope_paths == report.expected_scope_paths
|
||||
assert report.consumer_legacy_import_sites == ()
|
||||
assert report.consumer_legacy_writer_sites == ()
|
||||
assert report.forbidden_dynamic_sites == ()
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Task 10.1: isolate the live v1 contract and serialization owners."""
|
||||
"""Structural ownership tests for the temporary v1 compatibility implementation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,14 +6,8 @@ import ast
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
V1_FIXTURES = REPO / "tests/fixtures/detection_core/v1"
|
||||
PURE_COMPATIBILITY_MODULES = (
|
||||
"admission.py",
|
||||
"consumer.py",
|
||||
"trace.py",
|
||||
"verdict.py",
|
||||
"_legacy_factpack.py",
|
||||
)
|
||||
CORE = REPO / "src/openultrasast/core"
|
||||
PURE_COMPATIBILITY_MODULES = ("admission.py", "consumer.py", "trace.py", "verdict.py", "_legacy_factpack.py")
|
||||
MIXED_LEGACY_NAMES = {
|
||||
"budget": {"Budget", "Complexity"},
|
||||
"coverage": {"Coverage", "ResolutionOutcome", "check_reads"},
|
||||
@@ -30,27 +24,23 @@ MIXED_LEGACY_NAMES = {
|
||||
}
|
||||
|
||||
|
||||
def _trace() -> object:
|
||||
from openultrasast.core.legacy_v1 import contracts
|
||||
def _tree(path: Path) -> ast.Module:
|
||||
content = path.read_bytes()
|
||||
assert content, f"ownership instrument must read nonzero bytes from {path}"
|
||||
return ast.parse(content, filename=path.as_posix())
|
||||
|
||||
return contracts.TraceRecord(
|
||||
entry="src/app.py::handler",
|
||||
hops=(contracts.Hop(function="handler", call_line=10, complete=True),),
|
||||
operation_kind="protected_read",
|
||||
resource="credentials",
|
||||
authority="admin",
|
||||
witnesses=(contracts.Witness(kind="guard", provenance="fact:checkPermission", relation="all_paths"),),
|
||||
guards=(contracts.Guard(kind="permission_check", effect="branches"),),
|
||||
completeness_reasons=(),
|
||||
judgment=contracts.Judgment.UNGUARDED,
|
||||
digests={"input": "abc123"},
|
||||
)
|
||||
|
||||
def _defined(tree: ast.Module) -> set[str]:
|
||||
return {node.name for node in tree.body if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))}
|
||||
|
||||
|
||||
def _imported(tree: ast.Module) -> set[str]:
|
||||
return {alias.asname or alias.name for node in tree.body if isinstance(node, (ast.Import, ast.ImportFrom)) for alias in node.names}
|
||||
|
||||
|
||||
def test_legacy_contract_values_have_one_compatibility_owner() -> None:
|
||||
from openultrasast.core.legacy_v1 import contracts
|
||||
|
||||
names = {
|
||||
contracts = _tree(CORE / "legacy_v1/contracts.py")
|
||||
expected = {
|
||||
"AdmissionManifest",
|
||||
"AdmissionManifestError",
|
||||
"BoundedSilence",
|
||||
@@ -75,104 +65,59 @@ def test_legacy_contract_values_have_one_compatibility_owner() -> None:
|
||||
"TraceRecord",
|
||||
"Witness",
|
||||
}
|
||||
assert {getattr(contracts, name).__module__ for name in names} == {"openultrasast.core.legacy_v1.contracts"}
|
||||
assert expected <= _defined(contracts)
|
||||
|
||||
|
||||
def test_pure_compatibility_modules_define_no_contract_or_serializer() -> None:
|
||||
for name in PURE_COMPATIBILITY_MODULES:
|
||||
path = REPO / "src/openultrasast/core" / name
|
||||
content = path.read_bytes()
|
||||
assert content, f"ownership instrument must read nonzero bytes from {name}"
|
||||
tree = ast.parse(content, filename=path.as_posix())
|
||||
assert not [node.name for node in tree.body if isinstance(node, ast.ClassDef)]
|
||||
assert not [node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))]
|
||||
tree = _tree(CORE / name)
|
||||
assert not _defined(tree)
|
||||
assert not any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in tree.body)
|
||||
assert any(isinstance(node, ast.ImportFrom) and node.module == "openultrasast.core.legacy_v1.contracts" for node in tree.body)
|
||||
|
||||
|
||||
def test_writer_is_the_only_compatibility_json_serializer() -> None:
|
||||
namespace = REPO / "src/openultrasast/core/legacy_v1"
|
||||
namespace = CORE / "legacy_v1"
|
||||
paths = sorted(namespace.glob("*.py"))
|
||||
assert paths and all(path.read_bytes() for path in paths)
|
||||
assert {path.name for path in paths} == {"__init__.py", "contracts.py", "writer.py"}
|
||||
json_importers = []
|
||||
for path in paths:
|
||||
tree = ast.parse(path.read_bytes(), filename=path.as_posix())
|
||||
if any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in tree.body):
|
||||
json_importers.append(path.name)
|
||||
json_importers = [
|
||||
path.name
|
||||
for path in paths
|
||||
if any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in _tree(path).body)
|
||||
]
|
||||
assert json_importers == ["writer.py"]
|
||||
|
||||
|
||||
def test_namespace_and_v2_dependency_directions_are_closed() -> None:
|
||||
namespace = REPO / "src/openultrasast/core/legacy_v1"
|
||||
for path in namespace.glob("*.py"):
|
||||
tree = ast.parse(path.read_bytes(), filename=path.as_posix())
|
||||
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module}
|
||||
for path in (CORE / "legacy_v1").glob("*.py"):
|
||||
imports = {node.module for node in ast.walk(_tree(path)) if isinstance(node, ast.ImportFrom) and node.module}
|
||||
assert all(module.startswith("openultrasast.core.legacy_v1") for module in imports if module.startswith("openultrasast"))
|
||||
|
||||
compatibility = {*PURE_COMPATIBILITY_MODULES, "legacy.py", "__init__.py"}
|
||||
for path in (REPO / "src/openultrasast/core").glob("*.py"):
|
||||
if path.name in compatibility:
|
||||
continue
|
||||
content = path.read_bytes()
|
||||
assert content, f"dependency instrument must read nonzero bytes from {path.name}"
|
||||
tree = ast.parse(content, filename=path.as_posix())
|
||||
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module}
|
||||
assert not any(module.startswith("openultrasast.core.legacy_v1") for module in imports)
|
||||
|
||||
|
||||
def test_legacy_trace_write_remains_byte_identical() -> None:
|
||||
expected = (V1_FIXTURES / "trace.json").read_bytes()
|
||||
assert len(expected) == 401
|
||||
value = _trace()
|
||||
assert value.canonical() == expected.rstrip(b"\n") # type: ignore[attr-defined]
|
||||
for path in CORE.glob("*.py"):
|
||||
if path.name not in compatibility:
|
||||
imports = {node.module for node in ast.walk(_tree(path)) if isinstance(node, ast.ImportFrom) and node.module}
|
||||
assert not any(module.startswith("openultrasast.core.legacy_v1") for module in imports)
|
||||
|
||||
|
||||
def test_historical_decoder_stays_independent_from_compatibility_namespace() -> None:
|
||||
path = REPO / "src/openultrasast/core/legacy.py"
|
||||
content = path.read_bytes()
|
||||
assert content
|
||||
tree = ast.parse(content, filename=path.as_posix())
|
||||
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom)}
|
||||
imports = {node.module for node in ast.walk(_tree(CORE / "legacy.py")) if isinstance(node, ast.ImportFrom)}
|
||||
assert "openultrasast.core.legacy_v1" not in imports
|
||||
|
||||
|
||||
def test_top_level_v1_api_imports_only_from_closed_namespace() -> None:
|
||||
import openultrasast.core as public_core
|
||||
from openultrasast.core import legacy_v1
|
||||
|
||||
path = REPO / "src/openultrasast/core/__init__.py"
|
||||
content = path.read_bytes()
|
||||
assert content
|
||||
tree = ast.parse(content, filename=path.as_posix())
|
||||
legacy_imports = {
|
||||
node.module
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.ImportFrom) and any((alias.asname or alias.name) in public_core.__all__ for alias in node.names)
|
||||
}
|
||||
assert legacy_imports == {"openultrasast.core.legacy_v1"}
|
||||
assert all(getattr(public_core, name) is getattr(legacy_v1, name) for name in public_core.__all__)
|
||||
def test_top_level_v1_api_is_only_a_closed_namespace_shim() -> None:
|
||||
tree = _tree(CORE / "__init__.py")
|
||||
modules = {node.module for node in tree.body if isinstance(node, ast.ImportFrom)}
|
||||
assert modules == {"openultrasast.core.legacy_v1"}
|
||||
assert not _defined(tree)
|
||||
|
||||
|
||||
def test_mixed_v2_modules_expose_no_legacy_compatibility_names() -> None:
|
||||
import importlib
|
||||
|
||||
for module_name, legacy_names in MIXED_LEGACY_NAMES.items():
|
||||
module = importlib.import_module(f"openultrasast.core.{module_name}")
|
||||
assert legacy_names.isdisjoint(vars(module)), f"{module_name} still exposes {legacy_names & vars(module).keys()}"
|
||||
tree = _tree(CORE / f"{module_name}.py")
|
||||
assert legacy_names.isdisjoint(_defined(tree) | _imported(tree))
|
||||
|
||||
|
||||
def test_closed_namespace_is_the_only_owner_of_moved_mixed_names() -> None:
|
||||
from openultrasast.core import legacy_v1
|
||||
|
||||
contracts = _tree(CORE / "legacy_v1/contracts.py")
|
||||
expected = set().union(*MIXED_LEGACY_NAMES.values())
|
||||
assert all(hasattr(legacy_v1, name) for name in expected)
|
||||
operation_names = {
|
||||
"assert_no_shortcut",
|
||||
"check_reads",
|
||||
"require_admission_evidence",
|
||||
"reversed_pair_probe",
|
||||
"stamp",
|
||||
"verify_stamp",
|
||||
}
|
||||
assert {getattr(legacy_v1, name).__module__ for name in operation_names} == {"openultrasast.core.legacy_v1.contracts"}
|
||||
assert expected <= _defined(contracts)
|
||||
|
||||
@@ -66,11 +66,7 @@ def test_registered_production_entries_close_without_rekeying_other_consumers()
|
||||
assert set(production) == _PRODUCTION_ENTRY_IDS
|
||||
assert all(production[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED)
|
||||
assert all(production[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED)
|
||||
assert all(
|
||||
entry.disposition is MigrationDisposition.OPEN
|
||||
for entry in candidate.entries
|
||||
if entry.entry_id not in production and not entry.path.startswith("benchmarks/")
|
||||
)
|
||||
assert all(entry.disposition is not MigrationDisposition.OPEN for entry in candidate.entries)
|
||||
|
||||
frozen = replace(
|
||||
candidate,
|
||||
|
||||
@@ -59,19 +59,12 @@ def test_reader_audit_covers_direct_legacy_submodule_bindings(
|
||||
|
||||
|
||||
def test_reader_audit_covers_real_direct_legacy_submodule_reads() -> None:
|
||||
expected = {
|
||||
("tests/test_core_contract.py", "module_attr:openultrasast.core.verdict:__all__"),
|
||||
}
|
||||
paths = tuple(REPO / path for path in ("tests/test_core_budget_v2.py", "tests/test_core_contract.py", "tests/test_core_probes.py"))
|
||||
|
||||
report = audit_legacy_readers(paths)
|
||||
|
||||
assert report.files_read == 3 and report.bytes_read > 0
|
||||
assert expected <= set(report.capabilities)
|
||||
assert not any(
|
||||
capability.startswith(("module_attr:openultrasast.core.budget:", "module_attr:openultrasast.core.probes:"))
|
||||
for _path, capability in report.capabilities
|
||||
)
|
||||
assert report.capabilities == ()
|
||||
|
||||
|
||||
def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None:
|
||||
@@ -81,9 +74,9 @@ def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
|
||||
assert inventory.entries
|
||||
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 155
|
||||
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 25
|
||||
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 18
|
||||
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 0
|
||||
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 65
|
||||
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 133
|
||||
assert all(entry.entry_id == entry_identity(entry) for entry in inventory.entries)
|
||||
validate_inventory_sources(inventory, root=REPO)
|
||||
|
||||
@@ -174,33 +167,16 @@ def test_inventory_registers_the_active_legacy_probe_writer() -> None:
|
||||
|
||||
def test_independent_reader_audit_matches_split_inventory_capabilities() -> None:
|
||||
report = audit_legacy_readers(tracked_python())
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
reader_entries = tuple(entry for entry in inventory.entries if entry.dependency_kind.value == "reader")
|
||||
registered = {(entry.path, anchor) for entry in reader_entries for anchor in entry.capability_anchors}
|
||||
current = set(report.capabilities)
|
||||
open_registered = {
|
||||
(entry.path, anchor)
|
||||
for entry in reader_entries
|
||||
if entry.disposition is MigrationDisposition.OPEN
|
||||
for anchor in entry.capability_anchors
|
||||
}
|
||||
|
||||
assert report.files_read > 0 and report.bytes_read > 0
|
||||
assert ("tests/test_core_probes.py", "top_level:ProbeResult") in report.capabilities
|
||||
assert all(len(entry.capability_anchors) == 1 for entry in reader_entries)
|
||||
assert current <= registered, "the frozen inventory may never grow to admit a new reader"
|
||||
for path, _anchor in open_registered - current:
|
||||
source = (REPO / path).read_bytes()
|
||||
assert source, f"migration shim instrument must read nonzero bytes from {path}"
|
||||
if path in MIXED_V2_MODULES:
|
||||
assert b"openultrasast.core.legacy_v1" not in source
|
||||
continue
|
||||
assert b"openultrasast.core.legacy_v1" in source, f"open reader disappeared without an explicit compatibility shim: {path}"
|
||||
assert not any("legacy_v1" in capability for _path, capability in report.capabilities)
|
||||
assert not any(path in {"tests/test_core_contract.py", "tests/test_core_probes.py"} for path, _capability in report.capabilities)
|
||||
|
||||
|
||||
def test_inventory_targets_require_exact_capability_anchors(tmp_path: Path) -> None:
|
||||
frozen = load_migration_inventory(INVENTORY)
|
||||
entry = next(item for item in frozen.entries if item.disposition is MigrationDisposition.OPEN and item.symbol == "<module>")
|
||||
terminal = next(item for item in frozen.entries if item.symbol == "<module>")
|
||||
entry = replace(terminal, disposition=MigrationDisposition.OPEN)
|
||||
assert entry.capability_anchors
|
||||
substituted = tmp_path / entry.path
|
||||
substituted.parent.mkdir(parents=True)
|
||||
@@ -221,10 +197,10 @@ def test_historical_artifact_manifest_pins_exact_nonzero_bytes() -> None:
|
||||
|
||||
@pytest.mark.parametrize("terminal", (MigrationDisposition.V2_MIGRATED, MigrationDisposition.RETIRED))
|
||||
def test_inventory_entries_close_monotonically_without_identity_changes(terminal: MigrationDisposition) -> None:
|
||||
frozen = load_migration_inventory(INVENTORY)
|
||||
index = next(index for index, entry in enumerate(frozen.entries) if entry.disposition is MigrationDisposition.OPEN)
|
||||
changed_entry = replace(frozen.entries[index], disposition=terminal)
|
||||
candidate = replace(frozen, entries=(*frozen.entries[:index], changed_entry, *frozen.entries[index + 1 :]))
|
||||
candidate = load_migration_inventory(INVENTORY)
|
||||
index = next(index for index, entry in enumerate(candidate.entries) if entry.disposition is terminal)
|
||||
open_entry = replace(candidate.entries[index], disposition=MigrationDisposition.OPEN)
|
||||
frozen = replace(candidate, entries=(*candidate.entries[:index], open_entry, *candidate.entries[index + 1 :]))
|
||||
|
||||
validate_inventory_transition(frozen, candidate)
|
||||
|
||||
@@ -282,7 +258,7 @@ def test_inventory_refuses_schema_and_status_mutations(tmp_path: Path, field: st
|
||||
|
||||
def test_inventory_refuses_unreadable_zero_and_unknown_targets(tmp_path: Path) -> None:
|
||||
frozen = load_migration_inventory(INVENTORY)
|
||||
original = next(entry for entry in frozen.entries if entry.disposition is MigrationDisposition.OPEN)
|
||||
original = replace(frozen.entries[0], disposition=MigrationDisposition.OPEN)
|
||||
|
||||
missing = replace(original, path="missing.py", symbol="<module>")
|
||||
missing = replace(missing, entry_id=entry_identity(missing))
|
||||
|
||||
@@ -5,44 +5,9 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.trace import TraceRecord # noqa: E402
|
||||
from openultrasast.core.verdict import Judgment # noqa: E402
|
||||
from openultrasast.hunt.prover import Coverage, PairJudgmentCounts # noqa: E402
|
||||
|
||||
|
||||
def _legacy_record(*, judgment: Judgment = Judgment.UNGUARDED) -> TraceRecord:
|
||||
reasons = ("abstract_state_unset",) if judgment is Judgment.INDETERMINATE else ()
|
||||
return TraceRecord(
|
||||
entry="src/app.py::handler",
|
||||
hops=(),
|
||||
operation_kind="protected_read",
|
||||
resource="credentials",
|
||||
authority="admin",
|
||||
witnesses=(),
|
||||
guards=(),
|
||||
completeness_reasons=reasons,
|
||||
judgment=judgment,
|
||||
digests={"input": "abc123"},
|
||||
)
|
||||
|
||||
|
||||
def test_legacy_trace_contract_remains_available_only_as_tested_compatibility() -> None:
|
||||
record = _legacy_record()
|
||||
|
||||
assert isinstance(record, TraceRecord)
|
||||
assert record.entry == "src/app.py::handler"
|
||||
assert record.judgment is Judgment.UNGUARDED
|
||||
assert record.resource == "credentials"
|
||||
assert record.canonical() == _legacy_record().canonical()
|
||||
|
||||
|
||||
def test_legacy_indeterminate_record_stays_explicit() -> None:
|
||||
record = _legacy_record(judgment=Judgment.INDETERMINATE)
|
||||
|
||||
assert record.judgment is Judgment.INDETERMINATE
|
||||
assert record.completeness_reasons
|
||||
|
||||
|
||||
def test_hunt_coverage_retires_its_legacy_core_projection() -> None:
|
||||
coverage = Coverage(
|
||||
files_by_language={"java": 2},
|
||||
|
||||
@@ -2,14 +2,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from detection_core_fixtures import InMemoryContentResolver, stored_artifact
|
||||
|
||||
import openultrasast.core as public_core
|
||||
from openultrasast.core import canonical, evidence, probes, provenance, semantics
|
||||
from openultrasast.core.factpack import (
|
||||
IndependenceAttestation,
|
||||
@@ -28,14 +25,6 @@ from openultrasast.core.family import (
|
||||
derive_security_object_id,
|
||||
family_contract_digest,
|
||||
)
|
||||
from openultrasast.core.legacy_v1 import (
|
||||
FittedComponentBlocked,
|
||||
LegacyProbeResult,
|
||||
ShortcutDetected,
|
||||
assert_no_shortcut,
|
||||
require_admission_evidence,
|
||||
reversed_pair_probe,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -392,24 +381,3 @@ def test_observation_for_another_probe_is_refused() -> None:
|
||||
|
||||
with pytest.raises(probes.ProbeValidationError, match="probe identity"):
|
||||
probes.evaluate_probe(case.probe, substituted, case.contract, case.policy, case.resolver)
|
||||
|
||||
|
||||
def test_legacy_reversed_pair_probe_remains_compatible() -> None:
|
||||
forward: dict[str, Any] = {"lines": 10}
|
||||
reversed_change: dict[str, Any] = {"lines": 50}
|
||||
|
||||
def shape_score(change: Mapping[str, Any]) -> float:
|
||||
return float(change["lines"])
|
||||
|
||||
result = reversed_pair_probe(shape_score, forward, reversed_change)
|
||||
assert result.separated is True
|
||||
with pytest.raises(ShortcutDetected):
|
||||
assert_no_shortcut(shape_score, forward, reversed_change)
|
||||
|
||||
assert public_core.ProbeResult is LegacyProbeResult
|
||||
|
||||
|
||||
def test_legacy_admission_guard_remains_compatible() -> None:
|
||||
require_admission_evidence(holdout_passed=True, admission_decision="GO")
|
||||
with pytest.raises(FittedComponentBlocked):
|
||||
require_admission_evidence(holdout_passed=False, admission_decision="GO")
|
||||
|
||||
@@ -1,39 +1 @@
|
||||
"""Validation tests for detection-core tasks 7.1-7.2 (Req 5, 7, 11, 12)."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from test_core_native import _legacy_record
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core.admission import AdmissionManifestError, validate_report_schema # noqa: E402
|
||||
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
|
||||
from openultrasast.core.verdict import Judgment, ProvenClaim # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_committed_admission_report_satisfies_schema() -> None:
|
||||
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
|
||||
validate_report_schema(report)
|
||||
|
||||
|
||||
def test_schema_validator_refuses_incomplete_report() -> None:
|
||||
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
|
||||
del report["precision"]
|
||||
try:
|
||||
validate_report_schema(report)
|
||||
raise AssertionError("report missing precision must be refused")
|
||||
except AdmissionManifestError:
|
||||
pass
|
||||
|
||||
|
||||
def test_end_to_end_push_through_core_contracts() -> None:
|
||||
record = _legacy_record()
|
||||
claim = ProvenClaim(trace_id="t1", witness="fact:f", path="src/app.py", line=30, family="access_control")
|
||||
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), record, (claim,), ())
|
||||
assert verdict.claims[0].witness == "fact:f"
|
||||
assert record.judgment is Judgment.UNGUARDED
|
||||
assert record.canonical() == record.canonical()
|
||||
"""Retired v1 validation tests; family-neutral validation coverage uses v2."""
|
||||
|
||||
@@ -1,113 +1 @@
|
||||
"""Task 12.5 tests: consume and validate an externally reviewed admission manifest.
|
||||
|
||||
Reqs 12.1, 12.2, 12.3, 12.6, 12.8. Valid, malformed, exposed and spent fixture
|
||||
manifests must produce deterministic accept/refuse results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from openultrasast.core import legacy_v1 as prover_admission # noqa: E402
|
||||
|
||||
|
||||
def _manifest() -> dict[str, object]:
|
||||
return {
|
||||
"produced_by": "valid-detection-benchmark",
|
||||
"reviewed": True,
|
||||
"label_isolated": True,
|
||||
"spent": False,
|
||||
"population_role": "admission",
|
||||
"analyzer_revision": "a" * 40,
|
||||
"fact_set_digest": "b" * 12,
|
||||
"acquisition_record_digest": "c" * 12,
|
||||
"configuration_digest": "d" * 12,
|
||||
"population_digest": "e" * 12,
|
||||
"labels": (),
|
||||
}
|
||||
|
||||
|
||||
def test_valid_manifest_is_accepted() -> None:
|
||||
manifest = prover_admission.validate_admission_manifest(_manifest())
|
||||
assert manifest.produced_by == "valid-detection-benchmark"
|
||||
assert manifest.reviewed is True
|
||||
assert manifest.label_isolated is True
|
||||
assert manifest.population_role == "admission"
|
||||
|
||||
|
||||
def test_malformed_manifest_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
del payload["reviewed"]
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="malformed"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_unreviewed_manifest_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["reviewed"] = False
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="unreviewed"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_label_exposing_manifest_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["labels"] = ("vulnerability:case-9",)
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="label-exposure"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_not_label_isolated_manifest_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["label_isolated"] = False
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="label-isolated"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_non_sequence_labels_are_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["labels"] = None
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="label-exposure"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_changed_acquisition_output_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="acquisition-output-changed"):
|
||||
prover_admission.validate_admission_manifest(payload, expected_acquisition_digest="f" * 12)
|
||||
|
||||
|
||||
def test_matching_acquisition_digest_is_accepted() -> None:
|
||||
manifest = prover_admission.validate_admission_manifest(_manifest(), expected_acquisition_digest="c" * 12)
|
||||
assert manifest.acquisition_record_digest == "c" * 12
|
||||
|
||||
|
||||
def test_spent_population_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["spent"] = True
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="spent"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
|
||||
|
||||
def test_validation_is_deterministic() -> None:
|
||||
first = prover_admission.validate_admission_manifest(_manifest())
|
||||
second = prover_admission.validate_admission_manifest(copy.deepcopy(_manifest()))
|
||||
assert first == second
|
||||
|
||||
|
||||
def test_admission_manifest_is_frozen() -> None:
|
||||
manifest = prover_admission.validate_admission_manifest(_manifest())
|
||||
with pytest.raises(Exception): # noqa: B017
|
||||
manifest.reviewed = False # type: ignore[misc]
|
||||
|
||||
|
||||
def test_non_benchmark_producer_is_refused() -> None:
|
||||
payload = _manifest()
|
||||
payload["produced_by"] = "someone-else"
|
||||
with pytest.raises(prover_admission.AdmissionManifestError, match="producer"):
|
||||
prover_admission.validate_admission_manifest(payload)
|
||||
"""Retired parked-arm v1 admission tests; benchmark admission uses v2 replay."""
|
||||
|
||||
Reference in New Issue
Block a user