marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-11-13 01:48:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(detection-core): migrate test and operational consumers

Browse Source
This commit is contained in:
Marius Ciepluch
2026-11-04 08:53:21 +01:00
parent 11059c4cd4
commit 10e93e09b7
27 changed files with 5971 additions and 1407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.kiro/specs/detection-core/tasks.md
+3 -1
View File
@@ -390,7 +390,7 @@
- _Requirements: 8.1, 12.6, 14.3, 15.3_
- _Boundary: Benchmark Readiness and Admission Adapters_
- _Depends: 9.5, 9.6, 10.3_
- [ ] 10.6 Migrate or retire registered test and operational-tool consumers
- [x] 10.6 Migrate or retire registered test and operational-tool consumers
- Move authorization assumptions into legacy or adapter tests, keep family-neutral contract tests on v2, and retain frozen v1 bytes only for decoder regression.
- Move or retire registered operational-tool consumers without adding a second compatibility or write path.
- Mark the corresponding stable inventory entries `v2_migrated` or `retired` and require every original entry to reach a terminal status.
@@ -443,4 +443,6 @@
- The frozen migration inventory is a reviewed worklist with stable identities, not a semantic-use census. It may only close entries as `v2_migrated` or `retired`; it never grows, deletes entries, or proves absence by interpreting Python flow.
- Cutover is proved by explicit syntax checks plus deletion of the compatibility and v1 write capability. No task extends the discarded alias/data-flow interpreter.
- Task 10.5 debug established that executable owner probes cannot compensate for blanket source-closure acceptance; resumed hunt attempts also require collision-resistant attempt identity and create-only persistence before handoff validation is authoritative.
- Task 10.6 debug established that exact-byte successor authority must require canonical repository-relative paths and strict resolved-root containment, including rejection of outside-root symlink targets, before module digests can prove closure.
- Human review reopened Task 10.6 for one focused remediation cycle covering canonical path syntax, resolved-root containment/symlink refusal, and the task-local formatting gate.
- Human review reopened Task 10.5 for one focused remediation cycle covering exact source-closure evidence, collision-resistant create-only attempt persistence, and complexity-cap conformance.
ops/smoke_admission.py
+1 -68
View File
@@ -1,68 +1 @@
"""Verify that captured real comparison evidence remains diagnostic by default."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from openultrasast.model.contracts import ChangeContext, QuestionOutcome, ScopeDecision
from openultrasast.model.ladder import Rung
from openultrasast.model.pipeline import ModelFinding
from openultrasast.model.scan import ModelScanResult
from openultrasast.push.contracts import ComparisonAnalysis, PushComparison
from openultrasast.push.policy import AdmissionCandidate, CapabilityKey, admission_push_result, admit_candidates, compare_evidence
def restore_scan(data: dict) -> ModelScanResult:
"""Restore only the recorded evidence consumed by the comparison policy."""
return ModelScanResult(
findings=tuple(ModelFinding(**{**finding, "rung": Rung(finding["rung"])}) for finding in data["findings"]),
scope=ScopeDecision.from_payload(data["scope"]),
question_outcomes=tuple(QuestionOutcome.from_payload(outcome) for outcome in data["question_outcomes"]),
degradations=tuple(data["degradations"]),
)
def main() -> None:
artifact = Path("benchmarks/measurements/2026-09-13-delta-comparison-smoke.json")
raw = artifact.read_bytes()
assert raw, "comparison artifact is unreadable or empty"
recorded = json.loads(raw)
report = {
"purpose": "Current policy applied to captured actual engine evidence; no new engine execution or evaluated capability",
"input_bytes": len(raw),
"input_sha256": hashlib.sha256(raw).hexdigest(),
"cases": {},
}
for case, record in recorded["cases"].items():
evidence = record["evidence"]
head = restore_scan(evidence["head"])
base = restore_scan(evidence["delta"]["base_scan"])
context = ChangeContext.from_payload(evidence["context"])
comparison = PushComparison.from_payload(evidence["comparison"])
semantics = evidence["delta"]["semantics"]
deltas = compare_evidence(head, base, context=context, head_semantics=semantics, base_semantics=semantics)
assert deltas and all(delta.novelty == evidence["expected_novelty"] for delta in deltas)
candidates = []
for delta in deltas:
operation = delta.head_operation
assert operation is not None
key = CapabilityKey(
operation.question.language, "unspecified", "unspecified", delta.family, operation.mechanism, "unreviewed", semantics
)
candidates.append(AdmissionCandidate(delta, key, comparison))
admission = admit_candidates(candidates)
assert not admission.defects and len(admission.dispositions) == len(candidates)
assert all(not disposition.admitted for disposition in admission.dispositions)
assert "capability_unavailable" in admission.coverage_reasons
assert head.scope is not None
analysis = ComparisonAnalysis(comparison, (head.scope,), tuple(q.identity for q in head.question_outcomes), "complete_within_scope")
result = admission_push_result(admission, (analysis,))
assert result.finding_status == "none" and result.coverage_status == "incomplete" and result.push_disposition == "allow"
report["cases"][case] = {"admission": admission.to_payload(), "result": result.to_payload()}
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()
"""Retired v1 admission smoke tool; canonical v2 replay owns admission smoke."""
ops/smoke_report.py
+1 -177
View File
@@ -1,177 +1 @@
"""Packaged comparison/admission/report integration over captured and synthetic evidence."""
from __future__ import annotations
import hashlib
import json
import tempfile
import time
from dataclasses import asdict, replace
from pathlib import Path
from openultrasast.config import PushConfig
from openultrasast.model.contracts import ChangeContext, ExecutionBudget, FamilyCoverage, QuestionOutcome, RankedQuestion, ScopeDecision
from openultrasast.model.ladder import Rung
from openultrasast.model.partitions import PartitionCoverage
from openultrasast.model.pipeline import ModelFinding
from openultrasast.model.scan import ModelScanResult
from openultrasast.push.contracts import ComparisonAnalysis, PushComparison
from openultrasast.push.policy import (
AdmissionCandidate,
CapabilityKey,
admission_push_result,
admit_candidates,
compare_evidence,
)
from openultrasast.push.report import PushReport, ScanRecord, deliver_report
def restore(data: dict) -> ModelScanResult:
values = dict(data)
values["findings"] = tuple(ModelFinding(**{**f, "rung": Rung(f["rung"])}) for f in data["findings"])
values["scope"] = ScopeDecision.from_payload(data["scope"]) if data["scope"] else None
values["question_outcomes"] = tuple(QuestionOutcome.from_payload(q) for q in data["question_outcomes"])
values["change_context"] = ChangeContext.from_payload(data["change_context"]) if data["change_context"] else None
values["family_coverage"] = tuple(FamilyCoverage.from_payload(c) for c in data["family_coverage"])
values["partitions"] = tuple(
PartitionCoverage(**{**p, **{k: tuple(p[k]) for k in ("paths", "boundaries", "excluded_paths", "unshipped_paths")}})
for p in data["partitions"]
)
values["degradations"] = tuple(data["degradations"])
values["tiers"] = tuple(tuple(t) for t in data["tiers"])
values["tier_counts"] = {int(k): v for k, v in data["tier_counts"].items()}
result = ModelScanResult(**values)
assert json.loads(json.dumps(asdict(result))) == data, "record restoration lost evidence"
return result
def main() -> None:
measured = Path("benchmarks/measurements")
raw = (measured / "2026-09-13-delta-comparison-smoke.json").read_bytes()
controls = json.loads(raw)
synthetic_bytes = (measured / "2026-09-13-admission-policy-smoke.json").read_bytes()
assert raw and synthetic_bytes
synthetic = json.loads(synthetic_bytes)["synthetic_controls"]["javascript"]
policy = Path("src/openultrasast/push/policy.py").read_bytes()
reporter = Path("src/openultrasast/push/report.py").read_bytes()
provenance = {
"repository": "captured immutable Git fixture comparisons",
"engine": controls["runtime"]["image"],
"facts": "covered by recorded comparison semantics",
"queries": "covered by recorded comparison semantics",
"policy": hashlib.sha256(policy + reporter).hexdigest(),
}
summary = {
"purpose": "Packaged reporting over captured actual engine evidence and explicitly synthetic admission controls; no new Joern run",
"input_bytes": len(raw) + len(synthetic_bytes),
"comparison_artifact_sha256": hashlib.sha256(raw).hexdigest(),
"policy_sha256": hashlib.sha256(policy).hexdigest(),
"report_sha256": hashlib.sha256(reporter).hexdigest(),
"cases": {},
}
with tempfile.TemporaryDirectory(prefix="ousast-report-smoke-") as directory:
root = Path(directory)
def emit(name: str, report: PushReport, *, fail: bool = False) -> None:
target = root / (name + ".json")
if fail:
target.mkdir()
delivery = deliver_report(report, target, execution_budget=ExecutionBudget(time.monotonic() + 10, 1.0))
assert delivery.result == report.result and delivery.exit_code == int(report.result.push_disposition == "block")
assert delivery.text.count("Notice:") <= 1
artifact = json.loads(target.read_text()) if delivery.artifact else None
if fail:
assert delivery.error and delivery.artifact is None and target.is_dir()
else:
assert delivery.error is None and artifact is not None
assert artifact["result"] == report.result.to_payload()
assert artifact["admission"] == report.admission.to_payload()
assert artifact["scans"] == json.loads(json.dumps([asdict(s) for s in report.scans]))
summary["cases"][name] = {
"exit_code": delivery.exit_code,
"text": delivery.text,
"error": delivery.error,
"reporting_seconds": delivery.reporting_seconds,
"artifact": artifact,
}
for name, record in controls["cases"].items():
evidence = record["evidence"]
head = restore(evidence["head"])
base = restore(evidence["delta"]["base_scan"])
context = ChangeContext.from_payload(evidence["context"])
comparison = PushComparison.from_payload(evidence["comparison"])
semantics = evidence["delta"]["semantics"]
deltas = compare_evidence(head, base, context=context, head_semantics=semantics, base_semantics=semantics)
assert deltas and all(d.novelty == evidence["expected_novelty"] for d in deltas)
candidates = []
for delta in deltas:
op = delta.head_operation
assert op is not None
key = CapabilityKey(op.question.language, "unspecified", "unspecified", delta.family, op.mechanism, "unreviewed", semantics)
candidates.append(AdmissionCandidate(delta, key, comparison))
admission = admit_candidates(candidates)
assert not admission.defects and head.scope is not None
analysis = ComparisonAnalysis(
comparison, (head.scope,), tuple(q.identity for q in head.question_outcomes), "complete_within_scope"
)
result = admission_push_result(admission, (analysis,))
assert result.coverage_status == "incomplete"
report = PushReport(
result,
admission,
(ScanRecord(comparison, "head", head), ScanRecord(comparison, "base", base)),
{**provenance, "comparison_semantics": semantics},
{"total_seconds": record["completion"]["elapsed_seconds"]},
)
emit(name, report)
# These five normalized controls test display/decision separation only.
from openultrasast.push.policy import CandidateDisposition
disposition = CandidateDisposition.from_payload(synthetic["admission"]["dispositions"][0])
cap = disposition.capability_evaluations[0]
candidate = disposition.candidate
candidates = []
questions = []
findings = []
outcomes = []
for i in range(5):
old = candidate.delta.head_operation
assert old is not None
q = replace(old.question, path=f"handler{i}.src")
op = replace(old, question=q, path=q.path)
delta = replace(
candidate.delta,
defect_id=f"synthetic-defect-{i}",
head_operation=op,
site=f"{q.path}:{op.line}:handler",
change_evidence=(f"head:{q.path}:2-2",),
)
candidates.append(replace(candidate, delta=delta))
questions.append(q)
findings.append(ModelFinding(delta.site, delta.family, Rung(delta.rung), delta.witness or "synthetic witness"))
outcomes.append(QuestionOutcome(q, "completed", "synthetic-normalized-control", "[]"))
scope = ScopeDecision("synthetic-ranker", "static", True, tuple(RankedQuestion(q, 1.0, ()) for q in questions), (), ())
scan = ModelScanResult(findings=tuple(findings), scope=scope, question_outcomes=tuple(outcomes))
base_scan = replace(scan, findings=())
analysis = ComparisonAnalysis(candidate.comparison, (scope,), tuple(questions), "complete_within_scope")
admission = admit_candidates(candidates, capabilities=(cap,))
assert len(admission.defects) == 5
result = admission_push_result(admission, (analysis,), config=PushConfig(mode="blocking"))
report = PushReport(
result,
admission,
(ScanRecord(candidate.comparison, "head", scan), ScanRecord(candidate.comparison, "base", base_scan)),
{**provenance, "repository": "synthetic normalized policy controls", "engine": "synthetic; no engine execution"},
{"total_seconds": 0.0},
)
emit("synthetic-display-cap", report)
text = summary["cases"]["synthetic-display-cap"]["text"]
assert text.count("- injection at ") == 3 and "2 more" in text
emit("synthetic-write-failure", report, fail=True)
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
main()
"""Retired v1 report smoke tool; canonical v2 consumer tests own report smoke."""
tests/consumer_migration_evidence.py
+370
View File
@@ -0,0 +1,370 @@
"""Immutable reviewed closure authority for Task 10.6 migration consumers."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path, PurePosixPath, PureWindowsPath
from types import MappingProxyType
REPO = Path(__file__).resolve().parents[1]
EVIDENCE = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
SCHEMA = "openultrasast.detection-core.consumer-terminal-evidence.v1"
_FIELDS = frozenset(
{
"absence_anchors",
"artifact_family",
"binding_owner_ast_sha256",
"binding_owner_symbol",
"capability_anchors",
"closure_kind",
"dependency_kind",
"disposition",
"entry_id",
"owner_boundary",
"path",
"probe_ast_sha256",
"probe_symbol",
"pytest_node_id",
"semantic_binding",
"semantic_contract",
"semantic_local_name",
"successor_ast_sha256",
"successor_path",
"successor_symbol",
"symbol",
}
)
_SEMANTIC_CONTRACT_BINDINGS = {
"authorization_adapter_domain": frozenset({"import_from:openultrasast.semantic.obligations.trace:build_trace"}),
"benchmark_v2_artifact": frozenset(
{
"dynamic_import:benchmarks.unseen.calibrate",
"dynamic_import:benchmarks.unseen.proof",
"dynamic_import:benchmarks.unseen.replay",
"dynamic_import:benchmarks.unseen.score",
"dynamic_import:benchmarks.valid.hunter_arm",
"dynamic_import:benchmarks.valid.semgrep_arm",
"import_from:benchmarks.valid:arm_regression",
"import_from:benchmarks.valid:development_runner",
"import_from:benchmarks.valid.development_runner:DevelopmentRecord",
}
),
"hunt_adapter_artifact": frozenset(
{
"import_from:openultrasast.hunt.artifact:RunArtifact",
"import_from:openultrasast.hunt:contamination",
"import_from:openultrasast.hunt:runner",
}
),
"push_adapter_domain": frozenset(
{
"import_from:openultrasast.push.policy:admit_candidates",
"import_from:openultrasast.push.report:PushReport",
}
),
"v2_coverage_contract": frozenset({"import_from:openultrasast.core.coverage:InstrumentFailure"}),
}
_MODULE_FIELDS = frozenset({"path", "sha256"})
class ConsumerEvidenceError(ValueError):
"""The reviewed terminal-evidence authority is malformed."""
@dataclass(frozen=True)
class ReviewedSuccessorModule:
"""One manually reviewed successor module, authoritative down to exact bytes."""
path: str
sha256: str
@dataclass(frozen=True)
class ConsumerTerminalEvidence:
entry_id: str
path: str
symbol: str
dependency_kind: str
artifact_family: str
capability_anchors: tuple[str, ...]
owner_boundary: str
disposition: str
closure_kind: str
absence_anchors: tuple[str, ...]
successor_path: str | None
successor_symbol: str | None
semantic_contract: str | None
semantic_binding: str | None
semantic_local_name: str | None
binding_owner_symbol: str | None
binding_owner_ast_sha256: str | None
probe_symbol: str | None
probe_ast_sha256: str | None
successor_ast_sha256: str | None
pytest_node_id: str | None
@property
def stable_fields(self) -> tuple[str, str, str, str, tuple[str, ...], str]:
return (
self.path,
self.symbol,
self.dependency_kind,
self.artifact_family,
self.capability_anchors,
self.owner_boundary,
)
def _text(value: object, name: str) -> str:
if not isinstance(value, str) or not value:
raise ConsumerEvidenceError(f"{name} must be nonempty text")
return value
def _optional_text(value: object, name: str) -> str | None:
return None if value is None else _text(value, name)
def _repo_relative_posix(value: object, name: str) -> str:
if not isinstance(value, str) or not value:
raise ConsumerEvidenceError(f"{name} must be an exact canonical repository-relative POSIX path")
text = value
parts = text.split("/")
posix = PurePosixPath(text)
windows = PureWindowsPath(text)
if (
posix.is_absolute()
or windows.is_absolute()
or bool(windows.drive)
or "\0" in text
or "\\" in text
or any(part in {"", ".", ".."} for part in parts)
or posix.as_posix() != text
):
raise ConsumerEvidenceError(f"{name} must be an exact canonical repository-relative POSIX path")
return text
def _optional_repo_relative_posix(value: object, name: str) -> str | None:
return None if value is None else _repo_relative_posix(value, name)
def _optional_sha256(value: object, name: str) -> str | None:
text = _optional_text(value, name)
if text is not None and (len(text) != 64 or any(character not in "0123456789abcdef" for character in text)):
raise ConsumerEvidenceError(f"{name} must be a lowercase SHA-256 digest")
return text
def _sha256(value: object, name: str) -> str:
digest = _optional_sha256(value, name)
if digest is None:
raise ConsumerEvidenceError(f"{name} must be a lowercase SHA-256 digest")
return digest
def _texts(value: object, name: str) -> tuple[str, ...]:
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
raise ConsumerEvidenceError(f"{name} must be a text list")
result = tuple(value)
if len(result) != len(set(result)):
raise ConsumerEvidenceError(f"{name} contains duplicates")
return result
def _record(value: object, index: int) -> ConsumerTerminalEvidence:
if not isinstance(value, dict) or set(value) != _FIELDS:
raise ConsumerEvidenceError(f"entries[{index}] fields differ")
anchors = _texts(value["capability_anchors"], f"entries[{index}].capability_anchors")
absence = _texts(value["absence_anchors"], f"entries[{index}].absence_anchors")
record = ConsumerTerminalEvidence(
_text(value["entry_id"], f"entries[{index}].entry_id"),
_repo_relative_posix(value["path"], f"entries[{index}].path"),
_text(value["symbol"], f"entries[{index}].symbol"),
_text(value["dependency_kind"], f"entries[{index}].dependency_kind"),
_text(value["artifact_family"], f"entries[{index}].artifact_family"),
anchors,
_text(value["owner_boundary"], f"entries[{index}].owner_boundary"),
_text(value["disposition"], f"entries[{index}].disposition"),
_text(value["closure_kind"], f"entries[{index}].closure_kind"),
absence,
_optional_repo_relative_posix(value["successor_path"], f"entries[{index}].successor_path"),
_optional_text(value["successor_symbol"], f"entries[{index}].successor_symbol"),
_optional_text(value["semantic_contract"], f"entries[{index}].semantic_contract"),
_optional_text(value["semantic_binding"], f"entries[{index}].semantic_binding"),
_optional_text(value["semantic_local_name"], f"entries[{index}].semantic_local_name"),
_optional_text(value["binding_owner_symbol"], f"entries[{index}].binding_owner_symbol"),
_optional_sha256(value["binding_owner_ast_sha256"], f"entries[{index}].binding_owner_ast_sha256"),
_optional_text(value["probe_symbol"], f"entries[{index}].probe_symbol"),
_optional_sha256(value["probe_ast_sha256"], f"entries[{index}].probe_ast_sha256"),
_optional_sha256(value["successor_ast_sha256"], f"entries[{index}].successor_ast_sha256"),
_optional_text(value["pytest_node_id"], f"entries[{index}].pytest_node_id"),
)
_validate_record(record, index)
return record
def _successor_fields(record: ConsumerTerminalEvidence) -> tuple[str | None, ...]:
return (
record.successor_path,
record.successor_symbol,
record.successor_ast_sha256,
record.semantic_contract,
record.semantic_binding,
record.semantic_local_name,
record.binding_owner_symbol,
record.binding_owner_ast_sha256,
record.probe_symbol,
record.probe_ast_sha256,
record.pytest_node_id,
)
def _validate_retired(record: ConsumerTerminalEvidence, index: int) -> None:
if record.absence_anchors != record.capability_anchors:
raise ConsumerEvidenceError(f"entries[{index}] must govern every frozen anchor")
if any(_successor_fields(record)):
raise ConsumerEvidenceError(f"entries[{index}] retired evidence has a successor")
def _validate_migrated(record: ConsumerTerminalEvidence, index: int) -> None:
if record.absence_anchors or not all(_successor_fields(record)):
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence lacks semantic binding")
allowed = _SEMANTIC_CONTRACT_BINDINGS.get(record.semantic_contract or "", frozenset())
if record.semantic_binding not in allowed or not (record.successor_path or "").startswith("tests/"):
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence is outside its reviewed contract")
expected_node = f"{record.successor_path}::{record.probe_symbol}"
if record.pytest_node_id != expected_node or not (record.probe_symbol or "").startswith("test_"):
raise ConsumerEvidenceError(f"entries[{index}] migrated evidence lacks an exact pytest owner")
def _validate_record(record: ConsumerTerminalEvidence, index: int) -> None:
if len(record.entry_id) != 64 or any(character not in "0123456789abcdef" for character in record.entry_id):
raise ConsumerEvidenceError(f"entries[{index}].entry_id is not SHA-256")
identity = {
"artifact_family": record.artifact_family,
"capability_anchors": record.capability_anchors,
"dependency_kind": record.dependency_kind,
"owner_boundary": record.owner_boundary,
"path": record.path,
"symbol": record.symbol,
}
digest = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
if digest != record.entry_id:
raise ConsumerEvidenceError(f"entries[{index}].entry_id does not match frozen identity")
if record.disposition == "retired" and record.closure_kind == "retired_exact_absence":
_validate_retired(record, index)
return
if record.disposition == "v2_migrated" and record.closure_kind == "migrated_semantic_probe":
_validate_migrated(record, index)
return
raise ConsumerEvidenceError(f"entries[{index}] has invalid closure/disposition")
def _module_record(value: object, index: int) -> ReviewedSuccessorModule:
if not isinstance(value, dict) or set(value) != _MODULE_FIELDS:
raise ConsumerEvidenceError(f"successor_modules[{index}] fields differ")
return ReviewedSuccessorModule(
path=_repo_relative_posix(value["path"], f"successor_modules[{index}].path"),
sha256=_sha256(value["sha256"], f"successor_modules[{index}].sha256"),
)
def _reviewed_modules(value: object) -> tuple[ReviewedSuccessorModule, ...]:
if not isinstance(value, list) or not value:
raise ConsumerEvidenceError("successor module authority must be nonempty")
modules = tuple(_module_record(item, index) for index, item in enumerate(value))
paths = tuple(module.path for module in modules)
digests = tuple(module.sha256 for module in modules)
if paths != tuple(sorted(paths)):
raise ConsumerEvidenceError("successor module paths must be canonically ordered")
if len(paths) != len(set(paths)):
raise ConsumerEvidenceError("successor module paths are duplicated or conflicting")
if len(digests) != len(set(digests)):
raise ConsumerEvidenceError("successor module digests are duplicated or conflicting")
return modules
def _load_consumer_authority(
path: Path = EVIDENCE,
) -> tuple[Mapping[str, ConsumerTerminalEvidence], Mapping[str, ReviewedSuccessorModule]]:
try:
content = path.read_bytes()
except OSError as exc:
raise ConsumerEvidenceError("terminal evidence is not readable") from exc
if not content:
raise ConsumerEvidenceError("terminal evidence must contain nonzero bytes")
try:
document = json.loads(content)
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ConsumerEvidenceError("terminal evidence must be valid JSON") from exc
fields = {"schema_version", "entries", "successor_modules"}
if not isinstance(document, dict) or set(document) != fields or document["schema_version"] != SCHEMA:
raise ConsumerEvidenceError("terminal evidence schema is invalid")
values = document["entries"]
if not isinstance(values, list) or not values:
raise ConsumerEvidenceError("terminal evidence entries must be nonempty")
records = tuple(_record(value, index) for index, value in enumerate(values))
if len({record.entry_id for record in records}) != len(records):
raise ConsumerEvidenceError("terminal evidence identities are duplicated")
modules = _reviewed_modules(document["successor_modules"])
migrated_paths = {record.successor_path for record in records if record.disposition == "v2_migrated"}
module_paths = {module.path for module in modules}
if None in migrated_paths or migrated_paths != module_paths:
raise ConsumerEvidenceError("successor module authority must exactly cover migrated rows")
entries_by_id = MappingProxyType({record.entry_id: record for record in records})
modules_by_path = MappingProxyType({module.path: module for module in modules})
return entries_by_id, modules_by_path
def load_consumer_terminal_evidence(path: Path = EVIDENCE) -> Mapping[str, ConsumerTerminalEvidence]:
return _load_consumer_authority(path)[0]
def load_reviewed_successor_modules(path: Path = EVIDENCE) -> Mapping[str, ReviewedSuccessorModule]:
return _load_consumer_authority(path)[1]
def review_successor_modules(paths: tuple[str, ...], *, root: Path = REPO) -> tuple[ReviewedSuccessorModule, ...]:
"""Explicit helper for a human-reviewed manifest update; validation never calls it."""
reviewed = []
for relative in paths:
relative = _repo_relative_posix(relative, "successor module path")
try:
content = (root / relative).read_bytes()
except OSError as exc:
raise ConsumerEvidenceError(f"successor module is not readable: {relative}") from exc
if not content:
raise ConsumerEvidenceError(f"successor module must contain nonzero bytes: {relative}")
reviewed.append(ReviewedSuccessorModule(relative, hashlib.sha256(content).hexdigest()))
return tuple(reviewed)
CONSUMER_TERMINAL_EVIDENCE, REVIEWED_SUCCESSOR_MODULES = _load_consumer_authority()
def migrated_pytest_node_ids() -> tuple[str, ...]:
"""Return the reviewed executable selectors without duplicate helper owners."""
return tuple(
dict.fromkeys(
evidence.pytest_node_id
for evidence in CONSUMER_TERMINAL_EVIDENCE.values()
if evidence.disposition == "v2_migrated" and evidence.pytest_node_id is not None
)
)
__all__ = [
"CONSUMER_TERMINAL_EVIDENCE",
"REVIEWED_SUCCESSOR_MODULES",
"ConsumerEvidenceError",
"ConsumerTerminalEvidence",
"ReviewedSuccessorModule",
"load_consumer_terminal_evidence",
"load_reviewed_successor_modules",
"migrated_pytest_node_ids",
"review_successor_modules",
]
tests/core_migration_inventory.py
+336
View File
@@ -3,13 +3,17 @@
from __future__ import annotations
import ast
import errno
import hashlib
import json
import os
import stat
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, BenchmarkTerminalEvidence
from consumer_migration_evidence import CONSUMER_TERMINAL_EVIDENCE, REVIEWED_SUCCESSOR_MODULES, ConsumerTerminalEvidence
from openultrasast.core.legacy import LegacyArtifact, decode_legacy_artifact
@@ -491,7 +495,339 @@ def _benchmark_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Pat
return True
def _actual_definitions(tree: ast.Module) -> dict[str, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef]:
definitions: dict[str, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef] = {}
def visit(nodes: list[ast.stmt], parents: tuple[str, ...] = ()) -> None:
for node in nodes:
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
qualified = ".".join((*parents, node.name))
definitions[qualified] = node
visit(node.body, (*parents, node.name))
visit(tree.body)
return definitions
def _exact_anchor_present(anchor: str, tree: ast.Module) -> bool:
kind, separator, value = anchor.partition(":")
if not separator:
return False
if kind == "definition":
return value in _actual_definitions(tree)
return value in _capability_index(tree).get(kind, set())
def _retired_anchor_present(entry: LegacyInventoryEntry, anchor: str, tree: ast.Module) -> bool:
kind, separator, _ = anchor.partition(":")
if not separator or kind not in {"name", "constant"} or entry.symbol == "<module>":
return _exact_anchor_present(anchor, tree)
owner = _actual_definitions(tree).get(entry.symbol)
if not isinstance(owner, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
return False
scoped = ast.Module(body=owner.body, type_ignores=[])
return _exact_anchor_present(anchor, scoped)
def _target_names(node: ast.AST) -> tuple[str, ...]:
if isinstance(node, ast.Name):
return (node.id,)
if isinstance(node, (ast.List, ast.Tuple)):
return tuple(name for item in node.elts for name in _target_names(item))
if isinstance(node, ast.Starred):
return _target_names(node.value)
return ()
def _pattern_names(node: ast.pattern) -> tuple[str, ...]:
names = tuple(value for value in (getattr(node, "name", None), getattr(node, "rest", None)) if isinstance(value, str))
return (*names, *(name for child in ast.iter_child_nodes(node) if isinstance(child, ast.pattern) for name in _pattern_names(child)))
def _module_scope_nodes(node: ast.AST) -> tuple[ast.AST, ...]:
nodes = [node]
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)):
return tuple(nodes)
for child in ast.iter_child_nodes(node):
nodes.extend(_module_scope_nodes(child))
return tuple(nodes)
def _dynamic_import_signature(node: ast.Assign | ast.AnnAssign) -> str | None:
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
value = node.value
if len(targets) != 1 or not isinstance(targets[0], ast.Name) or not isinstance(value, ast.Call):
return None
if _call_leaf(value) != "import_module" or len(value.args) != 1 or value.keywords:
return None
module = value.args[0]
if not isinstance(module, ast.Constant) or not isinstance(module.value, str):
return None
return f"dynamic_import:{module.value}"
def _node_bindings(node: ast.AST) -> tuple[tuple[str, str], ...]:
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
return ((node.name, type(node).__name__),)
if isinstance(node, (ast.Assign, ast.AnnAssign)):
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
signature = _dynamic_import_signature(node) or type(node).__name__
return tuple((name, signature) for target in targets for name in _target_names(target))
if isinstance(node, (ast.AugAssign, ast.NamedExpr)):
return tuple((name, type(node).__name__) for name in _target_names(node.target))
if isinstance(node, ast.Delete):
return tuple((name, "Delete") for target in node.targets for name in _target_names(target))
if isinstance(node, (ast.For, ast.AsyncFor)):
return tuple((name, type(node).__name__) for name in _target_names(node.target))
if isinstance(node, (ast.With, ast.AsyncWith)):
return tuple((name, type(node).__name__) for item in node.items if item.optional_vars for name in _target_names(item.optional_vars))
if isinstance(node, ast.ExceptHandler) and node.name:
return ((node.name, "ExceptHandler"),)
if isinstance(node, ast.match_case):
return tuple((name, "MatchCase") for name in _pattern_names(node.pattern))
return _import_bindings(node)
def _import_bindings(node: ast.AST) -> tuple[tuple[str, str], ...]:
if isinstance(node, ast.Import):
return tuple((alias.asname or alias.name.split(".", 1)[0], f"import:{alias.name}") for alias in node.names)
if isinstance(node, ast.ImportFrom) and node.module is not None:
return tuple((alias.asname or alias.name, f"import_from:{node.module}:{alias.name}") for alias in node.names)
return ()
def _binding_events(tree: ast.Module, names: set[str]) -> dict[str, list[tuple[int, int, str]]]:
events: dict[str, list[tuple[int, int, str]]] = {name: [] for name in names}
for node in _module_scope_nodes(tree):
for name, signature in _node_bindings(node):
if name in events:
events[name].append((getattr(node, "lineno", 0), getattr(node, "col_offset", 0), signature))
return events
def _direct_reviewed_functions(tree: ast.Module, names: set[str]) -> dict[str, ast.FunctionDef]:
result: dict[str, ast.FunctionDef] = {}
for name in names:
matches = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name]
if len(matches) != 1:
raise LegacyInventoryError(f"migrated test owner binding terminal binding requires one direct FunctionDef: {name}")
result[name] = matches[0]
return result
def _marker_leaf(node: ast.AST) -> str:
value = node.func if isinstance(node, ast.Call) else node
if isinstance(value, ast.Attribute):
return value.attr
if isinstance(value, ast.Name):
return value.id
return ""
def _validate_pytest_outcome(tree: ast.Module, selectors: set[str], functions: dict[str, ast.FunctionDef]) -> None:
forbidden = {"skip", "skipif", "xfail"}
if any(
_marker_leaf(candidate) in forbidden
for name in selectors
for decorator in functions[name].decorator_list
for candidate in ast.walk(decorator)
):
raise LegacyInventoryError("migrated test pytest outcome is disabled by a reviewed-selector decorator")
for node in _module_scope_nodes(tree):
if isinstance(node, (ast.Assign, ast.Delete)):
targets = node.targets
elif isinstance(node, (ast.AnnAssign, ast.AugAssign)):
targets = [node.target]
else:
targets = []
bindings = _node_bindings(node)
if any(name == "pytestmark" for name, _ in bindings):
raise LegacyInventoryError("migrated test pytest outcome is disabled by module pytestmark")
if any(
isinstance(target, ast.Attribute) and target.attr == "pytestmark" and _dotted(target.value) in selectors for target in targets
):
raise LegacyInventoryError("migrated test pytest outcome is disabled by function pytestmark")
if isinstance(node, ast.Call) and _call_leaf(node) in {"globals", "locals", "setattr", "exec"}:
raise LegacyInventoryError("migrated test terminal binding has opaque namespace mutation")
if any(name in {"__getattr__", "__dir__"} for name, _ in bindings):
raise LegacyInventoryError("migrated test terminal binding has opaque module namespace hooks")
def _direct_semantic_event(tree: ast.Module, local_name: str, binding: str) -> tuple[int, int, str]:
matches = [
(node.lineno, node.col_offset, signature)
for node in tree.body
for name, signature in _node_bindings(node)
if name == local_name and signature == binding
]
if len(matches) != 1:
raise LegacyInventoryError(f"migrated test semantic evidence terminal binding lacks one direct reviewed import: {local_name}")
return matches[0]
def _validate_terminal_bindings(tree: ast.Module, evidence: ConsumerTerminalEvidence) -> dict[str, ast.FunctionDef]:
reviewed = (evidence,)
callable_names = {
name for item in reviewed for name in (item.successor_symbol, item.probe_symbol, item.binding_owner_symbol) if name is not None
}
semantic_bindings = {(item.semantic_local_name, item.semantic_binding) for item in reviewed}
if any(local is None or binding is None for local, binding in semantic_bindings):
raise LegacyInventoryError("migrated test terminal binding evidence is incomplete")
semantic_names = {local for local, _ in semantic_bindings if local is not None}
functions = _direct_reviewed_functions(tree, callable_names)
events = _binding_events(tree, callable_names | semantic_names)
for name, function in functions.items():
reviewed_event = (function.lineno, function.col_offset, "FunctionDef")
if not events[name] or events[name][-1] != reviewed_event:
raise LegacyInventoryError(f"migrated test terminal binding was replaced or removed: {name}")
for local_name, binding in semantic_bindings:
assert local_name is not None and binding is not None
reviewed_event = _direct_semantic_event(tree, local_name, binding)
if not events[local_name] or events[local_name][-1] != reviewed_event:
raise LegacyInventoryError(f"migrated test terminal binding replaced reviewed import: {local_name}")
selectors = {item.probe_symbol for item in reviewed if item.probe_symbol is not None}
_validate_pytest_outcome(tree, selectors, functions)
return functions
def _normalized_ast(value: object) -> object:
if isinstance(value, ast.AST):
fields = tuple((field, _normalized_ast(item)) for field, item in ast.iter_fields(value) if field != "type_params")
return type(value).__name__, fields
if isinstance(value, list):
return tuple(_normalized_ast(item) for item in value)
return value
def _callable_ast_digest(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
return hashlib.sha256(repr(_normalized_ast(node)).encode()).hexdigest()
def _references_binding(node: ast.FunctionDef | ast.AsyncFunctionDef, local_name: str) -> bool:
return any(isinstance(candidate, ast.Name) and candidate.id == local_name for candidate in ast.walk(node))
def _canonical_relative_parts(relative: str) -> tuple[str, ...]:
parts = tuple(relative.split("/"))
if not relative or relative.startswith("/") or "\0" in relative or "\\" in relative or any(part in {"", ".", ".."} for part in parts):
raise LegacyInventoryError(f"migrated test successor path is not canonical: {relative}")
if len(parts[0]) == 2 and parts[0][1] == ":":
raise LegacyInventoryError(f"migrated test successor path is not canonical: {relative}")
return parts
def _resolved_beneath_root(relative: str, root: Path) -> None:
try:
root_status = root.lstat()
except OSError as exc:
raise LegacyInventoryError("migrated test successor supplied root is not a real directory") from exc
if not stat.S_ISDIR(root_status.st_mode):
raise LegacyInventoryError("migrated test successor supplied root is not a real directory")
try:
resolved_root = root.resolve(strict=True)
resolved_candidate = (root / relative).resolve(strict=True)
except (OSError, RuntimeError) as exc:
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}") from exc
if not resolved_candidate.is_relative_to(resolved_root):
raise LegacyInventoryError(f"migrated test successor is not a symlink-free file beneath the supplied root: {relative}")
def _read_from_descriptor(descriptor: int, relative: str) -> bytes:
status = os.fstat(descriptor)
if not stat.S_ISREG(status.st_mode) or not status.st_mode & 0o444:
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}")
chunks = []
while chunk := os.read(descriptor, 1024 * 1024):
chunks.append(chunk)
return b"".join(chunks)
def _read_file_beneath_root(relative: str, *, root: Path) -> bytes:
parts = _canonical_relative_parts(relative)
_resolved_beneath_root(relative, root)
directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
file_flags = os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC
descriptors = []
try:
descriptors.append(os.open(root, directory_flags))
for part in parts[:-1]:
descriptors.append(os.open(part, directory_flags, dir_fd=descriptors[-1]))
descriptors.append(os.open(parts[-1], file_flags, dir_fd=descriptors[-1]))
return _read_from_descriptor(descriptors[-1], relative)
except LegacyInventoryError:
raise
except OSError as exc:
if exc.errno in {errno.EACCES, errno.ENOENT, errno.EPERM}:
raise LegacyInventoryError(f"migrated test successor is not readable: {relative}") from exc
raise LegacyInventoryError(f"migrated test successor is not a symlink-free file beneath the supplied root: {relative}") from exc
finally:
for descriptor in reversed(descriptors):
os.close(descriptor)
def _read_reviewed_successor_module(relative: str, *, root: Path) -> ast.Module:
reviewed = REVIEWED_SUCCESSOR_MODULES.get(relative)
if reviewed is None:
raise LegacyInventoryError(f"migrated test successor is outside module authority: {relative}")
content = _read_file_beneath_root(relative, root=root)
if not content:
raise LegacyInventoryError(f"migrated test successor must contain nonzero bytes: {relative}")
if hashlib.sha256(content).hexdigest() != reviewed.sha256:
raise LegacyInventoryError(f"migrated test successor module digest changed: {relative}")
try:
return ast.parse(content, filename=relative)
except (SyntaxError, UnicodeDecodeError) as exc:
raise LegacyInventoryError(f"migrated test successor is not parseable: {relative}") from exc
def _validate_migrated_evidence(entry: LegacyInventoryEntry, evidence: ConsumerTerminalEvidence, *, root: Path) -> None:
assert evidence.successor_path and evidence.successor_symbol and evidence.semantic_binding and evidence.probe_symbol
assert evidence.successor_ast_sha256 and evidence.probe_ast_sha256
assert evidence.binding_owner_symbol and evidence.binding_owner_ast_sha256 and evidence.semantic_local_name
successor = _read_reviewed_successor_module(evidence.successor_path, root=root)
definitions = _validate_terminal_bindings(successor, evidence)
owner = definitions.get(evidence.successor_symbol)
probe = definitions.get(evidence.probe_symbol)
binding_owner = definitions.get(evidence.binding_owner_symbol)
anchors_present = all(_exact_anchor_present(anchor, successor) for anchor in evidence.capability_anchors)
callables = (owner, probe, binding_owner)
if (
entry.owner_boundary != "tests"
or not anchors_present
or not all(isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) for node in callables)
):
raise LegacyInventoryError(f"migrated test owner binding is absent: {entry.entry_id}")
assert isinstance(owner, (ast.FunctionDef, ast.AsyncFunctionDef))
assert isinstance(probe, (ast.FunctionDef, ast.AsyncFunctionDef))
assert isinstance(binding_owner, (ast.FunctionDef, ast.AsyncFunctionDef))
digests = (_callable_ast_digest(owner), _callable_ast_digest(probe), _callable_ast_digest(binding_owner))
expected = (evidence.successor_ast_sha256, evidence.probe_ast_sha256, evidence.binding_owner_ast_sha256)
if digests != expected or not _references_binding(binding_owner, evidence.semantic_local_name):
raise LegacyInventoryError(f"migrated test semantic evidence does not match reviewed ownership: {entry.entry_id}")
def _consumer_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
evidence = CONSUMER_TERMINAL_EVIDENCE.get(entry.entry_id)
if evidence is None:
return False
if (
evidence.stable_fields != _stable_entry_fields(entry)
or evidence.disposition != entry.disposition.value
or entry.entry_id != entry_identity(entry)
):
raise LegacyInventoryError(f"consumer terminal evidence is absent or mismatched: {entry.entry_id}")
if evidence.closure_kind == "migrated_semantic_probe":
_validate_migrated_evidence(entry, evidence, root=root)
return True
present = tuple(anchor for anchor in evidence.absence_anchors if _retired_anchor_present(entry, anchor, tree))
if present:
raise LegacyInventoryError(f"retired consumer capability remains live: {entry.entry_id}:{present}")
return True
def _approved_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
if _consumer_terminal_source_closure(entry, root=root, tree=tree):
return True
if _benchmark_terminal_source_closure(entry, root=root, tree=tree):
return True
approved = _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition
tests/core_migration_scope.py
+4 -9
View File
@@ -9,11 +9,6 @@ from collections.abc import Sequence
from pathlib import Path, PurePosixPath
REPO = Path(__file__).resolve().parents[1]
_TASK_FILES = (
"tests/core_migration_boundary.py",
"tests/core_migration_scope.py",
"tests/test_core_migration_boundary.py",
)
_FIXTURE_PREFIXES = ("benchmarks/fixtures/", "tests/fixtures/")
COMPATIBILITY_PATHS = frozenset(
"""src/openultrasast/core/__init__.py src/openultrasast/core/_legacy_factpack.py
@@ -48,10 +43,10 @@ def _owned_python(relative: str) -> bool:
def boundary_python(root: Path = REPO) -> tuple[Path, ...]:
"""Return owned tracked Python, excluding only harvested fixture material."""
completed = subprocess.run(["git", "ls-files", "*.py"], cwd=root, check=True, capture_output=True, text=True)
relatives = {value for value in completed.stdout.splitlines() if _owned_python(value)}
relatives.update(path for path in _TASK_FILES if (root / path).is_file())
"""Return owned tracked and current untracked Python source."""
commands = (("git", "ls-files", "*.py"), ("git", "ls-files", "--others", "--exclude-standard", "--", "*.py"))
outputs = (subprocess.run(command, cwd=root, check=True, capture_output=True, text=True).stdout for command in commands)
relatives = {value for output in outputs for value in output.splitlines() if _owned_python(value)}
return tuple(root / relative for relative in sorted(relatives))
tests/fixtures/detection_core/v1/consumer_terminal_evidence.json
Vendored
+4410
View File
File diff suppressed because it is too large Load Diff
tests/fixtures/detection_core/v1/migration_inventory.json
Vendored
+155 -155
View File
File diff suppressed because it is too large Load Diff
tests/test_admission_scoring.py
+1 -191
View File
@@ -1,191 +1 @@
"""Paired scoring and the admission report (Task 12.6, Reqs 5.2, 12.4, 12.5, 12.7-12.9)."""
import json
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from benchmarks.valid.admission_scoring import (
AdmissionCase,
AdmissionScoringError,
OrdinaryChange,
admission_report,
)
from openultrasast.core.legacy_v1 import AdmissionManifest, validate_admission_manifest
def admission_manifest(role: str = "admission") -> AdmissionManifest:
payload = {
"produced_by": "valid-detection-benchmark",
"reviewed": True,
"label_isolated": True,
"spent": False,
"population_role": role,
"analyzer_revision": "analyzer-1",
"fact_set_digest": "facts-1",
"acquisition_record_digest": "acq-1",
"configuration_digest": "cfg-1",
"population_digest": "pop-1",
"labels": ("population-overview",),
}
return validate_admission_manifest(payload)
def correct_pair(case_id: str = "p1", *, seconds: float = 1.0) -> AdmissionCase:
return AdmissionCase(
case_id=case_id,
vulnerable_judgment="unguarded",
fixed_judgment="discharged",
location_verified=True,
seconds=seconds,
)
def quiet_ordinary(change_id: str = "o1") -> OrdinaryChange:
return OrdinaryChange(change_id=change_id, judgment="discharged", mislabeled_defect=False)
def test_vulnerable_only_pair_scores_correct() -> None:
report = admission_report(admission_manifest(), (correct_pair(),), (quiet_ordinary(),), coverage=1.0)
assert report["pairs"] == {"n": 1, "correct": 1}
assert report["pair_correctness"]["rate"] == 1.0
assert report["fix_recall"]["rate"] == 1.0
assert report["decision"] == "GO"
assert report["failed_gates"] == []
def test_fixed_side_regression_counts_pair_incorrect() -> None:
regressed = AdmissionCase(
case_id="p1",
vulnerable_judgment="unguarded",
fixed_judgment="unguarded",
location_verified=True,
seconds=1.0,
)
report = admission_report(admission_manifest(), (regressed,), (), coverage=1.0)
assert report["pairs"]["correct"] == 0
assert report["pair_correctness"]["rate"] == 0.0
assert report["fix_recall"]["rate"] == 1.0
def test_indeterminate_vulnerable_side_is_never_clean() -> None:
indeterminate = AdmissionCase(
case_id="p1",
vulnerable_judgment="indeterminate",
fixed_judgment="discharged",
location_verified=False,
seconds=1.0,
)
report = admission_report(admission_manifest(), (indeterminate,), (), coverage=1.0)
assert report["fix_recall"]["count"] == 0
assert report["fix_recall"]["rate"] == 0.0
assert report["indeterminate_rate"]["count"] == 1
assert report["indeterminate_rate"]["rate"] > 0.0
def test_mislabeled_negative_is_refused() -> None:
mislabeled = OrdinaryChange(change_id="o1", judgment="discharged", mislabeled_defect=True)
with pytest.raises(AdmissionScoringError, match="mislabeled"):
admission_report(admission_manifest(), (correct_pair(),), (mislabeled,), coverage=1.0)
def test_development_population_is_refused() -> None:
with pytest.raises(AdmissionScoringError, match="population-role"):
admission_report(admission_manifest(role="development"), (correct_pair(),), (), coverage=1.0)
def test_verified_location_recall_keeps_sample_count() -> None:
unverified = AdmissionCase(
case_id="p2",
vulnerable_judgment="unguarded",
fixed_judgment="discharged",
location_verified=False,
seconds=1.0,
)
report = admission_report(admission_manifest(), (correct_pair("p1"), unverified), (), coverage=1.0)
assert report["verified_location_recall"]["n"] == 2
assert report["verified_location_recall"]["count"] == 1
assert report["verified_location_recall"]["rate"] == 0.5
def test_flagged_ordinary_per_100_and_precision() -> None:
flagged = [OrdinaryChange(f"o{i}", "unguarded", False) for i in range(2)]
quiet = [OrdinaryChange(f"q{i}", "discharged", False) for i in range(2)]
pairs = tuple(correct_pair(f"p{i}") for i in range(3))
report = admission_report(admission_manifest(), pairs, tuple(flagged + quiet), coverage=1.0)
per_100 = report["flagged_ordinary_per_100"]
assert per_100["n"] == 4
assert per_100["count"] == 2
assert per_100["rate"] == 50.0
assert per_100["wilson95"][0] is not None and per_100["wilson95"][0] <= 50.0 <= per_100["wilson95"][1]
assert report["precision"]["rate"] == pytest.approx(3 / 5)
def test_budget_gate_returns_no_go() -> None:
slow = correct_pair("slow", seconds=121.0)
report = admission_report(admission_manifest(), (slow,), (), coverage=1.0)
assert report["decision"] == "NO-GO"
assert any("budget" in gate for gate in report["failed_gates"])
assert any("slow" in gate for gate in report["failed_gates"])
def test_instrument_gate_returns_no_go() -> None:
report = admission_report(admission_manifest(), (correct_pair(),), (), coverage=1.0, instrument_failures=1)
assert report["decision"] == "NO-GO"
assert any("instrument" in gate for gate in report["failed_gates"])
def test_pre_registered_criterion_failure_returns_no_go() -> None:
missed = AdmissionCase(
case_id="p1",
vulnerable_judgment="discharged",
fixed_judgment="discharged",
location_verified=False,
seconds=1.0,
)
report = admission_report(
admission_manifest(),
(missed,),
(),
coverage=1.0,
criteria={"min_fix_recall": 1.0},
)
assert report["decision"] == "NO-GO"
assert any("min_fix_recall" in gate for gate in report["failed_gates"])
def test_invalid_judgment_is_refused() -> None:
broken = AdmissionCase(
case_id="p1",
vulnerable_judgment="banana",
fixed_judgment="discharged",
location_verified=True,
seconds=1.0,
)
with pytest.raises(AdmissionScoringError, match="judgment"):
admission_report(admission_manifest(), (broken,), (), coverage=1.0)
def test_report_is_deterministic() -> None:
args = (
admission_manifest(),
(correct_pair("p1"), correct_pair("p2")),
(quiet_ordinary("o1"), OrdinaryChange("o2", "unguarded", False)),
)
first = admission_report(*args, coverage=0.75)
second = admission_report(*args, coverage=0.75)
assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)
def test_report_carries_provenance_without_labels() -> None:
report = admission_report(admission_manifest(), (correct_pair(),), (), coverage=1.0)
manifest = report["manifest"]
assert manifest["analyzer_revision"] == "analyzer-1"
assert manifest["population_digest"] == "pop-1"
assert manifest["population_role"] == "admission"
text = json.dumps(report, sort_keys=True)
for marker in ("vulnerability", "fix-location", "ordinary-change"):
assert marker not in text
"""Retired authorization-shaped v1 scorer tests; admission conformance uses v2."""
tests/test_core_admission.py
+1 -82
View File
@@ -1,82 +1 @@
"""Admission contract tests for detection-core tasks 4.1-4.2 (Req 7, 10, 12)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.admission import ( # noqa: E402
REPORT_SCHEMA_KEYS,
AdmissionManifest,
AdmissionManifestError,
validate_admission_manifest,
)
def _payload(**over: object) -> dict[str, object]:
payload: dict[str, object] = {
"produced_by": "valid-detection-benchmark",
"reviewed": True,
"label_isolated": True,
"spent": False,
"population_role": "admission",
"analyzer_revision": "d063167",
"fact_set_digest": "3eb5bf699434",
"acquisition_record_digest": "none",
"configuration_digest": "095b5a4905ad",
"population_digest": "c866c0e812c0",
"labels": [],
"families": ["access_control"],
}
payload.update(over)
return payload
def test_valid_manifest_accepted_with_family_scope() -> None:
manifest = validate_admission_manifest(_payload())
assert isinstance(manifest, AdmissionManifest)
assert manifest.population_role == "admission"
assert manifest.families == ("access_control",)
def test_refusal_order_is_deterministic() -> None:
cases = [
({}, "malformed"),
(_payload(produced_by="other"), "producer"),
(_payload(reviewed=False), "unreviewed"),
(_payload(label_isolated=False), "label-isolated"),
(_payload(labels=["vulnerability"]), "label-exposure"),
(_payload(spent=True), "spent"),
]
for payload, reason in cases:
try:
validate_admission_manifest(payload)
raise AssertionError(f"{reason} must be refused")
except AdmissionManifestError as error:
assert reason in str(error)
def test_acquisition_output_changed_refusal() -> None:
try:
validate_admission_manifest(_payload(), expected_acquisition_digest="different")
raise AssertionError("acquisition mismatch must be refused")
except AdmissionManifestError as error:
assert "acquisition-output-changed" in str(error)
def test_report_schema_requires_family_scope_and_counts() -> None:
for key in (
"manifest",
"families",
"pairs",
"fix_recall",
"flagged_ordinary_per_100",
"precision",
"indeterminate_rate",
"coverage",
"runtime",
"instrument_failures",
"failed_gates",
"decision",
):
assert key in REPORT_SCHEMA_KEYS
"""Retired v1 admission tests; family-neutral admission uses the v2 suites."""
tests/test_core_budget.py
+1 -60
View File
@@ -1,60 +1 @@
"""Measured-budget contract tests for detection-core task 2 (Req 4, 12)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.legacy_v1 import ( # noqa: E402
Budget,
BudgetGateFailure,
Complexity,
check_budget,
derive_budget,
)
SAMPLES = tuple((Complexity(regions=1 + i % 3, calls=10 * i, changed_bytes=100 * i), float(1 + i)) for i in range(20))
def test_derive_budget_publishes_derivation_metadata() -> None:
budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu")
assert isinstance(budget, Budget)
assert budget.workload_class == "dev-pool"
assert budget.n == 20
assert budget.percentile == 0.95
assert budget.hardware == "ci-runner-4cpu"
assert budget.seconds == 19.05 # linear-interpolated p95 of samples 1..20
def test_derive_budget_is_deterministic() -> None:
first = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h")
second = derive_budget(SAMPLES, workload_class="pilot", percentile=0.9, hardware="h")
assert first == second
def test_derive_budget_refuses_empty_samples() -> None:
try:
derive_budget((), workload_class="dev", percentile=0.95, hardware="h")
raise AssertionError("empty sample set must be refused")
except ValueError:
pass
def test_budget_is_configurable_per_class_not_a_constant() -> None:
dev = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="h")
pilot = derive_budget(tuple((c, s / 10) for c, s in SAMPLES), workload_class="pilot", percentile=0.95, hardware="h")
assert dev.seconds != pilot.seconds
assert dev.workload_class != pilot.workload_class
def test_budget_gate_names_change_elapsed_and_derivation() -> None:
budget = derive_budget(SAMPLES, workload_class="dev-pool", percentile=0.95, hardware="ci-runner-4cpu")
check_budget("case-1", 5.0, budget)
try:
check_budget("case-2", 999.0, budget)
raise AssertionError("over-budget change must fail the gate")
except BudgetGateFailure as error:
message = str(error)
assert "case-2" in message
assert "999" in message
assert "dev-pool" in message and "ci-runner-4cpu" in message and "n=20" in message
"""Retired v1 budget tests; family-neutral budget coverage uses v2."""
tests/test_core_budget_v2.py
-11
View File
@@ -12,10 +12,6 @@ from detection_core_fixtures import InMemoryContentResolver, stored_artifact
from openultrasast.core import budget as budget_v2
from openultrasast.core import canonical, evidence, provenance, semantics
from openultrasast.core.legacy_v1 import Budget as LegacyBudget
from openultrasast.core.legacy_v1 import Complexity as LegacyComplexity
from openultrasast.core.legacy_v1 import check_budget as legacy_check_budget
from openultrasast.core.legacy_v1 import derive_budget as legacy_derive_budget
def _digest(value: str) -> str:
@@ -343,10 +339,3 @@ def test_scope_revalidates_post_construction_tampering() -> None:
object.__setattr__(allowed.regions[0], "end_line", 0)
with pytest.raises((evidence.EvidenceValidationError, evidence.ScopeValidationError)):
evidence.validate_scope(_packet(), allowed)
def test_legacy_budget_api_remains_available() -> None:
samples = ((LegacyComplexity(1, 1, 10), 1.0), (LegacyComplexity(2, 2, 20), 2.0))
legacy = legacy_derive_budget(samples, workload_class="legacy", percentile=0.5, hardware="runner")
assert isinstance(legacy, LegacyBudget)
legacy_check_budget(change_id="change", seconds=1.0, budget=legacy)
tests/test_core_contract.py
+1 -144
View File
@@ -1,144 +1 @@
"""Core contract conformance tests for detection-core tasks 1.1-1.4 (Reqs 1, 2, 3, 5, 6, 9)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.legacy_v1 import ( # noqa: E402
BoundedSilence,
Coverage,
Guard,
Hop,
InstrumentFailure,
Judgment,
ProvenanceMismatch,
ProvenClaim,
TraceRecord,
Witness,
as_clean,
check_reads,
stamp,
verify_stamp,
)
def _trace() -> TraceRecord:
return TraceRecord(
entry="src/app.py::handler",
hops=(Hop(function="handler", call_line=10, complete=True),),
operation_kind="protected_read",
resource="credentials",
authority="admin",
witnesses=(Witness(kind="guard", provenance="fact:checkPermission", relation="all_paths"),),
guards=(Guard(kind="permission_check", effect="branches"),),
completeness_reasons=(),
judgment=Judgment.UNGUARDED,
digests={"input": "abc123"},
)
def test_trace_serializes_byte_identical_on_repeat() -> None:
first = _trace().canonical()
second = _trace().canonical()
assert first == second
assert isinstance(first, bytes)
def test_trace_record_pins_schema_fields() -> None:
record = _trace().record()
for field in (
"entry",
"hops",
"operation_kind",
"resource",
"authority",
"witnesses",
"guards",
"completeness_reasons",
"judgment",
"digests",
):
assert field in record
assert record["judgment"] == "unguarded"
def test_judgment_vocabulary_is_exactly_three_way() -> None:
assert {j.value for j in Judgment} == {"unguarded", "discharged", "indeterminate"}
def test_indeterminate_never_converts_to_clean() -> None:
assert as_clean(Judgment.DISCHARGED) == Judgment.DISCHARGED
try:
as_clean(Judgment.INDETERMINATE)
raise AssertionError("indeterminate must never convert to clean")
except ValueError:
pass
def test_proven_claim_requires_witness_and_location() -> None:
claim = ProvenClaim(trace_id="t1", witness="w1", path="src/app.py", line=10, family="access_control")
assert claim.family == "access_control"
silence = BoundedSilence(path="src/app.py", reason="unreached", coverage={"entry_points": 0})
assert silence.reason == "unreached"
def test_coverage_requires_exactly_one_bucket_per_call() -> None:
good = Coverage(entry_points=2, frameworks_recognised={"jenkins": 1}, calls=3, complete=1, incomplete=1, unresolved=1)
good.validate()
bad = Coverage(entry_points=1, frameworks_recognised={}, calls=3, complete=2, incomplete=1, unresolved=1)
try:
bad.validate()
raise AssertionError("two buckets for one call must be refused")
except ValueError:
pass
negative = Coverage(entry_points=1, frameworks_recognised={}, calls=3, complete=-1, incomplete=2, unresolved=2)
try:
negative.validate()
raise AssertionError("negative bucket counts must be refused")
except ValueError:
pass
def test_verdict_contract_exposes_no_model_role_path() -> None:
import openultrasast.core.verdict as verdict_module
public = verdict_module.__all__
assert not any("model" in name.lower() or "judge" in name.lower() for name in public), (
"the verdict contract must expose no model/judge path; models judge residuals zero-shot over a trace only"
)
def test_zero_byte_read_is_instrument_failure() -> None:
try:
check_reads({"src/app.py": 0, "src/lib.py": 12})
raise AssertionError("zero-byte read must fail the run")
except InstrumentFailure:
pass
check_reads({"src/app.py": 5})
def test_provenance_stamp_and_mismatch_refusal() -> None:
record = stamp(
analyzer_revision="d063167",
fact_set_digest="3eb5bf699434",
acquisition_record_digest="none",
configuration_digest="095b5a4905ad",
population_digest="c866c0e812c0",
change_id="case-1",
)
assert set(record) == {
"analyzer_revision",
"fact_set_digest",
"acquisition_record_digest",
"configuration_digest",
"population_digest",
"input_digest",
}
verify_stamp(record, record.copy())
tampered = {**record, "fact_set_digest": "ffffffffffff"}
try:
verify_stamp(tampered, record)
raise AssertionError("mismatched digest must be refused")
except ProvenanceMismatch:
pass
"""Retired authorization-shaped v1 contracts; family-neutral contracts use v2."""
tests/test_core_coverage_v2.py
+1 -9
View File
@@ -17,7 +17,6 @@ from openultrasast.core.coverage import (
diagnose_first_zero,
validate_instrument,
)
from openultrasast.core.legacy_v1 import InstrumentFailure as LegacyInstrumentFailure
from openultrasast.core.semantics import RelationStatus
DIGEST_A = "a" * 64
@@ -75,10 +74,7 @@ def test_input_manifest_and_receipts_validate_exactly() -> None:
validate_instrument(_manifest(), _receipts())
def test_v2_instrument_failure_is_distinct_from_legacy_and_owns_input_refusal() -> None:
from openultrasast.core.coverage import InstrumentFailure
assert InstrumentFailure is not LegacyInstrumentFailure
def test_v2_instrument_failure_owns_input_refusal() -> None:
receipt = InputReadReceipt("src/app.py", "src/app.py", False, 12, DIGEST_A)
with pytest.raises(InstrumentFailure, match="not opened"):
validate_instrument(_manifest(), (receipt, _receipts()[1]))
@@ -94,10 +90,6 @@ def test_zero_named_counts_are_allowed_for_measurements_but_not_unresolved_cover
)
def test_legacy_instrument_failure_is_available_only_through_compatibility_boundary() -> None:
assert LegacyInstrumentFailure.__module__ == "openultrasast.core.legacy_v1.contracts"
@pytest.mark.parametrize(
("opened", "byte_count", "digest", "match"),
[
tests/test_core_factpack_consumer.py
+1 -69
View File
@@ -1,69 +1 @@
"""Fact-pack and consumer contract tests for detection-core tasks 6.1-6.2 (Req 9, 10, 11)."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
from openultrasast.core.legacy_v1 import FactPackError, admit_fact_pack, verify_fact_pack # noqa: E402
from openultrasast.core.trace import TraceRecord # noqa: E402
from openultrasast.core.verdict import BoundedSilence, Judgment, ProvenClaim # noqa: E402
FACTS = ({"id": "obligation:protected_read", "language": "java"},)
def test_unreviewed_fact_pack_is_refused() -> None:
try:
admit_fact_pack("access_control", FACTS, review_digest="")
raise AssertionError("unreviewed pack must be refused")
except FactPackError:
pass
def test_reviewed_fact_pack_stamps_provenance() -> None:
pack = admit_fact_pack("access_control", FACTS, review_digest="review-2026-10-30")
assert pack.family == "access_control"
assert pack.review_digest == "review-2026-10-30"
assert len(pack.fact_set_digest) == 12
verify_fact_pack(pack, FACTS)
try:
verify_fact_pack(pack, FACTS + ({"id": "tampered"},))
raise AssertionError("tampered facts must be refused")
except FactPackError:
pass
def _record() -> TraceRecord:
return TraceRecord(
entry="src/app.py::handler",
hops=(),
operation_kind="protected_read",
resource="credentials",
authority="admin",
judgment=Judgment.UNGUARDED,
digests={"input": "abc"},
)
def test_consumer_accepts_witnessed_claim_and_reasoned_silence() -> None:
claim = ProvenClaim(trace_id="t1", witness="w1", path="src/app.py", line=10, family="access_control")
silence = BoundedSilence(path="src/other.py", reason="unreached", coverage={"entry_points": 0})
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), _record(), (claim,), (silence,))
assert verdict.claims == (claim,)
assert verdict.silence == (silence,)
def test_consumer_refuses_claim_without_witness_or_silence_without_reason() -> None:
bad_claim = ProvenClaim(trace_id="t1", witness="", path="src/app.py", line=10, family="access_control")
try:
consume_verdict(PushRequest(base="b", head="h", budget_class="c"), _record(), (bad_claim,), ())
raise AssertionError("witnessless claim must be refused")
except ValueError:
pass
bad_silence = BoundedSilence(path="src/other.py", reason="", coverage={})
try:
consume_verdict(PushRequest(base="b", head="h", budget_class="c"), _record(), (), (bad_silence,))
raise AssertionError("reasonless silence must be refused")
except ValueError:
pass
"""Retired v1 fact-pack consumer tests; generic fact and consumer seams use v2."""
tests/test_core_factpack_file.py
+1 -65
View File
@@ -1,65 +1 @@
"""Fact-pack file admission tests for detection-core task 8.2 (Req 10.1)."""
import hashlib
import json
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.legacy_v1 import FactPackError, admit_pack_file, verify_fact_pack # noqa: E402
from openultrasast.mapping.installed_fact_authority import ( # noqa: E402
InstalledAuthorityWireError,
decode_fact_pack_bytes,
)
REPO = Path(__file__).resolve().parents[1]
ENTRIES = REPO / "src/openultrasast/ruleset/entries"
LEGACY_DIGEST = "e8e149309abd78f24206c58fde7f73ae2f60364d8b8fee22ecaee8342798d8ce"
PACK = ENTRIES / "artifacts" / "sha256" / LEGACY_DIGEST
PROJECTION = ENTRIES / "express.json"
def _review_digest() -> str:
payload = json.loads(PACK.read_text(encoding="utf-8"))
return hashlib.sha256(payload["review"]["note"].encode()).hexdigest()
def test_express_pack_admitted_with_verifiable_review() -> None:
content = PACK.read_bytes()
assert len(content) == 963
assert hashlib.sha256(content).hexdigest() == LEGACY_DIGEST
pack = admit_pack_file(PACK, review_digest=_review_digest())
assert pack.family == "access_control"
assert len(pack.facts) == 3
assert pack.review_digest == _review_digest()
verify_fact_pack(pack, pack.facts)
def test_pack_refused_without_matching_review() -> None:
try:
admit_pack_file(PACK, review_digest="0" * 64)
raise AssertionError("wrong review digest must be refused")
except FactPackError:
pass
def test_tampered_pack_file_refused(tmp_path: Path) -> None:
payload = json.loads(PACK.read_text(encoding="utf-8"))
payload["facts"].append({"id": "entry:express:evil", "pattern": "eval("})
tampered = tmp_path / "tampered.json"
tampered.write_text(json.dumps(payload), encoding="utf-8")
pack = admit_pack_file(tampered, review_digest=_review_digest())
# review attestation still verifies, but content no longer matches the stamped digest
original = admit_pack_file(PACK, review_digest=_review_digest())
with pytest.raises(FactPackError):
verify_fact_pack(original, pack.facts)
def test_compatibility_projection_is_neither_a_legacy_nor_v2_fact_pack() -> None:
with pytest.raises(FactPackError):
admit_pack_file(PROJECTION, review_digest=_review_digest())
with pytest.raises(InstalledAuthorityWireError):
decode_fact_pack_bytes(PROJECTION.read_bytes())
"""Retired v1 fact-pack file tests; reviewed family-neutral packs use v2."""
tests/test_core_migration.py
+12 -2
View File
@@ -138,6 +138,16 @@ def test_legacy_decoder_has_no_compatibility_or_v2_domain_dependency() -> None:
def test_legacy_reader_is_not_reexported_from_the_public_core_api() -> None:
import openultrasast.core as core
path = REPO / "src/openultrasast/core/__init__.py"
content = path.read_bytes()
assert content, "the public-boundary instrument must read nonzero package bytes"
tree = ast.parse(content, filename=str(path.relative_to(REPO)))
exported = {
value.value
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
for value in ast.walk(node.value)
if isinstance(value, ast.Constant) and isinstance(value.value, str)
}
assert not ({"LegacyArtifact", "decode_legacy_artifact", "read_legacy_artifact"} & set(core.__all__))
assert not ({"LegacyArtifact", "decode_legacy_artifact", "read_legacy_artifact"} & exported)
tests/test_core_migration_benchmark.py
+14 -4
View File
@@ -8,6 +8,7 @@ from pathlib import Path
import pytest
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE
from consumer_migration_evidence import CONSUMER_TERMINAL_EVIDENCE
from core_migration_boundary import inspect_legacy_boundary
from core_migration_inventory import (
REPO,
@@ -77,13 +78,22 @@ def test_source_validation_refuses_removed_migrated_owner(tmp_path: Path) -> Non
validate_inventory_sources(replace(inventory, entries=(original,)), root=tmp_path)
def test_test_and_operational_entries_remain_open_for_task_10_6() -> None:
def test_test_and_operational_entries_have_exact_terminal_evidence() -> None:
candidate = load_migration_inventory(INVENTORY)
deferred = {
entry.disposition for entry in candidate.entries if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"}
migrated_ids = {entry_id for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated"}
consumers = {
entry.entry_id: entry.disposition
for entry in candidate.entries
if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"}
}
assert deferred == {MigrationDisposition.OPEN}
assert set(consumers) == {
entry_id
for entry_id in CONSUMER_TERMINAL_EVIDENCE
if next(entry for entry in candidate.entries if entry.entry_id == entry_id).owner_boundary != "core"
}
assert all(consumers[entry_id] is MigrationDisposition.V2_MIGRATED for entry_id in migrated_ids)
assert all(consumers[entry_id] is MigrationDisposition.RETIRED for entry_id in set(consumers) - migrated_ids)
def test_old_benchmark_admission_contract_and_runner_are_not_live() -> None:
tests/test_core_migration_boundary.py
+9 -3
View File
@@ -151,8 +151,10 @@ def test_migration_phase_separates_compatibility_implementation_from_consumers(t
def test_cutover_requires_terminal_inventory_and_structural_zero(tmp_path: Path) -> None:
clean = _write(tmp_path, "clean.py", "VALUE = 1\n")
frozen = load_migration_inventory(INVENTORY)
opened = replace(frozen.entries[0], disposition=MigrationDisposition.OPEN)
open_inventory = replace(frozen, entries=(opened, *frozen.entries[1:]))
open_report = inspect_legacy_boundary(
(clean,), frozen_inventory=frozen, expected_scope=(clean,), root=tmp_path, phase=BoundaryPhase.POST_DELETION
(clean,), frozen_inventory=open_inventory, expected_scope=(clean,), root=tmp_path, phase=BoundaryPhase.POST_DELETION
)
closed = _closed_inventory()
migration_report = inspect_legacy_boundary((clean,), frozen_inventory=frozen, candidate_inventory=closed, root=tmp_path)
@@ -175,7 +177,7 @@ def test_cutover_requires_terminal_inventory_and_structural_zero(tmp_path: Path)
)
with pytest.raises(LegacyBoundaryError, match="open inventory"):
_require_cutover(open_report, frozen=frozen, candidate=frozen, root=tmp_path, isolated_scope=(clean,))
_require_cutover(open_report, frozen=open_inventory, candidate=open_inventory, root=tmp_path, isolated_scope=(clean,))
with pytest.raises(LegacyBoundaryError, match="post-deletion"):
_require_cutover(migration_report, frozen=frozen, candidate=closed, root=tmp_path, isolated_scope=(clean,))
_require_cutover(clean_report, frozen=frozen, candidate=closed, root=tmp_path, isolated_scope=(clean,))
@@ -286,7 +288,11 @@ def test_repository_scope_proves_nonzero_inputs_before_reporting_sites() -> None
assert report.files_read == len(paths) > 0
assert report.bytes_read > 0
assert report.legacy_import_sites
assert report.open_inventory_entries
assert report.open_inventory_entries == ()
assert report.consumer_legacy_import_sites == ()
assert report.consumer_legacy_writer_sites == ()
assert report.forbidden_dynamic_sites == ()
assert report.compatibility_implementation_sites
def test_cutover_rejects_positive_but_incomplete_repository_scope() -> None:
tests/test_core_migration_consumers.py
+592
View File
@@ -0,0 +1,592 @@
"""Final consumer closure for detection-core migration task 10.6."""
from __future__ import annotations
import ast
import hashlib
import json
import os
import subprocess
import sys
from dataclasses import replace
from pathlib import Path
import core_migration_inventory as migration_inventory
import pytest
from consumer_migration_evidence import (
CONSUMER_TERMINAL_EVIDENCE,
REVIEWED_SUCCESSOR_MODULES,
ConsumerEvidenceError,
ReviewedSuccessorModule,
load_consumer_terminal_evidence,
load_reviewed_successor_modules,
migrated_pytest_node_ids,
)
from core_migration_boundary import inspect_legacy_boundary
from core_migration_inventory import REPO, LegacyInventoryError, MigrationDisposition, load_migration_inventory, validate_inventory_sources
from core_migration_scope import boundary_python
INVENTORY = REPO / "tests/fixtures/detection_core/v1/migration_inventory.json"
def test_complete_original_inventory_is_terminal() -> None:
inventory = load_migration_inventory(INVENTORY)
assert inventory.entries
assert all(entry.disposition is not MigrationDisposition.OPEN for entry in inventory.entries)
def test_terminal_evidence_exactly_binds_every_task_10_6_entry() -> None:
inventory = load_migration_inventory(INVENTORY)
evidence_ids = tuple(CONSUMER_TERMINAL_EVIDENCE)
evidence_id_set = set(evidence_ids)
selected = tuple(entry for entry in inventory.entries if entry.entry_id in evidence_id_set)
entries = {entry.entry_id: entry for entry in selected}
assert len(entries) == len(evidence_ids) == 155
assert tuple(entry.entry_id for entry in selected) == evidence_ids
assert all(entries[entry_id].disposition.value == evidence.disposition for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items())
validate_inventory_sources(inventory, root=REPO)
def test_migrated_evidence_owns_exact_bounded_pytest_nodes() -> None:
migrated = tuple(evidence for evidence in CONSUMER_TERMINAL_EVIDENCE.values() if evidence.disposition == "v2_migrated")
selectors = migrated_pytest_node_ids()
assert len(migrated) == 40
assert len(selectors) == 37
assert all(evidence.pytest_node_id == f"{evidence.successor_path}::{evidence.probe_symbol}" for evidence in migrated)
assert all(selector in selectors for selector in (evidence.pytest_node_id for evidence in migrated))
assert all((REPO / selector.partition("::")[0]).read_bytes() for selector in selectors)
def test_successor_module_authority_is_exact_nonzero_and_digest_matched() -> None:
migrated_paths = {evidence.successor_path for evidence in CONSUMER_TERMINAL_EVIDENCE.values() if evidence.disposition == "v2_migrated"}
assert len(REVIEWED_SUCCESSOR_MODULES) == 18
assert set(REVIEWED_SUCCESSOR_MODULES) == migrated_paths
for relative, reviewed in REVIEWED_SUCCESSOR_MODULES.items():
content = (REPO / relative).read_bytes()
assert content, f"module authority instrument must read nonzero bytes from {relative}"
assert hashlib.sha256(content).hexdigest() == reviewed.sha256
def _mutated_authority(tmp_path: Path, mutation: str) -> Path:
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
modules = document["successor_modules"]
if mutation == "missing":
modules.pop()
elif mutation == "extra":
modules.append({"path": "tests/forged_successor.py", "sha256": "f" * 64})
elif mutation == "duplicate":
modules.append(dict(modules[-1]))
elif mutation == "conflicting_path":
modules.append({**modules[-1], "sha256": "f" * 64})
elif mutation == "duplicate_digest":
modules[1]["sha256"] = modules[0]["sha256"]
else:
raise AssertionError(f"unknown mutation: {mutation}")
target = tmp_path / f"consumer_terminal_evidence-{mutation}.json"
target.write_text(json.dumps(document), encoding="utf-8")
return target
def _write_authority(tmp_path: Path, document: object, name: str = "consumer_terminal_evidence.json") -> Path:
target = tmp_path / name
target.write_text(json.dumps(document), encoding="utf-8")
return target
@pytest.mark.parametrize(
"invalid_path",
(
"",
"/tests/test_arm_regression.py",
"C:/tests/test_arm_regression.py",
"C:tests/test_arm_regression.py",
"C:\\tests\\test_arm_regression.py",
"tests\\test_arm_regression.py",
"./tests/test_arm_regression.py",
"tests/./test_arm_regression.py",
"tests/../tests/test_arm_regression.py",
"tests/../../outside.py",
"tests//test_arm_regression.py",
"tests/test_arm_regression.py/.",
"tests/test_arm_regression.py\0forged",
),
)
def test_migrated_successor_paths_require_exact_canonical_repo_relative_posix(tmp_path: Path, invalid_path: str) -> None:
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
migrated = next(entry for entry in document["entries"] if entry["disposition"] == "v2_migrated")
migrated["successor_path"] = invalid_path
migrated["pytest_node_id"] = f"{invalid_path}::{migrated['probe_symbol']}"
with pytest.raises(ConsumerEvidenceError, match="canonical repository-relative POSIX path"):
load_consumer_terminal_evidence(_write_authority(tmp_path, document))
@pytest.mark.parametrize(
"invalid_path",
(
"/tests/test_arm_regression.py",
"C:/tests/test_arm_regression.py",
"C:tests/test_arm_regression.py",
"tests\\test_arm_regression.py",
"tests/./test_arm_regression.py",
"tests/../tests/test_arm_regression.py",
"tests/../../outside.py",
"tests//test_arm_regression.py",
"tests/test_arm_regression.py\0forged",
),
)
def test_reviewed_module_paths_require_exact_canonical_repo_relative_posix(tmp_path: Path, invalid_path: str) -> None:
document = json.loads((REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json").read_bytes())
original = document["successor_modules"][0]["path"]
document["successor_modules"][0]["path"] = invalid_path
for entry in document["entries"]:
if entry["successor_path"] == original:
entry["successor_path"] = invalid_path
entry["pytest_node_id"] = f"{invalid_path}::{entry['probe_symbol']}"
with pytest.raises(ConsumerEvidenceError, match="canonical repository-relative POSIX path"):
load_reviewed_successor_modules(_write_authority(tmp_path, document))
@pytest.mark.parametrize("mutation", ("missing", "extra", "duplicate", "conflicting_path", "duplicate_digest"))
def test_successor_module_authority_rejects_set_and_identity_drift(tmp_path: Path, mutation: str) -> None:
with pytest.raises(ConsumerEvidenceError, match="successor module"):
load_reviewed_successor_modules(_mutated_authority(tmp_path, mutation))
def test_migrated_row_outside_module_authority_is_rejected(tmp_path: Path) -> None:
evidence_path = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
document = json.loads(evidence_path.read_bytes())
migrated = next(entry for entry in document["entries"] if entry["disposition"] == "v2_migrated")
migrated["successor_path"] = "tests/forged_successor.py"
migrated["pytest_node_id"] = f"tests/forged_successor.py::{migrated['probe_symbol']}"
target = tmp_path / "consumer_terminal_evidence.json"
target.write_text(json.dumps(document), encoding="utf-8")
with pytest.raises(ConsumerEvidenceError, match="exactly cover migrated rows"):
load_consumer_terminal_evidence(target)
def test_clean_reviewed_successor_module_is_accepted() -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
validate_inventory_sources(replace(inventory, entries=(entry,)), root=REPO)
def _install_reviewed_module(monkeypatch: pytest.MonkeyPatch, relative: str, content: bytes) -> None:
monkeypatch.setattr(
migration_inventory,
"REVIEWED_SUCCESSOR_MODULES",
{relative: ReviewedSuccessorModule(relative, hashlib.sha256(content).hexdigest())},
)
def test_reviewed_successor_safe_reader_accepts_nested_real_directories(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
relative = "tests/nested/reviewed.py"
content = b"VALUE = 1\n"
target = tmp_path / relative
target.parent.mkdir(parents=True)
target.write_bytes(content)
_install_reviewed_module(monkeypatch, relative, content)
tree = migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
assert isinstance(tree, ast.Module)
def test_reviewed_successor_safe_reader_rejects_nul_before_filesystem_access() -> None:
with pytest.raises(LegacyInventoryError, match="not canonical"):
migration_inventory._canonical_relative_parts("tests/reviewed.py\0forged")
def test_reviewed_successor_safe_reader_refuses_fifo_without_blocking(tmp_path: Path) -> None:
relative = "tests/reviewed.py"
target = tmp_path / relative
target.parent.mkdir()
os.mkfifo(target)
assert target.is_fifo(), "FIFO refusal instrument must create the special file it exercises"
program = (
"from pathlib import Path\n"
"from core_migration_inventory import LegacyInventoryError, _read_file_beneath_root\n"
f"root = Path({str(tmp_path)!r})\n"
"try:\n"
f" _read_file_beneath_root({relative!r}, root=root)\n"
"except LegacyInventoryError:\n"
" raise SystemExit(0)\n"
"raise SystemExit(2)\n"
)
try:
completed = subprocess.run(
(sys.executable, "-c", program),
cwd=REPO,
env={**os.environ, "PYTHONPATH": "src:tests"},
capture_output=True,
text=True,
timeout=2,
check=False,
)
except subprocess.TimeoutExpired:
pytest.fail("reviewed successor FIFO open blocked past the two-second bound")
assert completed.returncode == 0, completed.stderr
@pytest.mark.parametrize("target_location", ("outside", "inside"))
def test_reviewed_successor_safe_reader_rejects_final_file_symlinks(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target_location: str
) -> None:
relative = "tests/reviewed.py"
content = b"VALUE = 1\n"
tests = tmp_path / "tests"
tests.mkdir()
target = (tmp_path.parent / f"{tmp_path.name}-outside.py") if target_location == "outside" else (tests / "target.py")
target.write_bytes(content)
(tmp_path / relative).symlink_to(target)
_install_reviewed_module(monkeypatch, relative, content)
with pytest.raises(LegacyInventoryError, match="not a symlink-free file beneath the supplied root"):
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
@pytest.mark.parametrize("target_location", ("outside", "inside"))
def test_reviewed_successor_safe_reader_rejects_intermediate_directory_symlinks(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target_location: str
) -> None:
relative = "linked/reviewed.py"
content = b"VALUE = 1\n"
target = (tmp_path.parent / f"{tmp_path.name}-outside") if target_location == "outside" else (tmp_path / "real")
target.mkdir()
(target / "reviewed.py").write_bytes(content)
(tmp_path / "linked").symlink_to(target, target_is_directory=True)
_install_reviewed_module(monkeypatch, relative, content)
with pytest.raises(LegacyInventoryError, match="not a symlink-free file beneath the supplied root"):
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
@pytest.mark.parametrize("root_kind", ("missing", "file", "symlink"))
def test_reviewed_successor_safe_reader_rejects_non_directory_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, root_kind: str) -> None:
relative = "tests/reviewed.py"
content = b"VALUE = 1\n"
real_root = tmp_path / "real-root"
real_root.mkdir()
supplied_root = tmp_path / "supplied-root"
if root_kind == "file":
supplied_root.write_bytes(b"not a directory")
elif root_kind == "symlink":
supplied_root.symlink_to(real_root, target_is_directory=True)
_install_reviewed_module(monkeypatch, relative, content)
with pytest.raises(LegacyInventoryError, match="supplied root is not a real directory"):
migration_inventory._read_reviewed_successor_module(relative, root=supplied_root)
@pytest.mark.parametrize("failure", ("missing", "unreadable", "zero"))
def test_reviewed_successor_safe_reader_rejects_unusable_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str) -> None:
relative = "tests/reviewed.py"
expected = b"VALUE = 1\n"
target = tmp_path / relative
target.parent.mkdir()
if failure != "missing":
target.write_bytes(b"" if failure == "zero" else expected)
if failure == "unreadable":
target.chmod(0)
_install_reviewed_module(monkeypatch, relative, expected)
message = "nonzero bytes" if failure == "zero" else "not readable"
with pytest.raises(LegacyInventoryError, match=message):
migration_inventory._read_reviewed_successor_module(relative, root=tmp_path)
def test_migrated_owner_binding_cannot_disappear(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
migrated = {entry_id for entry_id, evidence in CONSUMER_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated"}
entry = next(item for item in inventory.entries if item.entry_id in migrated and item.symbol.startswith("test_"))
source = (REPO / entry.path).read_text(encoding="utf-8").replace(f"def {entry.symbol}(", f"def removed_{entry.symbol}(", 1)
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_retired_operational_owner_cannot_be_reintroduced(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.owner_boundary == "operational-tooling")
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text("import json\ndef main():\n return json.dumps({})\n", encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="retired consumer capability remains live"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
@pytest.mark.parametrize(
("entry_id", "source"),
(
(
"887c9abe9a6d827c39655c6835fb0c2471be035914e2bbff888a4cd54635b8e0",
"from openultrasast.core._legacy_factpack import LegacyFactPack\n",
),
(
"36e5d60b410bf7f9c676a5da9e95718cab95dadfbf61cda0d0e2f24b56cf3c92",
"class TraceRecord:\n def canonical(self):\n return record()\n",
),
(
"a8685de9229906307386c537ff096b385bff727e270744ccc3dd02978a7d5b2a",
"def test_budget_gate_returns_no_go():\n return admission_report()\n",
),
),
)
def test_retired_module_or_dotted_capability_cannot_be_reintroduced(tmp_path: Path, entry_id: str, source: str) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == entry_id)
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="retired consumer capability"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_dead_syntax_cannot_stand_in_for_migrated_test_semantics(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
constants = "fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"
source = f"def test_summarize_arm_publishes_budget_result_and_masks_repositories():\n if False:\n dumps({constants!r})\n"
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_pass_only_migrated_successor_cannot_claim_semantic_closure(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
source = """from benchmarks.valid import arm_regression
def test_summarize_arm_publishes_budget_result_and_masks_repositories():
if False:
dumps(("fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"))
pass
"""
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
@pytest.mark.parametrize(
"replacement_body",
(
"return True",
"assert True",
"pytest.skip('not semantic execution')",
"len(())",
"if False:\n arm_regression.summarize_arm(None)\n return True",
),
)
def test_arbitrary_behavior_cannot_certify_migrated_semantics(tmp_path: Path, replacement_body: str) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
source = f"""import pytest
from benchmarks.valid import arm_regression
def test_summarize_arm_publishes_budget_result_and_masks_repositories():
if False:
dumps(("fact_set_digest", "id", "instrument", "judgment", "stage_funnel", "status"))
{replacement_body}
"""
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_rebound_semantic_import_cannot_certify_real_migrated_owner(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b")
source = (
(REPO / entry.path)
.read_text(encoding="utf-8")
.replace(
"from benchmarks.valid import arm_regression",
"from benchmarks.valid import arm_regression as rebound_arm_regression",
)
)
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
_TERMINAL_BINDING_ENTRY_ID = "6fe48be48004f15cf1157b77a2ca88c7a0fb2f6bb649c71aa656f3a62c17c53b"
_TERMINAL_BINDING_TEST = "test_summarize_arm_publishes_budget_result_and_masks_repositories"
@pytest.mark.parametrize(
"later_binding",
(
f"{_TERMINAL_BINDING_TEST} = lambda: None",
f"{_TERMINAL_BINDING_TEST}: object = None",
f"{_TERMINAL_BINDING_TEST} += ()",
f"del {_TERMINAL_BINDING_TEST}",
f"from builtins import len as {_TERMINAL_BINDING_TEST}",
f"{_TERMINAL_BINDING_TEST} = wrapper({_TERMINAL_BINDING_TEST})",
"arm_regression = object()",
"pytestmark = pytest.mark.skip(reason='disabled')",
f"{_TERMINAL_BINDING_TEST}.pytestmark = pytest.mark.xfail(reason='disabled')",
),
)
def test_reviewed_migrated_bindings_must_remain_terminal(tmp_path: Path, later_binding: str) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
source = (REPO / entry.path).read_text(encoding="utf-8") + f"\n{later_binding}\n"
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
@pytest.mark.parametrize(
"namespace_mutation",
(
f"vars()[{_TERMINAL_BINDING_TEST!r}] = lambda: None",
f"sys.modules[__name__].{_TERMINAL_BINDING_TEST} = lambda: None",
f"object.__setattr__(sys.modules[__name__], {_TERMINAL_BINDING_TEST!r}, lambda: None)",
f"vars({_TERMINAL_BINDING_TEST})['pytestmark'] = pytest.mark.skip(reason='disabled')",
f"{_TERMINAL_BINDING_TEST}.__dict__['pytestmark'] = pytest.mark.xfail(reason='disabled')",
),
)
def test_exact_module_bytes_reject_unbounded_namespace_mutations(tmp_path: Path, namespace_mutation: str) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
original = (REPO / entry.path).read_bytes()
assert original, "namespace-mutation RED instrument must read nonzero successor bytes"
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_bytes(original + f"\n{namespace_mutation}\n".encode())
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_terminal_ast_diagnostics_remain_defense_in_depth(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
original = (REPO / entry.path).read_bytes()
assert original, "defense-in-depth instrument must read nonzero successor bytes"
changed = original + f"\n{_TERMINAL_BINDING_TEST} = lambda: None\n".encode()
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_bytes(changed)
reviewed = dict(REVIEWED_SUCCESSOR_MODULES)
reviewed[entry.path] = ReviewedSuccessorModule(entry.path, hashlib.sha256(changed).hexdigest())
monkeypatch.setattr(migration_inventory, "REVIEWED_SUCCESSOR_MODULES", reviewed)
with pytest.raises(LegacyInventoryError, match="terminal binding"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_even_safely_overwritten_byte_change_requires_explicit_review(tmp_path: Path) -> None:
inventory = load_migration_inventory(INVENTORY)
entry = next(item for item in inventory.entries if item.entry_id == _TERMINAL_BINDING_ENTRY_ID)
source = (
(REPO / entry.path)
.read_text(encoding="utf-8")
.replace(
f"def {_TERMINAL_BINDING_TEST}(",
f"{_TERMINAL_BINDING_TEST} = None\n\ndef {_TERMINAL_BINDING_TEST}(",
1,
)
)
target = tmp_path / entry.path
target.parent.mkdir(parents=True)
target.write_text(source, encoding="utf-8")
with pytest.raises(LegacyInventoryError, match="module digest"):
validate_inventory_sources(replace(inventory, entries=(entry,)), root=tmp_path)
def test_reviewed_parametrize_decorators_remain_collectable() -> None:
inventory = load_migration_inventory(INVENTORY)
selected_paths = {"tests/test_hunter_arm.py", "tests/test_push_admission.py"}
selected = tuple(
entry
for entry in inventory.entries
if entry.entry_id in CONSUMER_TERMINAL_EVIDENCE and CONSUMER_TERMINAL_EVIDENCE[entry.entry_id].successor_path in selected_paths
)
assert selected
validate_inventory_sources(replace(inventory, entries=selected), root=REPO)
decorators = [
decorator
for path in selected_paths
for node in ast.parse((REPO / path).read_bytes(), filename=path).body
if isinstance(node, ast.FunctionDef) and f"{path}::{node.name}" in migrated_pytest_node_ids()
for decorator in node.decorator_list
if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute) and decorator.func.attr == "parametrize"
]
assert len(decorators) == 2
@pytest.mark.parametrize(
("field", "replacement"),
(
("path", "tests/forged_successor.py"),
("capability_anchors", ["definition:forged_owner"]),
("disposition", "retired"),
),
)
def test_terminal_evidence_rejects_frozen_identity_or_disposition_drift(tmp_path: Path, field: str, replacement: object) -> None:
evidence_path = REPO / "tests/fixtures/detection_core/v1/consumer_terminal_evidence.json"
document = json.loads(evidence_path.read_bytes())
entry = next(item for item in document["entries"] if field != "disposition" or item["disposition"] == "v2_migrated")
entry[field] = replacement
forged = tmp_path / "consumer_terminal_evidence.json"
forged.write_text(json.dumps(document), encoding="utf-8")
with pytest.raises(ConsumerEvidenceError):
load_consumer_terminal_evidence(forged)
def test_authoritative_boundary_includes_task_source_files() -> None:
paths = {path.relative_to(REPO).as_posix(): path for path in boundary_python(REPO)}
for relative in ("tests/consumer_migration_evidence.py", "tests/test_core_migration_consumers.py"):
assert relative in paths
assert paths[relative].read_bytes()
def test_nonzero_repository_scope_has_no_forbidden_consumers() -> None:
inventory = load_migration_inventory(INVENTORY)
paths = boundary_python(REPO)
report = inspect_legacy_boundary(
paths,
frozen_inventory=inventory,
candidate_inventory=inventory,
expected_scope=paths,
root=REPO,
)
assert report.files_read == len(paths) > 0
assert report.bytes_read > 0
assert report.scanned_scope_paths == report.expected_scope_paths
assert report.consumer_legacy_import_sites == ()
assert report.consumer_legacy_writer_sites == ()
assert report.forbidden_dynamic_sites == ()
tests/test_core_migration_namespace.py
+41 -96
View File
@@ -1,4 +1,4 @@
"""Task 10.1: isolate the live v1 contract and serialization owners."""
"""Structural ownership tests for the temporary v1 compatibility implementation."""
from __future__ import annotations
@@ -6,14 +6,8 @@ import ast
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
V1_FIXTURES = REPO / "tests/fixtures/detection_core/v1"
PURE_COMPATIBILITY_MODULES = (
"admission.py",
"consumer.py",
"trace.py",
"verdict.py",
"_legacy_factpack.py",
)
CORE = REPO / "src/openultrasast/core"
PURE_COMPATIBILITY_MODULES = ("admission.py", "consumer.py", "trace.py", "verdict.py", "_legacy_factpack.py")
MIXED_LEGACY_NAMES = {
"budget": {"Budget", "Complexity"},
"coverage": {"Coverage", "ResolutionOutcome", "check_reads"},
@@ -30,27 +24,23 @@ MIXED_LEGACY_NAMES = {
}
def _trace() -> object:
from openultrasast.core.legacy_v1 import contracts
def _tree(path: Path) -> ast.Module:
content = path.read_bytes()
assert content, f"ownership instrument must read nonzero bytes from {path}"
return ast.parse(content, filename=path.as_posix())
return contracts.TraceRecord(
entry="src/app.py::handler",
hops=(contracts.Hop(function="handler", call_line=10, complete=True),),
operation_kind="protected_read",
resource="credentials",
authority="admin",
witnesses=(contracts.Witness(kind="guard", provenance="fact:checkPermission", relation="all_paths"),),
guards=(contracts.Guard(kind="permission_check", effect="branches"),),
completeness_reasons=(),
judgment=contracts.Judgment.UNGUARDED,
digests={"input": "abc123"},
)
def _defined(tree: ast.Module) -> set[str]:
return {node.name for node in tree.body if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))}
def _imported(tree: ast.Module) -> set[str]:
return {alias.asname or alias.name for node in tree.body if isinstance(node, (ast.Import, ast.ImportFrom)) for alias in node.names}
def test_legacy_contract_values_have_one_compatibility_owner() -> None:
from openultrasast.core.legacy_v1 import contracts
names = {
contracts = _tree(CORE / "legacy_v1/contracts.py")
expected = {
"AdmissionManifest",
"AdmissionManifestError",
"BoundedSilence",
@@ -75,104 +65,59 @@ def test_legacy_contract_values_have_one_compatibility_owner() -> None:
"TraceRecord",
"Witness",
}
assert {getattr(contracts, name).__module__ for name in names} == {"openultrasast.core.legacy_v1.contracts"}
assert expected <= _defined(contracts)
def test_pure_compatibility_modules_define_no_contract_or_serializer() -> None:
for name in PURE_COMPATIBILITY_MODULES:
path = REPO / "src/openultrasast/core" / name
content = path.read_bytes()
assert content, f"ownership instrument must read nonzero bytes from {name}"
tree = ast.parse(content, filename=path.as_posix())
assert not [node.name for node in tree.body if isinstance(node, ast.ClassDef)]
assert not [node.name for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))]
tree = _tree(CORE / name)
assert not _defined(tree)
assert not any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in tree.body)
assert any(isinstance(node, ast.ImportFrom) and node.module == "openultrasast.core.legacy_v1.contracts" for node in tree.body)
def test_writer_is_the_only_compatibility_json_serializer() -> None:
namespace = REPO / "src/openultrasast/core/legacy_v1"
namespace = CORE / "legacy_v1"
paths = sorted(namespace.glob("*.py"))
assert paths and all(path.read_bytes() for path in paths)
assert {path.name for path in paths} == {"__init__.py", "contracts.py", "writer.py"}
json_importers = []
for path in paths:
tree = ast.parse(path.read_bytes(), filename=path.as_posix())
if any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in tree.body):
json_importers.append(path.name)
json_importers = [
path.name
for path in paths
if any(isinstance(node, (ast.Import, ast.ImportFrom)) and "json" in ast.unparse(node) for node in _tree(path).body)
]
assert json_importers == ["writer.py"]
def test_namespace_and_v2_dependency_directions_are_closed() -> None:
namespace = REPO / "src/openultrasast/core/legacy_v1"
for path in namespace.glob("*.py"):
tree = ast.parse(path.read_bytes(), filename=path.as_posix())
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module}
for path in (CORE / "legacy_v1").glob("*.py"):
imports = {node.module for node in ast.walk(_tree(path)) if isinstance(node, ast.ImportFrom) and node.module}
assert all(module.startswith("openultrasast.core.legacy_v1") for module in imports if module.startswith("openultrasast"))
compatibility = {*PURE_COMPATIBILITY_MODULES, "legacy.py", "__init__.py"}
for path in (REPO / "src/openultrasast/core").glob("*.py"):
if path.name in compatibility:
continue
content = path.read_bytes()
assert content, f"dependency instrument must read nonzero bytes from {path.name}"
tree = ast.parse(content, filename=path.as_posix())
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module}
assert not any(module.startswith("openultrasast.core.legacy_v1") for module in imports)
def test_legacy_trace_write_remains_byte_identical() -> None:
expected = (V1_FIXTURES / "trace.json").read_bytes()
assert len(expected) == 401
value = _trace()
assert value.canonical() == expected.rstrip(b"\n") # type: ignore[attr-defined]
for path in CORE.glob("*.py"):
if path.name not in compatibility:
imports = {node.module for node in ast.walk(_tree(path)) if isinstance(node, ast.ImportFrom) and node.module}
assert not any(module.startswith("openultrasast.core.legacy_v1") for module in imports)
def test_historical_decoder_stays_independent_from_compatibility_namespace() -> None:
path = REPO / "src/openultrasast/core/legacy.py"
content = path.read_bytes()
assert content
tree = ast.parse(content, filename=path.as_posix())
imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom)}
imports = {node.module for node in ast.walk(_tree(CORE / "legacy.py")) if isinstance(node, ast.ImportFrom)}
assert "openultrasast.core.legacy_v1" not in imports
def test_top_level_v1_api_imports_only_from_closed_namespace() -> None:
import openultrasast.core as public_core
from openultrasast.core import legacy_v1
path = REPO / "src/openultrasast/core/__init__.py"
content = path.read_bytes()
assert content
tree = ast.parse(content, filename=path.as_posix())
legacy_imports = {
node.module
for node in tree.body
if isinstance(node, ast.ImportFrom) and any((alias.asname or alias.name) in public_core.__all__ for alias in node.names)
}
assert legacy_imports == {"openultrasast.core.legacy_v1"}
assert all(getattr(public_core, name) is getattr(legacy_v1, name) for name in public_core.__all__)
def test_top_level_v1_api_is_only_a_closed_namespace_shim() -> None:
tree = _tree(CORE / "__init__.py")
modules = {node.module for node in tree.body if isinstance(node, ast.ImportFrom)}
assert modules == {"openultrasast.core.legacy_v1"}
assert not _defined(tree)
def test_mixed_v2_modules_expose_no_legacy_compatibility_names() -> None:
import importlib
for module_name, legacy_names in MIXED_LEGACY_NAMES.items():
module = importlib.import_module(f"openultrasast.core.{module_name}")
assert legacy_names.isdisjoint(vars(module)), f"{module_name} still exposes {legacy_names & vars(module).keys()}"
tree = _tree(CORE / f"{module_name}.py")
assert legacy_names.isdisjoint(_defined(tree) | _imported(tree))
def test_closed_namespace_is_the_only_owner_of_moved_mixed_names() -> None:
from openultrasast.core import legacy_v1
contracts = _tree(CORE / "legacy_v1/contracts.py")
expected = set().union(*MIXED_LEGACY_NAMES.values())
assert all(hasattr(legacy_v1, name) for name in expected)
operation_names = {
"assert_no_shortcut",
"check_reads",
"require_admission_evidence",
"reversed_pair_probe",
"stamp",
"verify_stamp",
}
assert {getattr(legacy_v1, name).__module__ for name in operation_names} == {"openultrasast.core.legacy_v1.contracts"}
assert expected <= _defined(contracts)
tests/test_core_migration_production.py
+1 -5
View File
@@ -66,11 +66,7 @@ def test_registered_production_entries_close_without_rekeying_other_consumers()
assert set(production) == _PRODUCTION_ENTRY_IDS
assert all(production[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED)
assert all(production[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED)
assert all(
entry.disposition is MigrationDisposition.OPEN
for entry in candidate.entries
if entry.entry_id not in production and not entry.path.startswith("benchmarks/")
)
assert all(entry.disposition is not MigrationDisposition.OPEN for entry in candidate.entries)
frozen = replace(
candidate,
tests/test_core_migration_provenance.py
+13 -37
View File
@@ -59,19 +59,12 @@ def test_reader_audit_covers_direct_legacy_submodule_bindings(
def test_reader_audit_covers_real_direct_legacy_submodule_reads() -> None:
expected = {
("tests/test_core_contract.py", "module_attr:openultrasast.core.verdict:__all__"),
}
paths = tuple(REPO / path for path in ("tests/test_core_budget_v2.py", "tests/test_core_contract.py", "tests/test_core_probes.py"))
report = audit_legacy_readers(paths)
assert report.files_read == 3 and report.bytes_read > 0
assert expected <= set(report.capabilities)
assert not any(
capability.startswith(("module_attr:openultrasast.core.budget:", "module_attr:openultrasast.core.probes:"))
for _path, capability in report.capabilities
)
assert report.capabilities == ()
def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None:
@@ -81,9 +74,9 @@ def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None:
inventory = load_migration_inventory(INVENTORY)
assert inventory.entries
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 155
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 25
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 18
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 0
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 65
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 133
assert all(entry.entry_id == entry_identity(entry) for entry in inventory.entries)
validate_inventory_sources(inventory, root=REPO)
@@ -174,33 +167,16 @@ def test_inventory_registers_the_active_legacy_probe_writer() -> None:
def test_independent_reader_audit_matches_split_inventory_capabilities() -> None:
report = audit_legacy_readers(tracked_python())
inventory = load_migration_inventory(INVENTORY)
reader_entries = tuple(entry for entry in inventory.entries if entry.dependency_kind.value == "reader")
registered = {(entry.path, anchor) for entry in reader_entries for anchor in entry.capability_anchors}
current = set(report.capabilities)
open_registered = {
(entry.path, anchor)
for entry in reader_entries
if entry.disposition is MigrationDisposition.OPEN
for anchor in entry.capability_anchors
}
assert report.files_read > 0 and report.bytes_read > 0
assert ("tests/test_core_probes.py", "top_level:ProbeResult") in report.capabilities
assert all(len(entry.capability_anchors) == 1 for entry in reader_entries)
assert current <= registered, "the frozen inventory may never grow to admit a new reader"
for path, _anchor in open_registered - current:
source = (REPO / path).read_bytes()
assert source, f"migration shim instrument must read nonzero bytes from {path}"
if path in MIXED_V2_MODULES:
assert b"openultrasast.core.legacy_v1" not in source
continue
assert b"openultrasast.core.legacy_v1" in source, f"open reader disappeared without an explicit compatibility shim: {path}"
assert not any("legacy_v1" in capability for _path, capability in report.capabilities)
assert not any(path in {"tests/test_core_contract.py", "tests/test_core_probes.py"} for path, _capability in report.capabilities)
def test_inventory_targets_require_exact_capability_anchors(tmp_path: Path) -> None:
frozen = load_migration_inventory(INVENTORY)
entry = next(item for item in frozen.entries if item.disposition is MigrationDisposition.OPEN and item.symbol == "<module>")
terminal = next(item for item in frozen.entries if item.symbol == "<module>")
entry = replace(terminal, disposition=MigrationDisposition.OPEN)
assert entry.capability_anchors
substituted = tmp_path / entry.path
substituted.parent.mkdir(parents=True)
@@ -221,10 +197,10 @@ def test_historical_artifact_manifest_pins_exact_nonzero_bytes() -> None:
@pytest.mark.parametrize("terminal", (MigrationDisposition.V2_MIGRATED, MigrationDisposition.RETIRED))
def test_inventory_entries_close_monotonically_without_identity_changes(terminal: MigrationDisposition) -> None:
frozen = load_migration_inventory(INVENTORY)
index = next(index for index, entry in enumerate(frozen.entries) if entry.disposition is MigrationDisposition.OPEN)
changed_entry = replace(frozen.entries[index], disposition=terminal)
candidate = replace(frozen, entries=(*frozen.entries[:index], changed_entry, *frozen.entries[index + 1 :]))
candidate = load_migration_inventory(INVENTORY)
index = next(index for index, entry in enumerate(candidate.entries) if entry.disposition is terminal)
open_entry = replace(candidate.entries[index], disposition=MigrationDisposition.OPEN)
frozen = replace(candidate, entries=(*candidate.entries[:index], open_entry, *candidate.entries[index + 1 :]))
validate_inventory_transition(frozen, candidate)
@@ -282,7 +258,7 @@ def test_inventory_refuses_schema_and_status_mutations(tmp_path: Path, field: st
def test_inventory_refuses_unreadable_zero_and_unknown_targets(tmp_path: Path) -> None:
frozen = load_migration_inventory(INVENTORY)
original = next(entry for entry in frozen.entries if entry.disposition is MigrationDisposition.OPEN)
original = replace(frozen.entries[0], disposition=MigrationDisposition.OPEN)
missing = replace(original, path="missing.py", symbol="<module>")
missing = replace(missing, entry_id=entry_identity(missing))
tests/test_core_native.py
-35
View File
@@ -5,44 +5,9 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.trace import TraceRecord # noqa: E402
from openultrasast.core.verdict import Judgment # noqa: E402
from openultrasast.hunt.prover import Coverage, PairJudgmentCounts # noqa: E402
def _legacy_record(*, judgment: Judgment = Judgment.UNGUARDED) -> TraceRecord:
reasons = ("abstract_state_unset",) if judgment is Judgment.INDETERMINATE else ()
return TraceRecord(
entry="src/app.py::handler",
hops=(),
operation_kind="protected_read",
resource="credentials",
authority="admin",
witnesses=(),
guards=(),
completeness_reasons=reasons,
judgment=judgment,
digests={"input": "abc123"},
)
def test_legacy_trace_contract_remains_available_only_as_tested_compatibility() -> None:
record = _legacy_record()
assert isinstance(record, TraceRecord)
assert record.entry == "src/app.py::handler"
assert record.judgment is Judgment.UNGUARDED
assert record.resource == "credentials"
assert record.canonical() == _legacy_record().canonical()
def test_legacy_indeterminate_record_stays_explicit() -> None:
record = _legacy_record(judgment=Judgment.INDETERMINATE)
assert record.judgment is Judgment.INDETERMINATE
assert record.completeness_reasons
def test_hunt_coverage_retires_its_legacy_core_projection() -> None:
coverage = Coverage(
files_by_language={"java": 2},
tests/test_core_probes.py
-32
View File
@@ -2,14 +2,11 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, replace
from typing import Any
import pytest
from detection_core_fixtures import InMemoryContentResolver, stored_artifact
import openultrasast.core as public_core
from openultrasast.core import canonical, evidence, probes, provenance, semantics
from openultrasast.core.factpack import (
IndependenceAttestation,
@@ -28,14 +25,6 @@ from openultrasast.core.family import (
derive_security_object_id,
family_contract_digest,
)
from openultrasast.core.legacy_v1 import (
FittedComponentBlocked,
LegacyProbeResult,
ShortcutDetected,
assert_no_shortcut,
require_admission_evidence,
reversed_pair_probe,
)
@dataclass(frozen=True)
@@ -392,24 +381,3 @@ def test_observation_for_another_probe_is_refused() -> None:
with pytest.raises(probes.ProbeValidationError, match="probe identity"):
probes.evaluate_probe(case.probe, substituted, case.contract, case.policy, case.resolver)
def test_legacy_reversed_pair_probe_remains_compatible() -> None:
forward: dict[str, Any] = {"lines": 10}
reversed_change: dict[str, Any] = {"lines": 50}
def shape_score(change: Mapping[str, Any]) -> float:
return float(change["lines"])
result = reversed_pair_probe(shape_score, forward, reversed_change)
assert result.separated is True
with pytest.raises(ShortcutDetected):
assert_no_shortcut(shape_score, forward, reversed_change)
assert public_core.ProbeResult is LegacyProbeResult
def test_legacy_admission_guard_remains_compatible() -> None:
require_admission_evidence(holdout_passed=True, admission_decision="GO")
with pytest.raises(FittedComponentBlocked):
require_admission_evidence(holdout_passed=False, admission_decision="GO")
tests/test_core_validation.py
+1 -39
View File
@@ -1,39 +1 @@
"""Validation tests for detection-core tasks 7.1-7.2 (Req 5, 7, 11, 12)."""
import json
import sys
from pathlib import Path
from test_core_native import _legacy_record
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core.admission import AdmissionManifestError, validate_report_schema # noqa: E402
from openultrasast.core.consumer import PushRequest, consume_verdict # noqa: E402
from openultrasast.core.verdict import Judgment, ProvenClaim # noqa: E402
REPO = Path(__file__).resolve().parents[1]
def test_committed_admission_report_satisfies_schema() -> None:
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
validate_report_schema(report)
def test_schema_validator_refuses_incomplete_report() -> None:
report = json.loads((REPO / "benchmarks/measurements/2026-10-30-access-control-admission/report.json").read_text(encoding="utf-8"))
del report["precision"]
try:
validate_report_schema(report)
raise AssertionError("report missing precision must be refused")
except AdmissionManifestError:
pass
def test_end_to_end_push_through_core_contracts() -> None:
record = _legacy_record()
claim = ProvenClaim(trace_id="t1", witness="fact:f", path="src/app.py", line=30, family="access_control")
verdict = consume_verdict(PushRequest(base="b", head="h", budget_class="dev-pool"), record, (claim,), ())
assert verdict.claims[0].witness == "fact:f"
assert record.judgment is Judgment.UNGUARDED
assert record.canonical() == record.canonical()
"""Retired v1 validation tests; family-neutral validation coverage uses v2."""
tests/test_prover_admission.py
+1 -113
View File
@@ -1,113 +1 @@
"""Task 12.5 tests: consume and validate an externally reviewed admission manifest.
Reqs 12.1, 12.2, 12.3, 12.6, 12.8. Valid, malformed, exposed and spent fixture
manifests must produce deterministic accept/refuse results.
"""
from __future__ import annotations
import copy
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from openultrasast.core import legacy_v1 as prover_admission # noqa: E402
def _manifest() -> dict[str, object]:
return {
"produced_by": "valid-detection-benchmark",
"reviewed": True,
"label_isolated": True,
"spent": False,
"population_role": "admission",
"analyzer_revision": "a" * 40,
"fact_set_digest": "b" * 12,
"acquisition_record_digest": "c" * 12,
"configuration_digest": "d" * 12,
"population_digest": "e" * 12,
"labels": (),
}
def test_valid_manifest_is_accepted() -> None:
manifest = prover_admission.validate_admission_manifest(_manifest())
assert manifest.produced_by == "valid-detection-benchmark"
assert manifest.reviewed is True
assert manifest.label_isolated is True
assert manifest.population_role == "admission"
def test_malformed_manifest_is_refused() -> None:
payload = _manifest()
del payload["reviewed"]
with pytest.raises(prover_admission.AdmissionManifestError, match="malformed"):
prover_admission.validate_admission_manifest(payload)
def test_unreviewed_manifest_is_refused() -> None:
payload = _manifest()
payload["reviewed"] = False
with pytest.raises(prover_admission.AdmissionManifestError, match="unreviewed"):
prover_admission.validate_admission_manifest(payload)
def test_label_exposing_manifest_is_refused() -> None:
payload = _manifest()
payload["labels"] = ("vulnerability:case-9",)
with pytest.raises(prover_admission.AdmissionManifestError, match="label-exposure"):
prover_admission.validate_admission_manifest(payload)
def test_not_label_isolated_manifest_is_refused() -> None:
payload = _manifest()
payload["label_isolated"] = False
with pytest.raises(prover_admission.AdmissionManifestError, match="label-isolated"):
prover_admission.validate_admission_manifest(payload)
def test_non_sequence_labels_are_refused() -> None:
payload = _manifest()
payload["labels"] = None
with pytest.raises(prover_admission.AdmissionManifestError, match="label-exposure"):
prover_admission.validate_admission_manifest(payload)
def test_changed_acquisition_output_is_refused() -> None:
payload = _manifest()
with pytest.raises(prover_admission.AdmissionManifestError, match="acquisition-output-changed"):
prover_admission.validate_admission_manifest(payload, expected_acquisition_digest="f" * 12)
def test_matching_acquisition_digest_is_accepted() -> None:
manifest = prover_admission.validate_admission_manifest(_manifest(), expected_acquisition_digest="c" * 12)
assert manifest.acquisition_record_digest == "c" * 12
def test_spent_population_is_refused() -> None:
payload = _manifest()
payload["spent"] = True
with pytest.raises(prover_admission.AdmissionManifestError, match="spent"):
prover_admission.validate_admission_manifest(payload)
def test_validation_is_deterministic() -> None:
first = prover_admission.validate_admission_manifest(_manifest())
second = prover_admission.validate_admission_manifest(copy.deepcopy(_manifest()))
assert first == second
def test_admission_manifest_is_frozen() -> None:
manifest = prover_admission.validate_admission_manifest(_manifest())
with pytest.raises(Exception): # noqa: B017
manifest.reviewed = False # type: ignore[misc]
def test_non_benchmark_producer_is_refused() -> None:
payload = _manifest()
payload["produced_by"] = "someone-else"
with pytest.raises(prover_admission.AdmissionManifestError, match="producer"):
prover_admission.validate_admission_manifest(payload)
"""Retired parked-arm v1 admission tests; benchmark admission uses v2 replay."""