mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
feat(detection-core): replay admission evidence
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,341 @@
|
||||
"""Typed evidence and re-derived gates for final admission reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
from openultrasast.core import canonical
|
||||
from openultrasast.core.admission_case import (
|
||||
AdmissionCaseResult,
|
||||
derive_admission_case_result_digest,
|
||||
validate_admission_case_shape,
|
||||
)
|
||||
from openultrasast.core.admission_review import AdmissionThreshold, derive_threshold_set_digest
|
||||
from openultrasast.core.coverage import validate_instrument
|
||||
from openultrasast.core.measurement import (
|
||||
BaselineIdentity,
|
||||
Comparator,
|
||||
MetricId,
|
||||
MetricResult,
|
||||
Statistic,
|
||||
metric_passes,
|
||||
)
|
||||
from openultrasast.core.measurement_record import MeasurementRecordV2, ReadinessReference
|
||||
from openultrasast.core.readiness import StageResult
|
||||
|
||||
BUDGET_EVALUATION_SCHEMA = "openultrasast.detection-core.admission-budget-evaluation.v2"
|
||||
THRESHOLD_EVALUATION_SCHEMA = "openultrasast.detection-core.admission-threshold-evaluation.v2"
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_MANDATORY_METRICS = tuple(MetricId)[:6]
|
||||
|
||||
|
||||
class AdmissionReportError(ValueError):
|
||||
"""Admission evidence cannot form a valid final report."""
|
||||
|
||||
|
||||
class GateKind(Enum):
|
||||
INSTRUMENT = "instrument"
|
||||
BUDGET = "budget"
|
||||
QUALITY = "quality"
|
||||
READINESS = "readiness"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GateResult:
|
||||
kind: GateKind
|
||||
gate_id: str
|
||||
passed: bool
|
||||
reasons: tuple[str, ...]
|
||||
evidence_digests: tuple[str, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
validate_gate(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseBudgetEvaluation:
|
||||
case: AdmissionCaseResult
|
||||
case_result_digest: str
|
||||
budget_derivation_digest: str
|
||||
total_reply_ms: int
|
||||
budget_ms: int
|
||||
evaluation_digest: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
validate_budget_evaluation(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThresholdEvaluation:
|
||||
metric_id: MetricId
|
||||
statistic: Statistic
|
||||
comparator: Comparator
|
||||
target: float
|
||||
measurement_record_digest: str
|
||||
passed: bool
|
||||
evaluation_digest: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
validate_threshold_evaluation(self)
|
||||
|
||||
|
||||
def _digest(value: object, field: str) -> None:
|
||||
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
||||
raise AdmissionReportError(f"{field} must be a full lowercase SHA-256 digest")
|
||||
|
||||
|
||||
def _canonical_digests(value: object, field: str, *, allow_empty: bool = False) -> tuple[str, ...]:
|
||||
if not isinstance(value, tuple) or not all(isinstance(item, str) for item in value):
|
||||
raise AdmissionReportError(f"{field} must be a tuple of digests")
|
||||
for item in value:
|
||||
_digest(item, field.removesuffix("s"))
|
||||
if value != tuple(sorted(set(value))):
|
||||
raise AdmissionReportError(f"{field} must be unique and canonical")
|
||||
if not allow_empty and not value:
|
||||
raise AdmissionReportError(f"{field} must not be empty")
|
||||
return value
|
||||
|
||||
|
||||
def validate_gate(value: GateResult) -> None:
|
||||
if type(value) is not GateResult:
|
||||
raise AdmissionReportError("gate must be an exact GateResult")
|
||||
if not isinstance(value.kind, GateKind) or not isinstance(value.gate_id, str) or not value.gate_id:
|
||||
raise AdmissionReportError("gate kind and gate_id must use closed nonempty values")
|
||||
if not isinstance(value.passed, bool):
|
||||
raise AdmissionReportError("gate passed must be boolean")
|
||||
if not isinstance(value.reasons, tuple) or not all(isinstance(item, str) and item for item in value.reasons):
|
||||
raise AdmissionReportError("gate reasons must be a tuple of nonempty text")
|
||||
if value.reasons != tuple(sorted(set(value.reasons))):
|
||||
raise AdmissionReportError("gate reasons must be unique and canonical")
|
||||
if value.passed == bool(value.reasons):
|
||||
raise AdmissionReportError("passed gates have no reasons and failed gates require reasons")
|
||||
_canonical_digests(value.evidence_digests, "gate evidence digests")
|
||||
|
||||
|
||||
def gate_value(value: GateResult) -> dict[str, canonical.JsonValue]:
|
||||
validate_gate(value)
|
||||
return {
|
||||
"kind": value.kind.value,
|
||||
"gate_id": value.gate_id,
|
||||
"passed": value.passed,
|
||||
"reasons": value.reasons,
|
||||
"evidence_digests": value.evidence_digests,
|
||||
}
|
||||
|
||||
|
||||
def _make_gate(kind: GateKind, gate_id: str, reasons: tuple[str, ...], evidence: tuple[str, ...]) -> GateResult:
|
||||
canonical_reasons = tuple(sorted(set(reasons)))
|
||||
canonical_evidence = tuple(sorted(set(evidence)))
|
||||
return GateResult(kind, gate_id, not canonical_reasons, canonical_reasons, canonical_evidence)
|
||||
|
||||
|
||||
def budget_evaluation_value(value: CaseBudgetEvaluation) -> dict[str, canonical.JsonValue]:
|
||||
return {
|
||||
"schema_version": BUDGET_EVALUATION_SCHEMA,
|
||||
"case_result_digest": value.case_result_digest,
|
||||
"budget_derivation_digest": value.budget_derivation_digest,
|
||||
"total_reply_ms": value.total_reply_ms,
|
||||
"budget_ms": value.budget_ms,
|
||||
}
|
||||
|
||||
|
||||
def validate_budget_evaluation(value: CaseBudgetEvaluation) -> None:
|
||||
if type(value) is not CaseBudgetEvaluation:
|
||||
raise AdmissionReportError("budget evaluation must use the exact type")
|
||||
if type(value.case) is not AdmissionCaseResult:
|
||||
raise AdmissionReportError("budget evaluation must carry the exact admission case")
|
||||
validate_admission_case_shape(value.case)
|
||||
if value.case.result_digest != derive_admission_case_result_digest(value.case):
|
||||
raise AdmissionReportError("budget evaluation case result digest cannot be re-derived")
|
||||
_digest(value.case_result_digest, "case result digest")
|
||||
_digest(value.budget_derivation_digest, "budget derivation digest")
|
||||
for item, field in ((value.total_reply_ms, "total reply milliseconds"), (value.budget_ms, "budget milliseconds")):
|
||||
if isinstance(item, bool) or not isinstance(item, int) or item <= 0:
|
||||
raise AdmissionReportError(f"{field} must be a positive integer")
|
||||
_digest(value.evaluation_digest, "budget evaluation digest")
|
||||
expected = (
|
||||
value.case.result_digest,
|
||||
value.case.budget.derivation_digest,
|
||||
value.case.total_reply_ms,
|
||||
value.case.budget.budget_ms,
|
||||
)
|
||||
actual = (value.case_result_digest, value.budget_derivation_digest, value.total_reply_ms, value.budget_ms)
|
||||
if actual != expected:
|
||||
raise AdmissionReportError("budget evaluation does not match the exact case-to-budget derivation")
|
||||
if value.evaluation_digest != canonical.content_digest(budget_evaluation_value(value)):
|
||||
raise AdmissionReportError("budget evaluation digest does not match content")
|
||||
|
||||
|
||||
def make_budget_evaluations(cases: tuple[AdmissionCaseResult, ...]) -> tuple[CaseBudgetEvaluation, ...]:
|
||||
values = []
|
||||
for case in cases:
|
||||
fields = (
|
||||
case,
|
||||
case.result_digest,
|
||||
case.budget.derivation_digest,
|
||||
case.total_reply_ms,
|
||||
case.budget.budget_ms,
|
||||
)
|
||||
draft = object.__new__(CaseBudgetEvaluation)
|
||||
for field, item in zip(CaseBudgetEvaluation.__dataclass_fields__, (*fields, "0" * 64), strict=True):
|
||||
object.__setattr__(draft, field, item)
|
||||
digest = canonical.content_digest(budget_evaluation_value(draft))
|
||||
values.append(CaseBudgetEvaluation(*fields, digest))
|
||||
return tuple(sorted(values, key=lambda item: item.case_result_digest))
|
||||
|
||||
|
||||
def threshold_evaluation_value(value: ThresholdEvaluation) -> dict[str, canonical.JsonValue]:
|
||||
return {
|
||||
"schema_version": THRESHOLD_EVALUATION_SCHEMA,
|
||||
"metric_id": value.metric_id.value,
|
||||
"statistic": value.statistic.value,
|
||||
"comparator": value.comparator.value,
|
||||
"target": value.target,
|
||||
"measurement_record_digest": value.measurement_record_digest,
|
||||
"passed": value.passed,
|
||||
}
|
||||
|
||||
|
||||
def validate_threshold_evaluation(value: ThresholdEvaluation) -> None:
|
||||
if type(value) is not ThresholdEvaluation:
|
||||
raise AdmissionReportError("threshold evaluation must use the exact type")
|
||||
if not isinstance(value.metric_id, MetricId) or not isinstance(value.statistic, Statistic):
|
||||
raise AdmissionReportError("threshold evaluation metric and statistic must use closed enums")
|
||||
if not isinstance(value.comparator, Comparator):
|
||||
raise AdmissionReportError("threshold evaluation comparator must use the closed enum")
|
||||
if isinstance(value.target, bool) or not isinstance(value.target, float) or not math.isfinite(value.target):
|
||||
raise AdmissionReportError("threshold evaluation target must be a finite float")
|
||||
_digest(value.measurement_record_digest, "threshold measurement record digest")
|
||||
if not isinstance(value.passed, bool):
|
||||
raise AdmissionReportError("threshold evaluation passed must be boolean")
|
||||
_digest(value.evaluation_digest, "threshold evaluation digest")
|
||||
if value.evaluation_digest != canonical.content_digest(threshold_evaluation_value(value)):
|
||||
raise AdmissionReportError("threshold evaluation digest does not match content")
|
||||
|
||||
|
||||
def make_threshold_evaluations(
|
||||
thresholds: tuple[AdmissionThreshold, ...],
|
||||
measurement: MeasurementRecordV2,
|
||||
) -> tuple[ThresholdEvaluation, ...]:
|
||||
by_metric = {item.metric_id: item for item in measurement.metrics}
|
||||
values = []
|
||||
for threshold in thresholds:
|
||||
passed = metric_passes(by_metric[threshold.metric_id], threshold.statistic, threshold.comparator, threshold.target)
|
||||
fields = (
|
||||
threshold.metric_id,
|
||||
threshold.statistic,
|
||||
threshold.comparator,
|
||||
threshold.target,
|
||||
measurement.record_digest,
|
||||
passed,
|
||||
)
|
||||
draft = object.__new__(ThresholdEvaluation)
|
||||
for field, item in zip(ThresholdEvaluation.__dataclass_fields__, (*fields, "0" * 64), strict=True):
|
||||
object.__setattr__(draft, field, item)
|
||||
digest = canonical.content_digest(threshold_evaluation_value(draft))
|
||||
values.append(ThresholdEvaluation(*fields, digest))
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def _thresholds(evaluations: tuple[ThresholdEvaluation, ...]) -> tuple[AdmissionThreshold, ...]:
|
||||
return tuple(AdmissionThreshold(item.metric_id, item.statistic, item.comparator, item.target) for item in evaluations)
|
||||
|
||||
|
||||
def instrument_gate(
|
||||
case_result_digests: tuple[str, ...],
|
||||
cases: tuple[AdmissionCaseResult, ...],
|
||||
) -> GateResult:
|
||||
evidence = _canonical_digests(case_result_digests, "case result digests")
|
||||
identities = tuple(sorted(item.result_digest for item in cases))
|
||||
if identities != evidence or len(identities) != len(cases):
|
||||
raise AdmissionReportError("instrument cases must cover the exact complete case-result set")
|
||||
try:
|
||||
for case in cases:
|
||||
validate_instrument(case.input_manifest, case.input_receipts)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise AdmissionReportError(f"instrument gate evidence is invalid: {exc}") from exc
|
||||
return _make_gate(GateKind.INSTRUMENT, "instrument:all_cases", (), evidence)
|
||||
|
||||
|
||||
def budget_gate(
|
||||
case_result_digests: tuple[str, ...],
|
||||
evaluations: tuple[CaseBudgetEvaluation, ...],
|
||||
) -> GateResult:
|
||||
if not isinstance(evaluations, tuple) or not all(type(item) is CaseBudgetEvaluation for item in evaluations):
|
||||
raise AdmissionReportError("budget evaluations must use the exact typed tuple")
|
||||
for item in evaluations:
|
||||
validate_budget_evaluation(item)
|
||||
identities = tuple(item.case_result_digest for item in evaluations)
|
||||
if identities != case_result_digests or len(set(identities)) != len(identities):
|
||||
raise AdmissionReportError("budget evaluations must cover the exact complete case-result set")
|
||||
reasons = tuple(f"budget_exceeded:{item.case_result_digest}" for item in evaluations if item.total_reply_ms > item.budget_ms)
|
||||
evidence = tuple(item.evaluation_digest for item in evaluations)
|
||||
return _make_gate(GateKind.BUDGET, "budget:all_cases", reasons, evidence)
|
||||
|
||||
|
||||
def readiness_gate(references: tuple[ReadinessReference, ...]) -> GateResult:
|
||||
reasons = tuple(f"readiness_failed:{item.stage.value}" for item in references if item.result is not StageResult.PASSED)
|
||||
evidence = tuple(item.result_digest for item in references)
|
||||
return _make_gate(GateKind.READINESS, "readiness:prerequisites", reasons, evidence)
|
||||
|
||||
|
||||
def quality_gate(
|
||||
identity: BaselineIdentity,
|
||||
measurement: MeasurementRecordV2,
|
||||
evaluations: tuple[ThresholdEvaluation, ...],
|
||||
) -> GateResult:
|
||||
if not isinstance(evaluations, tuple) or not all(type(item) is ThresholdEvaluation for item in evaluations):
|
||||
raise AdmissionReportError("quality threshold evaluations must use the exact typed tuple")
|
||||
by_metric = {item.metric_id: item for item in measurement.metrics}
|
||||
reasons = _mandatory_metric_reasons(by_metric)
|
||||
model = by_metric[MetricId.MODEL_SUCCESS_RATE]
|
||||
reasons.extend(_model_reasons(identity, model, evaluations))
|
||||
for item in evaluations:
|
||||
validate_threshold_evaluation(item)
|
||||
if item.measurement_record_digest != measurement.record_digest:
|
||||
raise AdmissionReportError("threshold evaluation does not bind the exact measurement")
|
||||
expected = metric_passes(by_metric[item.metric_id], item.statistic, item.comparator, item.target)
|
||||
if item.passed is not expected:
|
||||
raise AdmissionReportError("threshold evaluation outcome does not match the exact metric")
|
||||
if not expected:
|
||||
reasons.append(f"threshold_failed:{item.metric_id.value}:{item.statistic.value}")
|
||||
thresholds = _thresholds(evaluations)
|
||||
try:
|
||||
threshold_set_digest = derive_threshold_set_digest(thresholds)
|
||||
except ValueError as exc:
|
||||
raise AdmissionReportError(f"quality threshold evaluations are invalid: {exc}") from exc
|
||||
if threshold_set_digest != identity.threshold_set_digest:
|
||||
raise AdmissionReportError("threshold evaluations do not match the exact threshold set")
|
||||
evidence = (identity.threshold_set_digest, measurement.record_digest, *(item.evaluation_digest for item in evaluations))
|
||||
return _make_gate(GateKind.QUALITY, "quality:threshold_set", tuple(reasons), evidence)
|
||||
|
||||
|
||||
def _model_reasons(
|
||||
identity: BaselineIdentity,
|
||||
model: MetricResult,
|
||||
evaluations: tuple[ThresholdEvaluation, ...],
|
||||
) -> list[str]:
|
||||
reasons: list[str] = []
|
||||
model_configured = bool(identity.model_program_digests)
|
||||
model_executed = model.denominator > 0
|
||||
model_applicable = model_configured or model_executed
|
||||
model_threshold_count = sum(item.metric_id is MetricId.MODEL_SUCCESS_RATE for item in evaluations)
|
||||
if model_applicable and model_threshold_count == 0:
|
||||
reasons.append("mandatory_model_success_threshold_missing")
|
||||
if model_configured and not model_executed:
|
||||
reasons.append("mandatory_model_success_has_no_attempted_calls")
|
||||
if model_executed and model.numerator < model.denominator:
|
||||
reasons.append("mandatory_model_call_failed")
|
||||
return reasons
|
||||
|
||||
|
||||
def _mandatory_metric_reasons(by_metric: dict[MetricId, MetricResult]) -> list[str]:
|
||||
reasons = []
|
||||
for metric in _MANDATORY_METRICS:
|
||||
result = by_metric.get(metric)
|
||||
if result is None or result.denominator == 0:
|
||||
reasons.append(f"mandatory_metric_missing_or_zero:{metric.value}")
|
||||
return reasons
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Immutable transport identity for authoritative admission replay inputs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from openultrasast.core import canonical
|
||||
from openultrasast.core.admission_case import (
|
||||
AdmissionCaseRelease,
|
||||
AdmissionCaseResult,
|
||||
derive_admission_case_release_digest,
|
||||
derive_admission_case_result_digest,
|
||||
)
|
||||
from openultrasast.core.admission_review import AdmissionManifestV2, derive_manifest_digest
|
||||
from openultrasast.core.admission_spend import AdmissionPreparation, derive_admission_preparation_digest
|
||||
from openultrasast.core.custody import (
|
||||
PopulationUseReceipt,
|
||||
derive_population_use_receipt_digest,
|
||||
)
|
||||
from openultrasast.core.measurement import DenominatorPolicy, derive_denominator_policy_digest
|
||||
from openultrasast.core.readiness import ReadinessResult, validate_readiness_result
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openultrasast.core.admission_report import AdmissionReportV2
|
||||
|
||||
ADMISSION_REPLAY_SCHEMA = "openultrasast.detection-core.admission-replay.v2"
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdmissionReplay:
|
||||
manifest: AdmissionManifestV2
|
||||
denominator: DenominatorPolicy
|
||||
readiness: tuple[ReadinessResult, ...]
|
||||
preparation: AdmissionPreparation
|
||||
label_release_receipt: PopulationUseReceipt
|
||||
case_release: AdmissionCaseRelease
|
||||
cases: tuple[AdmissionCaseResult, ...]
|
||||
report: AdmissionReportV2
|
||||
replay_digest: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
validate_admission_replay_structure(self)
|
||||
|
||||
|
||||
def _replay_value(value: AdmissionReplay) -> canonical.JsonValue:
|
||||
manifest_digest = derive_manifest_digest(value.manifest)
|
||||
denominator_digest = derive_denominator_policy_digest(value.denominator)
|
||||
preparation_digest = derive_admission_preparation_digest(value.preparation)
|
||||
receipt_digest = derive_population_use_receipt_digest(value.label_release_receipt)
|
||||
release_digest = derive_admission_case_release_digest(value.case_release)
|
||||
for item in value.readiness:
|
||||
validate_readiness_result(item)
|
||||
case_digests = tuple(derive_admission_case_result_digest(item) for item in value.cases)
|
||||
claimed = (
|
||||
value.manifest.manifest_digest,
|
||||
value.denominator.policy_digest,
|
||||
value.preparation.preparation_digest,
|
||||
value.label_release_receipt.receipt_digest,
|
||||
value.case_release.release_digest,
|
||||
tuple(item.result_digest for item in value.cases),
|
||||
)
|
||||
derived = (manifest_digest, denominator_digest, preparation_digest, receipt_digest, release_digest, case_digests)
|
||||
if claimed != derived:
|
||||
raise ValueError("replay component identity cannot be re-derived")
|
||||
return {
|
||||
"schema_version": ADMISSION_REPLAY_SCHEMA,
|
||||
"manifest_digest": manifest_digest,
|
||||
"denominator_policy_digest": denominator_digest,
|
||||
"readiness_result_digests": tuple(item.result_digest for item in value.readiness),
|
||||
"preparation_digest": preparation_digest,
|
||||
"label_release_receipt_digest": receipt_digest,
|
||||
"case_release_digest": release_digest,
|
||||
"case_result_digests": case_digests,
|
||||
"report_digest": value.report.report_digest,
|
||||
}
|
||||
|
||||
|
||||
def _validate_components(value: AdmissionReplay) -> None:
|
||||
exact_components = (
|
||||
(value.manifest, AdmissionManifestV2, "manifest"),
|
||||
(value.denominator, DenominatorPolicy, "denominator"),
|
||||
(value.preparation, AdmissionPreparation, "preparation"),
|
||||
(value.label_release_receipt, PopulationUseReceipt, "label release receipt"),
|
||||
(value.case_release, AdmissionCaseRelease, "case release"),
|
||||
)
|
||||
for item, expected, field in exact_components:
|
||||
if type(item) is not expected:
|
||||
raise ValueError(f"replay {field} must use the exact contract type")
|
||||
if not all(type(item) is ReadinessResult for item in value.readiness):
|
||||
raise ValueError("replay readiness must contain exact ReadinessResult values")
|
||||
if not all(type(item) is AdmissionCaseResult for item in value.cases):
|
||||
raise ValueError("replay cases must contain exact AdmissionCaseResult values")
|
||||
case_ids = tuple(item.case_id_digest for item in value.cases)
|
||||
if case_ids != tuple(sorted(set(case_ids))):
|
||||
raise ValueError("replay cases must use unique canonical case identity order")
|
||||
|
||||
|
||||
def validate_admission_replay_structure(value: AdmissionReplay) -> None:
|
||||
from openultrasast.core.admission_report import AdmissionReportError, AdmissionReportV2, _validate_report_structure
|
||||
|
||||
if type(value) is not AdmissionReplay:
|
||||
raise AdmissionReportError("replay must be an exact AdmissionReplay")
|
||||
if type(value.readiness) is not tuple or not value.readiness:
|
||||
raise AdmissionReportError("replay readiness must be an exact nonempty tuple")
|
||||
if type(value.cases) is not tuple or not value.cases:
|
||||
raise AdmissionReportError("replay cases must be an exact nonempty tuple")
|
||||
try:
|
||||
_validate_components(value)
|
||||
except (AttributeError, TypeError, ValueError) as exc:
|
||||
raise AdmissionReportError(str(exc)) from exc
|
||||
if type(value.report) is not AdmissionReportV2:
|
||||
raise AdmissionReportError("replay report must be an exact AdmissionReportV2")
|
||||
_validate_report_structure(value.report)
|
||||
if not isinstance(value.replay_digest, str) or _DIGEST.fullmatch(value.replay_digest) is None:
|
||||
raise AdmissionReportError("replay digest must be a full lowercase SHA-256 digest")
|
||||
try:
|
||||
expected = canonical.content_digest(_replay_value(value))
|
||||
except (AttributeError, TypeError, ValueError) as exc:
|
||||
raise AdmissionReportError(str(exc)) from exc
|
||||
if value.replay_digest != expected:
|
||||
raise AdmissionReportError("replay digest does not match authoritative input identities")
|
||||
|
||||
|
||||
def admission_replay_value(value: AdmissionReplay) -> canonical.JsonValue:
|
||||
validate_admission_replay_structure(value)
|
||||
return _replay_value(value)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADMISSION_REPLAY_SCHEMA",
|
||||
"AdmissionReplay",
|
||||
"admission_replay_value",
|
||||
]
|
||||
@@ -24,7 +24,7 @@ from openultrasast.core.admission_case import (
|
||||
from openultrasast.core.admission_context import AdmissionValidationContext
|
||||
from openultrasast.core.admission_review import AdmissionManifestV2, ScopeCensus, validate_reviewed_admission_payloads
|
||||
from openultrasast.core.admission_spend import AdmissionPreparation
|
||||
from openultrasast.core.budget import check_budget
|
||||
from openultrasast.core.budget import BudgetGateFailure, check_budget
|
||||
from openultrasast.core.coverage import CoverageFunnel, ReasonCount, validate_instrument
|
||||
from openultrasast.core.custody import PopulationCustodyPort, PopulationUseReceipt
|
||||
from openultrasast.core.evidence import EvidenceBasis, EvidenceRecord, NeutralOutcome, derive_packet_digest
|
||||
@@ -139,6 +139,8 @@ def _validated_records(
|
||||
raise AdmissionMetricsError(f"instrument validation failed: {exc}") from exc
|
||||
try:
|
||||
check_budget(case.complexity, case.total_reply_ms, case.budget, resolver)
|
||||
except BudgetGateFailure:
|
||||
pass
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise AdmissionMetricsError(f"budget validation failed: {exc}") from exc
|
||||
try:
|
||||
@@ -206,9 +208,7 @@ def _validate_population(cases: tuple[_ValidatedCase, ...], rows: tuple[ScopeCen
|
||||
for row in rows:
|
||||
selected = tuple(item for item in cases if item.binding.scope == row.scope)
|
||||
roles = Counter(item.binding.role for item in selected)
|
||||
model_calls = sum(
|
||||
value.model_result is not None for item in selected for value in item.supplied.evidence_inputs
|
||||
)
|
||||
model_calls = sum(value.model_result is not None for item in selected for value in item.supplied.evidence_inputs)
|
||||
actual = (
|
||||
len(selected),
|
||||
roles[AdmissionCaseRole.VULNERABLE],
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
"""Evidence-backed admission gates and canonical final reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
from openultrasast.core import canonical
|
||||
from openultrasast.core._admission_gates import (
|
||||
AdmissionReportError,
|
||||
CaseBudgetEvaluation,
|
||||
GateKind,
|
||||
GateResult,
|
||||
ThresholdEvaluation,
|
||||
budget_evaluation_value,
|
||||
budget_gate,
|
||||
gate_value,
|
||||
instrument_gate,
|
||||
make_budget_evaluations,
|
||||
make_threshold_evaluations,
|
||||
quality_gate,
|
||||
readiness_gate,
|
||||
threshold_evaluation_value,
|
||||
)
|
||||
from openultrasast.core._admission_replay import (
|
||||
ADMISSION_REPLAY_SCHEMA,
|
||||
AdmissionReplay,
|
||||
_replay_value,
|
||||
admission_replay_value,
|
||||
validate_admission_replay_structure,
|
||||
)
|
||||
from openultrasast.core.admission_case import AdmissionCaseRelease, AdmissionCaseResult
|
||||
from openultrasast.core.admission_context import AdmissionValidationContext
|
||||
from openultrasast.core.admission_metrics import AdmissionMetricEvidence, derive_admission_metrics
|
||||
from openultrasast.core.admission_review import AdmissionManifestV2
|
||||
from openultrasast.core.admission_spend import AdmissionPreparation
|
||||
from openultrasast.core.custody import PopulationCustodyPort, PopulationUseReceipt
|
||||
from openultrasast.core.measurement import (
|
||||
BaselineIdentity,
|
||||
DenominatorPolicy,
|
||||
derive_baseline_identity_digest,
|
||||
)
|
||||
from openultrasast.core.measurement_factory import create_measurement_record
|
||||
from openultrasast.core.measurement_record import MeasurementRecordV2, ReadinessReference, measurement_record_value
|
||||
from openultrasast.core.readiness import ReadinessResult, ReadinessStage
|
||||
|
||||
ADMISSION_REPORT_SCHEMA = "openultrasast.detection-core.admission-report.v2"
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_PREREQUISITE_STAGES = tuple(ReadinessStage)[:6]
|
||||
|
||||
|
||||
class AdmissionDecision(Enum):
|
||||
GO = "go"
|
||||
NO_GO = "no_go"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdmissionReportV2:
|
||||
identity: BaselineIdentity
|
||||
census_digest: str
|
||||
validation_context_digest: str
|
||||
authorization_digest: str
|
||||
label_release_receipt_digest: str
|
||||
case_release_digest: str
|
||||
case_result_digests: tuple[str, ...]
|
||||
in_scope_case_count: int
|
||||
whole_population_case_count: int
|
||||
budget_evaluations: tuple[CaseBudgetEvaluation, ...]
|
||||
quality_evaluations: tuple[ThresholdEvaluation, ...]
|
||||
measurement: MeasurementRecordV2
|
||||
instrument_results: tuple[GateResult, ...]
|
||||
budget_results: tuple[GateResult, ...]
|
||||
readiness_results: tuple[GateResult, ...]
|
||||
quality_results: tuple[GateResult, ...]
|
||||
decision: AdmissionDecision
|
||||
report_digest: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_report_structure(self)
|
||||
|
||||
|
||||
def _digest(value: object, field: str) -> None:
|
||||
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
||||
raise AdmissionReportError(f"{field} must be a full lowercase SHA-256 digest")
|
||||
|
||||
|
||||
def _case_digests(value: object, count: int) -> tuple[str, ...]:
|
||||
if not isinstance(value, tuple) or not all(isinstance(item, str) for item in value):
|
||||
raise AdmissionReportError("case result digests must be a tuple of digests")
|
||||
for item in value:
|
||||
_digest(item, "case result digest")
|
||||
if value != tuple(sorted(set(value))) or len(value) != count:
|
||||
raise AdmissionReportError("case result digests must identify every case exactly once in canonical order")
|
||||
return value
|
||||
|
||||
|
||||
def _report_value(value: AdmissionReportV2) -> canonical.JsonValue:
|
||||
return {
|
||||
"schema_version": ADMISSION_REPORT_SCHEMA,
|
||||
"baseline_identity_digest": derive_baseline_identity_digest(value.identity),
|
||||
"census_digest": value.census_digest,
|
||||
"validation_context_digest": value.validation_context_digest,
|
||||
"authorization_digest": value.authorization_digest,
|
||||
"label_release_receipt_digest": value.label_release_receipt_digest,
|
||||
"case_release_digest": value.case_release_digest,
|
||||
"case_result_digests": value.case_result_digests,
|
||||
"in_scope_case_count": value.in_scope_case_count,
|
||||
"whole_population_case_count": value.whole_population_case_count,
|
||||
"budget_evaluations": tuple(
|
||||
{**budget_evaluation_value(item), "evaluation_digest": item.evaluation_digest} for item in value.budget_evaluations
|
||||
),
|
||||
"quality_evaluations": tuple(
|
||||
{**threshold_evaluation_value(item), "evaluation_digest": item.evaluation_digest} for item in value.quality_evaluations
|
||||
),
|
||||
"measurement_record_digest": value.measurement.record_digest,
|
||||
"instrument_results": tuple(gate_value(item) for item in value.instrument_results),
|
||||
"budget_results": tuple(gate_value(item) for item in value.budget_results),
|
||||
"readiness_results": tuple(gate_value(item) for item in value.readiness_results),
|
||||
"quality_results": tuple(gate_value(item) for item in value.quality_results),
|
||||
"decision": value.decision.value,
|
||||
}
|
||||
|
||||
|
||||
def _validate_report_bindings(value: AdmissionReportV2) -> None:
|
||||
baseline_digest = derive_baseline_identity_digest(value.identity)
|
||||
if baseline_digest != derive_baseline_identity_digest(value.measurement.identity):
|
||||
raise AdmissionReportError("measurement baseline identity does not match report")
|
||||
measurement_record_value(value.measurement)
|
||||
for item, field in (
|
||||
(value.census_digest, "census digest"),
|
||||
(value.validation_context_digest, "validation context digest"),
|
||||
(value.authorization_digest, "authorization digest"),
|
||||
(value.label_release_receipt_digest, "label release receipt digest"),
|
||||
(value.case_release_digest, "case release digest"),
|
||||
):
|
||||
_digest(item, field)
|
||||
if isinstance(value.in_scope_case_count, bool) or not isinstance(value.in_scope_case_count, int):
|
||||
raise AdmissionReportError("in-scope case count must be an integer")
|
||||
if isinstance(value.whole_population_case_count, bool) or not isinstance(value.whole_population_case_count, int):
|
||||
raise AdmissionReportError("whole-population case count must be an integer")
|
||||
if not 0 < value.in_scope_case_count <= value.whole_population_case_count:
|
||||
raise AdmissionReportError("case counts must be positive and in-scope cannot exceed whole-population")
|
||||
_case_digests(value.case_result_digests, value.whole_population_case_count)
|
||||
|
||||
|
||||
def _require_exact_gate(value: object, expected: GateResult) -> tuple[GateResult, ...]:
|
||||
if not isinstance(value, tuple) or value != (expected,):
|
||||
raise AdmissionReportError(f"{expected.kind.value} gate does not match its derived evidence and outcome")
|
||||
return value
|
||||
|
||||
|
||||
def _derived_gates(value: AdmissionReportV2) -> tuple[GateResult, GateResult, GateResult, GateResult]:
|
||||
cases = tuple(item.case for item in value.budget_evaluations)
|
||||
return (
|
||||
instrument_gate(value.case_result_digests, cases),
|
||||
budget_gate(value.case_result_digests, value.budget_evaluations),
|
||||
readiness_gate(value.measurement.prerequisite_readiness),
|
||||
quality_gate(value.identity, value.measurement, value.quality_evaluations),
|
||||
)
|
||||
|
||||
|
||||
def _validate_report_fields(value: AdmissionReportV2) -> None:
|
||||
if type(value) is not AdmissionReportV2:
|
||||
raise AdmissionReportError("report must be an exact AdmissionReportV2")
|
||||
_validate_report_bindings(value)
|
||||
instrument, budget, readiness, quality = _derived_gates(value)
|
||||
actual = (
|
||||
_require_exact_gate(value.instrument_results, instrument),
|
||||
_require_exact_gate(value.budget_results, budget),
|
||||
_require_exact_gate(value.readiness_results, readiness),
|
||||
_require_exact_gate(value.quality_results, quality),
|
||||
)
|
||||
if not isinstance(value.decision, AdmissionDecision):
|
||||
raise AdmissionReportError("decision must use AdmissionDecision")
|
||||
all_passed = all(item.passed for group in actual for item in group)
|
||||
expected = AdmissionDecision.GO if all_passed else AdmissionDecision.NO_GO
|
||||
if value.decision is not expected:
|
||||
raise AdmissionReportError("decision does not match every re-derived mandatory gate")
|
||||
|
||||
|
||||
def _validate_report_structure(value: AdmissionReportV2) -> None:
|
||||
_validate_report_fields(value)
|
||||
_digest(value.report_digest, "report digest")
|
||||
if value.report_digest != canonical.content_digest(_report_value(value)):
|
||||
raise AdmissionReportError("report digest does not match admission content")
|
||||
|
||||
|
||||
def admission_report_value(value: AdmissionReportV2) -> canonical.JsonValue:
|
||||
_validate_report_structure(value)
|
||||
return _report_value(value)
|
||||
|
||||
|
||||
def _readiness_references(readiness: tuple[ReadinessResult, ...]) -> tuple[ReadinessReference, ...]:
|
||||
references = tuple(ReadinessReference(item.stage, item.result, item.result_digest) for item in readiness)
|
||||
if tuple(item.stage for item in references) != _PREREQUISITE_STAGES:
|
||||
raise AdmissionReportError("admission measurement requires instrument through development readiness")
|
||||
return references
|
||||
|
||||
|
||||
def _measurement(
|
||||
manifest: AdmissionManifestV2,
|
||||
evidence: AdmissionMetricEvidence,
|
||||
readiness: tuple[ReadinessResult, ...],
|
||||
) -> MeasurementRecordV2:
|
||||
return create_measurement_record(
|
||||
identity=manifest.identity,
|
||||
family_decision_vocabulary=evidence.family_decision_vocabulary,
|
||||
family_decision_counts=evidence.family_decision_counts,
|
||||
neutral_outcome_counts=evidence.neutral_outcome_counts,
|
||||
evidence_basis_counts=evidence.evidence_basis_counts,
|
||||
metrics=evidence.metrics,
|
||||
stage_funnel=evidence.stage_funnel,
|
||||
instrument_failures=(),
|
||||
model_outcome_counts=evidence.model_outcome_counts,
|
||||
model_failures=evidence.model_failures,
|
||||
runtime_ms=evidence.runtime_ms,
|
||||
prerequisite_readiness=_readiness_references(readiness),
|
||||
)
|
||||
|
||||
|
||||
def _quality_gate(manifest: AdmissionManifestV2, measurement: MeasurementRecordV2) -> GateResult:
|
||||
evaluations = make_threshold_evaluations(manifest.thresholds, measurement)
|
||||
return quality_gate(manifest.identity, measurement, evaluations)
|
||||
|
||||
|
||||
def _make_report(
|
||||
manifest: AdmissionManifestV2,
|
||||
preparation: AdmissionPreparation,
|
||||
receipt: PopulationUseReceipt,
|
||||
release: AdmissionCaseRelease,
|
||||
cases: tuple[AdmissionCaseResult, ...],
|
||||
evidence: AdmissionMetricEvidence,
|
||||
measurement: MeasurementRecordV2,
|
||||
) -> AdmissionReportV2:
|
||||
budgets = make_budget_evaluations(cases)
|
||||
quality = make_threshold_evaluations(manifest.thresholds, measurement)
|
||||
gates = (
|
||||
instrument_gate(evidence.case_result_digests, cases),
|
||||
budget_gate(evidence.case_result_digests, budgets),
|
||||
readiness_gate(measurement.prerequisite_readiness),
|
||||
quality_gate(manifest.identity, measurement, quality),
|
||||
)
|
||||
decision = AdmissionDecision.GO if all(item.passed for item in gates) else AdmissionDecision.NO_GO
|
||||
values = (
|
||||
manifest.identity,
|
||||
manifest.census.census_digest,
|
||||
preparation.authorization.validation_context_digest,
|
||||
preparation.authorization.authorization_digest,
|
||||
receipt.receipt_digest,
|
||||
release.release_digest,
|
||||
evidence.case_result_digests,
|
||||
evidence.in_scope_case_count,
|
||||
evidence.whole_population_case_count,
|
||||
budgets,
|
||||
quality,
|
||||
measurement,
|
||||
(gates[0],),
|
||||
(gates[1],),
|
||||
(gates[2],),
|
||||
(gates[3],),
|
||||
decision,
|
||||
)
|
||||
draft = object.__new__(AdmissionReportV2)
|
||||
fields = tuple(AdmissionReportV2.__dataclass_fields__)
|
||||
for field, item in zip(fields, (*values, "0" * 64), strict=True):
|
||||
object.__setattr__(draft, field, item)
|
||||
_validate_report_fields(draft)
|
||||
return AdmissionReportV2(*values, canonical.content_digest(_report_value(draft)))
|
||||
|
||||
|
||||
def score_admission(
|
||||
manifest: AdmissionManifestV2,
|
||||
denominator: DenominatorPolicy,
|
||||
readiness: Sequence[ReadinessResult],
|
||||
preparation: AdmissionPreparation,
|
||||
label_release_receipt: PopulationUseReceipt,
|
||||
case_release: AdmissionCaseRelease,
|
||||
cases: Sequence[AdmissionCaseResult],
|
||||
context: AdmissionValidationContext,
|
||||
custody: PopulationCustodyPort,
|
||||
resolver: canonical.ContentResolver,
|
||||
) -> AdmissionReplay:
|
||||
"""Revalidate exact admission evidence and freeze its authoritative replay."""
|
||||
frozen_readiness = tuple(readiness)
|
||||
frozen_cases = tuple(sorted(tuple(cases), key=lambda item: item.case_id_digest))
|
||||
report = _derive_report(
|
||||
manifest,
|
||||
denominator,
|
||||
frozen_readiness,
|
||||
preparation,
|
||||
label_release_receipt,
|
||||
case_release,
|
||||
frozen_cases,
|
||||
context,
|
||||
custody,
|
||||
resolver,
|
||||
)
|
||||
return _make_replay(
|
||||
manifest,
|
||||
denominator,
|
||||
frozen_readiness,
|
||||
preparation,
|
||||
label_release_receipt,
|
||||
case_release,
|
||||
frozen_cases,
|
||||
report,
|
||||
)
|
||||
|
||||
|
||||
def _derive_report(
|
||||
manifest: AdmissionManifestV2,
|
||||
denominator: DenominatorPolicy,
|
||||
readiness: tuple[ReadinessResult, ...],
|
||||
preparation: AdmissionPreparation,
|
||||
label_release_receipt: PopulationUseReceipt,
|
||||
case_release: AdmissionCaseRelease,
|
||||
cases: tuple[AdmissionCaseResult, ...],
|
||||
context: AdmissionValidationContext,
|
||||
custody: PopulationCustodyPort,
|
||||
resolver: canonical.ContentResolver,
|
||||
) -> AdmissionReportV2:
|
||||
evidence = derive_admission_metrics(
|
||||
manifest,
|
||||
denominator,
|
||||
readiness,
|
||||
preparation,
|
||||
label_release_receipt,
|
||||
case_release,
|
||||
cases,
|
||||
context,
|
||||
custody,
|
||||
resolver,
|
||||
)
|
||||
measurement = _measurement(manifest, evidence, readiness)
|
||||
return _make_report(
|
||||
manifest,
|
||||
preparation,
|
||||
label_release_receipt,
|
||||
case_release,
|
||||
cases,
|
||||
evidence,
|
||||
measurement,
|
||||
)
|
||||
|
||||
|
||||
def _make_replay(
|
||||
manifest: AdmissionManifestV2,
|
||||
denominator: DenominatorPolicy,
|
||||
readiness: tuple[ReadinessResult, ...],
|
||||
preparation: AdmissionPreparation,
|
||||
label_release_receipt: PopulationUseReceipt,
|
||||
case_release: AdmissionCaseRelease,
|
||||
cases: tuple[AdmissionCaseResult, ...],
|
||||
report: AdmissionReportV2,
|
||||
) -> AdmissionReplay:
|
||||
values = (manifest, denominator, readiness, preparation, label_release_receipt, case_release, cases, report)
|
||||
draft = object.__new__(AdmissionReplay)
|
||||
for field, item in zip(AdmissionReplay.__dataclass_fields__, (*values, "0" * 64), strict=True):
|
||||
object.__setattr__(draft, field, item)
|
||||
return AdmissionReplay(*values, canonical.content_digest(_replay_value(draft)))
|
||||
|
||||
|
||||
def validate_admission_replay(
|
||||
replay: AdmissionReplay,
|
||||
context: AdmissionValidationContext,
|
||||
custody: PopulationCustodyPort,
|
||||
resolver: canonical.ContentResolver,
|
||||
) -> AdmissionReportV2:
|
||||
"""Re-run every carried input against independently supplied current trust state."""
|
||||
validate_admission_replay_structure(replay)
|
||||
try:
|
||||
expected = _derive_report(
|
||||
replay.manifest,
|
||||
replay.denominator,
|
||||
replay.readiness,
|
||||
replay.preparation,
|
||||
replay.label_release_receipt,
|
||||
replay.case_release,
|
||||
replay.cases,
|
||||
context,
|
||||
custody,
|
||||
resolver,
|
||||
)
|
||||
except (AttributeError, LookupError, TypeError, ValueError) as exc:
|
||||
raise AdmissionReportError(f"authoritative admission replay refused: {exc}") from exc
|
||||
actual_bytes = canonical.canonical_bytes(_report_value(replay.report))
|
||||
expected_bytes = canonical.canonical_bytes(_report_value(expected))
|
||||
if replay.report != expected or actual_bytes != expected_bytes:
|
||||
raise AdmissionReportError("report does not match the authoritative admission inputs")
|
||||
expected_replay = _make_replay(
|
||||
replay.manifest,
|
||||
replay.denominator,
|
||||
replay.readiness,
|
||||
replay.preparation,
|
||||
replay.label_release_receipt,
|
||||
replay.case_release,
|
||||
replay.cases,
|
||||
expected,
|
||||
)
|
||||
if replay != expected_replay or canonical.canonical_bytes(_replay_value(replay)) != canonical.canonical_bytes(
|
||||
_replay_value(expected_replay)
|
||||
):
|
||||
raise AdmissionReportError("replay does not match the authoritative admission inputs")
|
||||
return expected
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADMISSION_REPLAY_SCHEMA",
|
||||
"ADMISSION_REPORT_SCHEMA",
|
||||
"AdmissionReplay",
|
||||
"AdmissionDecision",
|
||||
"AdmissionReportError",
|
||||
"AdmissionReportV2",
|
||||
"CaseBudgetEvaluation",
|
||||
"GateKind",
|
||||
"GateResult",
|
||||
"ThresholdEvaluation",
|
||||
"admission_replay_value",
|
||||
"admission_report_value",
|
||||
"score_admission",
|
||||
"validate_admission_replay",
|
||||
]
|
||||
@@ -20,7 +20,7 @@ from openultrasast.core.readiness import ReadinessEvidenceUse, ReadinessStage, S
|
||||
MEASUREMENT_RECORD_SCHEMA = "openultrasast.detection-core.measurement-record.v2"
|
||||
REPOSITORY_HOLDOUT_REPORT_SCHEMA = "openultrasast.detection-core.repository-holdout-report.v2"
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
_PREREQUISITE_STAGES = tuple(ReadinessStage)[:5]
|
||||
_PREREQUISITE_STAGES = tuple(ReadinessStage)[:6]
|
||||
|
||||
|
||||
class MeasurementRecordError(ValueError):
|
||||
|
||||
Reference in New Issue
Block a user