mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
feat(detection-core): account for instrument-safe coverage
This commit is contained in:
Marius Ciepluch
@@ -94,7 +94,7 @@
|
||||
- _Requirements: 3.5, 8.2_
|
||||
|
||||
- [ ] 4. Expose operational coverage, budget, and readiness gates
|
||||
- [ ] 4.1 (P) Account for instrument reads and every semantic-stage outcome
|
||||
- [x] 4.1 (P) Account for instrument reads and every semantic-stage outcome
|
||||
- Count reads, questions, sites considered and recognized, transitions, security objects, constraints, determinate outcomes, and findings.
|
||||
- Put every considered relation in exactly one complete, incomplete, or unresolved bucket.
|
||||
- Fail unopened or zero-byte tracked input and diagnose the last verified nonzero stage and first missing expected output.
|
||||
|
||||
@@ -8,9 +8,16 @@ instrument failure -- a zero from an unread input is a failed run, never a clean
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from collections.abc import Mapping
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from openultrasast.core.canonical import JsonValue, content_digest, require_unique_identities
|
||||
from openultrasast.core.semantics import RelationStatus
|
||||
|
||||
INPUT_MANIFEST_SCHEMA = "openultrasast.detection-core.input-manifest.v2"
|
||||
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
class InstrumentFailure(ValueError):
|
||||
"""The run's input was not read; any result from it is invalid."""
|
||||
@@ -54,4 +61,336 @@ def check_reads(reads: Mapping[str, int]) -> None:
|
||||
raise InstrumentFailure(f"zero-byte read of tracked input: {', '.join(zero)}")
|
||||
|
||||
|
||||
__all__ = ["Coverage", "InstrumentFailure", "ResolutionOutcome", "check_reads"]
|
||||
def _require_text(value: object, field_name: str) -> None:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"{field_name} must be nonempty text")
|
||||
|
||||
|
||||
def _require_digest(value: object, field_name: str) -> None:
|
||||
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
||||
raise ValueError(f"{field_name} must be a full lowercase SHA-256 digest")
|
||||
|
||||
|
||||
def _require_count(value: object, field_name: str) -> None:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise ValueError(f"{field_name} must be a non-negative integer")
|
||||
|
||||
|
||||
def _validate_input_ids(value: object) -> tuple[str, ...]:
|
||||
if not isinstance(value, tuple) or not value:
|
||||
raise ValueError("tracked_input_ids must be a nonempty tuple")
|
||||
for input_id in value:
|
||||
_require_text(input_id, "tracked input id")
|
||||
require_unique_identities(value, kind="tracked input")
|
||||
return value
|
||||
|
||||
|
||||
def derive_input_manifest_digest(tracked_input_ids: tuple[str, ...]) -> str:
|
||||
"""Derive the identity of the exact ordered tracked-input manifest."""
|
||||
inputs = _validate_input_ids(tracked_input_ids)
|
||||
payload: JsonValue = {"schema_version": INPUT_MANIFEST_SCHEMA, "tracked_input_ids": inputs}
|
||||
return content_digest(payload)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputManifest:
|
||||
tracked_input_ids: tuple[str, ...]
|
||||
manifest_digest: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_input_ids(self.tracked_input_ids)
|
||||
_require_digest(self.manifest_digest, "manifest_digest")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputReadReceipt:
|
||||
input_id: str
|
||||
path: str
|
||||
opened: bool
|
||||
byte_count: int
|
||||
content_digest: str | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_text(self.input_id, "input_id")
|
||||
_require_text(self.path, "path")
|
||||
if not isinstance(self.opened, bool):
|
||||
raise ValueError("opened must be a boolean")
|
||||
_require_count(self.byte_count, "byte_count")
|
||||
if self.content_digest is not None:
|
||||
_require_digest(self.content_digest, "content_digest")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RelationStatusCount:
|
||||
status: RelationStatus
|
||||
count: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_relation_status_count(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReasonCount:
|
||||
reason: str
|
||||
count: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_reason_count(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CoverageFunnel:
|
||||
inputs_read: int
|
||||
questions_formed: int
|
||||
sites_considered: int
|
||||
sites_recognized: int
|
||||
transitions_materialized: int
|
||||
security_objects_constructed: int
|
||||
constraints_evaluated: int
|
||||
determinate_outcomes: int
|
||||
findings_emitted: int
|
||||
relation_status_counts: tuple[RelationStatusCount, ...]
|
||||
unresolved_by_reason: tuple[ReasonCount, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_funnel(self)
|
||||
|
||||
|
||||
class CoverageStage(enum.Enum):
|
||||
INPUTS = "inputs"
|
||||
QUESTIONS = "questions"
|
||||
SITES_CONSIDERED = "sites_considered"
|
||||
SITES_RECOGNIZED = "sites_recognized"
|
||||
TRANSITIONS = "transitions"
|
||||
SECURITY_OBJECTS = "security_objects"
|
||||
CONSTRAINTS = "constraints"
|
||||
OUTCOMES = "outcomes"
|
||||
FINDINGS = "findings"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageExpectation:
|
||||
stage: CoverageStage
|
||||
expected_nonzero_input: bool
|
||||
expected_nonzero_output: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_stage_expectation(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageDiagnosis:
|
||||
last_verified_nonzero_stage: CoverageStage | None
|
||||
first_failed_output_stage: CoverageStage | None
|
||||
reasons: tuple[str, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value, field_name in (
|
||||
(self.last_verified_nonzero_stage, "last_verified_nonzero_stage"),
|
||||
(self.first_failed_output_stage, "first_failed_output_stage"),
|
||||
):
|
||||
if value is not None and not isinstance(value, CoverageStage):
|
||||
raise ValueError(f"{field_name} must be a CoverageStage or None")
|
||||
if not isinstance(self.reasons, tuple) or not all(isinstance(reason, str) and reason for reason in self.reasons):
|
||||
raise ValueError("reasons must be a tuple of nonempty text")
|
||||
|
||||
|
||||
_COUNT_FIELDS = (
|
||||
"inputs_read",
|
||||
"questions_formed",
|
||||
"sites_considered",
|
||||
"sites_recognized",
|
||||
"transitions_materialized",
|
||||
"security_objects_constructed",
|
||||
"constraints_evaluated",
|
||||
"determinate_outcomes",
|
||||
"findings_emitted",
|
||||
)
|
||||
|
||||
|
||||
def _validate_relation_status_count(value: RelationStatusCount) -> None:
|
||||
if not isinstance(value.status, RelationStatus):
|
||||
raise ValueError("status must be a RelationStatus")
|
||||
_require_count(value.count, "relation status count")
|
||||
|
||||
|
||||
def _validate_reason_count(value: ReasonCount) -> None:
|
||||
_require_text(value.reason, "reason")
|
||||
_require_count(value.count, "reason count")
|
||||
if value.count == 0:
|
||||
raise ValueError("reason count must be positive")
|
||||
|
||||
|
||||
def _validate_stage_expectation(value: StageExpectation) -> None:
|
||||
if not isinstance(value.stage, CoverageStage):
|
||||
raise ValueError("stage must be a CoverageStage")
|
||||
if not isinstance(value.expected_nonzero_input, bool):
|
||||
raise ValueError("expected_nonzero_input must be a boolean")
|
||||
if not isinstance(value.expected_nonzero_output, bool):
|
||||
raise ValueError("expected_nonzero_output must be a boolean")
|
||||
|
||||
|
||||
def _validate_funnel(funnel: CoverageFunnel) -> None:
|
||||
for field_name in _COUNT_FIELDS:
|
||||
_require_count(getattr(funnel, field_name), field_name)
|
||||
_validate_stage_inequalities(funnel)
|
||||
if not isinstance(funnel.relation_status_counts, tuple):
|
||||
raise ValueError("relation_status_counts must be a tuple")
|
||||
if not all(isinstance(item, RelationStatusCount) for item in funnel.relation_status_counts):
|
||||
raise ValueError("relation_status_counts must contain RelationStatusCount values")
|
||||
for item in funnel.relation_status_counts:
|
||||
_validate_relation_status_count(item)
|
||||
statuses = tuple(item.status for item in funnel.relation_status_counts)
|
||||
if len(statuses) != len(set(statuses)):
|
||||
raise ValueError("duplicate relation status bucket")
|
||||
if set(statuses) != set(RelationStatus):
|
||||
raise ValueError("relation_status_counts must contain every relation status")
|
||||
if sum(item.count for item in funnel.relation_status_counts) != funnel.transitions_materialized:
|
||||
raise ValueError("relation status buckets must equal transitions_materialized")
|
||||
_validate_reason_counts(funnel)
|
||||
|
||||
|
||||
def _validate_stage_inequalities(funnel: CoverageFunnel) -> None:
|
||||
inequalities = (
|
||||
(funnel.sites_recognized, funnel.sites_considered, "sites_recognized", "sites_considered"),
|
||||
(funnel.determinate_outcomes, funnel.questions_formed, "determinate_outcomes", "questions_formed"),
|
||||
(funnel.findings_emitted, funnel.determinate_outcomes, "findings_emitted", "determinate_outcomes"),
|
||||
)
|
||||
for downstream, upstream, downstream_name, upstream_name in inequalities:
|
||||
if downstream > upstream:
|
||||
raise ValueError(f"{downstream_name} cannot exceed {upstream_name}")
|
||||
|
||||
|
||||
def _validate_reason_counts(funnel: CoverageFunnel) -> None:
|
||||
if not isinstance(funnel.unresolved_by_reason, tuple):
|
||||
raise ValueError("unresolved_by_reason must be a tuple")
|
||||
if not all(isinstance(item, ReasonCount) for item in funnel.unresolved_by_reason):
|
||||
raise ValueError("unresolved_by_reason must contain ReasonCount values")
|
||||
for item in funnel.unresolved_by_reason:
|
||||
_validate_reason_count(item)
|
||||
reasons = tuple(item.reason for item in funnel.unresolved_by_reason)
|
||||
if len(reasons) != len(set(reasons)):
|
||||
raise ValueError("duplicate unresolved reason")
|
||||
unresolved = sum(item.count for item in funnel.relation_status_counts if item.status is not RelationStatus.COMPLETE)
|
||||
if sum(item.count for item in funnel.unresolved_by_reason) != unresolved:
|
||||
raise ValueError("reason counts must equal incomplete and unresolved relations")
|
||||
|
||||
|
||||
def _validate_manifest(manifest: InputManifest) -> None:
|
||||
inputs = _validate_input_ids(manifest.tracked_input_ids)
|
||||
_require_digest(manifest.manifest_digest, "manifest digest")
|
||||
if manifest.manifest_digest != derive_input_manifest_digest(inputs):
|
||||
raise ValueError("manifest digest cannot be re-derived")
|
||||
|
||||
|
||||
def _validate_receipt_shape(receipt: InputReadReceipt) -> None:
|
||||
_require_text(receipt.input_id, "input_id")
|
||||
_require_text(receipt.path, "path")
|
||||
if not isinstance(receipt.opened, bool):
|
||||
raise ValueError("opened must be a boolean")
|
||||
_require_count(receipt.byte_count, "byte_count")
|
||||
if receipt.content_digest is not None:
|
||||
_require_digest(receipt.content_digest, "content digest")
|
||||
|
||||
|
||||
def validate_instrument(manifest: InputManifest, receipts: Sequence[InputReadReceipt]) -> None:
|
||||
"""Refuse unread, zero-byte, missing, extra, or substituted tracked inputs."""
|
||||
try:
|
||||
if not isinstance(manifest, InputManifest):
|
||||
raise ValueError("manifest must be an InputManifest")
|
||||
_validate_manifest(manifest)
|
||||
if isinstance(receipts, (str, bytes)) or not isinstance(receipts, Sequence):
|
||||
raise ValueError("receipts must be a sequence")
|
||||
_validate_instrument_receipts(manifest, receipts)
|
||||
except InstrumentFailure:
|
||||
raise
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise InstrumentFailure(str(exc)) from exc
|
||||
|
||||
|
||||
def _validate_instrument_receipts(manifest: InputManifest, receipts: Sequence[InputReadReceipt]) -> None:
|
||||
by_id: dict[str, InputReadReceipt] = {}
|
||||
for receipt in receipts:
|
||||
if not isinstance(receipt, InputReadReceipt):
|
||||
raise InstrumentFailure("receipts must contain InputReadReceipt values")
|
||||
_validate_receipt_shape(receipt)
|
||||
if receipt.input_id in by_id:
|
||||
raise InstrumentFailure(f"duplicate input receipt: {receipt.input_id}")
|
||||
if receipt.input_id not in manifest.tracked_input_ids:
|
||||
raise InstrumentFailure(f"unexpected input receipt: {receipt.input_id}")
|
||||
by_id[receipt.input_id] = receipt
|
||||
missing = tuple(input_id for input_id in manifest.tracked_input_ids if input_id not in by_id)
|
||||
if missing:
|
||||
raise InstrumentFailure(f"missing input receipt: {', '.join(missing)}")
|
||||
for input_id in manifest.tracked_input_ids:
|
||||
receipt = by_id[input_id]
|
||||
if not receipt.opened:
|
||||
raise InstrumentFailure(f"tracked input not opened: {input_id} ({receipt.path})")
|
||||
if receipt.byte_count == 0:
|
||||
raise InstrumentFailure(f"zero-byte tracked input: {input_id} ({receipt.path})")
|
||||
if receipt.content_digest is None:
|
||||
raise InstrumentFailure(f"content digest missing for tracked input: {input_id}")
|
||||
|
||||
|
||||
def _stage_counts(funnel: CoverageFunnel) -> dict[CoverageStage, int]:
|
||||
return dict(zip(CoverageStage, (getattr(funnel, field_name) for field_name in _COUNT_FIELDS), strict=True))
|
||||
|
||||
|
||||
def _validate_expectations(expected: Sequence[StageExpectation]) -> tuple[StageExpectation, ...]:
|
||||
if isinstance(expected, (str, bytes)) or not isinstance(expected, Sequence):
|
||||
raise ValueError("expected must be a sequence")
|
||||
values = tuple(expected)
|
||||
if not all(isinstance(item, StageExpectation) for item in values):
|
||||
raise ValueError("expected must contain StageExpectation values")
|
||||
for item in values:
|
||||
_validate_stage_expectation(item)
|
||||
stages = tuple(item.stage for item in values)
|
||||
if len(stages) != len(set(stages)):
|
||||
raise ValueError("duplicate coverage stage expectation")
|
||||
if set(stages) != set(CoverageStage):
|
||||
raise ValueError("expectations must cover every coverage stage")
|
||||
if stages != tuple(CoverageStage):
|
||||
raise ValueError("expectations must use canonical coverage order")
|
||||
return values
|
||||
|
||||
|
||||
def diagnose_first_zero(funnel: CoverageFunnel, expected: Sequence[StageExpectation]) -> StageDiagnosis:
|
||||
"""Identify the first expected output that is zero without treating allowed zeros as failures."""
|
||||
if not isinstance(funnel, CoverageFunnel):
|
||||
raise ValueError("funnel must be a CoverageFunnel")
|
||||
_validate_funnel(funnel)
|
||||
expectations = _validate_expectations(expected)
|
||||
counts = _stage_counts(funnel)
|
||||
last_nonzero: CoverageStage | None = None
|
||||
previous_output: int | None = None
|
||||
for expectation in expectations:
|
||||
output = counts[expectation.stage]
|
||||
if expectation.expected_nonzero_input and (previous_output is None or previous_output == 0):
|
||||
reason = f"{expectation.stage.value}:expected_nonzero_input"
|
||||
return StageDiagnosis(last_nonzero, expectation.stage, (reason,))
|
||||
if expectation.expected_nonzero_output and output == 0:
|
||||
reason = f"{expectation.stage.value}:expected_nonzero_output"
|
||||
return StageDiagnosis(last_nonzero, expectation.stage, (reason,))
|
||||
if output > 0:
|
||||
last_nonzero = expectation.stage
|
||||
previous_output = output
|
||||
return StageDiagnosis(last_nonzero, None, ())
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Coverage",
|
||||
"CoverageFunnel",
|
||||
"CoverageStage",
|
||||
"InputManifest",
|
||||
"InputReadReceipt",
|
||||
"InstrumentFailure",
|
||||
"ReasonCount",
|
||||
"RelationStatusCount",
|
||||
"ResolutionOutcome",
|
||||
"StageDiagnosis",
|
||||
"StageExpectation",
|
||||
"check_reads",
|
||||
"derive_input_manifest_digest",
|
||||
"diagnose_first_zero",
|
||||
"validate_instrument",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Contract tests for the v2 coverage funnel and instrument gate."""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.core.coverage import (
|
||||
CoverageFunnel,
|
||||
CoverageStage,
|
||||
InputManifest,
|
||||
InputReadReceipt,
|
||||
InstrumentFailure,
|
||||
ReasonCount,
|
||||
RelationStatusCount,
|
||||
StageExpectation,
|
||||
derive_input_manifest_digest,
|
||||
diagnose_first_zero,
|
||||
validate_instrument,
|
||||
)
|
||||
from openultrasast.core.semantics import RelationStatus
|
||||
|
||||
DIGEST_A = "a" * 64
|
||||
DIGEST_B = "b" * 64
|
||||
|
||||
|
||||
def _manifest() -> InputManifest:
|
||||
inputs = ("src/app.py", "src/lib.py")
|
||||
return InputManifest(inputs, derive_input_manifest_digest(inputs))
|
||||
|
||||
|
||||
def _receipts() -> tuple[InputReadReceipt, ...]:
|
||||
return (
|
||||
InputReadReceipt("src/app.py", "src/app.py", True, 12, DIGEST_A),
|
||||
InputReadReceipt("src/lib.py", "src/lib.py", True, 8, DIGEST_B),
|
||||
)
|
||||
|
||||
|
||||
def _funnel(**overrides: object) -> CoverageFunnel:
|
||||
values: dict[str, object] = {
|
||||
"inputs_read": 2,
|
||||
"questions_formed": 2,
|
||||
"sites_considered": 4,
|
||||
"sites_recognized": 3,
|
||||
"transitions_materialized": 4,
|
||||
"security_objects_constructed": 2,
|
||||
"constraints_evaluated": 2,
|
||||
"determinate_outcomes": 1,
|
||||
"findings_emitted": 1,
|
||||
"relation_status_counts": (
|
||||
RelationStatusCount(RelationStatus.COMPLETE, 2),
|
||||
RelationStatusCount(RelationStatus.INCOMPLETE, 1),
|
||||
RelationStatusCount(RelationStatus.UNRESOLVED, 1),
|
||||
),
|
||||
"unresolved_by_reason": (ReasonCount("missing_constraint", 1), ReasonCount("ambiguous_call", 1)),
|
||||
}
|
||||
values.update(overrides)
|
||||
return CoverageFunnel(**values) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _expectations(*, findings_expected: bool = True) -> tuple[StageExpectation, ...]:
|
||||
return tuple(
|
||||
StageExpectation(stage, stage is not CoverageStage.INPUTS, findings_expected if stage is CoverageStage.FINDINGS else True)
|
||||
for stage in CoverageStage
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_v2_instrument_reads_nonzero_source() -> None:
|
||||
assert __file__
|
||||
with open(__file__, "rb") as source:
|
||||
assert len(source.read()) > 0
|
||||
|
||||
|
||||
def test_input_manifest_and_receipts_validate_exactly() -> None:
|
||||
validate_instrument(_manifest(), _receipts())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("opened", "byte_count", "digest", "match"),
|
||||
[
|
||||
(False, 12, DIGEST_A, "not opened"),
|
||||
(True, 0, None, "zero-byte"),
|
||||
],
|
||||
)
|
||||
def test_unopened_or_zero_byte_input_is_an_instrument_failure(opened: bool, byte_count: int, digest: str | None, match: str) -> None:
|
||||
receipt = InputReadReceipt("src/app.py", "src/app.py", opened, byte_count, digest)
|
||||
with pytest.raises(InstrumentFailure, match=match):
|
||||
validate_instrument(_manifest(), (receipt, _receipts()[1]))
|
||||
|
||||
|
||||
def test_instrument_refuses_missing_extra_and_duplicate_receipts() -> None:
|
||||
manifest = _manifest()
|
||||
receipts = _receipts()
|
||||
with pytest.raises(InstrumentFailure, match="missing input receipt"):
|
||||
validate_instrument(manifest, receipts[:1])
|
||||
extra = InputReadReceipt("src/extra.py", "src/extra.py", True, 1, DIGEST_A)
|
||||
with pytest.raises(InstrumentFailure, match="unexpected input receipt"):
|
||||
validate_instrument(manifest, receipts + (extra,))
|
||||
with pytest.raises(InstrumentFailure, match="duplicate input receipt"):
|
||||
validate_instrument(manifest, receipts + (receipts[0],))
|
||||
|
||||
|
||||
def test_instrument_revalidates_manifest_and_receipt_after_tampering() -> None:
|
||||
manifest = _manifest()
|
||||
object.__setattr__(manifest, "manifest_digest", DIGEST_A)
|
||||
with pytest.raises(InstrumentFailure, match="manifest digest"):
|
||||
validate_instrument(manifest, _receipts())
|
||||
receipt = _receipts()[0]
|
||||
object.__setattr__(receipt, "content_digest", "UPPER")
|
||||
with pytest.raises(InstrumentFailure, match="content digest"):
|
||||
validate_instrument(_manifest(), (receipt, _receipts()[1]))
|
||||
|
||||
|
||||
def test_manifest_rejects_duplicate_or_malformed_input_ids() -> None:
|
||||
with pytest.raises(ValueError, match="duplicate"):
|
||||
InputManifest(("src/app.py", "src/app.py"), DIGEST_A)
|
||||
with pytest.raises(ValueError, match="nonempty text"):
|
||||
InputManifest(("",), DIGEST_A)
|
||||
|
||||
|
||||
def test_funnel_accounts_every_relation_once() -> None:
|
||||
funnel = _funnel()
|
||||
assert sum(item.count for item in funnel.relation_status_counts) == funnel.transitions_materialized
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"updates",
|
||||
[
|
||||
{"relation_status_counts": (RelationStatusCount(RelationStatus.COMPLETE, 4),)},
|
||||
{
|
||||
"relation_status_counts": (
|
||||
RelationStatusCount(RelationStatus.COMPLETE, 2),
|
||||
RelationStatusCount(RelationStatus.COMPLETE, 1),
|
||||
RelationStatusCount(RelationStatus.UNRESOLVED, 1),
|
||||
)
|
||||
},
|
||||
{
|
||||
"relation_status_counts": (
|
||||
RelationStatusCount(RelationStatus.COMPLETE, 3),
|
||||
RelationStatusCount(RelationStatus.INCOMPLETE, 1),
|
||||
RelationStatusCount(RelationStatus.UNRESOLVED, 1),
|
||||
)
|
||||
},
|
||||
{"unresolved_by_reason": (ReasonCount("missing_constraint", 1),)},
|
||||
],
|
||||
)
|
||||
def test_funnel_refuses_missing_duplicate_or_mismatched_accounting(updates: dict[str, object]) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
_funnel(**updates)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["inputs_read", "questions_formed", "sites_considered", "findings_emitted"])
|
||||
def test_funnel_refuses_negative_or_boolean_counts(field: str) -> None:
|
||||
with pytest.raises(ValueError, match=field):
|
||||
_funnel(**{field: -1})
|
||||
with pytest.raises(ValueError, match=field):
|
||||
_funnel(**{field: True})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"updates",
|
||||
[
|
||||
{"sites_considered": 1, "sites_recognized": 2},
|
||||
{"questions_formed": 1, "determinate_outcomes": 2, "findings_emitted": 1},
|
||||
{"determinate_outcomes": 1, "findings_emitted": 2},
|
||||
],
|
||||
)
|
||||
def test_funnel_refuses_impossible_stage_counts_at_construction(updates: dict[str, object]) -> None:
|
||||
with pytest.raises(ValueError, match="cannot exceed"):
|
||||
_funnel(**updates)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "match"),
|
||||
[
|
||||
("sites_recognized", 5, "sites_recognized cannot exceed sites_considered"),
|
||||
("determinate_outcomes", 3, "determinate_outcomes cannot exceed questions_formed"),
|
||||
("findings_emitted", 2, "findings_emitted cannot exceed determinate_outcomes"),
|
||||
],
|
||||
)
|
||||
def test_diagnosis_revalidates_impossible_stage_counts_after_tampering(field: str, value: int, match: str) -> None:
|
||||
funnel = _funnel()
|
||||
object.__setattr__(funnel, field, value)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
diagnose_first_zero(funnel, _expectations())
|
||||
|
||||
|
||||
def test_valid_zero_is_not_diagnosed_as_a_failure() -> None:
|
||||
funnel = _funnel(findings_emitted=0)
|
||||
diagnosis = diagnose_first_zero(funnel, _expectations(findings_expected=False))
|
||||
assert diagnosis.first_failed_output_stage is None
|
||||
assert diagnosis.last_verified_nonzero_stage is CoverageStage.OUTCOMES
|
||||
assert diagnosis.reasons == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_stage", tuple(CoverageStage))
|
||||
def test_each_injected_first_zero_names_the_failing_stage(failed_stage: CoverageStage) -> None:
|
||||
counts = {stage: 1 for stage in CoverageStage}
|
||||
failed_index = tuple(CoverageStage).index(failed_stage)
|
||||
for stage in tuple(CoverageStage)[failed_index:]:
|
||||
counts[stage] = 0
|
||||
funnel = CoverageFunnel(
|
||||
inputs_read=counts[CoverageStage.INPUTS],
|
||||
questions_formed=counts[CoverageStage.QUESTIONS],
|
||||
sites_considered=counts[CoverageStage.SITES_CONSIDERED],
|
||||
sites_recognized=counts[CoverageStage.SITES_RECOGNIZED],
|
||||
transitions_materialized=counts[CoverageStage.TRANSITIONS],
|
||||
security_objects_constructed=counts[CoverageStage.SECURITY_OBJECTS],
|
||||
constraints_evaluated=counts[CoverageStage.CONSTRAINTS],
|
||||
determinate_outcomes=counts[CoverageStage.OUTCOMES],
|
||||
findings_emitted=counts[CoverageStage.FINDINGS],
|
||||
relation_status_counts=(
|
||||
RelationStatusCount(RelationStatus.COMPLETE, counts[CoverageStage.TRANSITIONS]),
|
||||
RelationStatusCount(RelationStatus.INCOMPLETE, 0),
|
||||
RelationStatusCount(RelationStatus.UNRESOLVED, 0),
|
||||
),
|
||||
unresolved_by_reason=(),
|
||||
)
|
||||
diagnosis = diagnose_first_zero(funnel, _expectations())
|
||||
assert diagnosis.first_failed_output_stage is failed_stage
|
||||
assert diagnosis.last_verified_nonzero_stage is (None if failed_index == 0 else tuple(CoverageStage)[failed_index - 1])
|
||||
assert diagnosis.reasons == (f"{failed_stage.value}:expected_nonzero_output",)
|
||||
|
||||
|
||||
def test_expectations_must_be_complete_unique_and_ordered() -> None:
|
||||
funnel = _funnel()
|
||||
expectations = _expectations()
|
||||
with pytest.raises(ValueError, match="every coverage stage"):
|
||||
diagnose_first_zero(funnel, expectations[:-1])
|
||||
with pytest.raises(ValueError, match="canonical coverage order"):
|
||||
diagnose_first_zero(funnel, tuple(reversed(expectations)))
|
||||
with pytest.raises(ValueError, match="duplicate"):
|
||||
diagnose_first_zero(funnel, expectations[:-1] + (expectations[0],))
|
||||
|
||||
|
||||
def test_diagnosis_revalidates_nested_counts_and_expectations_after_tampering() -> None:
|
||||
funnel = _funnel()
|
||||
object.__setattr__(funnel.relation_status_counts[0], "count", -1)
|
||||
with pytest.raises(ValueError, match="relation status count"):
|
||||
diagnose_first_zero(funnel, _expectations())
|
||||
expectation = _expectations()[0]
|
||||
object.__setattr__(expectation, "expected_nonzero_output", 1)
|
||||
with pytest.raises(ValueError, match="expected_nonzero_output"):
|
||||
diagnose_first_zero(_funnel(), (expectation,) + _expectations()[1:])
|
||||
|
||||
|
||||
def test_stage_contracts_reject_wrong_runtime_types() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
StageExpectation("inputs", True, True) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError):
|
||||
RelationStatusCount("complete", 1) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError):
|
||||
ReasonCount("", 1)
|
||||
with pytest.raises(ValueError):
|
||||
replace(_expectations()[0], expected_nonzero_output=1) # type: ignore[arg-type]
|
||||
Reference in New Issue
Block a user