diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index 6b1c4b3..451aa27 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -94,7 +94,7 @@ - _Requirements: 3.5, 8.2_ - [ ] 4. Expose operational coverage, budget, and readiness gates -- [ ] 4.1 (P) Account for instrument reads and every semantic-stage outcome +- [x] 4.1 (P) Account for instrument reads and every semantic-stage outcome - Count reads, questions, sites considered and recognized, transitions, security objects, constraints, determinate outcomes, and findings. - Put every considered relation in exactly one complete, incomplete, or unresolved bucket. - Fail unopened or zero-byte tracked input and diagnose the last verified nonzero stage and first missing expected output. diff --git a/src/openultrasast/core/coverage.py b/src/openultrasast/core/coverage.py index 55e2339..d64266d 100644 --- a/src/openultrasast/core/coverage.py +++ b/src/openultrasast/core/coverage.py @@ -8,9 +8,16 @@ instrument failure -- a zero from an unread input is a failed run, never a clean from __future__ import annotations import enum -from collections.abc import Mapping +import re +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field +from openultrasast.core.canonical import JsonValue, content_digest, require_unique_identities +from openultrasast.core.semantics import RelationStatus + +INPUT_MANIFEST_SCHEMA = "openultrasast.detection-core.input-manifest.v2" +_DIGEST = re.compile(r"[0-9a-f]{64}") + class InstrumentFailure(ValueError): """The run's input was not read; any result from it is invalid.""" @@ -54,4 +61,336 @@ def check_reads(reads: Mapping[str, int]) -> None: raise InstrumentFailure(f"zero-byte read of tracked input: {', '.join(zero)}") -__all__ = ["Coverage", "InstrumentFailure", "ResolutionOutcome", "check_reads"] +def _require_text(value: object, field_name: str) -> None: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be nonempty text") + + +def _require_digest(value: object, field_name: str) -> None: + if not isinstance(value, str) or _DIGEST.fullmatch(value) is None: + raise ValueError(f"{field_name} must be a full lowercase SHA-256 digest") + + +def _require_count(value: object, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a non-negative integer") + + +def _validate_input_ids(value: object) -> tuple[str, ...]: + if not isinstance(value, tuple) or not value: + raise ValueError("tracked_input_ids must be a nonempty tuple") + for input_id in value: + _require_text(input_id, "tracked input id") + require_unique_identities(value, kind="tracked input") + return value + + +def derive_input_manifest_digest(tracked_input_ids: tuple[str, ...]) -> str: + """Derive the identity of the exact ordered tracked-input manifest.""" + inputs = _validate_input_ids(tracked_input_ids) + payload: JsonValue = {"schema_version": INPUT_MANIFEST_SCHEMA, "tracked_input_ids": inputs} + return content_digest(payload) + + +@dataclass(frozen=True) +class InputManifest: + tracked_input_ids: tuple[str, ...] + manifest_digest: str + + def __post_init__(self) -> None: + _validate_input_ids(self.tracked_input_ids) + _require_digest(self.manifest_digest, "manifest_digest") + + +@dataclass(frozen=True) +class InputReadReceipt: + input_id: str + path: str + opened: bool + byte_count: int + content_digest: str | None + + def __post_init__(self) -> None: + _require_text(self.input_id, "input_id") + _require_text(self.path, "path") + if not isinstance(self.opened, bool): + raise ValueError("opened must be a boolean") + _require_count(self.byte_count, "byte_count") + if self.content_digest is not None: + _require_digest(self.content_digest, "content_digest") + + +@dataclass(frozen=True) +class RelationStatusCount: + status: RelationStatus + count: int + + def __post_init__(self) -> None: + _validate_relation_status_count(self) + + +@dataclass(frozen=True) +class ReasonCount: + reason: str + count: int + + def __post_init__(self) -> None: + _validate_reason_count(self) + + +@dataclass(frozen=True) +class CoverageFunnel: + inputs_read: int + questions_formed: int + sites_considered: int + sites_recognized: int + transitions_materialized: int + security_objects_constructed: int + constraints_evaluated: int + determinate_outcomes: int + findings_emitted: int + relation_status_counts: tuple[RelationStatusCount, ...] + unresolved_by_reason: tuple[ReasonCount, ...] + + def __post_init__(self) -> None: + _validate_funnel(self) + + +class CoverageStage(enum.Enum): + INPUTS = "inputs" + QUESTIONS = "questions" + SITES_CONSIDERED = "sites_considered" + SITES_RECOGNIZED = "sites_recognized" + TRANSITIONS = "transitions" + SECURITY_OBJECTS = "security_objects" + CONSTRAINTS = "constraints" + OUTCOMES = "outcomes" + FINDINGS = "findings" + + +@dataclass(frozen=True) +class StageExpectation: + stage: CoverageStage + expected_nonzero_input: bool + expected_nonzero_output: bool + + def __post_init__(self) -> None: + _validate_stage_expectation(self) + + +@dataclass(frozen=True) +class StageDiagnosis: + last_verified_nonzero_stage: CoverageStage | None + first_failed_output_stage: CoverageStage | None + reasons: tuple[str, ...] + + def __post_init__(self) -> None: + for value, field_name in ( + (self.last_verified_nonzero_stage, "last_verified_nonzero_stage"), + (self.first_failed_output_stage, "first_failed_output_stage"), + ): + if value is not None and not isinstance(value, CoverageStage): + raise ValueError(f"{field_name} must be a CoverageStage or None") + if not isinstance(self.reasons, tuple) or not all(isinstance(reason, str) and reason for reason in self.reasons): + raise ValueError("reasons must be a tuple of nonempty text") + + +_COUNT_FIELDS = ( + "inputs_read", + "questions_formed", + "sites_considered", + "sites_recognized", + "transitions_materialized", + "security_objects_constructed", + "constraints_evaluated", + "determinate_outcomes", + "findings_emitted", +) + + +def _validate_relation_status_count(value: RelationStatusCount) -> None: + if not isinstance(value.status, RelationStatus): + raise ValueError("status must be a RelationStatus") + _require_count(value.count, "relation status count") + + +def _validate_reason_count(value: ReasonCount) -> None: + _require_text(value.reason, "reason") + _require_count(value.count, "reason count") + if value.count == 0: + raise ValueError("reason count must be positive") + + +def _validate_stage_expectation(value: StageExpectation) -> None: + if not isinstance(value.stage, CoverageStage): + raise ValueError("stage must be a CoverageStage") + if not isinstance(value.expected_nonzero_input, bool): + raise ValueError("expected_nonzero_input must be a boolean") + if not isinstance(value.expected_nonzero_output, bool): + raise ValueError("expected_nonzero_output must be a boolean") + + +def _validate_funnel(funnel: CoverageFunnel) -> None: + for field_name in _COUNT_FIELDS: + _require_count(getattr(funnel, field_name), field_name) + _validate_stage_inequalities(funnel) + if not isinstance(funnel.relation_status_counts, tuple): + raise ValueError("relation_status_counts must be a tuple") + if not all(isinstance(item, RelationStatusCount) for item in funnel.relation_status_counts): + raise ValueError("relation_status_counts must contain RelationStatusCount values") + for item in funnel.relation_status_counts: + _validate_relation_status_count(item) + statuses = tuple(item.status for item in funnel.relation_status_counts) + if len(statuses) != len(set(statuses)): + raise ValueError("duplicate relation status bucket") + if set(statuses) != set(RelationStatus): + raise ValueError("relation_status_counts must contain every relation status") + if sum(item.count for item in funnel.relation_status_counts) != funnel.transitions_materialized: + raise ValueError("relation status buckets must equal transitions_materialized") + _validate_reason_counts(funnel) + + +def _validate_stage_inequalities(funnel: CoverageFunnel) -> None: + inequalities = ( + (funnel.sites_recognized, funnel.sites_considered, "sites_recognized", "sites_considered"), + (funnel.determinate_outcomes, funnel.questions_formed, "determinate_outcomes", "questions_formed"), + (funnel.findings_emitted, funnel.determinate_outcomes, "findings_emitted", "determinate_outcomes"), + ) + for downstream, upstream, downstream_name, upstream_name in inequalities: + if downstream > upstream: + raise ValueError(f"{downstream_name} cannot exceed {upstream_name}") + + +def _validate_reason_counts(funnel: CoverageFunnel) -> None: + if not isinstance(funnel.unresolved_by_reason, tuple): + raise ValueError("unresolved_by_reason must be a tuple") + if not all(isinstance(item, ReasonCount) for item in funnel.unresolved_by_reason): + raise ValueError("unresolved_by_reason must contain ReasonCount values") + for item in funnel.unresolved_by_reason: + _validate_reason_count(item) + reasons = tuple(item.reason for item in funnel.unresolved_by_reason) + if len(reasons) != len(set(reasons)): + raise ValueError("duplicate unresolved reason") + unresolved = sum(item.count for item in funnel.relation_status_counts if item.status is not RelationStatus.COMPLETE) + if sum(item.count for item in funnel.unresolved_by_reason) != unresolved: + raise ValueError("reason counts must equal incomplete and unresolved relations") + + +def _validate_manifest(manifest: InputManifest) -> None: + inputs = _validate_input_ids(manifest.tracked_input_ids) + _require_digest(manifest.manifest_digest, "manifest digest") + if manifest.manifest_digest != derive_input_manifest_digest(inputs): + raise ValueError("manifest digest cannot be re-derived") + + +def _validate_receipt_shape(receipt: InputReadReceipt) -> None: + _require_text(receipt.input_id, "input_id") + _require_text(receipt.path, "path") + if not isinstance(receipt.opened, bool): + raise ValueError("opened must be a boolean") + _require_count(receipt.byte_count, "byte_count") + if receipt.content_digest is not None: + _require_digest(receipt.content_digest, "content digest") + + +def validate_instrument(manifest: InputManifest, receipts: Sequence[InputReadReceipt]) -> None: + """Refuse unread, zero-byte, missing, extra, or substituted tracked inputs.""" + try: + if not isinstance(manifest, InputManifest): + raise ValueError("manifest must be an InputManifest") + _validate_manifest(manifest) + if isinstance(receipts, (str, bytes)) or not isinstance(receipts, Sequence): + raise ValueError("receipts must be a sequence") + _validate_instrument_receipts(manifest, receipts) + except InstrumentFailure: + raise + except (TypeError, ValueError) as exc: + raise InstrumentFailure(str(exc)) from exc + + +def _validate_instrument_receipts(manifest: InputManifest, receipts: Sequence[InputReadReceipt]) -> None: + by_id: dict[str, InputReadReceipt] = {} + for receipt in receipts: + if not isinstance(receipt, InputReadReceipt): + raise InstrumentFailure("receipts must contain InputReadReceipt values") + _validate_receipt_shape(receipt) + if receipt.input_id in by_id: + raise InstrumentFailure(f"duplicate input receipt: {receipt.input_id}") + if receipt.input_id not in manifest.tracked_input_ids: + raise InstrumentFailure(f"unexpected input receipt: {receipt.input_id}") + by_id[receipt.input_id] = receipt + missing = tuple(input_id for input_id in manifest.tracked_input_ids if input_id not in by_id) + if missing: + raise InstrumentFailure(f"missing input receipt: {', '.join(missing)}") + for input_id in manifest.tracked_input_ids: + receipt = by_id[input_id] + if not receipt.opened: + raise InstrumentFailure(f"tracked input not opened: {input_id} ({receipt.path})") + if receipt.byte_count == 0: + raise InstrumentFailure(f"zero-byte tracked input: {input_id} ({receipt.path})") + if receipt.content_digest is None: + raise InstrumentFailure(f"content digest missing for tracked input: {input_id}") + + +def _stage_counts(funnel: CoverageFunnel) -> dict[CoverageStage, int]: + return dict(zip(CoverageStage, (getattr(funnel, field_name) for field_name in _COUNT_FIELDS), strict=True)) + + +def _validate_expectations(expected: Sequence[StageExpectation]) -> tuple[StageExpectation, ...]: + if isinstance(expected, (str, bytes)) or not isinstance(expected, Sequence): + raise ValueError("expected must be a sequence") + values = tuple(expected) + if not all(isinstance(item, StageExpectation) for item in values): + raise ValueError("expected must contain StageExpectation values") + for item in values: + _validate_stage_expectation(item) + stages = tuple(item.stage for item in values) + if len(stages) != len(set(stages)): + raise ValueError("duplicate coverage stage expectation") + if set(stages) != set(CoverageStage): + raise ValueError("expectations must cover every coverage stage") + if stages != tuple(CoverageStage): + raise ValueError("expectations must use canonical coverage order") + return values + + +def diagnose_first_zero(funnel: CoverageFunnel, expected: Sequence[StageExpectation]) -> StageDiagnosis: + """Identify the first expected output that is zero without treating allowed zeros as failures.""" + if not isinstance(funnel, CoverageFunnel): + raise ValueError("funnel must be a CoverageFunnel") + _validate_funnel(funnel) + expectations = _validate_expectations(expected) + counts = _stage_counts(funnel) + last_nonzero: CoverageStage | None = None + previous_output: int | None = None + for expectation in expectations: + output = counts[expectation.stage] + if expectation.expected_nonzero_input and (previous_output is None or previous_output == 0): + reason = f"{expectation.stage.value}:expected_nonzero_input" + return StageDiagnosis(last_nonzero, expectation.stage, (reason,)) + if expectation.expected_nonzero_output and output == 0: + reason = f"{expectation.stage.value}:expected_nonzero_output" + return StageDiagnosis(last_nonzero, expectation.stage, (reason,)) + if output > 0: + last_nonzero = expectation.stage + previous_output = output + return StageDiagnosis(last_nonzero, None, ()) + + +__all__ = [ + "Coverage", + "CoverageFunnel", + "CoverageStage", + "InputManifest", + "InputReadReceipt", + "InstrumentFailure", + "ReasonCount", + "RelationStatusCount", + "ResolutionOutcome", + "StageDiagnosis", + "StageExpectation", + "check_reads", + "derive_input_manifest_digest", + "diagnose_first_zero", + "validate_instrument", +] diff --git a/tests/test_core_coverage_v2.py b/tests/test_core_coverage_v2.py new file mode 100644 index 0000000..50014fc --- /dev/null +++ b/tests/test_core_coverage_v2.py @@ -0,0 +1,254 @@ +"""Contract tests for the v2 coverage funnel and instrument gate.""" + +from dataclasses import replace + +import pytest + +from openultrasast.core.coverage import ( + CoverageFunnel, + CoverageStage, + InputManifest, + InputReadReceipt, + InstrumentFailure, + ReasonCount, + RelationStatusCount, + StageExpectation, + derive_input_manifest_digest, + diagnose_first_zero, + validate_instrument, +) +from openultrasast.core.semantics import RelationStatus + +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 + + +def _manifest() -> InputManifest: + inputs = ("src/app.py", "src/lib.py") + return InputManifest(inputs, derive_input_manifest_digest(inputs)) + + +def _receipts() -> tuple[InputReadReceipt, ...]: + return ( + InputReadReceipt("src/app.py", "src/app.py", True, 12, DIGEST_A), + InputReadReceipt("src/lib.py", "src/lib.py", True, 8, DIGEST_B), + ) + + +def _funnel(**overrides: object) -> CoverageFunnel: + values: dict[str, object] = { + "inputs_read": 2, + "questions_formed": 2, + "sites_considered": 4, + "sites_recognized": 3, + "transitions_materialized": 4, + "security_objects_constructed": 2, + "constraints_evaluated": 2, + "determinate_outcomes": 1, + "findings_emitted": 1, + "relation_status_counts": ( + RelationStatusCount(RelationStatus.COMPLETE, 2), + RelationStatusCount(RelationStatus.INCOMPLETE, 1), + RelationStatusCount(RelationStatus.UNRESOLVED, 1), + ), + "unresolved_by_reason": (ReasonCount("missing_constraint", 1), ReasonCount("ambiguous_call", 1)), + } + values.update(overrides) + return CoverageFunnel(**values) # type: ignore[arg-type] + + +def _expectations(*, findings_expected: bool = True) -> tuple[StageExpectation, ...]: + return tuple( + StageExpectation(stage, stage is not CoverageStage.INPUTS, findings_expected if stage is CoverageStage.FINDINGS else True) + for stage in CoverageStage + ) + + +def test_coverage_v2_instrument_reads_nonzero_source() -> None: + assert __file__ + with open(__file__, "rb") as source: + assert len(source.read()) > 0 + + +def test_input_manifest_and_receipts_validate_exactly() -> None: + validate_instrument(_manifest(), _receipts()) + + +@pytest.mark.parametrize( + ("opened", "byte_count", "digest", "match"), + [ + (False, 12, DIGEST_A, "not opened"), + (True, 0, None, "zero-byte"), + ], +) +def test_unopened_or_zero_byte_input_is_an_instrument_failure(opened: bool, byte_count: int, digest: str | None, match: str) -> None: + receipt = InputReadReceipt("src/app.py", "src/app.py", opened, byte_count, digest) + with pytest.raises(InstrumentFailure, match=match): + validate_instrument(_manifest(), (receipt, _receipts()[1])) + + +def test_instrument_refuses_missing_extra_and_duplicate_receipts() -> None: + manifest = _manifest() + receipts = _receipts() + with pytest.raises(InstrumentFailure, match="missing input receipt"): + validate_instrument(manifest, receipts[:1]) + extra = InputReadReceipt("src/extra.py", "src/extra.py", True, 1, DIGEST_A) + with pytest.raises(InstrumentFailure, match="unexpected input receipt"): + validate_instrument(manifest, receipts + (extra,)) + with pytest.raises(InstrumentFailure, match="duplicate input receipt"): + validate_instrument(manifest, receipts + (receipts[0],)) + + +def test_instrument_revalidates_manifest_and_receipt_after_tampering() -> None: + manifest = _manifest() + object.__setattr__(manifest, "manifest_digest", DIGEST_A) + with pytest.raises(InstrumentFailure, match="manifest digest"): + validate_instrument(manifest, _receipts()) + receipt = _receipts()[0] + object.__setattr__(receipt, "content_digest", "UPPER") + with pytest.raises(InstrumentFailure, match="content digest"): + validate_instrument(_manifest(), (receipt, _receipts()[1])) + + +def test_manifest_rejects_duplicate_or_malformed_input_ids() -> None: + with pytest.raises(ValueError, match="duplicate"): + InputManifest(("src/app.py", "src/app.py"), DIGEST_A) + with pytest.raises(ValueError, match="nonempty text"): + InputManifest(("",), DIGEST_A) + + +def test_funnel_accounts_every_relation_once() -> None: + funnel = _funnel() + assert sum(item.count for item in funnel.relation_status_counts) == funnel.transitions_materialized + + +@pytest.mark.parametrize( + "updates", + [ + {"relation_status_counts": (RelationStatusCount(RelationStatus.COMPLETE, 4),)}, + { + "relation_status_counts": ( + RelationStatusCount(RelationStatus.COMPLETE, 2), + RelationStatusCount(RelationStatus.COMPLETE, 1), + RelationStatusCount(RelationStatus.UNRESOLVED, 1), + ) + }, + { + "relation_status_counts": ( + RelationStatusCount(RelationStatus.COMPLETE, 3), + RelationStatusCount(RelationStatus.INCOMPLETE, 1), + RelationStatusCount(RelationStatus.UNRESOLVED, 1), + ) + }, + {"unresolved_by_reason": (ReasonCount("missing_constraint", 1),)}, + ], +) +def test_funnel_refuses_missing_duplicate_or_mismatched_accounting(updates: dict[str, object]) -> None: + with pytest.raises(ValueError): + _funnel(**updates) + + +@pytest.mark.parametrize("field", ["inputs_read", "questions_formed", "sites_considered", "findings_emitted"]) +def test_funnel_refuses_negative_or_boolean_counts(field: str) -> None: + with pytest.raises(ValueError, match=field): + _funnel(**{field: -1}) + with pytest.raises(ValueError, match=field): + _funnel(**{field: True}) + + +@pytest.mark.parametrize( + "updates", + [ + {"sites_considered": 1, "sites_recognized": 2}, + {"questions_formed": 1, "determinate_outcomes": 2, "findings_emitted": 1}, + {"determinate_outcomes": 1, "findings_emitted": 2}, + ], +) +def test_funnel_refuses_impossible_stage_counts_at_construction(updates: dict[str, object]) -> None: + with pytest.raises(ValueError, match="cannot exceed"): + _funnel(**updates) + + +@pytest.mark.parametrize( + ("field", "value", "match"), + [ + ("sites_recognized", 5, "sites_recognized cannot exceed sites_considered"), + ("determinate_outcomes", 3, "determinate_outcomes cannot exceed questions_formed"), + ("findings_emitted", 2, "findings_emitted cannot exceed determinate_outcomes"), + ], +) +def test_diagnosis_revalidates_impossible_stage_counts_after_tampering(field: str, value: int, match: str) -> None: + funnel = _funnel() + object.__setattr__(funnel, field, value) + with pytest.raises(ValueError, match=match): + diagnose_first_zero(funnel, _expectations()) + + +def test_valid_zero_is_not_diagnosed_as_a_failure() -> None: + funnel = _funnel(findings_emitted=0) + diagnosis = diagnose_first_zero(funnel, _expectations(findings_expected=False)) + assert diagnosis.first_failed_output_stage is None + assert diagnosis.last_verified_nonzero_stage is CoverageStage.OUTCOMES + assert diagnosis.reasons == () + + +@pytest.mark.parametrize("failed_stage", tuple(CoverageStage)) +def test_each_injected_first_zero_names_the_failing_stage(failed_stage: CoverageStage) -> None: + counts = {stage: 1 for stage in CoverageStage} + failed_index = tuple(CoverageStage).index(failed_stage) + for stage in tuple(CoverageStage)[failed_index:]: + counts[stage] = 0 + funnel = CoverageFunnel( + inputs_read=counts[CoverageStage.INPUTS], + questions_formed=counts[CoverageStage.QUESTIONS], + sites_considered=counts[CoverageStage.SITES_CONSIDERED], + sites_recognized=counts[CoverageStage.SITES_RECOGNIZED], + transitions_materialized=counts[CoverageStage.TRANSITIONS], + security_objects_constructed=counts[CoverageStage.SECURITY_OBJECTS], + constraints_evaluated=counts[CoverageStage.CONSTRAINTS], + determinate_outcomes=counts[CoverageStage.OUTCOMES], + findings_emitted=counts[CoverageStage.FINDINGS], + relation_status_counts=( + RelationStatusCount(RelationStatus.COMPLETE, counts[CoverageStage.TRANSITIONS]), + RelationStatusCount(RelationStatus.INCOMPLETE, 0), + RelationStatusCount(RelationStatus.UNRESOLVED, 0), + ), + unresolved_by_reason=(), + ) + diagnosis = diagnose_first_zero(funnel, _expectations()) + assert diagnosis.first_failed_output_stage is failed_stage + assert diagnosis.last_verified_nonzero_stage is (None if failed_index == 0 else tuple(CoverageStage)[failed_index - 1]) + assert diagnosis.reasons == (f"{failed_stage.value}:expected_nonzero_output",) + + +def test_expectations_must_be_complete_unique_and_ordered() -> None: + funnel = _funnel() + expectations = _expectations() + with pytest.raises(ValueError, match="every coverage stage"): + diagnose_first_zero(funnel, expectations[:-1]) + with pytest.raises(ValueError, match="canonical coverage order"): + diagnose_first_zero(funnel, tuple(reversed(expectations))) + with pytest.raises(ValueError, match="duplicate"): + diagnose_first_zero(funnel, expectations[:-1] + (expectations[0],)) + + +def test_diagnosis_revalidates_nested_counts_and_expectations_after_tampering() -> None: + funnel = _funnel() + object.__setattr__(funnel.relation_status_counts[0], "count", -1) + with pytest.raises(ValueError, match="relation status count"): + diagnose_first_zero(funnel, _expectations()) + expectation = _expectations()[0] + object.__setattr__(expectation, "expected_nonzero_output", 1) + with pytest.raises(ValueError, match="expected_nonzero_output"): + diagnose_first_zero(_funnel(), (expectation,) + _expectations()[1:]) + + +def test_stage_contracts_reject_wrong_runtime_types() -> None: + with pytest.raises(ValueError): + StageExpectation("inputs", True, True) # type: ignore[arg-type] + with pytest.raises(ValueError): + RelationStatusCount("complete", 1) # type: ignore[arg-type] + with pytest.raises(ValueError): + ReasonCount("", 1) + with pytest.raises(ValueError): + replace(_expectations()[0], expected_nonzero_output=1) # type: ignore[arg-type]