feat(access-control-prover): 12.4 four-arm development comparison

tests/test_four_arm.py

@@ -0,0 +1,102 @@
+"""Task 12.4 -- four-arm development comparison (Reqs 5.2, 5.5, 5.7, 11.8).
+
+Baseline/candidate analyzer crossed with baseline/candidate facts over identical
+inputs; fixture arms isolate fact-only lift from analyzer-only lift. No admission
+language in any report field.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import cast
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from benchmarks.valid import development_runner  # noqa: E402
+
+
+def _record(
+    *,
+    change_id: str = "change-1",
+    unguarded: int = 1,
+    indeterminate: int = 0,
+    seconds: float = 1.0,
+    judgment_transition: str = "same",
+) -> development_runner.DevelopmentRecord:
+    return development_runner.DevelopmentRecord(
+        change_id=change_id,
+        analyzer_revision="a" * 12,
+        fact_set_digest="b" * 12,
+        acquisition_record_digest="none",
+        configuration_digest="c" * 12,
+        population_digest="d" * 12,
+        input_digest="e" * 12,
+        stage_funnel={"judgment": {"unguarded": unguarded, "discharged": 0, "indeterminate": indeterminate}},
+        judgment_transition=judgment_transition,
+        seconds=seconds,
+    )
+
+
+def test_four_arm_report_contains_all_arms() -> None:
+    baseline_analyzer_baseline_facts = (_record(),)
+    baseline_analyzer_candidate_facts = (_record(unguarded=0, indeterminate=1),)
+    candidate_analyzer_baseline_facts = (_record(unguarded=0, indeterminate=1),)
+    candidate_analyzer_candidate_facts = (_record(unguarded=0, indeterminate=1),)
+    report = development_runner.four_arm_report(
+        baseline_analyzer_baseline_facts,
+        baseline_analyzer_candidate_facts,
+        candidate_analyzer_baseline_facts,
+        candidate_analyzer_candidate_facts,
+    )
+    assert report["arms"] == (
+        "baseline-analyzer/baseline-facts",
+        "baseline-analyzer/candidate-facts",
+        "candidate-analyzer/baseline-facts",
+        "candidate-analyzer/candidate-facts",
+    )
+
+
+def test_four_arm_report_isolates_fact_only_and_analyzer_only_lift() -> None:
+    """Done-when: fixture arms isolate fact-only lift from analyzer-only lift."""
+    # baseline/baseline: unguarded; only-facts-changed: discharged; only-analyzer-changed: discharged.
+    bab = (_record(unguarded=1),)
+    bacf = (_record(unguarded=0, indeterminate=0),)
+    cabf = (_record(unguarded=0, indeterminate=0),)
+    cacfon = (_record(unguarded=0, indeterminate=0),)
+    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
+    assert report["fact_only_lift"] == {"unguarded": -1, "discharged": 0, "indeterminate": 0}
+    assert report["analyzer_only_lift"] == {"unguarded": -1, "discharged": 0, "indeterminate": 0}
+
+
+def test_four_arm_report_includes_judgment_transitions() -> None:
+    bab = (_record(judgment_transition="same"),)
+    bacf = (_record(judgment_transition="unguarded->indeterminate"),)
+    cabf = (_record(judgment_transition="same"),)
+    cacfon = (_record(judgment_transition="unguarded->discharged"),)
+    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
+    transitions = cast("tuple[str, ...]", report["transitions"])
+    assert "unguarded->indeterminate" in transitions
+    assert "unguarded->discharged" in transitions
+
+
+def test_four_arm_report_contains_no_admission_language() -> None:
+    bab = bacf = cabf = cacfon = (_record(),)
+    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
+    text = repr(report)
+    for word in ("admission", "vulnerable", "ordinary", "label"):
+        assert word not in text.lower()
+
+
+def test_four_arm_report_includes_metric_and_funnel_deltas() -> None:
+    bab = (_record(unguarded=1, seconds=2.0),)
+    bacf = (_record(unguarded=0, seconds=3.0),)
+    cabf = (_record(unguarded=0, seconds=2.5),)
+    cacfon = (_record(unguarded=0, seconds=3.5),)
+    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
+    fact_delta = cast("dict[str, int]", report["judgment_delta_fact_only"])
+    analyzer_delta = cast("dict[str, int]", report["judgment_delta_analyzer_only"])
+    funnel_delta = cast("dict[str, float]", report["funnel_delta_fact_only"])
+    assert fact_delta["unguarded"] == -1
+    assert analyzer_delta["unguarded"] == -1
+    assert "seconds" in funnel_delta