Open_Ultra_SAST/tests/test_four_arm.py

"""Task 12.4 -- four-arm development comparison (Reqs 5.2, 5.5, 5.7, 11.8).

Baseline/candidate analyzer crossed with baseline/candidate facts over identical
inputs; fixture arms isolate fact-only lift from analyzer-only lift. No admission
language in any report field.
"""

from __future__ import annotations

import sys
from pathlib import Path
from typing import cast

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from benchmarks.valid import development_runner  # noqa: E402


def _record(
    *,
    change_id: str = "change-1",
    unguarded: int = 1,
    indeterminate: int = 0,
    seconds: float = 1.0,
    judgment_transition: str = "same",
) -> development_runner.DevelopmentRecord:
    return development_runner.DevelopmentRecord(
        change_id=change_id,
        analyzer_revision="a" * 12,
        fact_set_digest="b" * 12,
        acquisition_record_digest="none",
        configuration_digest="c" * 12,
        population_digest="d" * 12,
        input_digest="e" * 12,
        stage_funnel={"judgment": {"unguarded": unguarded, "discharged": 0, "indeterminate": indeterminate}},
        judgment_transition=judgment_transition,
        seconds=seconds,
    )


def test_four_arm_report_contains_all_arms() -> None:
    baseline_analyzer_baseline_facts = (_record(),)
    baseline_analyzer_candidate_facts = (_record(unguarded=0, indeterminate=1),)
    candidate_analyzer_baseline_facts = (_record(unguarded=0, indeterminate=1),)
    candidate_analyzer_candidate_facts = (_record(unguarded=0, indeterminate=1),)
    report = development_runner.four_arm_report(
        baseline_analyzer_baseline_facts,
        baseline_analyzer_candidate_facts,
        candidate_analyzer_baseline_facts,
        candidate_analyzer_candidate_facts,
    )
    assert report["arms"] == (
        "baseline-analyzer/baseline-facts",
        "baseline-analyzer/candidate-facts",
        "candidate-analyzer/baseline-facts",
        "candidate-analyzer/candidate-facts",
    )


def test_four_arm_report_isolates_fact_only_and_analyzer_only_lift() -> None:
    """Done-when: fixture arms isolate fact-only lift from analyzer-only lift."""
    # baseline/baseline: unguarded; only-facts-changed: discharged; only-analyzer-changed: discharged.
    bab = (_record(unguarded=1),)
    bacf = (_record(unguarded=0, indeterminate=0),)
    cabf = (_record(unguarded=0, indeterminate=0),)
    cacfon = (_record(unguarded=0, indeterminate=0),)
    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
    assert report["fact_only_lift"] == {"unguarded": -1, "discharged": 0, "indeterminate": 0}
    assert report["analyzer_only_lift"] == {"unguarded": -1, "discharged": 0, "indeterminate": 0}


def test_four_arm_report_includes_judgment_transitions() -> None:
    bab = (_record(judgment_transition="same"),)
    bacf = (_record(judgment_transition="unguarded->indeterminate"),)
    cabf = (_record(judgment_transition="same"),)
    cacfon = (_record(judgment_transition="unguarded->discharged"),)
    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
    transitions = cast("tuple[str, ...]", report["transitions"])
    assert "unguarded->indeterminate" in transitions
    assert "unguarded->discharged" in transitions


def test_four_arm_report_contains_no_admission_language() -> None:
    bab = bacf = cabf = cacfon = (_record(),)
    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
    text = repr(report)
    for word in ("admission", "vulnerable", "ordinary", "label"):
        assert word not in text.lower()


def test_four_arm_report_includes_metric_and_funnel_deltas() -> None:
    bab = (_record(unguarded=1, seconds=2.0),)
    bacf = (_record(unguarded=0, seconds=3.0),)
    cabf = (_record(unguarded=0, seconds=2.5),)
    cacfon = (_record(unguarded=0, seconds=3.5),)
    report = development_runner.four_arm_report(bab, bacf, cabf, cacfon)
    fact_delta = cast("dict[str, int]", report["judgment_delta_fact_only"])
    analyzer_delta = cast("dict[str, int]", report["judgment_delta_analyzer_only"])
    funnel_delta = cast("dict[str, float]", report["funnel_delta_fact_only"])
    assert fact_delta["unguarded"] == -1
    assert analyzer_delta["unguarded"] == -1
    assert "seconds" in funnel_delta