Open_Ultra_SAST/tests/test_hunt_accept.py

"""Each acceptance clause has an independent numerical counterexample."""

import json
from copy import deepcopy
from dataclasses import replace

import pytest

from openultrasast.hunt.accept import FoldResult, Rule, decide, record_rejection
from openultrasast.hunt.artifact import RunArtifact
from openultrasast.hunt.propose import Candidate


def estimate(diff=0.2):
    """A synthetic paired estimate with counts and a finite interval."""
    return {"diff": diff, "ci": [diff - 0.1, diff + 0.1], "units": 20, "groups": 4}


@pytest.fixture
def candidate(tmp_path):
    """A single instruction edit, with its baseline available for comparison."""
    base = {"hunt": {"signature": {"instructions": "old", "fields": []}, "demos": []}}
    state = deepcopy(base)
    state["hunt"]["signature"]["instructions"] = "new"
    return Candidate(
        "candidate",
        "instruction",
        {"instruction": {"hunt": "new"}},
        {"recall": 0.7, "ci": [0.5, 0.9]},
        state,
        "current",
        base,
        (),
        (),
        str(tmp_path),
    )


@pytest.fixture
def results():
    """Two scopes containing pooled and family differences."""
    return (
        {family: FoldResult("compile", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
        {family: FoldResult("holdout", family, estimate(), estimate(0)) for family in ("pooled", "injection")},
    )


def rule(**kwargs):
    """A fixed rule with no permitted family regression by default."""
    return Rule(**({"tolerance": 0.01, "regression_budget": {"default": 0, "injection": -0.1}, "instruction_cap": 100} | kwargs))


def difference(results, scope, family, value):
    """Change one measured recall difference."""
    results[scope][family] = replace(results[scope][family], recall=estimate(value))


@pytest.mark.parametrize("scope,reason", [(0, "compile_recall_negative"), (1, "holdout_recall_negative")])
def test_negative_pooled_difference_rejects(candidate, results, scope, reason):
    """Either negative pooled result defeats an otherwise acceptable proposal."""
    assert decide("current", candidate, *results, rule()).accepted
    difference(results, scope, "pooled", -0.01)
    decision = decide("current", candidate, *results, rule())
    assert not decision.accepted and reason in decision.reasons


def test_both_zero_rejects(candidate, results):
    """Zero gain in both scopes is insufficient and is a held-out tie."""
    difference(results, 0, "pooled", 0)
    difference(results, 1, "pooled", 0)
    decision = decide("current", candidate, *results, rule())
    assert not decision.accepted
    assert {"no_positive_recall_difference", "holdout_tie"} <= set(decision.reasons)


def test_compile_zero_holdout_positive_accepts(candidate, results):
    """A positive held-out gain can accept when compile recall is unchanged."""
    difference(results, 0, "pooled", 0)
    assert decide("current", candidate, *results, rule()).accepted


def test_heldout_tolerance_rejects_even_with_compile_gain(candidate, results):
    """The tolerance boundary is inclusive and applies to held-out recall."""
    difference(results, 1, "pooled", 0.01)
    assert "holdout_tie" in decide("current", candidate, *results, rule()).reasons
    difference(results, 1, "pooled", 0.0101)
    assert decide("current", candidate, *results, rule()).accepted


def test_family_budget_overrides_pooled_gain(candidate, results):
    """A family at its allowed floor passes; below it rejects."""
    difference(results, 1, "injection", -0.1)
    assert decide("current", candidate, *results, rule()).accepted
    difference(results, 1, "injection", -0.101)
    assert "family_regression:injection" in decide("current", candidate, *results, rule()).reasons


def test_unlisted_family_uses_default_budget(candidate, results):
    """Families without an override still have a regression floor."""
    for scope, name in enumerate(("compile", "holdout")):
        results[scope]["path"] = FoldResult(name, "path", estimate(-0.01), estimate(0))
    assert "family_regression:path" in decide("current", candidate, *results, rule()).reasons
    assert decide("current", candidate, *results, rule(regression_budget={"default": -0.02})).accepted


def test_instruction_cap_uses_saved_instruction(candidate, results):
    """The state is authoritative even when the reported change is short."""
    assert decide("current", candidate, *results, rule(instruction_cap=3)).accepted
    decision = decide("current", candidate, *results, rule(instruction_cap=2))
    assert not decision.accepted and "instruction_cap" in decision.reasons


@pytest.mark.parametrize("scope,family,metric", [(0, "pooled", "recall"), (1, "injection", "recall"), (1, "pooled", "suspicions_per_push")])
def test_missing_interval_rejects(candidate, results, scope, family, metric):
    """Neither a family nor a suspicion point estimate can stand in for an interval."""
    result = results[scope][family]
    results[scope][family] = replace(result, **{metric: {**getattr(result, metric), "ci": None}})
    assert not decide("current", candidate, *results, rule()).accepted
    assert any(r.startswith("missing_interval:") for r in decide("current", candidate, *results, rule()).reasons)


def test_multiple_changes_reject_even_when_kind_claims_instruction(candidate, results):
    """Saved demos reveal an undeclared second edit."""
    state = deepcopy(candidate.state)
    state["hunt"]["demos"] = [{"findings": []}]
    decision = decide("current", replace(candidate, state=state), *results, rule())
    assert not decision.accepted and "multiple_or_invalid_changes" in decision.reasons


def test_comparison_is_saved_and_fits_existing_artifact(candidate, results, tmp_path):
    """Both IDs, scopes and the rule survive an artifact round trip."""
    decision = decide("current", candidate, *results, rule())
    assert decision.comparison["current_id"] == "current"
    assert decision.comparison["candidate_id"] == "candidate"
    assert decision.comparison["compile_results"]["pooled"]["recall"]["diff"] == 0.2
    assert decision.comparison["holdout_results"]["injection"]["recall"]["ci"]
    assert decision.comparison["rule"]["version"] == "accept-v1"
    assert json.loads((tmp_path / "comparison.json").read_text()) == decision.comparison
    artifact = RunArtifact("candidate", 1, (), {}, [], {}, {}, "fixture")
    decision.record(tmp_path, artifact=artifact)
    assert artifact.configurations == [{"comparison": decision.comparison}]
    path = artifact.write(tmp_path)
    assert RunArtifact.read(path).configurations == artifact.configurations


def test_rejection_omits_heldout_numbers(candidate, results, tmp_path):
    """The rejection journal discloses reasons and compile measurements only."""
    difference(results, 1, "pooled", -0.987654)
    decision = decide("current", candidate, *results, rule())
    record_rejection(tmp_path, candidate, decision)
    record_rejection(tmp_path, candidate, decision)
    rows = [json.loads(line) for line in (tmp_path / "rejected.jsonl").read_text().splitlines()]
    assert len(rows) == 2 and rows[0]["reasons"] == decision.reasons
    assert rows[0]["compile_score"] == candidate.compile_score
    assert "-0.987654" not in json.dumps(rows)
    assert set(rows[0]) == {"program_id", "kind", "change", "compile_score", "reasons"}


def test_missing_family_and_wrong_scope_cannot_accept(candidate, results):
    """Incomplete or mislabelled comparisons are not usable evidence."""
    del results[1]["injection"]
    assert not decide("current", candidate, *results, rule()).accepted
    results[1]["pooled"] = replace(results[1]["pooled"], scope="compile")
    assert not decide("current", candidate, *results, rule()).accepted