Open_Ultra_SAST/tests/test_plane_tiebreak.py

"""The tie-break of the first increment's revisit (ai-service-plane Requirement 6.4, task 8.1): `verify` restricted
to the candidates passes a and b disputed (`OUSAST_INPUT_ONLY`), and `agree` deciding 2-of-3 where pass c asked."""

from __future__ import annotations

import json
from pathlib import Path

import pytest
from test_plane_verify import CANDIDATES, FACTS, FLASH, HuntClient, _two_passes, _workspace

from openultrasast.plane.budget import MeteredClient
from openultrasast.plane.tasks import agree, verify
from openultrasast.tool_hunter import CLIENT_ENV

SITES = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"], "ranges": {"app.py": [[40, 41]]}}


def _verify(root: Path, out: Path, client: HuntClient, only: set[str] | None, label: str = "c") -> dict:
    metered = MeteredClient(client, prices=FLASH)  # type: ignore[arg-type]
    return verify.run(root, out, CANDIDATES, FACTS, client=client, budget=metered, model="deepseek-chat", pass_label=label, only=only)


def test_only_restricts_the_hunt_to_the_named_candidates(tmp_path: Path) -> None:
    client = HuntClient({"app.py::other"})
    summary = _verify(_workspace(tmp_path), tmp_path / "c", client, {"app.py::other", "gone.py::f"})
    assert summary["status"] == "done" and summary["units_total"] == 1 and summary["only"] == 1 and summary["pass"] == "c"
    prompts = client.prompts()
    assert prompts and all("`other`" in p and "`run`" not in p and "db.py" not in p for p in prompts)
    rows = [json.loads(line) for line in (tmp_path / "c" / "units.jsonl").read_text().splitlines()]
    assert [r["candidates"] for r in rows] == [[["app.py", "other", 5]]]
    assert [f["candidate"] for f in rows[0]["flagged"]] == ["app.py::other"]


def test_only_names_reads_disputed_json_rows_and_names() -> None:
    disputed = [{"candidate": "a.py::f", "site": "a.py:1:f", "passes": 1}]
    assert verify.only_names(disputed) == {"a.py::f"}
    assert verify.only_names({"disputed": disputed}) == {"a.py::f"}
    assert verify.only_names([["b.py", "g", 3], "c.py::h"]) == {"b.py::g", "c.py::h"}
    with pytest.raises(ValueError):
        verify.only_names("a.py::f")


def test_an_empty_restriction_is_done_with_zero_units_and_no_call(tmp_path: Path) -> None:
    client = HuntClient({"app.py::run"})
    summary = _verify(_workspace(tmp_path), tmp_path / "c", client, set())
    assert summary["status"] == "done" and (summary["units_done"], summary["units_total"], summary["calls"]) == (0, 0, 0)
    assert client.calls == [] and (tmp_path / "c" / "units.jsonl").read_text() == ""


def test_main_with_an_empty_only_list_needs_no_model_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    root, out = _workspace(tmp_path), tmp_path / "out"
    (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES))
    (tmp_path / "disputed.json").write_text("[]\n")
    for name in ("DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", CLIENT_ENV, "OUSAST_CLIENT"):
        monkeypatch.delenv(name, raising=False)
    env = {
        "OUSAST_WORKSPACE_DIR": str(root), "OUSAST_OUTPUT_DIR": str(out), "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"),
        "OUSAST_INPUT_ONLY": str(tmp_path / "disputed.json"), "OUSAST_MODEL": "deepseek-chat", "OUSAST_PASS": "c",
    }  # fmt: skip
    for key, value in env.items():
        monkeypatch.setenv(key, value)
    assert verify.main() == 0
    summary = json.loads((out / "summary.json").read_text())
    assert (summary["status"], summary["units_total"], summary["calls"], summary["only"]) == ("done", 0, 0, 0)


def _pass_c(tmp_path: Path, root: Path, disputed: Path, flag: set[str]) -> list[dict]:
    only = verify.only_names(json.loads(disputed.read_text()))
    _verify(root, tmp_path / "pass-c", HuntClient(flag), only)
    return agree.load_pass(tmp_path / "pass-c")


def _final(tmp_path: Path, a: Path, b: Path, c: list[dict] | None) -> dict:
    return agree.run(tmp_path / "final", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES, pass_c=c)


def test_a_and_c_agreeing_flips_a_disputed_candidate_to_agreed(tmp_path: Path) -> None:
    root, a, b = _two_passes(tmp_path)  # a: run, query; b: run, other -> disputed: other (b), query (a)
    first = agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
    c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query", "app.py::run"})
    summary = _final(tmp_path, a, b, c)
    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
    rows = {r["candidate"]: (r["a"], r["b"], r["c"], r["agreed"]) for r in report["candidates"]}
    assert rows == {
        "app.py::run": (True, True, None, True),  # not asked in c: the a/b rule
        "app.py::other": (False, True, False, False),  # b alone
        "db.py::query": (True, False, True, True),  # a + c
    }
    assert [f["candidate"] for f in report["agreed"]] == ["app.py::run", "db.py::query"]
    assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"]
    metrics = summary["metrics"]
    assert metrics["tiebreak"]["asked"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1
    assert metrics["tiebreak"]["flipped_to_rejected"] == 0 and "2 of passes a, b, c" in metrics["tiebreak"]["rule"]
    assert metrics["agreed"] == 2 and metrics["disputed"] == 1
    assert first["metrics"]["declared_sites_matched_agreed"] == 1 and metrics["declared_sites_matched_agreed"] == 2
    assert metrics["hunts"] == first["metrics"]["hunts"] + 2 and metrics["usd_total"] > first["metrics"]["usd_total"]
    assert set(first["metrics"]) - {"tiebreak"} <= set(metrics)


def test_b_and_c_agreeing_flips_to_agreed_and_a_lone_a_is_rejected(tmp_path: Path) -> None:
    root, a, b = _two_passes(tmp_path)
    agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
    c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"app.py::other"})
    report_metrics = _final(tmp_path, a, b, c)["metrics"]
    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
    assert [f["candidate"] for f in report["agreed"]] == ["app.py::other", "app.py::run"]
    assert [f["candidate"] for f in report["disputed"]] == ["db.py::query"]
    assert report_metrics["declared_sites_matched_agreed"] == 1 and report_metrics["tiebreak"]["flipped_to_agreed"] == 1


def test_without_pass_c_the_a_b_rule_and_numbers_are_unchanged(tmp_path: Path) -> None:
    _, a, b = _two_passes(tmp_path)
    before = agree.run(tmp_path / "ab", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)["metrics"]
    empty = _final(tmp_path, a, b, [])["metrics"]
    assert before["tiebreak"] == {"asked": 0, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_A_B}
    assert empty["tiebreak"]["rule"] == agree.RULE_2_OF_3 and empty["tiebreak"]["asked"] == 0
    assert {k: v for k, v in empty.items() if k != "tiebreak"} == {k: v for k, v in before.items() if k != "tiebreak"}


def test_pass_c_alone_never_decides(tmp_path: Path) -> None:
    root = _workspace(tmp_path)
    a, b = tmp_path / "pass-a", tmp_path / "pass-b"
    for out in (a, b):
        _verify(root, out, HuntClient({"app.py::run"}), None, label=out.name[-1])
    c = tmp_path / "pass-c"
    _verify(root, c, HuntClient({"app.py::other"}), None)  # asked about everything, flags only what a and b rejected
    metrics = _final(tmp_path, a, b, agree.load_pass(c))["metrics"]
    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
    assert [f["candidate"] for f in report["agreed"]] == ["app.py::run"]  # 2 of 3 without c's vote
    assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"]  # 1 of 3
    assert metrics["tiebreak"] == {"asked": 3, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_2_OF_3}


def test_agree_main_reads_pass_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    root, a, b = _two_passes(tmp_path)
    agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
    _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query"})
    (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES))
    (tmp_path / "case.json").write_text(json.dumps(SITES))
    env = {
        "OUSAST_OUTPUT_DIR": str(tmp_path / "final"), "OUSAST_INPUT_PASS_A": str(a / "units.jsonl"),
        "OUSAST_INPUT_PASS_B": str(b / "units.jsonl"), "OUSAST_INPUT_PASS_C": str(tmp_path / "pass-c" / "units.jsonl"),
        "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"), "OUSAST_INPUT_SITES": str(tmp_path / "case.json"),
    }  # fmt: skip
    for key, value in env.items():
        monkeypatch.setenv(key, value)
    assert agree.main() == 0
    metrics = json.loads((tmp_path / "final" / "summary.json").read_text())["metrics"]
    assert metrics["agreed"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1