feat(plane): tie-break pass on disputed candidates; 2-of-3 agreement

The design revisit after plane-increment-1.json (14/20 agreed, 19/20 in either pass): a third verify pass on the candidates passes a and b dispute, and agree decides 2-of-3 where pass c asked (the a/b rule elsewhere; c alone never decides). - verify: optional OUSAST_INPUT_ONLY restricts the hunt to the named candidates (agree's disputed.json); an empty list is done with 0 units and no model client. - agree: optional OUSAST_INPUT_PASS_C; metrics keep every key and add tiebreak {asked, flipped_to_agreed, flipped_to_rejected, rule}; usd_total, hunts and declared-site metrics are on the final decision. - generate: per case <case>-vc and <case>-final appended to validation-46; the first 60 entries stay byte-identical, so a rerun keeps their done state. - Offline replay over the recorded outputs with an empty pass c reproduces the recorded agree summaries exactly (14/20 agreed, 23 agreed, 10 disputed). - Spec: Requirement 6.4, design Measurement, task 8.1. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
2026-10-16 03:49:26 +00:00 · 2026-09-30 08:19:46 +02:00
parent c5e6ed3221
commit 6e6bcfc40c
11 changed files with 1197 additions and 19 deletions
@@ -2,6 +2,7 @@

 from __future__ import annotations

+import hashlib
 import json
 import re
 import shutil
@@ -53,7 +54,7 @@ def test_model_prices_are_the_endpoint_prices() -> None:

 def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
    run, manifests = reconciler.load_run(RUN)
-    assert len(run.tasks) == 60
+    assert len(run.tasks) == 90
    population = {c["id"]: c for c in tomllib.loads(POPULATION.read_text())["case"]}
    ax_names: set[str] = set()
    for task in manifests.tasks.values():
@@ -69,7 +70,7 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
            for name, sha in workspace.pins.items():
                case_id = binding.name.removesuffix("-vulnerable")
                assert (name, sha) == ("repo", population[case_id]["vulnerable"])
-        if entry.name.endswith(("-va", "-vb")):
+        if entry.name.endswith(("-va", "-vb", "-vc")):
            assert model == "deepseek-flash" and entry.budget and entry.budget.usd and entry.budget.calls
            env = {e.name: e.value for e in task.env}
            assert env["OUSAST_PASS"] == entry.name[-1]
@@ -83,6 +84,34 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
        ax_names.add(docs[-1]["metadata"]["name"])


+# sha256 of the Run's entries as the first measurement ran them (plane-increment-1.json): the text after
+# ``  tasks:`` of plane/runs/validation-46.yaml at c5e6ed3. A rerun keeps their ``done`` state only while they match.
+FIRST_RUN_ENTRIES = "412ad6171cf096d5d7f6b8e7358374077e2beabf5f4dff35a2801d37c341b97e"
+
+
+def test_tiebreak_appends_thirty_tasks_and_keeps_the_first_sixty_byte_identical() -> None:
+    text = RUN.read_text().split("  tasks:\n", 1)[1]
+    first = text.split("  - name: budibase-mongo-template-nosqli-vc\n", 1)[0]
+    assert hashlib.sha256(first.encode()).hexdigest() == FIRST_RUN_ENTRIES
+    run, manifests = reconciler.load_run(RUN)
+    old, new = run.tasks[:60], run.tasks[60:]
+    assert [e.name.rsplit("-", 1)[1] for e in old] == ["facts", "va", "vb", "agree"] * 15
+    assert [e.name.rsplit("-", 1)[1] for e in new] == ["vc", "final"] * 15
+    for vc, final in zip(new[::2], new[1::2], strict=True):
+        case = vc.name.removesuffix("-vc")
+        agree_entry = run.task(f"{case}-agree")
+        assert vc.inputs == {"facts": f"{case}-facts/facts.json", "only": f"{case}-agree/disputed.json"}
+        assert vc.depends_on == (f"{case}-agree",) and vc.budget == run.task(f"{case}-va").budget
+        env = {e.name: e.value for e in manifests.tasks[vc.task].env}
+        assert env["OUSAST_PASS"] == "c" and env["OUSAST_INPUT_CANDIDATES"] == "/workspace/inputs/candidates.json"
+        assert final.task == agree_entry.task, "the final agree binds the same static inputs as the first"
+        assert final.inputs == {**agree_entry.inputs, "pass_c": f"{case}-vc/units.jsonl"}
+        assert set(final.producers) == {f"{case}-va", f"{case}-vb", f"{case}-vc"}
+        for entry in (vc, final):
+            name = reconciler._ax_name(run.metadata.name, entry.name, limit=reconciler.TASK_NAME_LIMIT)
+            assert len(name) <= reconciler.TASK_NAME_LIMIT
+
+
 def test_inputs_carry_the_validation_set() -> None:
    manifests = load_manifests(sorted((PLANE / "workspaces").glob("*-inputs.yaml")))
    assert len(manifests.workspaces) == 15
@@ -448,6 +448,30 @@ def test_unfinished_stops_dependants_and_rerun_skips_done(fake: Fake, tmp_path:
    assert statuses("resume") == {"repo-facts": "done", "verify": "done", "agree": "done"}


+TIEBREAK = """
+    - name: tiebreak
+      task: verify
+      dependsOn: [agree]
+      inputs: {only: verify/agreed.json, facts: repo-facts/facts.json}
+      outputs: [agreed.json]
+      budget: {usd: 1, calls: 8}
+"""
+
+
+def test_a_rerun_of_a_run_that_gained_tasks_runs_only_the_new_ones(fake: Fake, tmp_path: Path) -> None:
+    manifest = write_run(tmp_path, "grow")
+    assert reconciler.run(manifest, ax=str(fake.ax)) == "done"
+    first = state_of("grow")["tasks"]
+    (fake.root / "ax.log").unlink()
+    fake.script({**SCRIPT, "tiebreak": SCRIPT["verify"]})
+    write_run(tmp_path, "grow", CHAIN + TIEBREAK)  # the same Run, one task appended (the tie-break pass)
+    assert reconciler.run(manifest, ax=str(fake.ax)) == "done"
+    assert [name for _, name in fake.events("apply")] == ["tiebreak"], "done tasks keep their state; only the new task runs"
+    assert statuses("grow") == {"repo-facts": "done", "verify": "done", "agree": "done", "tiebreak": "done"}
+    assert {k: v for k, v in state_of("grow")["tasks"].items() if k != "tiebreak"} == first
+    assert sorted(n for _, n in fake.events("fetched")) == ["tiebreak FACTS 200", "tiebreak ONLY 200"], "inputs of done producers"
+
+
 def test_failed_task_halts_the_run(fake: Fake, tmp_path: Path) -> None:
    script = json.loads(json.dumps(SCRIPT))
    failed = {"status": "failed", "units_done": 0, "units_total": 3, "reason": "402 insufficient balance"}
@@ -0,0 +1,150 @@
+"""The tie-break of the first increment's revisit (ai-service-plane Requirement 6.4, task 8.1): `verify` restricted
+to the candidates passes a and b disputed (`OUSAST_INPUT_ONLY`), and `agree` deciding 2-of-3 where pass c asked."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from test_plane_verify import CANDIDATES, FACTS, FLASH, HuntClient, _two_passes, _workspace
+
+from openultrasast.plane.budget import MeteredClient
+from openultrasast.plane.tasks import agree, verify
+from openultrasast.tool_hunter import CLIENT_ENV
+
+SITES = {"id": "case", "family": "injection", "sites": ["app.py::run", "db.py::query"], "ranges": {"app.py": [[40, 41]]}}
+
+
+def _verify(root: Path, out: Path, client: HuntClient, only: set[str] | None, label: str = "c") -> dict:
+    metered = MeteredClient(client, prices=FLASH)  # type: ignore[arg-type]
+    return verify.run(root, out, CANDIDATES, FACTS, client=client, budget=metered, model="deepseek-chat", pass_label=label, only=only)
+
+
+def test_only_restricts_the_hunt_to_the_named_candidates(tmp_path: Path) -> None:
+    client = HuntClient({"app.py::other"})
+    summary = _verify(_workspace(tmp_path), tmp_path / "c", client, {"app.py::other", "gone.py::f"})
+    assert summary["status"] == "done" and summary["units_total"] == 1 and summary["only"] == 1 and summary["pass"] == "c"
+    prompts = client.prompts()
+    assert prompts and all("`other`" in p and "`run`" not in p and "db.py" not in p for p in prompts)
+    rows = [json.loads(line) for line in (tmp_path / "c" / "units.jsonl").read_text().splitlines()]
+    assert [r["candidates"] for r in rows] == [[["app.py", "other", 5]]]
+    assert [f["candidate"] for f in rows[0]["flagged"]] == ["app.py::other"]
+
+
+def test_only_names_reads_disputed_json_rows_and_names() -> None:
+    disputed = [{"candidate": "a.py::f", "site": "a.py:1:f", "passes": 1}]
+    assert verify.only_names(disputed) == {"a.py::f"}
+    assert verify.only_names({"disputed": disputed}) == {"a.py::f"}
+    assert verify.only_names([["b.py", "g", 3], "c.py::h"]) == {"b.py::g", "c.py::h"}
+    with pytest.raises(ValueError):
+        verify.only_names("a.py::f")
+
+
+def test_an_empty_restriction_is_done_with_zero_units_and_no_call(tmp_path: Path) -> None:
+    client = HuntClient({"app.py::run"})
+    summary = _verify(_workspace(tmp_path), tmp_path / "c", client, set())
+    assert summary["status"] == "done" and (summary["units_done"], summary["units_total"], summary["calls"]) == (0, 0, 0)
+    assert client.calls == [] and (tmp_path / "c" / "units.jsonl").read_text() == ""
+
+
+def test_main_with_an_empty_only_list_needs_no_model_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    root, out = _workspace(tmp_path), tmp_path / "out"
+    (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES))
+    (tmp_path / "disputed.json").write_text("[]\n")
+    for name in ("DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", CLIENT_ENV, "OUSAST_CLIENT"):
+        monkeypatch.delenv(name, raising=False)
+    env = {
+        "OUSAST_WORKSPACE_DIR": str(root), "OUSAST_OUTPUT_DIR": str(out), "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"),
+        "OUSAST_INPUT_ONLY": str(tmp_path / "disputed.json"), "OUSAST_MODEL": "deepseek-chat", "OUSAST_PASS": "c",
+    }  # fmt: skip
+    for key, value in env.items():
+        monkeypatch.setenv(key, value)
+    assert verify.main() == 0
+    summary = json.loads((out / "summary.json").read_text())
+    assert (summary["status"], summary["units_total"], summary["calls"], summary["only"]) == ("done", 0, 0, 0)
+
+
+def _pass_c(tmp_path: Path, root: Path, disputed: Path, flag: set[str]) -> list[dict]:
+    only = verify.only_names(json.loads(disputed.read_text()))
+    _verify(root, tmp_path / "pass-c", HuntClient(flag), only)
+    return agree.load_pass(tmp_path / "pass-c")
+
+
+def _final(tmp_path: Path, a: Path, b: Path, c: list[dict] | None) -> dict:
+    return agree.run(tmp_path / "final", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES, pass_c=c)
+
+
+def test_a_and_c_agreeing_flips_a_disputed_candidate_to_agreed(tmp_path: Path) -> None:
+    root, a, b = _two_passes(tmp_path)  # a: run, query; b: run, other -> disputed: other (b), query (a)
+    first = agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
+    c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query", "app.py::run"})
+    summary = _final(tmp_path, a, b, c)
+    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
+    rows = {r["candidate"]: (r["a"], r["b"], r["c"], r["agreed"]) for r in report["candidates"]}
+    assert rows == {
+        "app.py::run": (True, True, None, True),  # not asked in c: the a/b rule
+        "app.py::other": (False, True, False, False),  # b alone
+        "db.py::query": (True, False, True, True),  # a + c
+    }
+    assert [f["candidate"] for f in report["agreed"]] == ["app.py::run", "db.py::query"]
+    assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"]
+    metrics = summary["metrics"]
+    assert metrics["tiebreak"]["asked"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1
+    assert metrics["tiebreak"]["flipped_to_rejected"] == 0 and "2 of passes a, b, c" in metrics["tiebreak"]["rule"]
+    assert metrics["agreed"] == 2 and metrics["disputed"] == 1
+    assert first["metrics"]["declared_sites_matched_agreed"] == 1 and metrics["declared_sites_matched_agreed"] == 2
+    assert metrics["hunts"] == first["metrics"]["hunts"] + 2 and metrics["usd_total"] > first["metrics"]["usd_total"]
+    assert set(first["metrics"]) - {"tiebreak"} <= set(metrics)
+
+
+def test_b_and_c_agreeing_flips_to_agreed_and_a_lone_a_is_rejected(tmp_path: Path) -> None:
+    root, a, b = _two_passes(tmp_path)
+    agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
+    c = _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"app.py::other"})
+    report_metrics = _final(tmp_path, a, b, c)["metrics"]
+    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
+    assert [f["candidate"] for f in report["agreed"]] == ["app.py::other", "app.py::run"]
+    assert [f["candidate"] for f in report["disputed"]] == ["db.py::query"]
+    assert report_metrics["declared_sites_matched_agreed"] == 1 and report_metrics["tiebreak"]["flipped_to_agreed"] == 1
+
+
+def test_without_pass_c_the_a_b_rule_and_numbers_are_unchanged(tmp_path: Path) -> None:
+    _, a, b = _two_passes(tmp_path)
+    before = agree.run(tmp_path / "ab", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)["metrics"]
+    empty = _final(tmp_path, a, b, [])["metrics"]
+    assert before["tiebreak"] == {"asked": 0, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_A_B}
+    assert empty["tiebreak"]["rule"] == agree.RULE_2_OF_3 and empty["tiebreak"]["asked"] == 0
+    assert {k: v for k, v in empty.items() if k != "tiebreak"} == {k: v for k, v in before.items() if k != "tiebreak"}
+
+
+def test_pass_c_alone_never_decides(tmp_path: Path) -> None:
+    root = _workspace(tmp_path)
+    a, b = tmp_path / "pass-a", tmp_path / "pass-b"
+    for out in (a, b):
+        _verify(root, out, HuntClient({"app.py::run"}), None, label=out.name[-1])
+    c = tmp_path / "pass-c"
+    _verify(root, c, HuntClient({"app.py::other"}), None)  # asked about everything, flags only what a and b rejected
+    metrics = _final(tmp_path, a, b, agree.load_pass(c))["metrics"]
+    report = json.loads((tmp_path / "final" / "agreed.json").read_text())
+    assert [f["candidate"] for f in report["agreed"]] == ["app.py::run"]  # 2 of 3 without c's vote
+    assert [f["candidate"] for f in report["disputed"]] == ["app.py::other"]  # 1 of 3
+    assert metrics["tiebreak"] == {"asked": 3, "flipped_to_agreed": 0, "flipped_to_rejected": 0, "rule": agree.RULE_2_OF_3}
+
+
+def test_agree_main_reads_pass_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    root, a, b = _two_passes(tmp_path)
+    agree.run(tmp_path / "agree", CANDIDATES, agree.load_pass(a), agree.load_pass(b), sites=SITES)
+    _pass_c(tmp_path, root, tmp_path / "agree" / "disputed.json", {"db.py::query"})
+    (tmp_path / "candidates.json").write_text(json.dumps(CANDIDATES))
+    (tmp_path / "case.json").write_text(json.dumps(SITES))
+    env = {
+        "OUSAST_OUTPUT_DIR": str(tmp_path / "final"), "OUSAST_INPUT_PASS_A": str(a / "units.jsonl"),
+        "OUSAST_INPUT_PASS_B": str(b / "units.jsonl"), "OUSAST_INPUT_PASS_C": str(tmp_path / "pass-c" / "units.jsonl"),
+        "OUSAST_INPUT_CANDIDATES": str(tmp_path / "candidates.json"), "OUSAST_INPUT_SITES": str(tmp_path / "case.json"),
+    }  # fmt: skip
+    for key, value in env.items():
+        monkeypatch.setenv(key, value)
+    assert agree.main() == 0
+    metrics = json.loads((tmp_path / "final" / "summary.json").read_text())["metrics"]
+    assert metrics["agreed"] == 2 and metrics["tiebreak"]["flipped_to_agreed"] == 1