mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 20:49:28 +00:00
feat(plane): tie-break pass on disputed candidates; 2-of-3 agreement
The design revisit after plane-increment-1.json (14/20 agreed, 19/20 in either
pass): a third verify pass on the candidates passes a and b dispute, and agree
decides 2-of-3 where pass c asked (the a/b rule elsewhere; c alone never decides).
- verify: optional OUSAST_INPUT_ONLY restricts the hunt to the named candidates
(agree's disputed.json); an empty list is done with 0 units and no model client.
- agree: optional OUSAST_INPUT_PASS_C; metrics keep every key and add tiebreak
{asked, flipped_to_agreed, flipped_to_rejected, rule}; usd_total, hunts and
declared-site metrics are on the final decision.
- generate: per case <case>-vc and <case>-final appended to validation-46; the
first 60 entries stay byte-identical, so a rerun keeps their done state.
- Offline replay over the recorded outputs with an empty pass c reproduces the
recorded agree summaries exactly (14/20 agreed, 23 agreed, 10 disputed).
- Spec: Requirement 6.4, design Measurement, task 8.1.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c5e6ed3221
commit
6e6bcfc40c
@@ -233,12 +233,31 @@ def verify_budget(item: CaseInputs) -> dict[str, float | int]:
|
||||
return {"usd": math.ceil(HEADROOM * share * 100 - 1e-9) / 100, "calls": max(hunts, 1) * CALLS_PER_HUNT}
|
||||
|
||||
|
||||
def _tiebreak_entries(item: CaseInputs) -> list[dict[str, Any]]:
|
||||
"""The tie-break (Requirement 6.4): verify pass c on the candidates a and b disputed, then agree 2-of-3. Fresh
|
||||
budget dicts, so YAML adds no anchor to the entries before them."""
|
||||
case_id = item.case["id"]
|
||||
return [
|
||||
{
|
||||
"name": f"{case_id}-vc", "task": f"verify-c-{case_id}", "dependsOn": [f"{case_id}-agree"],
|
||||
"inputs": {"facts": f"{case_id}-facts/facts.json", "only": f"{case_id}-agree/disputed.json"},
|
||||
"outputs": ["units.jsonl", "summary.json"], "budget": verify_budget(item),
|
||||
},
|
||||
{
|
||||
"name": f"{case_id}-final", "task": f"agree-{case_id}",
|
||||
"inputs": {pass_: f"{case_id}-v{pass_[-1]}/units.jsonl" for pass_ in ("pass_a", "pass_b", "pass_c")},
|
||||
"outputs": ["agreed.json", "disputed.json"], "budget": {"usd": 0, "calls": 0},
|
||||
},
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest."""
|
||||
header = f"# generated by `{command}`; do not edit, regenerate.\n"
|
||||
out: dict[str, str] = {}
|
||||
tasks: list[dict[str, Any]] = []
|
||||
entries: list[dict[str, Any]] = []
|
||||
tiebreak: list[dict[str, Any]] = []
|
||||
for item in cases:
|
||||
case_id = item.case["id"]
|
||||
pinned, inputs = _workspaces(item)
|
||||
@@ -270,6 +289,11 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name:
|
||||
"inputs": {"pass_a": f"{case_id}-va/units.jsonl", "pass_b": f"{case_id}-vb/units.jsonl"},
|
||||
"outputs": ["agreed.json", "disputed.json"], "budget": free,
|
||||
}) # fmt: skip
|
||||
env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json", "OUSAST_PASS": "c"}
|
||||
tasks.append(_task(templates["verify"], f"verify-c-{case_id}", env, repo))
|
||||
tiebreak.extend(_tiebreak_entries(item))
|
||||
tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first
|
||||
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
|
||||
out[f"tasks/{run_name}.yaml"] = _dump(header, tasks)
|
||||
run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": {"name": run_name}, "spec": {"tasks": entries}}
|
||||
total = sum(float(e["budget"]["usd"]) for e in entries)
|
||||
|
||||
Reference in New Issue
Block a user