feat(plane): scoring parity with the reference -- sites, ranges, triage cost

agree matched declared sites through benchmarks/independent/evaluate.py,
which the runner image does not carry, so site matching would have been
"not assessed". The generator now resolves the fix's old-side ranges
(evaluate.hunks, line for line) into case.json, with the declared sites in
the set and the reference's cost, and agree.matches_declared reproduces
matches_v2; a parity test runs both over the recorded outputs of the 15
validation cases (200 probes, identical verdicts) and agree on those outputs
returns the reference's 16 of 20 and $0.022.

agree reports cost_per_candidate (plane spend) and
cost_per_candidate_with_recorded_triage; Requirement 6.2 is judged on the
latter (design.md). --runner-image re-pins the templates from ops/ax/up.sh's
pin file; the committed manifests are regenerated at the current digest.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/plane/tasks/agree.py

@@ -3,31 +3,65 @@
 The rule is `verify_batched.agreement`: a candidate is reported when every pass flagged it; a candidate flagged
 by one pass is `disputed`. Beside the verdicts go the measurement's numbers: per-candidate cost (a hunt's usd
 split evenly over the candidates it asked about, summed over the passes), tool turns per hunt, and -- when the
-case's declared sites are given -- the site match by `benchmarks/independent/evaluate.matches_v2`, the rule
-protocol v2 scores by, imported from the benchmark tree rather than copied.
+case's declared sites and fix ranges are given -- the site match by :func:`matches_declared`, which is
+`benchmarks/independent/evaluate.matches_v2` (the rule protocol v2 and `score_batched.py` score by) reproduced
+here so the runner image needs no benchmark tree; `tests/test_plane_scoring_parity.py` proves identical verdicts
+on the recorded reference outputs. The ranges are the reference's own (`evaluate.hunks(case, "old")`), resolved
+by the generator into `case.json`, and sites are anchored to the candidate as `score_batched.anchored` does.
+
+Cost is reported twice: `cost_per_candidate` is the plane's spend alone; `cost_per_candidate_with_recorded_triage`
+adds the recorded triage cost of the case (`case.json` `cost.recorded_triage_usd`), because the reference's
+$0.022 per candidate for two passes paid for triage and the plane applies the recorded triage instead. Both
+divide by the candidates before triage when `case.json` gives that count, as the reference did.
 
 Env contract: `OUSAST_INPUT_PASS_A`, `OUSAST_INPUT_PASS_B` (a verify output directory or its `units.jsonl`),
-`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (a JSON case record with `family`
-and `sites` in the `population-v2.toml` shape, or that TOML itself with `OUSAST_CASE_ID` naming the case) and
-`OUSAST_BENCHMARKS_DIR` (where `independent/evaluate.py` lives when the package is not run from the repository).
-Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
+`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (the generator's `case.json`:
+`family`, `sites`, `ranges`, `sites_in_set`, `cost`; or a `population-v2.toml`-shaped TOML with `OUSAST_CASE_ID`
+naming the case). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
 """
 
 from __future__ import annotations
 
-import importlib.util
 import json
 import os
 import sys
 import tomllib
 import traceback
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any
 
 from .verify import EXIT_CODES, USAGE_FIELDS, load_candidates, read_units
 
-Matcher = Callable[[dict[str, Any], dict[str, Any], dict[str, list[tuple[int, int]]]], bool]
+LINE_WINDOW = 10  # evaluate.LINE_WINDOW: a finding within 10 lines of a changed range matches
+
+
+def matches_declared(finding: Mapping[str, Any], case: Mapping[str, Any], ranges: Mapping[str, Sequence[Sequence[int]]]) -> bool:
+    """`evaluate.matches_v2`: the finding's family is the case's, and its `path:line:function` site is a declared
+    `path::function`, or lies in a file the fix changed and names a declared site's function or is within
+    `LINE_WINDOW` lines of a changed range (`ranges`: the old side of the fix diff, per file)."""
+    path, _, rest = str(finding["site"]).partition(":")
+    line_text, _, function = rest.partition(":")
+    if finding.get("family") != case["family"]:
+        return False
+    sites = {str(s) for s in case.get("sites", [])}
+    if f"{path}::{function}" in sites:
+        return True
+    if path not in ranges:
+        return False
+    if function in {site.partition("::")[2] for site in sites}:
+        return True
+    line = int(line_text) if line_text.isdigit() else -1
+    return any(int(first) - LINE_WINDOW <= line <= int(last) + LINE_WINDOW for first, last in ranges[path])
+
+
+def anchor_site(candidate: str, site: str) -> str:
+    """`score_batched.anchored`: the site is the candidate's file and function; the reported line is kept only
+    when the site is in the candidate's file (otherwise 0)."""
+    path, _, function = candidate.partition("::")
+    site_path, _, rest = site.partition(":")
+    line = rest.split(":")[0] if site_path == path else "0"
+    return f"{path}:{line}:{function}"
 
 
 def agreement(passes: Sequence[Sequence[Mapping[str, Any]]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
@@ -51,7 +85,8 @@ def load_pass(source: Path) -> list[dict[str, Any]]:
 
 
 def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
-    """A case record with `family`, `sites` and optional `ranges` (`{path: [[first, last], ...]}`)."""
+    """A case record with `family`, `sites` and `ranges` (`{path: [[first, last], ...]}`); without `ranges`
+    site matching is reported as not assessed."""
     if path is None:
         return None
     if path.suffix == ".toml":
@@ -66,24 +101,6 @@ def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
     return record
 
 
-def load_matcher(benchmarks_dir: Path | None = None) -> Matcher | None:
-    """`evaluate.matches_v2` from the benchmark tree, loaded by file so the package never imports it by name;
-    None when the tree is not present (site matching is then reported as not assessed, never as a miss)."""
-    roots = [benchmarks_dir] if benchmarks_dir is not None else []
-    roots += [p / "benchmarks" for p in (Path.cwd(), *Path(__file__).resolve().parents)]
-    for root in roots:
-        script = root / "independent" / "evaluate.py"
-        if script.is_file():
-            spec = importlib.util.spec_from_file_location("ousast_benchmark_evaluate", script)
-            if spec is None or spec.loader is None:
-                continue
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-            matcher: Matcher = module.matches_v2
-            return matcher
-    return None
-
-
 def _per_candidate_cost(rows: Sequence[Mapping[str, Any]]) -> tuple[dict[str, float | None], dict[str, int]]:
     """A hunt's usd split evenly over its candidates; its turns and calls counted once per candidate."""
     usd: dict[str, float | None] = {}
@@ -101,6 +118,10 @@ def _mean(values: Sequence[float]) -> float | None:
     return round(sum(values) / len(values), 3) if values else None
 
 
+def _per_candidate(total: float | None, count: int) -> float | None:
+    return None if total is None or not count else round(total / count, 6)
+
+
 def run(
     output_dir: Path,
     candidates: object,
@@ -108,7 +129,6 @@ def run(
     pass_b: Sequence[Mapping[str, Any]],
     *,
     sites: Mapping[str, Any] | None = None,
-    matcher: Matcher | None = None,
 ) -> dict[str, Any]:
     output_dir.mkdir(parents=True, exist_ok=True)
     family, unique = load_candidates(candidates)
@@ -121,18 +141,17 @@ def run(
     case = dict(sites) if sites else None
     if case is not None and "family" not in case and family:
         case["family"] = family
-    ranges = {p: [(int(a), int(b)) for a, b in spans] for p, spans in (case or {}).get("ranges", {}).items()}
-    match_state = (
-        "not assessed: no sites given" if case is None else ("not assessed: evaluate.py not found" if matcher is None else "assessed")
-    )
+    ranges = (case or {}).get("ranges")
+    assessed = case is not None and isinstance(ranges, Mapping)
+    match_state = "assessed" if assessed else ("not assessed: no sites given" if case is None else "not assessed: no fix ranges given")
     rows: list[dict[str, Any]] = []
     for name in names:
         in_a, in_b = (name in flagged[0] if name in asked[0] else None), (name in flagged[1] if name in asked[1] else None)
         finding = flagged[0].get(name) or flagged[1].get(name)
-        site = finding["site"] if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
+        site = anchor_site(name, finding["site"]) if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
         site_match = None
-        if case is not None and matcher is not None:
-            site_match = matcher({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
+        if assessed and case is not None and isinstance(ranges, Mapping):
+            site_match = matches_declared({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
         cost = usd_by.get(name)
         rows.append({
             "candidate": name, "site": site, "a": in_a, "b": in_b, "agreed": bool(in_a and in_b), "disputed": bool(in_a) != bool(in_b),
@@ -142,12 +161,20 @@ def run(
     declared = [r for r in rows if r["site_match"]]
     hunts = [*pass_a, *pass_b]
     usd_total = None if any(r.get("usd") is None for r in hunts) else round(sum(float(r["usd"]) for r in hunts), 6)
+    reference: Mapping[str, Any] = (case or {}).get("cost") or {}
+    before = int(reference.get("candidates_before_triage") or len(names))
+    triage_usd = float(reference["recorded_triage_usd"]) if reference.get("recorded_triage_usd") is not None else None
+    with_triage = None if usd_total is None or triage_usd is None else round(usd_total + triage_usd, 6)
     metrics = {
-        "candidates": len(names), "asked_in_both": len(names) - len(unasked), "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]),
-        "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
-        "declared_sites": len(set((case or {}).get("sites", []))), "declared_sites_matched": len(declared),
-        "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]), "site_matching": match_state,
-        "usd_total": usd_total, "cost_per_candidate": (None if usd_total is None or not names else round(usd_total / len(names), 6)),
+        "candidates": len(names), "candidates_before_triage": before, "asked_in_both": len(names) - len(unasked),
+        "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]), "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
+        "declared_sites": len(set((case or {}).get("sites", []))),
+        "declared_sites_in_set": (len(case["sites_in_set"]) if case and "sites_in_set" in case else None),
+        "declared_sites_matched": len(declared), "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]),
+        "site_matching": match_state,
+        "usd_total": usd_total, "cost_per_candidate": _per_candidate(usd_total, before),
+        "recorded_triage_usd": triage_usd, "recorded_triage_basis": reference.get("recorded_triage_basis"),
+        "usd_total_with_recorded_triage": with_triage, "cost_per_candidate_with_recorded_triage": _per_candidate(with_triage, before),
         "turns_per_hunt": _mean([float(r.get("turns") or 0) for r in hunts]),
         "calls_per_hunt": _mean([float((r.get("usage") or {}).get("calls") or 0) for r in hunts]),
     }  # fmt: skip
@@ -174,8 +201,7 @@ def main() -> int:
         candidates = json.loads(Path(os.environ["OUSAST_INPUT_CANDIDATES"]).read_text())
         sites_path = Path(os.environ["OUSAST_INPUT_SITES"]) if os.environ.get("OUSAST_INPUT_SITES") else None
         sites = load_sites(sites_path, os.environ.get("OUSAST_CASE_ID"))
-        benchmarks = Path(os.environ["OUSAST_BENCHMARKS_DIR"]) if os.environ.get("OUSAST_BENCHMARKS_DIR") else None
-        summary = run(output_dir, candidates, pass_a, pass_b, sites=sites, matcher=load_matcher(benchmarks) if sites else None)
+        summary = run(output_dir, candidates, pass_a, pass_b, sites=sites)
     except Exception:  # noqa: BLE001 -- a crash is `failed` with its traceback
         summary = {
             "status": "failed", "units_done": 0, "units_total": 0, "usd": 0, "calls": 0, "usage": dict.fromkeys(USAGE_FIELDS, 0),
@@ -188,7 +214,7 @@ def main() -> int:
     return EXIT_CODES.get(str(summary["status"]), 2)
 
 
-__all__ = ["agreement", "load_matcher", "load_pass", "load_sites", "main", "run"]
+__all__ = ["LINE_WINDOW", "agreement", "anchor_site", "load_pass", "load_sites", "main", "matches_declared", "run"]
 
 
 if __name__ == "__main__":