merge(plane): scoring parity with the reference -- sites, ranges, triage cost

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/cli.py

@@ -256,6 +256,13 @@ def main(argv: list[str] | None = None) -> int:
     plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
     plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name")
     plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root")
+    plane_ws.add_argument(
+        "--repos", type=Path, default=Path.home() / ".cache" / "openultrasast" / "independent",
+        help="with --validation-set: one clone per case id, for the fix ranges agree scores by (evaluate.hunks)",
+    )  # fmt: skip
+    plane_ws.add_argument(
+        "--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
+    )
 
     args = parser.parse_args(argv)
     # Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
@@ -1324,11 +1331,18 @@ def _plane_increment(args: argparse.Namespace) -> int:
         print("--validation-set needs --candidates and --triage", file=sys.stderr)
         return 2
     home = str(Path.home())
-    shown = [str(p).replace(home, "~", 1) for p in (args.population, args.validation_set, args.candidates, args.triage)]
-    command = f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]} --run {args.run}"
-    written = increment(
-        args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command
+    paths = (args.population, args.validation_set, args.candidates, args.triage, args.repos)
+    shown = [str(p).replace(home, "~", 1) for p in paths]
+    command = (
+        f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]}"
+        f" --repos {shown[4]} --run {args.run}"
     )
+    if args.runner_image is not None:
+        command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
+    written = increment(
+        args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
+        repos=args.repos, runner_image=args.runner_image,
+    )  # fmt: skip
     print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
     return 0
 

src/openultrasast/plane/generate.py

@@ -8,8 +8,11 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
   ``openultrasast.io/git-commits`` annotation, as :mod:`.workspaces` writes it);
 - ``workspaces/<case>-inputs.yaml``: a files-only Workspace with the inputs no task produces -- the recorded
   candidates (``candidates.json`` in the ``independent-v2-modelsinks`` shape for ``verify`` and ``agree``,
-  ``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), the case's declared sites
-  (``case.json`` for ``agree``) and the recorded triage (``triage.json``, for the measurement);
+  ``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), everything ``agree`` scores by
+  (``case.json``: the declared sites, the old-side fix ranges exactly as ``evaluate.hunks(case, "old")`` computes
+  them from ``--repos``, the declared sites among the set's candidates -- the reference's "of 20" -- and the
+  reference's cost: candidates before triage, recorded usd, and the triage share of it) and the recorded triage
+  (``triage.json``, for the measurement);
 - ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
   Workspaces and naming the files as env;
 - ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets.
@@ -24,12 +27,21 @@ Candidates are the set's ``(path, function)`` pairs with the line of the case re
 ``benchmarks/independent/verify_batched.py`` does (sorted, the last line of a repeated pair wins), and filtered
 by the recorded triage (the ``kept`` lists of the batched-check scans): ``verify`` has no triage stage, so the
 hunts see exactly the candidates the reference hunts saw. Output is a pure function of the inputs.
+
+The recorded triage cost is derived per file: a batched-check record's ``usd`` paid for the triage call and the
+hunts, and its ``usage`` counts only the hunts (``verify_sinks._Recording`` slices the client's usage from the
+first hunt on), so the triage share is ``usd`` minus that usage priced at the reference model's list price.
+
+``--runner-image FILE`` (``~/.cache/ousast/ax-src/runner-image``, written by ``ops/ax/up.sh``) re-pins the
+templates' ``image:`` to the digest in that file before generating, so a rebuilt image is one command.
 """
 
 from __future__ import annotations
 
 import json
 import math
+import re
+import subprocess
 import tomllib
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
@@ -38,13 +50,21 @@ from typing import Any
 
 import yaml
 
+from ..model.endpoint import price_of
+from .budget import cost_of
 from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
 from .reconciler import _ax_name, _doc
 from .tasks.verify import MAX_STEPS, units_of
 
-__all__ = ["CaseInputs", "increment", "read_cases", "render"]
+__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
 
 REFERENCE_USD_PER_CANDIDATE = 0.022  # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes
+REFERENCE_MODEL = "deepseek-flash"  # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check
+TRIAGE_BASIS = (
+    "derived per file from the batched-check scans: recorded usd (triage + hunts) minus the recorded hunt usage "
+    f"priced at {REFERENCE_MODEL} list prices; the triage call's own usage was not recorded"
+)
+RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, old side: benchmarks/independent/evaluate.hunks(case, 'old')"
 HEADROOM = 1.5
 CALLS_PER_HUNT = MAX_STEPS + 2  # every tool step, the answer, one empty-content retry
 ATESPACE = "default"
@@ -62,6 +82,37 @@ class CaseInputs:
     kept: tuple[Candidate, ...]
     triage: Mapping[str, str]
     recorded_usd: float
+    triage_usd: float = 0.0
+    ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
+
+
+def fix_ranges(repo: Path, vulnerable: str, fixed: str) -> dict[str, list[tuple[int, int]]]:
+    """Changed line ranges per file on the vulnerable side of the fix: `evaluate.hunks(case, "old")`, line for
+    line (including its attribution of a hunk under ``--- /dev/null`` to the file before it)."""
+    if not (repo / ".git").exists() and not (repo / "HEAD").is_file():
+        raise ValueError(f"no repository at {repo}: the fix ranges cannot be computed")
+    diff = subprocess.run(["git", "-C", str(repo), "diff", "-U0", vulnerable, fixed], capture_output=True, text=True, check=True).stdout
+    ranges: dict[str, list[tuple[int, int]]] = {}
+    current = ""
+    for line in diff.splitlines():
+        if line.startswith("--- a/"):
+            current = line[6:]
+        elif line.startswith("@@") and current:
+            match = re.search(r"-(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?", line)
+            if match:
+                first, length = int(match.group(1)), int(match.group(2)) if match.group(2) is not None else 1
+                ranges.setdefault(current, []).append((first, first + max(length, 1) - 1))
+    if not ranges:
+        raise ValueError(f"{repo}: the diff {vulnerable[:12]}..{fixed[:12]} changed no line on the vulnerable side")
+    return ranges
+
+
+def _triage_usd(rows: Sequence[Mapping[str, Any]]) -> float:
+    """The triage share of the recorded spend: per file, usd minus the recorded hunt usage at list price."""
+    prices = price_of(REFERENCE_MODEL)
+    if prices is None:
+        raise ValueError(f"no recorded price for {REFERENCE_MODEL}")
+    return round(sum(max(float(r.get("usd") or 0.0) - cost_of(r.get("usage") or {}, prices), 0.0) for r in rows), 6)
 
 
 def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
@@ -75,8 +126,11 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
     return rows
 
 
-def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path) -> list[CaseInputs]:
-    """The set's cases in id order, each with its candidates, the triage-kept subset and the recorded cost."""
+def read_cases(
+    validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None
+) -> list[CaseInputs]:
+    """The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its
+    triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges."""
     subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8"))
     records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]}
     cases: list[CaseInputs] = []
@@ -104,6 +158,8 @@ def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, pop
                 kept=tuple(c for c in chosen if (c[0], c[1]) in kept_keys),
                 triage=dict(sorted(triage.items())),
                 recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6),
+                triage_usd=_triage_usd(rows),
+                ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]),
             )
         )
     return cases
@@ -117,6 +173,23 @@ def _json(payload: object) -> str:
     return json.dumps(payload, indent=1, sort_keys=True) + "\n"
 
 
+def _case_record(item: CaseInputs) -> dict[str, Any]:
+    """What `agree` scores by: sites, fix ranges (absent without ``--repos``), sites in the set, reference cost."""
+    case = item.case
+    sites = [str(s) for s in case.get("sites", [])]
+    in_set = {f"{p}::{fn}" for p, fn, _ in item.candidates}
+    record: dict[str, Any] = {"id": case["id"], "family": case["family"], "sites": sites}
+    if item.ranges is not None:
+        record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()}
+        record["ranges_basis"] = RANGES_BASIS
+    record["sites_in_set"] = [s for s in sites if s in in_set]
+    record["cost"] = {
+        "candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd,
+        "recorded_triage_usd": item.triage_usd, "recorded_triage_basis": TRIAGE_BASIS,
+    }  # fmt: skip
+    return record
+
+
 def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
     case = item.case
     git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
@@ -125,7 +198,7 @@ def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
     files = {
         "candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}),
         "functions.json": _json([{"path": p, "function": fn, "line": ln} for p, fn, ln in item.kept]),
-        "case.json": _json({"id": case["id"], "family": case["family"], "sites": list(case.get("sites", []))}),
+        "case.json": _json(_case_record(item)),
         "triage.json": _json(
             {"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)}
         ),
@@ -207,12 +280,43 @@ def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name:
     return out
 
 
+_IMAGE_LINE = re.compile(r'^(\s*image:\s*)"?[^"\s]+"?\s*$', re.MULTILINE)
+
+
+def repin_templates(plane: Path, image_file: Path) -> str:
+    """Set every template's ``image:`` to the digest-pinned reference in ``image_file`` (as ``ops/ax/up.sh``
+    writes it); comments and layout are kept. Returns the reference."""
+    image = image_file.read_text(encoding="utf-8").strip()
+    if not re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", image):
+        raise ValueError(f"{image_file}: expected a digest-pinned image reference, got {image!r}")
+    for name in TEMPLATES:
+        path = plane / "tasks" / f"{name}.yaml"
+        text = path.read_text(encoding="utf-8")
+        pinned, count = _IMAGE_LINE.subn(lambda m: f'{m.group(1)}"{image}"', text)
+        if count != 1:
+            raise ValueError(f"{path}: {count} image lines, expected one")
+        path.write_text(pinned, encoding="utf-8")
+    return image
+
+
 def increment(
-    population: Path, validation_set: Path, candidates_dir: Path, triage_dir: Path, *, plane: Path, run_name: str, command: str
+    population: Path,
+    validation_set: Path,
+    candidates_dir: Path,
+    triage_dir: Path,
+    *,
+    plane: Path,
+    run_name: str,
+    command: str,
+    repos: Path | None = None,
+    runner_image: Path | None = None,
 ) -> list[Path]:
-    """Write the generated manifests under ``plane`` from the templates in ``plane/tasks``; returns the paths."""
+    """Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
+    ``runner_image`` is given); returns the paths."""
+    if runner_image is not None:
+        repin_templates(plane, runner_image)
     loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
-    files = render(read_cases(validation_set, candidates_dir, triage_dir, population), loaded.tasks, run_name, command)
+    files = render(read_cases(validation_set, candidates_dir, triage_dir, population, repos), loaded.tasks, run_name, command)
     written: list[Path] = []
     for relative, text in files.items():
         path = plane / relative

src/openultrasast/plane/tasks/agree.py

@@ -3,31 +3,65 @@
 The rule is `verify_batched.agreement`: a candidate is reported when every pass flagged it; a candidate flagged
 by one pass is `disputed`. Beside the verdicts go the measurement's numbers: per-candidate cost (a hunt's usd
 split evenly over the candidates it asked about, summed over the passes), tool turns per hunt, and -- when the
-case's declared sites are given -- the site match by `benchmarks/independent/evaluate.matches_v2`, the rule
-protocol v2 scores by, imported from the benchmark tree rather than copied.
+case's declared sites and fix ranges are given -- the site match by :func:`matches_declared`, which is
+`benchmarks/independent/evaluate.matches_v2` (the rule protocol v2 and `score_batched.py` score by) reproduced
+here so the runner image needs no benchmark tree; `tests/test_plane_scoring_parity.py` proves identical verdicts
+on the recorded reference outputs. The ranges are the reference's own (`evaluate.hunks(case, "old")`), resolved
+by the generator into `case.json`, and sites are anchored to the candidate as `score_batched.anchored` does.
+
+Cost is reported twice: `cost_per_candidate` is the plane's spend alone; `cost_per_candidate_with_recorded_triage`
+adds the recorded triage cost of the case (`case.json` `cost.recorded_triage_usd`), because the reference's
+$0.022 per candidate for two passes paid for triage and the plane applies the recorded triage instead. Both
+divide by the candidates before triage when `case.json` gives that count, as the reference did.
 
 Env contract: `OUSAST_INPUT_PASS_A`, `OUSAST_INPUT_PASS_B` (a verify output directory or its `units.jsonl`),
-`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (a JSON case record with `family`
-and `sites` in the `population-v2.toml` shape, or that TOML itself with `OUSAST_CASE_ID` naming the case) and
-`OUSAST_BENCHMARKS_DIR` (where `independent/evaluate.py` lives when the package is not run from the repository).
-Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
+`OUSAST_INPUT_CANDIDATES`, `OUSAST_OUTPUT_DIR`, optional `OUSAST_INPUT_SITES` (the generator's `case.json`:
+`family`, `sites`, `ranges`, `sites_in_set`, `cost`; or a `population-v2.toml`-shaped TOML with `OUSAST_CASE_ID`
+naming the case). Writes `agreed.json`, `disputed.json`, `summary.json`. Exit 0 done, 2 failed, 3 unfinished.
 """
 
 from __future__ import annotations
 
-import importlib.util
 import json
 import os
 import sys
 import tomllib
 import traceback
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Any
 
 from .verify import EXIT_CODES, USAGE_FIELDS, load_candidates, read_units
 
-Matcher = Callable[[dict[str, Any], dict[str, Any], dict[str, list[tuple[int, int]]]], bool]
+LINE_WINDOW = 10  # evaluate.LINE_WINDOW: a finding within 10 lines of a changed range matches
+
+
+def matches_declared(finding: Mapping[str, Any], case: Mapping[str, Any], ranges: Mapping[str, Sequence[Sequence[int]]]) -> bool:
+    """`evaluate.matches_v2`: the finding's family is the case's, and its `path:line:function` site is a declared
+    `path::function`, or lies in a file the fix changed and names a declared site's function or is within
+    `LINE_WINDOW` lines of a changed range (`ranges`: the old side of the fix diff, per file)."""
+    path, _, rest = str(finding["site"]).partition(":")
+    line_text, _, function = rest.partition(":")
+    if finding.get("family") != case["family"]:
+        return False
+    sites = {str(s) for s in case.get("sites", [])}
+    if f"{path}::{function}" in sites:
+        return True
+    if path not in ranges:
+        return False
+    if function in {site.partition("::")[2] for site in sites}:
+        return True
+    line = int(line_text) if line_text.isdigit() else -1
+    return any(int(first) - LINE_WINDOW <= line <= int(last) + LINE_WINDOW for first, last in ranges[path])
+
+
+def anchor_site(candidate: str, site: str) -> str:
+    """`score_batched.anchored`: the site is the candidate's file and function; the reported line is kept only
+    when the site is in the candidate's file (otherwise 0)."""
+    path, _, function = candidate.partition("::")
+    site_path, _, rest = site.partition(":")
+    line = rest.split(":")[0] if site_path == path else "0"
+    return f"{path}:{line}:{function}"
 
 
 def agreement(passes: Sequence[Sequence[Mapping[str, Any]]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
@@ -51,7 +85,8 @@ def load_pass(source: Path) -> list[dict[str, Any]]:
 
 
 def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
-    """A case record with `family`, `sites` and optional `ranges` (`{path: [[first, last], ...]}`)."""
+    """A case record with `family`, `sites` and `ranges` (`{path: [[first, last], ...]}`); without `ranges`
+    site matching is reported as not assessed."""
     if path is None:
         return None
     if path.suffix == ".toml":
@@ -66,24 +101,6 @@ def load_sites(path: Path | None, case_id: str | None) -> dict[str, Any] | None:
     return record
 
 
-def load_matcher(benchmarks_dir: Path | None = None) -> Matcher | None:
-    """`evaluate.matches_v2` from the benchmark tree, loaded by file so the package never imports it by name;
-    None when the tree is not present (site matching is then reported as not assessed, never as a miss)."""
-    roots = [benchmarks_dir] if benchmarks_dir is not None else []
-    roots += [p / "benchmarks" for p in (Path.cwd(), *Path(__file__).resolve().parents)]
-    for root in roots:
-        script = root / "independent" / "evaluate.py"
-        if script.is_file():
-            spec = importlib.util.spec_from_file_location("ousast_benchmark_evaluate", script)
-            if spec is None or spec.loader is None:
-                continue
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-            matcher: Matcher = module.matches_v2
-            return matcher
-    return None
-
-
 def _per_candidate_cost(rows: Sequence[Mapping[str, Any]]) -> tuple[dict[str, float | None], dict[str, int]]:
     """A hunt's usd split evenly over its candidates; its turns and calls counted once per candidate."""
     usd: dict[str, float | None] = {}
@@ -101,6 +118,10 @@ def _mean(values: Sequence[float]) -> float | None:
     return round(sum(values) / len(values), 3) if values else None
 
 
+def _per_candidate(total: float | None, count: int) -> float | None:
+    return None if total is None or not count else round(total / count, 6)
+
+
 def run(
     output_dir: Path,
     candidates: object,
@@ -108,7 +129,6 @@ def run(
     pass_b: Sequence[Mapping[str, Any]],
     *,
     sites: Mapping[str, Any] | None = None,
-    matcher: Matcher | None = None,
 ) -> dict[str, Any]:
     output_dir.mkdir(parents=True, exist_ok=True)
     family, unique = load_candidates(candidates)
@@ -121,18 +141,17 @@ def run(
     case = dict(sites) if sites else None
     if case is not None and "family" not in case and family:
         case["family"] = family
-    ranges = {p: [(int(a), int(b)) for a, b in spans] for p, spans in (case or {}).get("ranges", {}).items()}
-    match_state = (
-        "not assessed: no sites given" if case is None else ("not assessed: evaluate.py not found" if matcher is None else "assessed")
-    )
+    ranges = (case or {}).get("ranges")
+    assessed = case is not None and isinstance(ranges, Mapping)
+    match_state = "assessed" if assessed else ("not assessed: no sites given" if case is None else "not assessed: no fix ranges given")
     rows: list[dict[str, Any]] = []
     for name in names:
         in_a, in_b = (name in flagged[0] if name in asked[0] else None), (name in flagged[1] if name in asked[1] else None)
         finding = flagged[0].get(name) or flagged[1].get(name)
-        site = finding["site"] if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
+        site = anchor_site(name, finding["site"]) if finding else f"{name.partition('::')[0]}:{lines[name]}:{name.partition('::')[2]}"
         site_match = None
-        if case is not None and matcher is not None:
-            site_match = matcher({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
+        if assessed and case is not None and isinstance(ranges, Mapping):
+            site_match = matches_declared({"site": site, "family": (finding or {}).get("family", family)}, case, ranges)
         cost = usd_by.get(name)
         rows.append({
             "candidate": name, "site": site, "a": in_a, "b": in_b, "agreed": bool(in_a and in_b), "disputed": bool(in_a) != bool(in_b),
@@ -142,12 +161,20 @@ def run(
     declared = [r for r in rows if r["site_match"]]
     hunts = [*pass_a, *pass_b]
     usd_total = None if any(r.get("usd") is None for r in hunts) else round(sum(float(r["usd"]) for r in hunts), 6)
+    reference: Mapping[str, Any] = (case or {}).get("cost") or {}
+    before = int(reference.get("candidates_before_triage") or len(names))
+    triage_usd = float(reference["recorded_triage_usd"]) if reference.get("recorded_triage_usd") is not None else None
+    with_triage = None if usd_total is None or triage_usd is None else round(usd_total + triage_usd, 6)
     metrics = {
-        "candidates": len(names), "asked_in_both": len(names) - len(unasked), "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]),
-        "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
-        "declared_sites": len(set((case or {}).get("sites", []))), "declared_sites_matched": len(declared),
-        "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]), "site_matching": match_state,
-        "usd_total": usd_total, "cost_per_candidate": (None if usd_total is None or not names else round(usd_total / len(names), 6)),
+        "candidates": len(names), "candidates_before_triage": before, "asked_in_both": len(names) - len(unasked),
+        "flagged_a": len(flagged[0]), "flagged_b": len(flagged[1]), "agreed": len(agreed), "disputed": len(disputed), "hunts": len(hunts),
+        "declared_sites": len(set((case or {}).get("sites", []))),
+        "declared_sites_in_set": (len(case["sites_in_set"]) if case and "sites_in_set" in case else None),
+        "declared_sites_matched": len(declared), "declared_sites_matched_agreed": sum(1 for r in declared if r["agreed"]),
+        "site_matching": match_state,
+        "usd_total": usd_total, "cost_per_candidate": _per_candidate(usd_total, before),
+        "recorded_triage_usd": triage_usd, "recorded_triage_basis": reference.get("recorded_triage_basis"),
+        "usd_total_with_recorded_triage": with_triage, "cost_per_candidate_with_recorded_triage": _per_candidate(with_triage, before),
         "turns_per_hunt": _mean([float(r.get("turns") or 0) for r in hunts]),
         "calls_per_hunt": _mean([float((r.get("usage") or {}).get("calls") or 0) for r in hunts]),
     }  # fmt: skip
@@ -174,8 +201,7 @@ def main() -> int:
         candidates = json.loads(Path(os.environ["OUSAST_INPUT_CANDIDATES"]).read_text())
         sites_path = Path(os.environ["OUSAST_INPUT_SITES"]) if os.environ.get("OUSAST_INPUT_SITES") else None
         sites = load_sites(sites_path, os.environ.get("OUSAST_CASE_ID"))
-        benchmarks = Path(os.environ["OUSAST_BENCHMARKS_DIR"]) if os.environ.get("OUSAST_BENCHMARKS_DIR") else None
-        summary = run(output_dir, candidates, pass_a, pass_b, sites=sites, matcher=load_matcher(benchmarks) if sites else None)
+        summary = run(output_dir, candidates, pass_a, pass_b, sites=sites)
     except Exception:  # noqa: BLE001 -- a crash is `failed` with its traceback
         summary = {
             "status": "failed", "units_done": 0, "units_total": 0, "usd": 0, "calls": 0, "usage": dict.fromkeys(USAGE_FIELDS, 0),
@@ -188,7 +214,7 @@ def main() -> int:
     return EXIT_CODES.get(str(summary["status"]), 2)
 
 
-__all__ = ["agreement", "load_matcher", "load_pass", "load_sites", "main", "run"]
+__all__ = ["LINE_WINDOW", "agreement", "anchor_site", "load_pass", "load_sites", "main", "matches_declared", "run"]
 
 
 if __name__ == "__main__":