merge(plane): scoring parity with the reference -- sites, ranges, triage cost

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_plane_increment.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import re
 import shutil
+import subprocess
 import tomllib
 from pathlib import Path
 
@@ -15,13 +16,14 @@ from test_plane_reconciler import AX_FIELDS, non_ax_fields
 from openultrasast.model.endpoint import price_of
 from openultrasast.plane import reconciler
 from openultrasast.plane.budget import prices_from
-from openultrasast.plane.generate import TEMPLATES, increment, read_cases, render
+from openultrasast.plane.generate import TEMPLATES, fix_ranges, increment, read_cases, render, repin_templates
 from openultrasast.plane.manifests import load_manifests
 
 PLANE = Path("plane")
 RUN = PLANE / "runs" / "validation-46.yaml"
 POPULATION = Path("benchmarks/independent/population-v2.toml")
 RESULTS = Path.home() / "ousast-results"
+REPOS = Path.home() / ".cache" / "openultrasast" / "independent"
 PIN = "0123456789abcdef0123456789abcdef01234567"
 
 
@@ -81,14 +83,30 @@ def test_validation_run_resolves_and_renders_every_task(tmp_path: Path) -> None:
 def test_inputs_carry_the_validation_set() -> None:
     manifests = load_manifests(sorted((PLANE / "workspaces").glob("*-inputs.yaml")))
     assert len(manifests.workspaces) == 15
-    candidates = kept = 0
+    candidates = kept = in_set = before = 0
+    usd = triage = 0.0
     for workspace in manifests.workspaces.values():
         files = {f.path: json.loads(f.content) for f in workspace.files}
         assert set(files) == {"candidates.json", "functions.json", "case.json", "triage.json"}
         candidates += len(files["triage.json"]["candidates"])
         kept += len(files["candidates.json"]["candidates"])
         assert [[r["path"], r["function"], r["line"]] for r in files["functions.json"]] == files["candidates.json"]["candidates"]
-    assert (candidates, kept) == (46, 43)
+        case = files["case.json"]
+        assert case["ranges"] and all(first <= last for spans in case["ranges"].values() for first, last in spans), case["id"]
+        assert set(case["sites_in_set"]) <= set(case["sites"])
+        in_set += len(case["sites_in_set"])
+        before += case["cost"]["candidates_before_triage"]
+        usd += case["cost"]["recorded_usd"]
+        triage += case["cost"]["recorded_triage_usd"]
+        assert 0 < case["cost"]["recorded_triage_usd"] <= case["cost"]["recorded_usd"], case["id"]
+    assert (candidates, kept, in_set, before) == (46, 43, 20, 46)
+    # the reference's $1.01 for 46 candidates, two passes, triage included; triage was about 9% of it
+    assert round(usd / before, 3) == 0.022 and 0.05 < triage / usd < 0.15
+
+
+def test_templates_are_pinned_to_one_runner_image() -> None:
+    images = {task.image for task in load_manifests(sorted((PLANE / "tasks").glob("*.yaml"))).tasks.values()}
+    assert len(images) == 1 and re.fullmatch(r"\S+@sha256:[0-9a-f]{64}", images.pop() or "")
 
 
 def _committed_command() -> str:
@@ -98,7 +116,9 @@ def _committed_command() -> str:
     return found.group(1)
 
 
-@pytest.mark.skipif(not (RESULTS / "independent-v2-batched-check" / "scans").is_dir(), reason="recorded results not on this host")
+@pytest.mark.skipif(
+    not (RESULTS / "independent-v2-batched-check" / "scans").is_dir() or not REPOS.is_dir(), reason="recorded results not on this host"
+)
 def test_committed_manifests_match_the_generator() -> None:
     templates = load_manifests([PLANE / "tasks" / f"{name}.yaml" for name in TEMPLATES]).tasks
     cases = read_cases(
@@ -106,6 +126,7 @@ def test_committed_manifests_match_the_generator() -> None:
         RESULTS / "independent-v2-modelsinks",
         RESULTS / "independent-v2-batched-check" / "scans",
         POPULATION,
+        REPOS,
     )
     for relative, text in render(cases, templates, "validation-46", _committed_command()).items():
         assert (PLANE / relative).read_text() == text, relative
@@ -173,3 +194,43 @@ def test_generator_refuses_a_set_the_recorded_scans_did_not_ask(tmp_path: Path)
     (paths["scans"] / "alpha--vulnerable_a.jsonl").unlink()
     with pytest.raises(ValueError, match="no recorded triage"):
         _generate(paths, paths["plane"])
+
+
+def _git(repo: Path, *args: str) -> str:
+    done = subprocess.run(
+        ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, text=True, check=True
+    )
+    return done.stdout.strip()
+
+
+def test_fix_ranges_are_the_old_side_of_the_fix_diff(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    _git(repo, "init", "-q")
+    (repo / "a.py").write_text("".join(f"line {i}\n" for i in range(1, 31)))
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-qm", "vulnerable")
+    vulnerable = _git(repo, "rev-parse", "HEAD")
+    (repo / "a.py").write_text("".join(f"line {i}\n" if i not in (3, 4, 20) else "fixed\n" for i in range(1, 31)) + "added\n")
+    _git(repo, "commit", "-qam", "fixed")
+    assert fix_ranges(repo, vulnerable, _git(repo, "rev-parse", "HEAD")) == {"a.py": [(3, 4), (20, 20), (30, 30)]}
+    with pytest.raises(ValueError, match="no repository"):
+        fix_ranges(tmp_path / "absent", vulnerable, vulnerable)
+
+
+def test_case_record_carries_sites_in_set_and_cost_and_the_image_is_repinned(tmp_path: Path) -> None:
+    paths = _fixture(tmp_path)
+    digest = "localhost:5001/ousast-runner@sha256:" + "ab" * 32
+    (tmp_path / "runner-image").write_text(digest + "\n")
+    increment(
+        paths["population"], paths["set"], paths["candidates"], paths["scans"], plane=paths["plane"], run_name="check",
+        command="ousast plane test", runner_image=tmp_path / "runner-image",
+    )  # fmt: skip
+    manifests = load_manifests(sorted(paths["plane"].rglob("*.yaml")))
+    assert {t.image for t in manifests.tasks.values()} == {digest}
+    case = json.loads(next(f.content for f in manifests.workspaces["alpha-inputs"].files if f.path == "case.json"))
+    assert "ranges" not in case and case["sites_in_set"] == ["a.py::run"]  # no --repos: agree reports not assessed
+    assert case["cost"]["candidates_before_triage"] == 2 and case["cost"]["recorded_triage_usd"] == 0.5  # no hunt usage recorded
+    (tmp_path / "runner-image").write_text("localhost:5001/ousast-runner:dev\n")
+    with pytest.raises(ValueError, match="digest-pinned"):
+        repin_templates(paths["plane"], tmp_path / "runner-image")