feat(plane): increment manifests for the 46-candidate validation run (task 7)

Model deepseek-flash (declared provider extension, endpoint prices in
parameters, secretKey naming the DEEPSEEK_API_KEY secret), digest-pinned
Task templates repo-facts / verify / agree, and a generator
(plane/generate.py, `ousast plane workspaces --validation-set ...`) that
emits per case the vulnerable-pin Workspace, a files-only inputs Workspace
(recorded candidates after the recorded triage, declared sites, triage),
per-case Tasks and one Run validation-46: facts -> verify a, verify b ->
agree for the 15 cases, 46 candidates (43 kept), budgets 1.5x the recorded
per-pass share. A Run entry cannot add a binding or env, and Run.inputs
names only task artifacts, so the case is fixed by per-case Tasks.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/cli.py

@@ -251,6 +251,11 @@ def main(argv: list[str] | None = None) -> int:
     plane_ws = plane_sub.add_parser("workspaces", help="one Workspace manifest per case pin of a population file")
     plane_ws.add_argument("population", type=Path)
     plane_ws.add_argument("--out", type=Path, default=Path("plane/workspaces"))
+    plane_ws.add_argument("--validation-set", type=Path, help="JSON {case: [[path, function], ...]}: emit that set's increment instead")
+    plane_ws.add_argument("--candidates", type=Path, help="with --validation-set: per-case candidate records (independent-v2-modelsinks)")
+    plane_ws.add_argument("--triage", type=Path, help="with --validation-set: recorded batched-check scans (<case>--vulnerable_a.jsonl)")
+    plane_ws.add_argument("--run", default="validation-46", help="with --validation-set: the Run name")
+    plane_ws.add_argument("--plane", type=Path, default=Path("plane"), help="with --validation-set: the manifest root")
 
     args = parser.parse_args(argv)
     # Diagnosis needs the stage costs, and nothing configures logging, so the default root level of
@@ -1303,6 +1308,8 @@ def _plane(args: argparse.Namespace) -> int:
         for name, ok, text in checks:
             print(f"[{'ok' if ok else 'FAIL'}] {name}: {text}")
         return 0 if all(ok for _, ok, _ in checks) else 1
+    if args.validation_set is not None:
+        return _plane_increment(args)
     from .plane.workspaces import workspaces
 
     written = workspaces(args.population, args.out)
@@ -1310,6 +1317,22 @@ def _plane(args: argparse.Namespace) -> int:
     return 0
 
 
+def _plane_increment(args: argparse.Namespace) -> int:
+    from .plane.generate import increment
+
+    if args.candidates is None or args.triage is None:
+        print("--validation-set needs --candidates and --triage", file=sys.stderr)
+        return 2
+    home = str(Path.home())
+    shown = [str(p).replace(home, "~", 1) for p in (args.population, args.validation_set, args.candidates, args.triage)]
+    command = f"ousast plane workspaces {shown[0]} --validation-set {shown[1]} --candidates {shown[2]} --triage {shown[3]} --run {args.run}"
+    written = increment(
+        args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command
+    )
+    print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
+    return 0
+
+
 def _model_candidates(args: argparse.Namespace) -> int:
     """What the candidate enumerator can reach, per family and per slice. No model is called anywhere here.
 

src/openultrasast/model/audit.py

@@ -62,6 +62,7 @@ SPEC_OWNED = {
     "plane.tasks.verify": "ai-service-plane Req 5.3/2.1-2.3/7.2: the batched hunt with known callers, metered and resumable per file hunt",
     "plane.tasks.agree": "ai-service-plane Req 5.3/2.1: agreement across two verify passes with per-candidate cost, turns and site match",
     "plane.reconciler": "ai-service-plane Req 3.1-3.5, 7.3: ax-backed Run execution, status attribution and doctor; `ousast plane`",
+    "plane.generate": "ai-service-plane task 7: an increment's Workspaces, per-case Tasks and Run from a recorded set",
     "plane.workspaces": "ai-service-plane task 4: Workspace manifests per case pin of a population; `ousast plane workspaces`",
     "plane.runner": "ai-service-plane Req 4.1-4.4: the ax-task-runner entrypoint (PID 1 of the task image); delivers the output directory",
 }

src/openultrasast/plane/generate.py

@@ -0,0 +1,222 @@
+"""Manifests of a recorded measurement as a Run (ai-service-plane task 7).
+
+``ousast plane workspaces <population.toml> --validation-set <set.json> --candidates <dir> --triage <dir>`` writes,
+under ``--plane`` (default ``plane/``), everything a Run over the set needs beside the hand-written templates
+(``tasks/repo-facts.yaml``, ``tasks/verify.yaml``, ``tasks/agree.yaml``, ``models/*.yaml``):
+
+- ``workspaces/<case>-vulnerable.yaml``: the case's repository at its vulnerable pin (commit in the
+  ``openultrasast.io/git-commits`` annotation, as :mod:`.workspaces` writes it);
+- ``workspaces/<case>-inputs.yaml``: a files-only Workspace with the inputs no task produces -- the recorded
+  candidates (``candidates.json`` in the ``independent-v2-modelsinks`` shape for ``verify`` and ``agree``,
+  ``functions.json`` as ``{path, function, line}`` rows for ``repo-facts``), the case's declared sites
+  (``case.json`` for ``agree``) and the recorded triage (``triage.json``, for the measurement);
+- ``tasks/<run>.yaml``: per case, one Task per template (two for ``verify``: ``OUSAST_PASS=a|b``) binding the two
+  Workspaces and naming the files as env;
+- ``runs/<run>.yaml``: the Run, per case ``facts -> verify a, verify b -> agree``, with budgets.
+
+Why per-case Tasks rather than one Task per step: an ax Task binds its Workspaces by name and a Run entry adds
+neither a binding nor env (it names a Task, artifacts and a budget), and ``Run.inputs`` accepts only
+``<task>/<artifact>``. So an input that no task produces is a Workspace ``files`` entry, and the case a step works
+on is fixed by the Task it runs. One Run covers every case so the measurement has one state and one attribution
+table.
+
+Candidates are the set's ``(path, function)`` pairs with the line of the case record, deduplicated the way
+``benchmarks/independent/verify_batched.py`` does (sorted, the last line of a repeated pair wins), and filtered
+by the recorded triage (the ``kept`` lists of the batched-check scans): ``verify`` has no triage stage, so the
+hunts see exactly the candidates the reference hunts saw. Output is a pure function of the inputs.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import tomllib
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
+from .reconciler import _ax_name, _doc
+from .tasks.verify import MAX_STEPS, units_of
+
+__all__ = ["CaseInputs", "increment", "read_cases", "render"]
+
+REFERENCE_USD_PER_CANDIDATE = 0.022  # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes
+HEADROOM = 1.5
+CALLS_PER_HUNT = MAX_STEPS + 2  # every tool step, the answer, one empty-content retry
+ATESPACE = "default"
+CASE_PATH = "/workspace/case"
+INPUTS_PATH = "/workspace/inputs"
+TEMPLATES = ("repo-facts", "verify", "agree")
+
+Candidate = tuple[str, str, int]
+
+
+@dataclass(frozen=True)
+class CaseInputs:
+    case: Mapping[str, Any]
+    candidates: tuple[Candidate, ...]
+    kept: tuple[Candidate, ...]
+    triage: Mapping[str, str]
+    recorded_usd: float
+
+
+def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
+    path = triage_dir / f"{case_id}--vulnerable_a.jsonl"
+    if not path.is_file():
+        raise ValueError(f"{case_id}: no recorded triage at {path}")
+    rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+    rows = [r for r in rows if "unverified" not in r]
+    if not rows:
+        raise ValueError(f"{path}: no completed file record")
+    return rows
+
+
+def read_cases(validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path) -> list[CaseInputs]:
+    """The set's cases in id order, each with its candidates, the triage-kept subset and the recorded cost."""
+    subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8"))
+    records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]}
+    cases: list[CaseInputs] = []
+    for case_id in sorted(subset):
+        if case_id not in records:
+            raise ValueError(f"{validation_set}: case {case_id!r} is not in {population}")
+        record = json.loads((candidates_dir / f"{case_id}.json").read_text(encoding="utf-8"))
+        unique = {(str(p), str(fn)): (str(p), str(fn), int(ln)) for p, fn, ln in sorted(record["candidates"])}
+        wanted = [(str(p), str(fn)) for p, fn, *_ in subset[case_id]]
+        missing = [f"{p}::{fn}" for p, fn in wanted if (p, fn) not in unique]
+        if missing:
+            raise ValueError(f"{case_id}: set candidates absent from {candidates_dir / case_id}.json: {missing[:5]}")
+        wanted_keys = set(wanted)
+        chosen = [c for key, c in unique.items() if key in wanted_keys]
+        rows = _scan_rows(triage_dir, case_id)
+        asked = {(str(c[0]), str(c[1])): (str(c[0]), str(c[1]), int(c[2])) for r in rows for c in r["candidates"]}
+        if sorted(asked.values()) != sorted(chosen):
+            raise ValueError(f"{case_id}: the recorded scans asked {sorted(asked.values())[:3]}..., the set gives {chosen[:3]}...")
+        kept_keys = {(str(r["path"]), str(fn)) for r in rows for fn in r.get("kept", [c[1] for c in r["candidates"]])}
+        triage = {f"{r['path']}::{fn}": str(label) for r in rows for fn, label in sorted((r.get("triage") or {}).items())}
+        cases.append(
+            CaseInputs(
+                case=records[case_id],
+                candidates=tuple(chosen),
+                kept=tuple(c for c in chosen if (c[0], c[1]) in kept_keys),
+                triage=dict(sorted(triage.items())),
+                recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6),
+            )
+        )
+    return cases
+
+
+def _dump(header: str, docs: Sequence[Mapping[str, Any]]) -> str:
+    return header + str(yaml.safe_dump_all(list(docs), sort_keys=False, width=1000))
+
+
+def _json(payload: object) -> str:
+    return json.dumps(payload, indent=1, sort_keys=True) + "\n"
+
+
+def _workspaces(item: CaseInputs) -> tuple[dict[str, Any], dict[str, Any]]:
+    case = item.case
+    git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
+    pinned = _doc("Workspace", _ax_name(case["id"], "vulnerable"), {"git": git}, ATESPACE)
+    pinned["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['vulnerable']}"}
+    files = {
+        "candidates.json": _json({"id": case["id"], "family": case["family"], "candidates": [list(c) for c in item.kept]}),
+        "functions.json": _json([{"path": p, "function": fn, "line": ln} for p, fn, ln in item.kept]),
+        "case.json": _json({"id": case["id"], "family": case["family"], "sites": list(case.get("sites", []))}),
+        "triage.json": _json(
+            {"candidates": [list(c) for c in item.candidates], "kept": [list(c) for c in item.kept], "triage": dict(item.triage)}
+        ),
+    }
+    inputs = _doc("Workspace", _ax_name(case["id"], "inputs"), {"files": [{"path": k, "content": v} for k, v in files.items()]}, ATESPACE)
+    return pinned, inputs
+
+
+def _task(template: Task, name: str, env: Mapping[str, str], bindings: Sequence[tuple[str, str]]) -> dict[str, Any]:
+    spec: dict[str, Any] = {}
+    if template.image:
+        spec["image"] = template.image
+    spec["command"] = list(template.command)
+    spec["env"] = [{"name": e.name, "value": e.value} for e in template.env] + [{"name": k, "value": v} for k, v in env.items()]
+    if template.resources:
+        kinds = {kind: values for kind, values in vars(template.resources).items() if values is not None}
+        spec["resources"] = {kind: {k: v for k, v in vars(values).items() if v is not None} for kind, values in kinds.items()}
+    spec["workspaces"] = [{"name": n, "path": p} for n, p in bindings]
+    if template.debug:
+        spec["debug"] = True
+    doc = _doc("Task", name, spec, template.metadata.atespace)
+    if template.metadata.annotations:
+        doc["metadata"]["annotations"] = dict(template.metadata.annotations)
+    return doc
+
+
+def verify_budget(item: CaseInputs) -> dict[str, float | int]:
+    """One verify pass: 1.5x its share of the case's recorded two-pass cost (at least the set's per-candidate
+    average times the case's candidates), rounded up to a cent; calls: every hunt at its step ceiling."""
+    share = max(item.recorded_usd, REFERENCE_USD_PER_CANDIDATE * len(item.candidates)) / 2
+    hunts = len(units_of(list(item.kept)))
+    return {"usd": math.ceil(HEADROOM * share * 100 - 1e-9) / 100, "calls": max(hunts, 1) * CALLS_PER_HUNT}
+
+
+def render(cases: Sequence[CaseInputs], templates: Mapping[str, Task], run_name: str, command: str) -> dict[str, str]:
+    """Relative path under the plane root -> file text, for every generated manifest."""
+    header = f"# generated by `{command}`; do not edit, regenerate.\n"
+    out: dict[str, str] = {}
+    tasks: list[dict[str, Any]] = []
+    entries: list[dict[str, Any]] = []
+    for item in cases:
+        case_id = item.case["id"]
+        pinned, inputs = _workspaces(item)
+        for doc in (pinned, inputs):
+            out[f"workspaces/{doc['metadata']['name']}.yaml"] = _dump(header, [doc])
+        repo = [(pinned["metadata"]["name"], CASE_PATH), (inputs["metadata"]["name"], INPUTS_PATH)]
+        on_repo = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"}
+        facts_env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/functions.json"}
+        tasks.append(_task(templates["repo-facts"], f"repo-facts-{case_id}", facts_env, repo))
+        for label in ("a", "b"):
+            env = {**on_repo, "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json", "OUSAST_PASS": label}
+            tasks.append(_task(templates["verify"], f"verify-{label}-{case_id}", env, repo))
+        agree_env = {
+            "OUSAST_INPUT_CANDIDATES": f"{INPUTS_PATH}/candidates.json",
+            "OUSAST_INPUT_SITES": f"{INPUTS_PATH}/case.json",
+            "OUSAST_CASE_ID": case_id,
+        }
+        tasks.append(_task(templates["agree"], f"agree-{case_id}", agree_env, repo[1:]))
+        free = {"usd": 0, "calls": 0}
+        facts = {"name": f"{case_id}-facts", "task": f"repo-facts-{case_id}", "outputs": ["facts.json"], "budget": free}
+        entries.append({**facts, "serialize": "repo-facts"})
+        for label in ("a", "b"):
+            entries.append({
+                "name": f"{case_id}-v{label}", "task": f"verify-{label}-{case_id}", "inputs": {"facts": f"{case_id}-facts/facts.json"},
+                "outputs": ["units.jsonl", "summary.json"], "budget": verify_budget(item),
+            })  # fmt: skip
+        entries.append({
+            "name": f"{case_id}-agree", "task": f"agree-{case_id}",
+            "inputs": {"pass_a": f"{case_id}-va/units.jsonl", "pass_b": f"{case_id}-vb/units.jsonl"},
+            "outputs": ["agreed.json", "disputed.json"], "budget": free,
+        })  # fmt: skip
+    out[f"tasks/{run_name}.yaml"] = _dump(header, tasks)
+    run = {"apiVersion": RUN_API_VERSION, "kind": "Run", "metadata": {"name": run_name}, "spec": {"tasks": entries}}
+    total = sum(float(e["budget"]["usd"]) for e in entries)
+    candidates = sum(len(c.candidates) for c in cases)
+    kept = sum(len(c.kept) for c in cases)
+    note = f"# {len(cases)} cases, {candidates} candidates ({kept} kept by the recorded triage); usd ceiling {total:.2f}.\n"
+    out[f"runs/{run_name}.yaml"] = _dump(header + note, [run])
+    return out
+
+
+def increment(
+    population: Path, validation_set: Path, candidates_dir: Path, triage_dir: Path, *, plane: Path, run_name: str, command: str
+) -> list[Path]:
+    """Write the generated manifests under ``plane`` from the templates in ``plane/tasks``; returns the paths."""
+    loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
+    files = render(read_cases(validation_set, candidates_dir, triage_dir, population), loaded.tasks, run_name, command)
+    written: list[Path] = []
+    for relative, text in files.items():
+        path = plane / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+        written.append(path)
+    return written