feat(learn): compile, calibrate, evaluate (decision-engine 6.5)

learn/compile.toml: the pre-registered spec -- balanced Brier score of the
raw score on C_val (cells by family x label, repositories capped at 5,
unsure = 0.5, ties within 0.005 to fewer prompt tokens), seed, the
compile split (25%, boot 60/40, compile-split-only memory), bootstrap
(150 candidates, confidence >= 0.7, 4 sets of 3), search (6 proposals +
baseline, 20 errors shown, 60 validation candidates), k = 1, the
retrieval settings, and the $3 / 1,200-call ceiling.

learn/compile.py: bootstrap few-shot keeps only right, confident
rationales that cite a line and name no repository, path or CVE/GHSA id;
seeded demo sets (positive, negative, any; families mixed) with
other-framework alternates; one proposal call; 7 instructions scored with
set 1, the best with sets 2-4. The artifact is canonical JSON stored as
programs/<program_id>.json, program_id = sha256 of the artifact without
its id; it records the spec sha, memory/compile-groups/folds digests and
the response-cache digest, with spend priced from cached usage, so a
rerun from the cache is byte-identical at $0 (tested on FileStore and
MinioStore through the fake client; real MinIO gated as today). A ceiling
ends the compile `unfinished` with no artifact. packageable() keeps only
permissively licensed demonstrations (MIT, BSD, Apache-2.0, ISC, zlib),
swapping in a permissive alternate or dropping, under a new program id.

learn/calibrate.py: Platt scaling on logit(s), pure Python Newton, per
family / pooled with family intercepts / insufficient data by the floors;
cross-fitted (fold j's map is fitted on the other folds only); the check
(5-bin reliability, slope and intercept with repository-bootstrap CIs,
ECE <= 0.05), ADVISORY only when it fails.

learn/evaluate.py: predictions under each fold's boundary, families below
the floor never asked; operating points read from the calibrated,
cross-fitted predictions of the other folds (BLOCK: Wilson lower bound
>= 0.95, never on a majority unsure; ADVISORY: recall >= 0.90);
repository-cluster bootstrap intervals; strata with "insufficient data";
>5% parse failures is an instrument failure; explanations keep only
signals the record holds; version mismatch names the fallback reason;
the memory-size learning curve.

`ousast learn compile|evaluate|curve`: paid runs need --budget-usd (exit
3 at the ceiling), --replay-only runs from the cache. Nothing was spent:
every test and the dry run used scripted clients.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/learn/cli.py

@@ -34,6 +34,24 @@ def add_commands(learn_sub: Any) -> None:
     embed.add_argument("--budget-calls", type=int)
     embed.add_argument("--batch", type=int, default=32)
     embed.add_argument("--dry-run", action="store_true", help="count and estimate the tokens; no call")
+    for name, text in (
+        ("compile", "compile the program on the compile split (bootstrap few-shot, instruction search)"),
+        ("evaluate", "evaluate a compiled program on the outer folds: cross-fitted calibration, nested operating points"),
+        ("curve", "the program's metric over memory size on a fixed candidate subset"),
+    ):
+        sub = learn_sub.add_parser(name, help=text)
+        sub.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
+        sub.add_argument("--profile", choices=("static", "plane"), default="static")
+        sub.add_argument("--spec", type=Path, help="the pre-registered compile spec (default: the packaged learn/compile.toml)")
+        sub.add_argument("--manifest", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model with the prices")
+        sub.add_argument("--budget-usd", type=float, help="the ceiling of a paid run; required unless --replay-only")
+        sub.add_argument("--budget-calls", type=int)
+        sub.add_argument("--replay-only", action="store_true", help="no client: every response must come from the cache")
+        sub.add_argument("--out", type=Path, help="write the report (counts and intervals only) here")
+        if name != "compile":
+            sub.add_argument("--program", required=True, help="the compiled program id (programs/<id>.json in the store)")
+        if name == "curve":
+            sub.add_argument("--subset", type=int, default=200)
 
 
 def run(args: argparse.Namespace) -> int:
@@ -41,10 +59,104 @@ def run(args: argparse.Namespace) -> int:
         return _memory_build(args)
     if args.learn_command == "memory" and args.memory_command == "embed":
         return _memory_embed(args)
+    if args.learn_command in ("compile", "evaluate", "curve"):
+        return _program_command(args)
     print(f"learn: unknown command {args.learn_command}", file=sys.stderr)
     return 2
 
 
+def _caller(args: argparse.Namespace, store: Any, model: str) -> Any:
+    """The metered, cached caller; ``None`` client under --replay-only. Exits 2 without a ceiling or a key."""
+    from ..plane.budget import MeteredClient
+    from .program import Caller
+
+    parameters = _model_parameters(args.manifest)
+    if args.replay_only:
+        return Caller(None, model, parameters, store=store)
+    if args.budget_usd is None:
+        raise SystemExit("learn: --budget-usd is required (a paid run starts only under a ceiling), or --replay-only")
+    from ..config import load_config
+    from ..model.endpoint import resolve_chat_endpoint
+
+    resolved = resolve_chat_endpoint(load_config(None))
+    if resolved is None:
+        raise SystemExit("learn: no chat endpoint configured (DEEPSEEK_API_KEY); nothing was spent")
+    client = MeteredClient(resolved[0], prices=parameters, budget_usd=args.budget_usd, budget_calls=args.budget_calls)
+    return Caller(client, model, parameters, store=store)
+
+
+def _identities(store: Any) -> dict[str, tuple[str, ...]]:
+    """Per example id, the strings its demonstration's rationale must not name: group, repository and path parts."""
+    from .compile import group_identities
+    from .examples import EXAMPLE_KIND
+
+    out: dict[str, tuple[str, ...]] = {}
+    for record in store.rows(kind=EXAMPLE_KIND):
+        row = record.row
+        path = str(row.get("candidate", "")).partition("::")[0]
+        out[str(row["id"])] = (*group_identities(str(row.get("group") or "")), str(row["repo"]), path, path.rsplit("/", 1)[-1])
+    return out
+
+
+def _program_command(args: argparse.Namespace) -> int:
+    """compile | evaluate | curve. Exit 3 when the ceiling stops the run (``unfinished``, no artifact, no number)."""
+    from datetime import date
+
+    from ..plane.budget import BudgetExhausted
+    from ..plane.memory import open_store
+    from .compile import DEFAULT_SPEC, compile_program, load_program, load_spec, spec_of
+    from .embeddings import EmbeddingCache
+    from .evaluate import candidate_from_example, evaluate, learning_curve, predict, targets, unevaluable_families
+    from .examples import load_examples
+    from .folds import compile_split, outer_folds
+    from .program import Program
+
+    store = open_store(args.memory)
+    spec = load_spec(args.spec or DEFAULT_SPEC)
+    memory = load_examples(store, args.profile)
+    cache = EmbeddingCache(store)
+    vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None
+
+    def excerpt_text(sha: str) -> str | None:
+        data = store.get_blob("excerpts", sha)
+        return data.decode("utf-8") if data is not None else None
+
+    model = str(spec["classify"].get("model", "deepseek-flash"))
+    caller = _caller(args, store, model)
+    report: dict[str, Any]
+    try:
+        if args.learn_command == "compile":
+            ids = _identities(store)
+            result = compile_program(
+                memory, caller, excerpt_text, spec=spec, profile=args.profile, vectors=vectors,
+                identities=lambda e: ids.get(e.id, ()), store=store, created=date.today().isoformat(),
+            )  # fmt: skip
+            report = {"status": result.status, "program_id": result.program_id, "calls": result.calls, "replayed": result.replayed,
+                      "reason": result.reason, "bootstrap": result.bootstrap}  # fmt: skip
+        else:
+            artifact = load_program(store, args.program)
+            program = Program(spec_of(artifact), memory, excerpt_text, vectors)
+            split = compile_split(memory, fraction=float(spec["split"].get("compile_fraction", 0.25)), seed=spec.seed)
+            folds = outer_folds(memory, seed=spec.seed, exclude=split.groups)
+            pairs = targets(folds, memory, skip=unevaluable_families(memory))
+            candidate_of = candidate_from_example(excerpt_text, vectors)
+            if args.learn_command == "evaluate":
+                report = evaluate(predict(program, folds, caller, candidate_of, seed=spec.seed, pairs=pairs), seed=spec.seed)
+            else:
+                report = {"curve": learning_curve(program, pairs[: args.subset], caller, candidate_of, seed=spec.seed)}
+            report["program_id"] = args.program
+    except BudgetExhausted as exc:
+        print(f"learn {args.learn_command}: unfinished -- {exc}", file=sys.stderr)
+        return 3
+    report["spend"] = {"usd": caller.usd(), "client_calls": caller.calls, "replayed": caller.replayed}
+    text = json.dumps(report, indent=2, sort_keys=True, default=str) + "\n"
+    if args.out is not None:
+        args.out.parent.mkdir(parents=True, exist_ok=True)
+        args.out.write_text(text, encoding="utf-8")
+    print(text, end="")
+    return 0
+
+
 def _model_parameters(path: Path) -> dict[str, Any]:
     from ..plane.manifests import load_manifests