learn: exp-005 graph slice -- slice in the prompt (opt-in), program-arm fill, manifest and units

Implemented with Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/openultrasast/learn/cli.py

@@ -60,7 +60,7 @@ def add_commands(learn_sub: Any) -> None:
         ("register", "record the manifest's and the units file's sha256 as the experiment row (both committed at HEAD)"),
         ("units", "freeze the units of a program experiment from the memory (no model): the families' evaluation candidates"),
         ("run", "run every arm on every unit, paired; arm A replays from the response cache when the manifest says so"),
-        ("fill", "fill a rule experiment's response cache under a separate budget, or estimate with --dry-run"),
+        ("fill", "fill an experiment's response cache under a separate budget, or estimate with --dry-run"),
         ("analyse", "paired cluster bootstrap, exact McNemar, the two looks and the adoption verdict (counts and intervals only)"),
     ):
         sub = experiment_sub.add_parser(name, help=text)
@@ -73,8 +73,10 @@ def add_commands(learn_sub: Any) -> None:
                 "--manifest-model", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model (prices)"
             )
         if name == "run":
+            sub.add_argument("--replay-only", action="store_true", help="replay every arm without resolving a model endpoint")
             sub.add_argument("--no-early-stop", action="store_true", help="take no first look during the run")
         if name == "fill":
+            sub.add_argument("--arm", action="append", help="arm to fill (repeatable; default: non-replay program arms)")
             sub.add_argument("--budget-usd", type=float, help="required for paid fill; includes the smoke call")
             sub.add_argument("--dry-run", action="store_true", help="inspect and estimate only; no calls or writes, including --out")
             sub.add_argument("--out", type=Path, help="also write the fill report as JSON (ignored for --dry-run)")
@@ -117,7 +119,7 @@ def _experiment(args: argparse.Namespace) -> int:
     from ..plane.budget import MeteredClient
     from ..plane.memory import open_store
     from . import experiments as ex
-    from .program import Caller
+    from .program import Caller, ReplayMiss
 
     store = open_store(args.memory)
     try:
@@ -162,7 +164,7 @@ def _experiment(args: argparse.Namespace) -> int:
 
         if args.experiment_command == "fill":
             from ..tool_hunter import ChatClient
-            from .fill import fill_rule
+            from .fill import fill_experiment
 
             def resolve_fill_client() -> ChatClient:
                 from ..config import load_config
@@ -173,7 +175,7 @@ def _experiment(args: argparse.Namespace) -> int:
                     raise ex.ExperimentError("no chat endpoint configured; nothing was spent")
                 return resolved[0]
 
-            report = fill_rule(
+            report = fill_experiment(
                 store,
                 manifest,
                 memory,
@@ -183,6 +185,7 @@ def _experiment(args: argparse.Namespace) -> int:
                 client_factory=resolve_fill_client,
                 budget_usd=args.budget_usd,
                 dry_run=args.dry_run,
+                arms=args.arm,
             )
             _write(report, None if args.dry_run else args.out)
             return {"done": 0, "dry_run": 0, "unfinished": 3, "failed": 1}[report["status"]]
@@ -194,7 +197,7 @@ def _experiment(args: argparse.Namespace) -> int:
             return 0
         callers: dict[str, Caller] = {}
         for name, arm in manifest.arms.items():
-            if arm.replay_only:
+            if arm.replay_only or args.replay_only:
                 callers[name] = Caller(None, _arm_model(manifest, arm, store), parameters, store=store)
                 continue
             from ..config import load_config
@@ -208,7 +211,7 @@ def _experiment(args: argparse.Namespace) -> int:
 
         early = not args.no_early_stop
         summary = ex.run(store, manifest, memory, callers, excerpt_text, vectors, stop_early=early, resamples=args.resamples)
-    except ex.ExperimentError as exc:
+    except (ex.ExperimentError, ReplayMiss) as exc:
         print(f"learn experiment {args.experiment_command}: {exc}", file=sys.stderr)
         return 2
     _write(summary.as_dict(), args.out)

src/openultrasast/learn/experiments.py

@@ -97,6 +97,8 @@ ADVISORY_VERDICT = "vulnerable"
 METRICS: Mapping[str, tuple[int, bool]] = {
     "auc": (1, False),
     "within_pair_auc": (1, False),
+    "top_30_precision": (1, False),
+    "unsure_rate": (-1, True),
     "advisory_recall": (1, True),
     "advisory_precision": (1, False),
     "canary_agreement": (1, True),
@@ -598,10 +600,10 @@ def auc(rows: Sequence[ArmUnit]) -> float | None:
 def within_pair_auc(rows: Sequence[ArmUnit]) -> float | None:
     """Of the candidate pairs (vulnerable and fixed side both present), the share where the vulnerable side scores
     higher (ties one half)."""
-    pairs: dict[str, dict[int, float]] = {}
+    pairs: dict[tuple[str, str, str], dict[int, float]] = {}
     for r in rows:
         if r.pair and r.s is not None:
-            pairs.setdefault(r.pair, {})[r.label] = r.s
+            pairs.setdefault((r.group, r.family, r.pair), {})[r.label] = r.s
     done = [(p[1], p[0]) for p in pairs.values() if 0 in p and 1 in p]
     if not done:
         return None
@@ -616,6 +618,11 @@ def metric_value(metric: Metric, rows: Sequence[ArmUnit]) -> float | None:
         return auc(rows)
     if name == "within_pair_auc":
         return within_pair_auc(rows)
+    if name == "top_30_precision":
+        ranked = sorted((r for r in rows if r.s is not None), key=lambda r: (-r.s if r.s is not None else 0.0, r.unit))
+        return _mean([float(r.label) for r in ranked[:30]]) if len(ranked) >= 30 else None
+    if name == "unsure_rate":
+        return _mean([float(r.verdict == "unsure") for r in rows])
     if name == "advisory_recall":
         return _mean([float(r.flag) for r in rows if r.label == 1])
     if name == "advisory_precision":
@@ -639,6 +646,8 @@ def binary_value(metric: Metric, row: ArmUnit) -> bool | None:
         return row.agree
     if metric.name == "flag_rate":
         return row.flag
+    if metric.name == "unsure_rate":
+        return row.verdict == "unsure"
     return None
 
 
@@ -676,7 +685,13 @@ def paired_bootstrap(
     rng = random.Random(seed)
     values: list[float] = []
     for _ in range(resamples if names else 0):
-        sample = [p for g in (rng.choice(names) for _ in names) for p in by_group[g]]
+        # Give each drawn cluster its own identity: repeated draws must retain
+        # repeated candidate pairs when within_pair_auc groups the two sides.
+        sample = [
+            (replace(a, group=f"{draw}:{g}"), replace(b, group=f"{draw}:{g}"))
+            for draw, g in enumerate(rng.choice(names) for _ in names)
+            for a, b in by_group[g]
+        ]
         a = metric_value(metric, [p[0] for p in sample])
         b = metric_value(metric, [p[1] for p in sample])
         if a is not None and b is not None:
@@ -785,6 +800,9 @@ def decide(manifest: Manifest, finals: Mapping[str, Estimate], cost: Estimate |
         return "reject", f"the CI of {', '.join(bad)} excludes 0 against B"
     first = manifest.primary[0].name
     if sides[first] == "favour":
+        max_ratio = (manifest.raw.get("decision") or {}).get("max_cost_ratio")
+        if max_ratio is not None and (cost is None or cost.a is None or cost.b is None or cost.b > float(max_ratio) * cost.a):
+            return "inconclusive", f"B's cost per candidate is unavailable or exceeds {max_ratio}x A; the incumbent stays"
         return "adopt", f"the CI of {first} excludes 0 in B's favour and no primary CI excludes 0 against B"
     if manifest.cost_only and cost is not None and tost is not None and tost.ci is not None and cost.ci is not None:
         within = -manifest.tost_margin <= tost.ci[0] and tost.ci[1] <= manifest.tost_margin

src/openultrasast/learn/fill.py

@@ -1,4 +1,4 @@
-"""Fill rule-experiment responses without writing scores or changing the frozen population.
+"""Fill experiment responses without writing scores or changing the frozen population.
 
 Estimates use the family's mean billed cached response cost when usage is available.
 Otherwise they use Prompt.estimated_tokens and 300 output tokens. Unknown answers may require one retry;
@@ -81,7 +81,7 @@ class FillMeter(MeteredClient):
         return response
 
 
-def fill_rule(
+def fill_experiment(
     store: MemoryStore,
     manifest: ex.Manifest,
     memory: Sequence[Example],
@@ -93,6 +93,7 @@ def fill_rule(
     client_factory: Callable[[], ChatClient] | None = None,
     budget_usd: float | None = None,
     dry_run: bool = False,
+    arms: Sequence[str] | None = None,
 ) -> dict[str, Any]:
     """Inspect or fill every frozen unit. The first cache miss is the smoke call.
 
@@ -102,19 +103,32 @@ def fill_rule(
     """
     from .evaluate import candidate_from_example
 
-    if manifest.kind != "rule":
-        raise ex.ExperimentError("experiment fill supports kind: rule only")
+    if manifest.kind not in ("rule", "program"):
+        raise ex.ExperimentError("experiment fill supports kind: rule or program only")
+    selected = list(dict.fromkeys(arms if arms is not None else (["A"] if manifest.kind == "rule" else manifest.paid_arms)))
+    if not selected or any(name not in manifest.arms for name in selected):
+        raise ex.ExperimentError("fill requires at least one known arm")
+    # Rule arms share exactly one scoring pass, regardless of the selected rule.
+    scoring = selected[:1] if manifest.kind == "rule" else selected
     if not dry_run and (budget_usd is None or not math.isfinite(budget_usd) or budget_usd < 0):
         raise ex.ExperimentError("--budget-usd must be a finite non-negative number (or use --dry-run)")
+    if manifest.kind == "program" and not dry_run:
+        ceiling = min(manifest.budget_total_usd, sum(manifest.arms[name].budget_usd for name in selected))
+        if budget_usd is not None and budget_usd > ceiling:
+            raise ex.ExperimentError(f"--budget-usd exceeds the selected program arms' manifest ceiling ({ceiling})")
     prices = prices_from(parameters)
     if prices is None or any(not math.isfinite(p) or p < 0 for p in (prices.input_per_m, prices.output_per_m, prices.cache_hit_per_m)):
         raise ex.ExperimentError("fill requires finite, non-negative Model prices")
     ex.check_registered(store, manifest)
     units, folds, _ = ex._checked_units(manifest, memory)
-    programs = {f: Program(ex.arm_spec(manifest, manifest.arms["A"], f, store), memory, excerpt_text, vectors) for f in manifest.families}
+    programs = {
+        (name, f): Program(ex.arm_spec(manifest, manifest.arms[name], f, store), memory, excerpt_text, vectors)
+        for name in scoring
+        for f in manifest.families
+    }
     models = {p.spec.model for p in programs.values()}
     if len(models) != 1:
-        raise ex.ExperimentError("the rule programs must use the same model")
+        raise ex.ExperimentError("the selected programs must use the same model")
     caller = Caller(None, next(iter(models)), parameters, store=store)
     index = {e.id: e for e in memory}
     candidate_of = candidate_from_example(excerpt_text, vectors)
@@ -123,10 +137,13 @@ def fill_rule(
         candidate = candidate_of(index[unit.unit])
         if candidate is None:
             raise ex.ExperimentError(f"{manifest.id}: unit {unit.unit[:12]} has no excerpt in the store")
-        for prompt, _, temperature, sample in programs[unit.family].prepare_samples(
-            candidate, folds[unit.fold], seed=int(manifest.folds.get("seed", 0))
-        ):
-            requests.append(Request(unit.family, prompt, temperature, sample))
+        for name in scoring:
+            prepared = programs[name, unit.family].prepare_samples(candidate, folds[unit.fold], seed=int(manifest.folds.get("seed", 0)))
+            repeats = manifest.arms[name].repeats if manifest.kind == "program" else 1
+            for replicate in range(repeats):
+                salt = f"r{replicate}:" if replicate else ""
+                for prompt, _, temperature, sample in prepared:
+                    requests.append(Request(unit.family, prompt, temperature, f"{salt}{sample}"))
 
     # Request counts are distinct cache keys per family, even when multiple units share one.
     def inventory() -> dict[str, dict[str, Request]]:
@@ -167,6 +184,7 @@ def fill_rule(
         }
     report: dict[str, Any] = {
         "experiment": manifest.id,
+        "arms": selected,
         "status": "dry_run" if dry_run else "done",
         "families": families,
         "client_calls": 0,
@@ -205,3 +223,7 @@ def fill_rule(
     if meter is not None:
         report.update(client_calls=meter.calls, usd=meter.usd)
     return report
+
+
+# Backwards-compatible name for callers of the original rule-only API.
+fill_rule = fill_experiment

src/openultrasast/learn/program.py

@@ -38,6 +38,7 @@ from .examples import Example
 from .folds import Fold
 from .retrieve import Retrieval, Target, assert_boundary, demonstrations_for, retrieve
 from .schema import DEFAULT_INPUTS, features_for, instruments_for, validate_x, withheld
+from .slice import function_slice, render_slice
 
 VERDICTS = ("vulnerable", "not_vulnerable", "unsure")
 UNSURE = "unsure"
@@ -150,9 +151,11 @@ def _case(
     profile: str,
     language: str,
     inputs: str = DEFAULT_INPUTS,
+    slice: bool = False,
 ) -> str:
     signals = render_signals(x, instruments, profile, inputs)
-    return f"Family: {family}\nLanguage: {language}\nCode:\n{code.rstrip()}\nSignals:\n{signals}\nRoles:\n{render_roles(roles)}"
+    flow = "\n" + render_slice(function_slice(code, language, family), language) if slice else ""
+    return f"Family: {family}\nLanguage: {language}\nCode:\n{code.rstrip()}{flow}\nSignals:\n{signals}\nRoles:\n{render_roles(roles)}"
 
 
 def label_word(label: int) -> str:
@@ -204,7 +207,7 @@ class ProgramSpec:
     priors: str = "off"
     inputs: str = DEFAULT_INPUTS  # the input profile: which instruments the classifier never sees (v1: the engine)
     max_output_tokens: int = MAX_OUTPUT_TOKENS
-    slice: bool = False  # reserved experiment switch; rendering stays disabled until the leak audit is reviewed
+    slice: bool = False  # opt-in in-function def-use evidence, computed from the same excerpt
     sampling: str = SAMPLING[0]  # temperature | retrieval_ensemble (an experiment arm's setting; never compiled)
 
 
@@ -253,7 +256,9 @@ def render_prefix(spec: ProgramSpec, demos: Sequence[Demo], examples: Mapping[st
                   "cited_lines": list(demo.cited_lines)}  # fmt: skip
         parts.append(
             f"Demonstration {number}:\n"
-            + _case(code, example.x, example.instruments, example.roles, example.family, spec.profile, example.language, spec.inputs)
+            + _case(
+                code, example.x, example.instruments, example.roles, example.family, spec.profile, example.language, spec.inputs, spec.slice
+            )
             + f"\nAnswer: {json.dumps(answer, sort_keys=True)}"
         )
     return "\n\n".join(parts)
@@ -274,11 +279,14 @@ def render(
         code = excerpt_text(example.excerpt_sha) or ""
         blocks.append(
             f"Example {number} (label: {label_word(example.label)}):\n"
-            + _case(code, example.x, example.instruments, example.roles, example.family, spec.profile, example.language, spec.inputs)
+            + _case(
+                code, example.x, example.instruments, example.roles, example.family, spec.profile, example.language, spec.inputs, spec.slice
+            )
         )
     t = candidate.target
     blocks.append(
-        "Candidate:\n" + _case(candidate.code, t.x, t.instruments, candidate.roles, t.family, spec.profile, candidate.language, spec.inputs)
+        "Candidate:\n"
+        + _case(candidate.code, t.x, t.instruments, candidate.roles, t.family, spec.profile, candidate.language, spec.inputs, spec.slice)
     )
     messages: list[dict[str, object]] = [
         {"role": "system", "content": render_prefix(spec, demos, examples, excerpt_text)},