mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(learn): compile, calibrate, evaluate (decision-engine 6.5)
learn/compile.toml: the pre-registered spec -- balanced Brier score of the raw score on C_val (cells by family x label, repositories capped at 5, unsure = 0.5, ties within 0.005 to fewer prompt tokens), seed, the compile split (25%, boot 60/40, compile-split-only memory), bootstrap (150 candidates, confidence >= 0.7, 4 sets of 3), search (6 proposals + baseline, 20 errors shown, 60 validation candidates), k = 1, the retrieval settings, and the $3 / 1,200-call ceiling. learn/compile.py: bootstrap few-shot keeps only right, confident rationales that cite a line and name no repository, path or CVE/GHSA id; seeded demo sets (positive, negative, any; families mixed) with other-framework alternates; one proposal call; 7 instructions scored with set 1, the best with sets 2-4. The artifact is canonical JSON stored as programs/<program_id>.json, program_id = sha256 of the artifact without its id; it records the spec sha, memory/compile-groups/folds digests and the response-cache digest, with spend priced from cached usage, so a rerun from the cache is byte-identical at $0 (tested on FileStore and MinioStore through the fake client; real MinIO gated as today). A ceiling ends the compile `unfinished` with no artifact. packageable() keeps only permissively licensed demonstrations (MIT, BSD, Apache-2.0, ISC, zlib), swapping in a permissive alternate or dropping, under a new program id. learn/calibrate.py: Platt scaling on logit(s), pure Python Newton, per family / pooled with family intercepts / insufficient data by the floors; cross-fitted (fold j's map is fitted on the other folds only); the check (5-bin reliability, slope and intercept with repository-bootstrap CIs, ECE <= 0.05), ADVISORY only when it fails. learn/evaluate.py: predictions under each fold's boundary, families below the floor never asked; operating points read from the calibrated, cross-fitted predictions of the other folds (BLOCK: Wilson lower bound >= 0.95, never on a majority unsure; ADVISORY: recall >= 0.90); repository-cluster bootstrap intervals; strata with "insufficient data"; >5% parse failures is an instrument failure; explanations keep only signals the record holds; version mismatch names the fallback reason; the memory-size learning curve. `ousast learn compile|evaluate|curve`: paid runs need --budget-usd (exit 3 at the ceiling), --replay-only runs from the cache. Nothing was spent: every test and the dry run used scripted clients. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
ea61dc1bb3
commit
7f6bbacb6e
@@ -0,0 +1,230 @@
|
||||
"""Calibration of the program's score on held-out repositories (learned-decision-engine design 4.4, Req 3.5, 7.4).
|
||||
|
||||
Pure Python, no fitted classifier: the raw score ``s`` (the vote mean) is mapped by **Platt scaling** on
|
||||
``logit(clip(s, 0.01, 0.99))``, a two-parameter logistic fitted by Newton's method with Platt's smoothed targets.
|
||||
|
||||
- **Cross-fitted**: the map applied to fold j is fitted on the out-of-fold predictions of the other folds only
|
||||
(:func:`cross_fit`), so no map is checked on the fold it was fitted on.
|
||||
- Per family when the family reaches the per-family floor of positive and negative repository groups; pooled with
|
||||
a per-family intercept between the floors; otherwise ``insufficient data`` (no calibrated probability).
|
||||
- **The check** (:func:`check`): a reliability table (5 quantile bins), the calibration slope and intercept (a
|
||||
logistic regression of the label on ``logit(p)``) with repository-bootstrap 95% intervals, and the expected
|
||||
calibration error. It holds when the slope interval contains 1, the intercept interval contains 0 and ECE <= 0.05;
|
||||
otherwise only ADVISORY is offered (``block.offered = false, reason = "calibration"``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .labels import PER_FAMILY_FLOOR, POOLED_FLOOR
|
||||
|
||||
CLIP = 0.01
|
||||
ECE_LIMIT = 0.05
|
||||
BINS = 5
|
||||
|
||||
|
||||
def logit(p: float) -> float:
|
||||
q = min(max(p, CLIP), 1 - CLIP)
|
||||
return math.log(q / (1 - q))
|
||||
|
||||
|
||||
def sigmoid(z: float) -> float:
|
||||
if z >= 0:
|
||||
return 1 / (1 + math.exp(-z))
|
||||
e = math.exp(z)
|
||||
return e / (1 + e)
|
||||
|
||||
|
||||
def _solve(a: list[list[float]], b: list[float]) -> list[float]:
|
||||
"""Gaussian elimination with partial pivoting (the systems here are 2 to ~10 wide)."""
|
||||
n = len(b)
|
||||
m = [row[:] + [b[i]] for i, row in enumerate(a)]
|
||||
for col in range(n):
|
||||
pivot = max(range(col, n), key=lambda r: abs(m[r][col]))
|
||||
m[col], m[pivot] = m[pivot], m[col]
|
||||
if abs(m[col][col]) < 1e-12:
|
||||
continue
|
||||
for r in range(n):
|
||||
if r != col:
|
||||
f = m[r][col] / m[col][col]
|
||||
for c in range(col, n + 1):
|
||||
m[r][c] -= f * m[col][c]
|
||||
return [m[i][n] / m[i][i] if abs(m[i][i]) >= 1e-12 else 0.0 for i in range(n)]
|
||||
|
||||
|
||||
def fit_logistic(xs: Sequence[Sequence[float]], ys: Sequence[float], *, l2: float = 1e-3, iterations: int = 100) -> list[float]:
|
||||
"""Weights of ``y ~ sigmoid(w . x)`` by Newton's method with a small ridge (separable data stays finite)."""
|
||||
d = len(xs[0]) if xs else 0
|
||||
w = [0.0] * d
|
||||
for _ in range(iterations):
|
||||
grad = [-l2 * wi for wi in w]
|
||||
hess = [[(l2 if i == j else 0.0) for j in range(d)] for i in range(d)]
|
||||
for x, y in zip(xs, ys, strict=True):
|
||||
p = sigmoid(sum(wi * xi for wi, xi in zip(w, x, strict=True)))
|
||||
for i in range(d):
|
||||
grad[i] += (y - p) * x[i]
|
||||
for j in range(d):
|
||||
hess[i][j] += p * (1 - p) * x[i] * x[j]
|
||||
step = _solve(hess, grad)
|
||||
w = [wi + si for wi, si in zip(w, step, strict=True)]
|
||||
if max((abs(s) for s in step), default=0.0) < 1e-9:
|
||||
break
|
||||
return w
|
||||
|
||||
|
||||
def platt_targets(ys: Sequence[int]) -> list[float]:
|
||||
pos = sum(1 for y in ys if y == 1)
|
||||
neg = len(ys) - pos
|
||||
hi, lo = (pos + 1) / (pos + 2), 1 / (neg + 2)
|
||||
return [hi if y == 1 else lo for y in ys]
|
||||
|
||||
|
||||
def fit_platt(scores: Sequence[float], labels: Sequence[int]) -> tuple[float, float]:
|
||||
"""(a, b) of ``p = sigmoid(a logit(s) + b)``."""
|
||||
a, b = fit_logistic([[logit(s), 1.0] for s in scores], platt_targets(labels))
|
||||
return a, b
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Row:
|
||||
"""One out-of-fold prediction: its fold, family, repository group, label and raw score."""
|
||||
|
||||
fold: str
|
||||
family: str
|
||||
group: str
|
||||
label: int
|
||||
s: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CalibrationMap:
|
||||
per_family: Mapping[str, tuple[float, float]] = field(default_factory=dict)
|
||||
pooled: tuple[float, float] | None = None # slope and baseline intercept of the pooled map
|
||||
intercepts: Mapping[str, float] = field(default_factory=dict) # per-family intercept shift of the pooled map
|
||||
insufficient: tuple[str, ...] = ()
|
||||
|
||||
def apply(self, family: str, s: float) -> float | None:
|
||||
if family in self.per_family:
|
||||
a, b = self.per_family[family]
|
||||
return sigmoid(a * logit(s) + b)
|
||||
if self.pooled is not None and family in self.intercepts:
|
||||
a, b = self.pooled
|
||||
return sigmoid(a * logit(s) + b + self.intercepts[family])
|
||||
return None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"per_family": {f: list(v) for f, v in sorted(self.per_family.items())}, "pooled": list(self.pooled) if self.pooled else None,
|
||||
"intercepts": dict(sorted(self.intercepts.items())), "insufficient": list(self.insufficient),
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def _groups(rows: Sequence[Row], family: str, label: int) -> int:
|
||||
return len({r.group for r in rows if r.family == family and r.label == label})
|
||||
|
||||
|
||||
def fit_map(rows: Sequence[Row], *, per_family_floor: int = PER_FAMILY_FLOOR, pooled_floor: int = POOLED_FLOOR) -> CalibrationMap:
|
||||
families = sorted({r.family for r in rows})
|
||||
level = {f: min(_groups(rows, f, 1), _groups(rows, f, 0)) for f in families}
|
||||
own = [f for f in families if level[f] >= per_family_floor]
|
||||
pooled = [f for f in families if pooled_floor <= level[f] < per_family_floor]
|
||||
per_family = {f: fit_platt([r.s for r in rows if r.family == f], [r.label for r in rows if r.family == f]) for f in own}
|
||||
pooled_map: tuple[float, float] | None = None
|
||||
intercepts: dict[str, float] = {}
|
||||
if pooled:
|
||||
chosen = [r for r in rows if r.family in pooled]
|
||||
xs = [[logit(r.s), 1.0] + [1.0 if r.family == f else 0.0 for f in pooled[1:]] for r in chosen]
|
||||
w = fit_logistic(xs, platt_targets([r.label for r in chosen]))
|
||||
pooled_map = (w[0], w[1])
|
||||
intercepts = {f: (w[2 + i - 1] if i else 0.0) for i, f in enumerate(pooled)}
|
||||
return CalibrationMap(per_family, pooled_map, intercepts, tuple(f for f in families if level[f] < pooled_floor))
|
||||
|
||||
|
||||
def cross_fit(rows: Sequence[Row], **floors: int) -> tuple[list[float | None], dict[str, CalibrationMap]]:
|
||||
"""Calibrated ``p`` per row, each from a map fitted on the other folds' rows only; and the map per fold."""
|
||||
maps = {fold: fit_map([r for r in rows if r.fold != fold], **floors) for fold in sorted({r.fold for r in rows})}
|
||||
return [maps[r.fold].apply(r.family, r.s) for r in rows], maps
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Check:
|
||||
reliability: tuple[Mapping[str, float], ...]
|
||||
slope: float
|
||||
slope_ci: tuple[float, float]
|
||||
intercept: float
|
||||
intercept_ci: tuple[float, float]
|
||||
ece: float
|
||||
holds: bool
|
||||
n: int
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"reliability": [dict(b) for b in self.reliability], "slope": round(self.slope, 4),
|
||||
"slope_ci": [round(v, 4) for v in self.slope_ci], "intercept": round(self.intercept, 4),
|
||||
"intercept_ci": [round(v, 4) for v in self.intercept_ci], "ece": round(self.ece, 4), "holds": self.holds, "n": self.n,
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def _slope_intercept(ps: Sequence[float], ys: Sequence[int]) -> tuple[float, float]:
|
||||
a, b = fit_logistic([[logit(p), 1.0] for p in ps], [float(y) for y in ys])
|
||||
return a, b
|
||||
|
||||
|
||||
def reliability(ps: Sequence[float], ys: Sequence[int], bins: int = BINS) -> tuple[list[dict[str, float]], float]:
|
||||
order = sorted(range(len(ps)), key=lambda i: ps[i])
|
||||
table, ece = [], 0.0
|
||||
for b in range(bins):
|
||||
idx = order[b * len(order) // bins : (b + 1) * len(order) // bins]
|
||||
if not idx:
|
||||
continue
|
||||
mean_p = sum(ps[i] for i in idx) / len(idx)
|
||||
rate = sum(ys[i] for i in idx) / len(idx)
|
||||
table.append({"n": len(idx), "mean_p": round(mean_p, 4), "positive_rate": round(rate, 4)})
|
||||
ece += len(idx) / len(ps) * abs(mean_p - rate)
|
||||
return table, ece
|
||||
|
||||
|
||||
def _percentile(values: Sequence[float], q: float) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[min(len(ordered) - 1, max(0, round(q * (len(ordered) - 1))))]
|
||||
|
||||
|
||||
def check(ps: Sequence[float], ys: Sequence[int], groups: Sequence[str], *, resamples: int = 200, seed: int = 0) -> Check:
|
||||
"""The calibration check on held-out predictions, with repository-cluster bootstrap intervals."""
|
||||
table, ece = reliability(ps, ys)
|
||||
slope, intercept = _slope_intercept(ps, ys)
|
||||
by_group: dict[str, list[int]] = {}
|
||||
for i, g in enumerate(groups):
|
||||
by_group.setdefault(g, []).append(i)
|
||||
names = sorted(by_group)
|
||||
rng = random.Random(seed)
|
||||
slopes, intercepts = [], []
|
||||
for _ in range(resamples):
|
||||
idx = [i for g in (rng.choice(names) for _ in names) for i in by_group[g]]
|
||||
sample_y = [ys[i] for i in idx]
|
||||
if len(set(sample_y)) < 2:
|
||||
continue
|
||||
a, b = _slope_intercept([ps[i] for i in idx], sample_y)
|
||||
slopes.append(a)
|
||||
intercepts.append(b)
|
||||
slope_ci = (_percentile(slopes, 0.025), _percentile(slopes, 0.975)) if slopes else (math.nan, math.nan)
|
||||
intercept_ci = (_percentile(intercepts, 0.025), _percentile(intercepts, 0.975)) if intercepts else (math.nan, math.nan)
|
||||
holds = bool(slopes) and slope_ci[0] <= 1 <= slope_ci[1] and intercept_ci[0] <= 0 <= intercept_ci[1] and ece <= ECE_LIMIT
|
||||
return Check(tuple(table), slope, slope_ci, intercept, intercept_ci, ece, holds, len(ps))
|
||||
|
||||
|
||||
def offered(result: Check) -> dict[str, dict[str, Any]]:
|
||||
"""What a family is offered: ADVISORY always; BLOCK only when calibration holds."""
|
||||
block = {"offered": True, "reason": None} if result.holds else {"offered": False, "reason": "calibration"}
|
||||
return {"block": block, "advisory": {"offered": True, "reason": None}}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ECE_LIMIT", "CalibrationMap", "Check", "Row", "check", "cross_fit", "fit_logistic", "fit_map", "fit_platt", "logit", "offered",
|
||||
"reliability", "sigmoid",
|
||||
] # fmt: skip
|
||||
@@ -34,6 +34,24 @@ def add_commands(learn_sub: Any) -> None:
|
||||
embed.add_argument("--budget-calls", type=int)
|
||||
embed.add_argument("--batch", type=int, default=32)
|
||||
embed.add_argument("--dry-run", action="store_true", help="count and estimate the tokens; no call")
|
||||
for name, text in (
|
||||
("compile", "compile the program on the compile split (bootstrap few-shot, instruction search)"),
|
||||
("evaluate", "evaluate a compiled program on the outer folds: cross-fitted calibration, nested operating points"),
|
||||
("curve", "the program's metric over memory size on a fixed candidate subset"),
|
||||
):
|
||||
sub = learn_sub.add_parser(name, help=text)
|
||||
sub.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
|
||||
sub.add_argument("--profile", choices=("static", "plane"), default="static")
|
||||
sub.add_argument("--spec", type=Path, help="the pre-registered compile spec (default: the packaged learn/compile.toml)")
|
||||
sub.add_argument("--manifest", type=Path, default=Path("plane/models/deepseek-flash.yaml"), help="the chat Model with the prices")
|
||||
sub.add_argument("--budget-usd", type=float, help="the ceiling of a paid run; required unless --replay-only")
|
||||
sub.add_argument("--budget-calls", type=int)
|
||||
sub.add_argument("--replay-only", action="store_true", help="no client: every response must come from the cache")
|
||||
sub.add_argument("--out", type=Path, help="write the report (counts and intervals only) here")
|
||||
if name != "compile":
|
||||
sub.add_argument("--program", required=True, help="the compiled program id (programs/<id>.json in the store)")
|
||||
if name == "curve":
|
||||
sub.add_argument("--subset", type=int, default=200)
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
@@ -41,10 +59,104 @@ def run(args: argparse.Namespace) -> int:
|
||||
return _memory_build(args)
|
||||
if args.learn_command == "memory" and args.memory_command == "embed":
|
||||
return _memory_embed(args)
|
||||
if args.learn_command in ("compile", "evaluate", "curve"):
|
||||
return _program_command(args)
|
||||
print(f"learn: unknown command {args.learn_command}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def _caller(args: argparse.Namespace, store: Any, model: str) -> Any:
|
||||
"""The metered, cached caller; ``None`` client under --replay-only. Exits 2 without a ceiling or a key."""
|
||||
from ..plane.budget import MeteredClient
|
||||
from .program import Caller
|
||||
|
||||
parameters = _model_parameters(args.manifest)
|
||||
if args.replay_only:
|
||||
return Caller(None, model, parameters, store=store)
|
||||
if args.budget_usd is None:
|
||||
raise SystemExit("learn: --budget-usd is required (a paid run starts only under a ceiling), or --replay-only")
|
||||
from ..config import load_config
|
||||
from ..model.endpoint import resolve_chat_endpoint
|
||||
|
||||
resolved = resolve_chat_endpoint(load_config(None))
|
||||
if resolved is None:
|
||||
raise SystemExit("learn: no chat endpoint configured (DEEPSEEK_API_KEY); nothing was spent")
|
||||
client = MeteredClient(resolved[0], prices=parameters, budget_usd=args.budget_usd, budget_calls=args.budget_calls)
|
||||
return Caller(client, model, parameters, store=store)
|
||||
|
||||
|
||||
def _identities(store: Any) -> dict[str, tuple[str, ...]]:
|
||||
"""Per example id, the strings its demonstration's rationale must not name: group, repository and path parts."""
|
||||
from .compile import group_identities
|
||||
from .examples import EXAMPLE_KIND
|
||||
|
||||
out: dict[str, tuple[str, ...]] = {}
|
||||
for record in store.rows(kind=EXAMPLE_KIND):
|
||||
row = record.row
|
||||
path = str(row.get("candidate", "")).partition("::")[0]
|
||||
out[str(row["id"])] = (*group_identities(str(row.get("group") or "")), str(row["repo"]), path, path.rsplit("/", 1)[-1])
|
||||
return out
|
||||
|
||||
|
||||
def _program_command(args: argparse.Namespace) -> int:
|
||||
"""compile | evaluate | curve. Exit 3 when the ceiling stops the run (``unfinished``, no artifact, no number)."""
|
||||
from datetime import date
|
||||
|
||||
from ..plane.budget import BudgetExhausted
|
||||
from ..plane.memory import open_store
|
||||
from .compile import DEFAULT_SPEC, compile_program, load_program, load_spec, spec_of
|
||||
from .embeddings import EmbeddingCache
|
||||
from .evaluate import candidate_from_example, evaluate, learning_curve, predict, targets, unevaluable_families
|
||||
from .examples import load_examples
|
||||
from .folds import compile_split, outer_folds
|
||||
from .program import Program
|
||||
|
||||
store = open_store(args.memory)
|
||||
spec = load_spec(args.spec or DEFAULT_SPEC)
|
||||
memory = load_examples(store, args.profile)
|
||||
cache = EmbeddingCache(store)
|
||||
vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None
|
||||
|
||||
def excerpt_text(sha: str) -> str | None:
|
||||
data = store.get_blob("excerpts", sha)
|
||||
return data.decode("utf-8") if data is not None else None
|
||||
|
||||
model = str(spec["classify"].get("model", "deepseek-flash"))
|
||||
caller = _caller(args, store, model)
|
||||
report: dict[str, Any]
|
||||
try:
|
||||
if args.learn_command == "compile":
|
||||
ids = _identities(store)
|
||||
result = compile_program(
|
||||
memory, caller, excerpt_text, spec=spec, profile=args.profile, vectors=vectors,
|
||||
identities=lambda e: ids.get(e.id, ()), store=store, created=date.today().isoformat(),
|
||||
) # fmt: skip
|
||||
report = {"status": result.status, "program_id": result.program_id, "calls": result.calls, "replayed": result.replayed,
|
||||
"reason": result.reason, "bootstrap": result.bootstrap} # fmt: skip
|
||||
else:
|
||||
artifact = load_program(store, args.program)
|
||||
program = Program(spec_of(artifact), memory, excerpt_text, vectors)
|
||||
split = compile_split(memory, fraction=float(spec["split"].get("compile_fraction", 0.25)), seed=spec.seed)
|
||||
folds = outer_folds(memory, seed=spec.seed, exclude=split.groups)
|
||||
pairs = targets(folds, memory, skip=unevaluable_families(memory))
|
||||
candidate_of = candidate_from_example(excerpt_text, vectors)
|
||||
if args.learn_command == "evaluate":
|
||||
report = evaluate(predict(program, folds, caller, candidate_of, seed=spec.seed, pairs=pairs), seed=spec.seed)
|
||||
else:
|
||||
report = {"curve": learning_curve(program, pairs[: args.subset], caller, candidate_of, seed=spec.seed)}
|
||||
report["program_id"] = args.program
|
||||
except BudgetExhausted as exc:
|
||||
print(f"learn {args.learn_command}: unfinished -- {exc}", file=sys.stderr)
|
||||
return 3
|
||||
report["spend"] = {"usd": caller.usd(), "client_calls": caller.calls, "replayed": caller.replayed}
|
||||
text = json.dumps(report, indent=2, sort_keys=True, default=str) + "\n"
|
||||
if args.out is not None:
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(text, encoding="utf-8")
|
||||
print(text, end="")
|
||||
return 0
|
||||
|
||||
|
||||
def _model_parameters(path: Path) -> dict[str, Any]:
|
||||
from ..plane.manifests import load_manifests
|
||||
|
||||
|
||||
@@ -0,0 +1,526 @@
|
||||
"""Compilation of the program, DSPy-style and in-house (learned-decision-engine design 4.3, 5, Req 3.3).
|
||||
|
||||
``ousast learn compile --profile P --spec learn/compile.toml`` produces a compiled program from a frozen memory
|
||||
snapshot. Nothing is fitted; the instruction and the demonstrations are *chosen* against the pre-registered metric
|
||||
of ``compile.toml`` (the balanced Brier score of the raw score on ``C_val``):
|
||||
|
||||
1. **Compile split** (:func:`.folds.compile_split`): 25% of the groups, never evaluated, split into ``C_boot`` and
|
||||
``C_val``; during the compile the memory is the compile split only, so no evaluated repository shapes it.
|
||||
2. **Bootstrap few-shot**: the program without demonstrations (retrieval on, ``k = 1``, temperature 0) on up to 150
|
||||
candidates of ``C_boot``; the kept demonstrations are right, confident (>= 0.7), cite a line in range and name no
|
||||
identity -- no repository, no path, no CVE/GHSA id (:func:`identity_hits`). The rationale was produced without
|
||||
the label, so a demonstration teaches reasoning, not the answer. Four seeded sets of three (a positive, a
|
||||
negative, one of either), each demonstration with an alternate of another framework for leave-one-framework-out.
|
||||
3. **Instruction search**: one proposal call (the signature, a signal glossary, 20 summarised bootstrap errors -- no
|
||||
code) for 6 instructions; with the baseline, 7 are scored with demo set 1, then the best with sets 2-4.
|
||||
4. The **artifact** is one canonical JSON (sorted keys), stored as the keyed blob ``programs/<program_id>.json``,
|
||||
where ``program_id`` is the sha256 of the artifact without its own id. It records the spec's sha256, the
|
||||
memory snapshot digest, the folds digest and the response-cache digest; spend is priced from the cached usage,
|
||||
so a replay writes the identical artifact. Whether a run replayed is reported beside it, never inside it.
|
||||
|
||||
A compile that reaches its ceiling (``BudgetExhausted``) is ``unfinished`` and writes no artifact.
|
||||
|
||||
**Packaging** (maintainer decision 2026-10-01): only demonstrations from permissively licensed code (MIT, BSD,
|
||||
Apache-2.0, ISC, zlib) may ship in the package; :func:`packageable` swaps the others for a permissive alternate or
|
||||
drops them, and the result is a new program (its own id) to be re-evaluated as its own variant.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import tomllib
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..plane.budget import BudgetExhausted
|
||||
from ..plane.memory import MemoryStore
|
||||
from .examples import Example
|
||||
from .folds import CompileSplit, Fold, assert_disjoint, compile_split, folds_digest, outer_folds
|
||||
from .labels import _ADVISORY
|
||||
from .program import (
|
||||
BASELINE_INSTRUCTION,
|
||||
SIGNATURE,
|
||||
Caller,
|
||||
Candidate,
|
||||
Decision,
|
||||
Demo,
|
||||
Program,
|
||||
ProgramSpec,
|
||||
label_word,
|
||||
)
|
||||
from .schema import SCHEMA_VERSION, feature_set_digest
|
||||
|
||||
DEFAULT_SPEC = Path(__file__).resolve().parent / "compile.toml"
|
||||
KIND = "llm-program"
|
||||
PERMISSIVE = ("mit", "bsd", "0bsd", "apache-2.0", "apache 2.0", "apache2", "apache-2", "isc", "zlib")
|
||||
_PATH = re.compile(r"(?<![\w.-])[\w.-]+/[\w./-]+|\b[\w-]+\.(?:py|php|js|jsx|ts|tsx|java|c|cc|cpp|h|hpp|go|rb|cs|kt|scala|groovy)\b")
|
||||
|
||||
|
||||
class CompileError(RuntimeError):
|
||||
"""A compile that cannot start: no demonstration pool, no validation candidates, or folds that overlap."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileSpec:
|
||||
data: Mapping[str, Any]
|
||||
sha256: str
|
||||
|
||||
def __getitem__(self, section: str) -> Mapping[str, Any]:
|
||||
return dict(self.data.get(section) or {})
|
||||
|
||||
@property
|
||||
def seed(self) -> int:
|
||||
return int(self.data.get("seed", 0))
|
||||
|
||||
|
||||
def load_spec(path: Path = DEFAULT_SPEC) -> CompileSpec:
|
||||
text = Path(path).read_bytes()
|
||||
return CompileSpec(tomllib.loads(text.decode("utf-8")), hashlib.sha256(text).hexdigest())
|
||||
|
||||
|
||||
# --- the metric ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def balanced_brier(rows: Sequence[tuple[str, int, str, float]], *, repository_cap: int = 5) -> float | None:
|
||||
"""Mean over (family, label) cells of the cell's Brier score, each repository capped at ``repository_cap``
|
||||
effective rows per cell. ``rows`` are ``(family, label, group, s)``."""
|
||||
cells: dict[tuple[str, int], list[tuple[str, float]]] = {}
|
||||
for family, label, group, s in rows:
|
||||
cells.setdefault((family, label), []).append((group, (s - label) ** 2))
|
||||
scores = []
|
||||
for members in cells.values():
|
||||
per_group: dict[str, int] = {}
|
||||
for group, _ in members:
|
||||
per_group[group] = per_group.get(group, 0) + 1
|
||||
weights = [min(1.0, repository_cap / per_group[g]) for g, _ in members]
|
||||
scores.append(sum(w * e for w, (_, e) in zip(weights, members, strict=True)) / sum(weights))
|
||||
return sum(scores) / len(scores) if scores else None
|
||||
|
||||
|
||||
def secondary(rows: Sequence[tuple[int, float, str]], precision: float = 0.90) -> dict[str, float | None]:
|
||||
"""Recall at the lowest threshold reaching ``precision`` and the share of ``unsure`` (``rows``: label, s, verdict)."""
|
||||
positives = sum(label for label, _, _ in rows)
|
||||
recall = None
|
||||
for t in sorted({s for _, s, _ in rows}):
|
||||
chosen = [label for label, s, _ in rows if s >= t]
|
||||
if chosen and sum(chosen) / len(chosen) >= precision:
|
||||
recall = sum(chosen) / positives if positives else None
|
||||
break
|
||||
share = sum(1 for _, _, v in rows if v == "unsure") / len(rows) if rows else None
|
||||
return {"recall_at_precision_0.90": recall, "unsure_share": share}
|
||||
|
||||
|
||||
# --- identities ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def identity_hits(rationale: str, identities: Iterable[str] = ()) -> list[str]:
|
||||
"""Identities a rationale names: CVE/GHSA ids, paths and source file names, and any given repository string."""
|
||||
hits = [m.group(0) for m in _ADVISORY.finditer(rationale)] + [m.group(0) for m in _PATH.finditer(rationale)]
|
||||
folded = rationale.casefold()
|
||||
hits += [i for i in identities if len(i) >= 4 and i.casefold() in folded]
|
||||
return hits
|
||||
|
||||
|
||||
def group_identities(group: str) -> tuple[str, ...]:
|
||||
parts = [p for p in re.split(r"[/:]", group) if p]
|
||||
return (group, *parts)
|
||||
|
||||
|
||||
def permissive(license_id: str) -> bool:
|
||||
text = license_id.strip().casefold().removesuffix(" license")
|
||||
return bool(text) and any(text == p or text.startswith(p + "-") or text.startswith(p + " ") for p in PERMISSIVE)
|
||||
|
||||
|
||||
# --- bootstrap, demo sets, proposals ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _order(items: Iterable[Example], seed: int, salt: str) -> list[Example]:
|
||||
return sorted(items, key=lambda e: hashlib.sha256(f"{seed}\x00{salt}\x00{e.id}".encode()).hexdigest())
|
||||
|
||||
|
||||
@dataclass
|
||||
class Bootstrap:
|
||||
pool: list[Demo] = field(default_factory=list)
|
||||
errors: list[dict[str, Any]] = field(default_factory=list)
|
||||
rejected: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def reject(self, why: str) -> None:
|
||||
self.rejected[why] = self.rejected.get(why, 0) + 1
|
||||
|
||||
|
||||
def _signal_summary(example: Example) -> dict[str, Any]:
|
||||
ran = {k: v for k, v in example.x.items() if v not in (None, 0, False) and not k.startswith("lang.")}
|
||||
return dict(sorted(ran.items())[:12])
|
||||
|
||||
|
||||
def bootstrap(
|
||||
program: Program,
|
||||
examples: Sequence[Example],
|
||||
fold: Fold,
|
||||
caller: Caller,
|
||||
candidate_of: Callable[[Example], Candidate | None],
|
||||
*,
|
||||
min_confidence: float,
|
||||
identities: Callable[[Example], Iterable[str]],
|
||||
seed: int = 0,
|
||||
) -> Bootstrap:
|
||||
out = Bootstrap()
|
||||
for example in examples:
|
||||
candidate = candidate_of(example)
|
||||
if candidate is None:
|
||||
out.reject("no excerpt")
|
||||
continue
|
||||
decision = program.decide(candidate, fold, caller, seed=seed)
|
||||
answer = decision.answers[0]
|
||||
right = decision.verdict == label_word(example.label)
|
||||
if not right:
|
||||
out.errors.append({"family": example.family, "label": label_word(example.label), "verdict": decision.verdict,
|
||||
"confidence": answer.confidence, "signals": _signal_summary(example)}) # fmt: skip
|
||||
out.reject("wrong or unsure")
|
||||
continue
|
||||
if answer.parse_failed or answer.confidence < min_confidence:
|
||||
out.reject("not confident")
|
||||
continue
|
||||
if identity_hits(answer.rationale, identities(example)):
|
||||
out.reject("rationale names an identity")
|
||||
continue
|
||||
out.pool.append(
|
||||
Demo(
|
||||
example.id,
|
||||
example.excerpt_sha,
|
||||
example.label,
|
||||
example.family,
|
||||
answer.verdict,
|
||||
answer.confidence,
|
||||
answer.rationale,
|
||||
answer.cited_lines,
|
||||
) # fmt: skip
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def draw_demo_sets(
|
||||
pool: Sequence[Demo], examples: Mapping[str, Example], *, sets: int = 4, size: int = 3, seed: int = 0
|
||||
) -> list[tuple[Demo, ...]]:
|
||||
"""``sets`` seeded sets of ``size``: one positive, one negative, then any, preferring a family not yet in the set;
|
||||
each demonstration carries an alternate of the same label from no shared framework (or ``None``)."""
|
||||
|
||||
def ordered(salt: str, items: Iterable[Demo]) -> list[Demo]:
|
||||
return sorted(items, key=lambda d: hashlib.sha256(f"{seed}\x00{salt}\x00{d.example_id}".encode()).hexdigest())
|
||||
|
||||
out: list[tuple[Demo, ...]] = []
|
||||
for n in range(sets):
|
||||
positives = ordered(f"pos{n}", [d for d in pool if d.label == 1])
|
||||
negatives = ordered(f"neg{n}", [d for d in pool if d.label == 0])
|
||||
chosen: list[Demo] = [*positives[:1], *negatives[:1]]
|
||||
families = {d.family for d in chosen}
|
||||
rest = [d for d in ordered(f"any{n}", pool) if d not in chosen]
|
||||
rest.sort(key=lambda d: d.family in families) # stable: other families first
|
||||
chosen += rest[: max(0, size - len(chosen))]
|
||||
with_alternates = []
|
||||
for demo in chosen:
|
||||
frameworks = set(examples[demo.example_id].frameworks)
|
||||
alternate = next(
|
||||
(a for a in ordered(f"alt{n}:{demo.example_id}", pool)
|
||||
if a.label == demo.label and a not in chosen and not frameworks & set(examples[a.example_id].frameworks)),
|
||||
None,
|
||||
) # fmt: skip
|
||||
with_alternates.append(replace(demo, alternate=alternate))
|
||||
if with_alternates:
|
||||
out.append(tuple(with_alternates))
|
||||
return out
|
||||
|
||||
|
||||
PROPOSAL_INSTRUCTION = (
|
||||
"Propose instructions for a classifier. It reads one function's numbered code, the signals static instruments\n"
|
||||
"emitted for it and similar labelled examples, and answers a JSON verdict (vulnerable, not_vulnerable, unsure)\n"
|
||||
'with a confidence and a rationale citing lines. Answer one JSON object {"instructions": [<strings>]}.'
|
||||
)
|
||||
|
||||
|
||||
def glossary() -> str:
|
||||
from .schema import FEATURES
|
||||
|
||||
return "\n".join(f"{s.name}: {s.type}{' (prior)' if s.prior else ''} from {s.instrument}" for s in FEATURES)
|
||||
|
||||
|
||||
def propose(caller: Caller, errors: Sequence[Mapping[str, Any]], *, n: int, shown: int) -> list[str]:
|
||||
user = (
|
||||
f"Signature fields: {json.dumps([f.__dict__ for f in SIGNATURE.fields])}\nSignal glossary:\n{glossary()}\n"
|
||||
f"Mistakes of the baseline instruction (signals and verdicts only):\n{json.dumps(list(errors)[:shown], sort_keys=True)}\n"
|
||||
f"Baseline instruction:\n{BASELINE_INSTRUCTION}\nPropose {n} different instructions that would avoid these mistakes."
|
||||
)
|
||||
messages: list[dict[str, object]] = [{"role": "system", "content": PROPOSAL_INSTRUCTION}, {"role": "user", "content": user}]
|
||||
from ..provider.openrouter import OpenRouterError, parse_json_content
|
||||
|
||||
try:
|
||||
data = parse_json_content(caller.ask(messages, temperature=0.7, sample="propose"))
|
||||
except OpenRouterError:
|
||||
return []
|
||||
found = data.get("instructions") if isinstance(data, dict) else None
|
||||
texts = [t.strip() for t in found if isinstance(t, str) and t.strip()] if isinstance(found, list) else []
|
||||
return list(dict.fromkeys(texts))[:n]
|
||||
|
||||
|
||||
# --- scoring and the artifact -------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Scored:
|
||||
instruction: int # 0 = baseline, n = proposal n
|
||||
demo_set: int
|
||||
score: float | None
|
||||
secondary: Mapping[str, float | None]
|
||||
prompt_tokens: int
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"instruction": self.instruction,
|
||||
"demo_set": self.demo_set,
|
||||
"score": None if self.score is None else round(self.score, 6),
|
||||
"secondary": {k: None if v is None else round(v, 6) for k, v in self.secondary.items()},
|
||||
"prompt_tokens": self.prompt_tokens,
|
||||
}
|
||||
|
||||
|
||||
def score(
|
||||
program: Program, examples: Sequence[Example], fold: Fold, caller: Caller, candidate_of: Callable[[Example], Candidate | None], *,
|
||||
repository_cap: int, seed: int, instruction: int, demo_set: int,
|
||||
) -> Scored: # fmt: skip
|
||||
rows: list[tuple[str, int, str, float]] = []
|
||||
second: list[tuple[int, float, str]] = []
|
||||
tokens = 0
|
||||
for example in examples:
|
||||
candidate = candidate_of(example)
|
||||
if candidate is None:
|
||||
continue
|
||||
prompt, _ = program.prepare(candidate, fold, seed=seed)
|
||||
tokens = max(tokens, round(len(prompt.prefix) / 4))
|
||||
decision: Decision = program.decide(candidate, fold, caller, seed=seed)
|
||||
rows.append((example.family, example.label, example.group, decision.s))
|
||||
second.append((example.label, decision.s, decision.verdict))
|
||||
return Scored(instruction, demo_set, balanced_brier(rows, repository_cap=repository_cap), secondary(second), tokens)
|
||||
|
||||
|
||||
def best(configurations: Sequence[Scored], tolerance: float) -> Scored:
|
||||
valid = [c for c in configurations if c.score is not None]
|
||||
if not valid:
|
||||
raise CompileError("no configuration produced a score")
|
||||
top = min(c.score for c in valid if c.score is not None)
|
||||
close = [c for c in valid if c.score is not None and c.score <= top + tolerance]
|
||||
return min(close, key=lambda c: (c.prompt_tokens, configurations.index(c)))
|
||||
|
||||
|
||||
def canonical(artifact: Mapping[str, Any]) -> bytes:
|
||||
return json.dumps(artifact, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def program_id(artifact: Mapping[str, Any]) -> str:
|
||||
return hashlib.sha256(canonical({k: v for k, v in artifact.items() if k != "program_id"})).hexdigest()
|
||||
|
||||
|
||||
def memory_snapshot_digest(examples: Iterable[Example]) -> str:
|
||||
return hashlib.sha256("\n".join(sorted(f"{e.id}:{e.label}:{e.excerpt_sha}" for e in examples)).encode()).hexdigest()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompileResult:
|
||||
status: str # done | unfinished
|
||||
artifact: dict[str, Any] | None
|
||||
program_id: str | None
|
||||
calls: int # client calls made by this run (0 on a full replay)
|
||||
replayed: int
|
||||
reason: str | None = None
|
||||
bootstrap: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def compile_program(
|
||||
memory: Sequence[Example],
|
||||
caller: Caller,
|
||||
excerpt_text: Callable[[str], str | None],
|
||||
*,
|
||||
spec: CompileSpec,
|
||||
profile: str = "static",
|
||||
vectors: Mapping[str, Sequence[float]] | None = None,
|
||||
identities: Callable[[Example], Iterable[str]] | None = None,
|
||||
families: Sequence[str] = (),
|
||||
store: MemoryStore | None = None,
|
||||
created: str = "",
|
||||
code_commit: str = "",
|
||||
) -> CompileResult:
|
||||
"""Compile one profile's program from ``memory`` (module docstring). Deterministic given the memory, the spec and
|
||||
the response cache."""
|
||||
seed = spec.seed
|
||||
pool = [e for e in memory if e.profile == profile]
|
||||
split: CompileSplit = compile_split(pool, fraction=float(spec["split"].get("compile_fraction", 0.25)),
|
||||
boot=float(spec["split"].get("boot_fraction", 0.6)), seed=seed) # fmt: skip
|
||||
evaluation = outer_folds(pool, seed=seed, exclude=split.groups)
|
||||
assert_disjoint(split, evaluation)
|
||||
compile_memory = [e for e in pool if e.group in split.groups]
|
||||
fold = Fold("compile", "compile", frozenset(), phase="compile")
|
||||
boot = _order([e for e in compile_memory if e.group in split.boot], seed, "boot")[: int(spec["bootstrap"].get("max_candidates", 150))]
|
||||
val = _order([e for e in compile_memory if e.group in split.val], seed, "val")[: int(spec["search"].get("val_candidates", 60))]
|
||||
if not boot or not val:
|
||||
raise CompileError(f"compile split too small: {len(boot)} bootstrap and {len(val)} validation candidates")
|
||||
retrieval, classify = spec["retrieval"], spec["classify"]
|
||||
base = ProgramSpec(
|
||||
profile=profile, model=str(classify.get("model", "deepseek-flash")), k=int(classify.get("k", 1)), k_ret=int(retrieval.get("k", 6)),
|
||||
lam=float(retrieval.get("lam", 0.5)), n1=int(retrieval.get("n1", 40)), priors=str(retrieval.get("priors", "off")),
|
||||
max_output_tokens=int(classify.get("max_output_tokens", 300)),
|
||||
) # fmt: skip
|
||||
from .evaluate import candidate_from_example
|
||||
|
||||
candidate_of = candidate_from_example(excerpt_text, vectors)
|
||||
ids = identities or (lambda e: group_identities(e.group))
|
||||
tolerance = float(spec["metric"].get("tie_tolerance", 0.005))
|
||||
cap = int(spec["metric"].get("repository_cap", 5))
|
||||
try:
|
||||
boot_run = bootstrap(
|
||||
Program(base, compile_memory, excerpt_text, vectors, families), boot, fold, caller, candidate_of,
|
||||
min_confidence=float(spec["bootstrap"].get("min_confidence", 0.7)), identities=ids, seed=seed,
|
||||
) # fmt: skip
|
||||
index = {e.id: e for e in compile_memory}
|
||||
sets = draw_demo_sets(boot_run.pool, index, sets=int(spec["bootstrap"].get("demo_sets", 4)),
|
||||
size=int(spec["bootstrap"].get("demos_per_set", 3)), seed=seed) # fmt: skip
|
||||
if not sets:
|
||||
raise CompileError(f"bootstrap kept no demonstration ({boot_run.rejected})")
|
||||
shown = int(spec["search"].get("errors_shown", 20))
|
||||
instructions = [BASELINE_INSTRUCTION, *propose(caller, boot_run.errors, n=int(spec["search"].get("proposals", 6)), shown=shown)]
|
||||
configurations: list[Scored] = []
|
||||
|
||||
def run(i: int, d: int) -> Scored:
|
||||
program = Program(replace(base, instruction=instructions[i], demos=sets[d]), compile_memory,
|
||||
excerpt_text, vectors, families) # fmt: skip
|
||||
result = score(program, val, fold, caller, candidate_of, repository_cap=cap, seed=seed, instruction=i, demo_set=d)
|
||||
configurations.append(result)
|
||||
return result
|
||||
|
||||
for i in range(len(instructions)):
|
||||
run(i, 0)
|
||||
chosen_instruction = best(configurations, tolerance).instruction
|
||||
for d in range(1, len(sets)):
|
||||
run(chosen_instruction, d)
|
||||
winner = best(configurations, tolerance)
|
||||
except BudgetExhausted as exc:
|
||||
return CompileResult("unfinished", None, None, caller.calls, caller.replayed, reason=str(exc))
|
||||
artifact: dict[str, Any] = {
|
||||
"kind": KIND,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"feature_set_digest": feature_set_digest(),
|
||||
"profile": profile,
|
||||
"signature": {"digest": SIGNATURE.digest, "fields": [f.__dict__ for f in SIGNATURE.fields]},
|
||||
"instruction": instructions[winner.instruction],
|
||||
"instruction_source": "baseline" if winner.instruction == 0 else f"proposed:{winner.instruction}",
|
||||
"demos": [d.as_dict() for d in sets[winner.demo_set]],
|
||||
"retrieval": {
|
||||
"distance": "gower-v1",
|
||||
"n1": base.n1,
|
||||
"k": base.k_ret,
|
||||
"lam": base.lam,
|
||||
"priors": base.priors,
|
||||
"near_duplicate_cos": float(retrieval.get("near_duplicate_cos", 0.98)),
|
||||
"embedding_model": "openai/text-embedding-3-small" if vectors else None,
|
||||
"balance": {"per_label": base.k_ret // 2, "per_group": 2, "other_families": 2},
|
||||
"excerpt": {"candidate_lines": 80, "candidate_chars": 4000, "example_lines": 40, "example_chars": 2000},
|
||||
},
|
||||
"classify": {
|
||||
"model": base.model,
|
||||
"model_params_digest": caller.params_digest,
|
||||
"k": base.k,
|
||||
"temperature": [0.0, 0.7],
|
||||
"confidence": "vote_mean",
|
||||
"max_output_tokens": base.max_output_tokens,
|
||||
},
|
||||
"calibration": None,
|
||||
"operating_points": None,
|
||||
"priors": base.priors,
|
||||
"roles": "deterministic",
|
||||
"data": {
|
||||
"memory_snapshot_digest": memory_snapshot_digest(pool),
|
||||
"compile_groups_digest": hashlib.sha256("\n".join(sorted(split.groups)).encode()).hexdigest(),
|
||||
"folds_digest": folds_digest(split, evaluation),
|
||||
"sources": sorted({e.source for e in pool}),
|
||||
"examples": len(pool),
|
||||
"groups": len({e.group for e in pool}),
|
||||
},
|
||||
"compile": {
|
||||
"spec_sha256": spec.sha256,
|
||||
"metric": str(spec["metric"].get("name", "balanced_brier_v1")),
|
||||
"configurations": [c.as_dict() for c in configurations],
|
||||
"usd": caller.usd(),
|
||||
"requests": len(set(caller.keys)),
|
||||
"usage": dict(caller.usage),
|
||||
"response_cache_digest": caller.cache_digest(),
|
||||
"bootstrap": {"kept": len(boot_run.pool), "rejected": dict(sorted(boot_run.rejected.items())), "candidates": len(boot)},
|
||||
},
|
||||
"seed": seed,
|
||||
"code_commit": code_commit,
|
||||
"created": created,
|
||||
}
|
||||
artifact["program_id"] = program_id(artifact)
|
||||
if store is not None:
|
||||
store.put_blob("programs", canonical(artifact), name=artifact["program_id"])
|
||||
return CompileResult(
|
||||
"done", artifact, artifact["program_id"], caller.calls, caller.replayed, bootstrap=artifact["compile"]["bootstrap"]
|
||||
)
|
||||
|
||||
|
||||
def load_program(store: MemoryStore, pid: str) -> dict[str, Any]:
|
||||
data = store.get_blob("programs", pid, verify=False)
|
||||
if data is None:
|
||||
raise CompileError(f"no program {pid[:12]} in {store.describe()}")
|
||||
artifact = dict(json.loads(data))
|
||||
if program_id(artifact) != pid:
|
||||
raise CompileError(f"program {pid[:12]} does not hash to its id: the artifact changed")
|
||||
return artifact
|
||||
|
||||
|
||||
def spec_of(artifact: Mapping[str, Any]) -> ProgramSpec:
|
||||
retrieval, classify = artifact["retrieval"], artifact["classify"]
|
||||
return ProgramSpec(
|
||||
instruction=str(artifact["instruction"]),
|
||||
demos=tuple(Demo.from_dict(d) for d in artifact["demos"]),
|
||||
profile=str(artifact["profile"]),
|
||||
model=str(classify["model"]),
|
||||
k=int(classify["k"]),
|
||||
k_ret=int(retrieval["k"]),
|
||||
lam=float(retrieval["lam"]),
|
||||
n1=int(retrieval["n1"]),
|
||||
priors=str(retrieval.get("priors", "off")),
|
||||
max_output_tokens=int(classify.get("max_output_tokens", 300)),
|
||||
)
|
||||
|
||||
|
||||
def packageable(artifact: Mapping[str, Any], examples: Mapping[str, Example]) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
"""The artifact with only permissively licensed demonstrations: a demonstration from other code is replaced by
|
||||
its alternate when that one is permissive, dropped otherwise. A new program id; re-evaluate before adoption."""
|
||||
kept: list[dict[str, Any]] = []
|
||||
tally = {"kept": 0, "swapped": 0, "dropped": 0}
|
||||
for demo in artifact["demos"]:
|
||||
example = examples.get(str(demo["example_id"]))
|
||||
alternate = demo.get("alternate")
|
||||
alt_example = examples.get(str(alternate["example_id"])) if isinstance(alternate, Mapping) else None
|
||||
if example is not None and permissive(example.license):
|
||||
kept.append({**demo, "alternate": None})
|
||||
tally["kept"] += 1
|
||||
elif isinstance(alternate, Mapping) and alt_example is not None and permissive(alt_example.license):
|
||||
kept.append({**alternate, "alternate": None})
|
||||
tally["swapped"] += 1
|
||||
else:
|
||||
tally["dropped"] += 1
|
||||
packaged = {k: v for k, v in artifact.items() if k != "program_id"}
|
||||
packaged["demos"] = kept
|
||||
packaged["packaged"] = {"from": artifact.get("program_id"), "licences": list(PERMISSIVE), **tally}
|
||||
packaged["program_id"] = program_id(packaged)
|
||||
return packaged, tally
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_SPEC", "PERMISSIVE", "Bootstrap", "CompileError", "CompileResult", "CompileSpec", "Scored", "balanced_brier", "best",
|
||||
"bootstrap", "canonical", "compile_program", "draw_demo_sets", "group_identities", "identity_hits", "load_program", "load_spec",
|
||||
"memory_snapshot_digest", "packageable", "permissive", "program_id", "propose", "score", "secondary", "spec_of",
|
||||
] # fmt: skip
|
||||
@@ -0,0 +1,44 @@
|
||||
# The pre-registered compile specification of the decision engine's program (learned-decision-engine design 4.3).
|
||||
# Committed before any compile runs: the metric, the seeds, the search space and the budget are fixed here, and the
|
||||
# compiled artifact records this file's sha256. Changing anything below after looking at a result is a new spec.
|
||||
version = 1
|
||||
seed = 0
|
||||
|
||||
[metric]
|
||||
name = "balanced_brier_v1" # Brier score of the raw score s on C_val, each (family, label) cell weighted equally
|
||||
repository_cap = 5 # each repository counts at most 5 effective rows per cell
|
||||
unsure_score = 0.5 # unsure and parse_failed answers score as s = 0.5
|
||||
tie_tolerance = 0.005 # within this, fewer prompt tokens wins, then the earlier configuration
|
||||
secondary = ["recall_at_precision_0.90", "unsure_share"] # reported, never optimised
|
||||
|
||||
[split]
|
||||
compile_fraction = 0.25 # of the repository groups, per primary family; never an evaluation fold
|
||||
boot_fraction = 0.6 # C_boot (demonstrations) vs C_val (scoring)
|
||||
retrieval = "compile_only" # during the compile, memory is the compile split only: no evaluated group shapes it
|
||||
|
||||
[bootstrap]
|
||||
max_candidates = 150
|
||||
min_confidence = 0.7
|
||||
demo_sets = 4
|
||||
demos_per_set = 3 # one positive, one negative, one of either; families mixed
|
||||
|
||||
[search]
|
||||
proposals = 6 # instructions proposed in one call, scored with the hand-written baseline (7)
|
||||
errors_shown = 20 # summarised bootstrap errors in the proposal prompt (signals and verdicts, no code)
|
||||
val_candidates = 60
|
||||
|
||||
[classify]
|
||||
model = "deepseek-flash"
|
||||
k = 1 # k and lam are A/B variants (design section 6), not searched here
|
||||
max_output_tokens = 300
|
||||
|
||||
[retrieval]
|
||||
n1 = 40
|
||||
k = 6
|
||||
lam = 0.5
|
||||
near_duplicate_cos = 0.98
|
||||
priors = "off"
|
||||
|
||||
[budget]
|
||||
usd = 3.0 # a compile that reaches it is `unfinished` and writes no artifact
|
||||
calls = 1200
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Evaluation of a compiled program on repositories it never saw (learned-decision-engine design 4.4-4.5, Req 5.2,
|
||||
7.3, 7.4).
|
||||
|
||||
For each evaluation fold the program decides every candidate of the fold's groups with memory and demonstrations
|
||||
filtered by the fold's boundary. The raw scores are then **cross-fitted** (:mod:`.calibrate`), and the operating
|
||||
points are chosen on the *other* folds' calibrated predictions (nested), never on the fold reported:
|
||||
|
||||
- **BLOCK**: the lowest threshold whose precision has a Wilson 95% lower bound >= 0.95 (delta units when the data
|
||||
has them), counting only candidates whose majority verdict is not ``unsure``; none -> ``precision_unreachable``;
|
||||
- **ADVISORY**: the highest threshold whose recall is >= 0.90; its precision is reported, not assumed;
|
||||
- **DROP** below ADVISORY.
|
||||
|
||||
Metrics carry repository-cluster bootstrap 95% intervals. A family or language below the pooled floor of positive
|
||||
and negative repository groups is reported as ``insufficient data``, never as a number. More than 5% parse failures
|
||||
make the run an instrument failure, not a result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import random
|
||||
import re
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
from .calibrate import Row, check, cross_fit, offered
|
||||
from .examples import Example
|
||||
from .folds import Fold
|
||||
from .labels import PER_FAMILY_FLOOR, POOLED_FLOOR
|
||||
from .program import Caller, Candidate, Program
|
||||
from .retrieve import Target
|
||||
from .schema import BY_NAME, SCHEMA_VERSION, feature_set_digest
|
||||
|
||||
TARGET_PRECISION = 0.95
|
||||
TARGET_RECALL = 0.90
|
||||
PARSE_FAILURE_LIMIT = 0.05
|
||||
BOOTSTRAP = 2000
|
||||
_SIGNAL = re.compile(r"\b[a-z]+(?:\.[a-z_]+)+\b")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Prediction:
|
||||
fold: str
|
||||
family: str
|
||||
group: str
|
||||
label: int
|
||||
s: float
|
||||
verdict: str
|
||||
unit: str = "pin"
|
||||
language: str = "other"
|
||||
source: str = ""
|
||||
parse_failed: int = 0
|
||||
p: float | None = None
|
||||
point: str | None = None # BLOCK | ADVISORY | DROP, from points chosen on the other folds
|
||||
|
||||
@property
|
||||
def blockable(self) -> bool:
|
||||
return self.verdict != "unsure"
|
||||
|
||||
|
||||
def wilson_lower(successes: int, n: int, z: float = 1.959964) -> float:
|
||||
if n == 0:
|
||||
return 0.0
|
||||
phat = successes / n
|
||||
centre = phat + z * z / (2 * n)
|
||||
margin = z * math.sqrt(phat * (1 - phat) / n + z * z / (4 * n * n))
|
||||
return (centre - margin) / (1 + z * z / n)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Points:
|
||||
block: float | None
|
||||
block_offered: bool
|
||||
block_reason: str | None
|
||||
advisory: float | None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"block": {
|
||||
"threshold": self.block,
|
||||
"offered": self.block_offered,
|
||||
"reason": self.block_reason,
|
||||
"target_precision": TARGET_PRECISION,
|
||||
},
|
||||
"advisory": {"threshold": self.advisory, "target_recall": TARGET_RECALL},
|
||||
}
|
||||
|
||||
|
||||
def choose_points(preds: Sequence[Prediction], *, precision: float = TARGET_PRECISION, recall: float = TARGET_RECALL) -> Points:
|
||||
"""Operating points on calibrated predictions (``p`` set); rows without ``p`` are not decided."""
|
||||
rows = [r for r in preds if r.p is not None]
|
||||
thresholds = sorted({float(r.p) for r in rows if r.p is not None})
|
||||
deltas = [r for r in rows if r.unit == "delta"]
|
||||
blockable = [r for r in (deltas or rows) if r.blockable]
|
||||
block = None
|
||||
for t in thresholds:
|
||||
chosen = [r for r in blockable if (r.p or 0.0) >= t]
|
||||
if chosen and wilson_lower(sum(r.label for r in chosen), len(chosen)) >= precision:
|
||||
block = t
|
||||
break
|
||||
positives = sum(r.label for r in rows)
|
||||
advisory = None
|
||||
for t in reversed(thresholds):
|
||||
if positives and sum(r.label for r in rows if (r.p or 0.0) >= t) / positives >= recall:
|
||||
advisory = t
|
||||
break
|
||||
return Points(block, block is not None, None if block is not None else "precision_unreachable", advisory)
|
||||
|
||||
|
||||
def nested_points(preds: Sequence[Prediction]) -> tuple[list[Prediction], dict[str, Points]]:
|
||||
"""Each fold's rows decided with the points chosen on the other folds' calibrated predictions."""
|
||||
points = {fold: choose_points([r for r in preds if r.fold != fold]) for fold in sorted({r.fold for r in preds})}
|
||||
out = []
|
||||
for r in preds:
|
||||
pts = points[r.fold]
|
||||
if r.p is None:
|
||||
out.append(r)
|
||||
continue
|
||||
if pts.block is not None and r.blockable and r.p >= pts.block:
|
||||
point = "BLOCK"
|
||||
elif pts.advisory is not None and r.p >= pts.advisory:
|
||||
point = "ADVISORY"
|
||||
else:
|
||||
point = "DROP"
|
||||
out.append(replace(r, point=point))
|
||||
return out, points
|
||||
|
||||
|
||||
def _at(rows: Sequence[Prediction], level: str) -> dict[str, float | None]:
|
||||
flagged = [r for r in rows if r.point in (("BLOCK",) if level == "block" else ("BLOCK", "ADVISORY"))]
|
||||
positives = sum(r.label for r in rows)
|
||||
benign = [r for r in rows if r.label == 0 and r.unit == "delta"]
|
||||
return {
|
||||
"recall": sum(r.label for r in flagged) / positives if positives else None,
|
||||
"precision": sum(r.label for r in flagged) / len(flagged) if flagged else None,
|
||||
"false_per_100_benign_deltas": 100 * sum(1 for r in benign if r.point == "BLOCK") / len(benign)
|
||||
if benign and level == "block"
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
def cluster_bootstrap(
|
||||
rows: Sequence[Prediction], stat: Callable[[Sequence[Prediction]], float | None], *, resamples: int = BOOTSTRAP, seed: int = 0
|
||||
) -> tuple[float, float] | None:
|
||||
by_group: dict[str, list[Prediction]] = {}
|
||||
for r in rows:
|
||||
by_group.setdefault(r.group, []).append(r)
|
||||
names = sorted(by_group)
|
||||
if not names:
|
||||
return None
|
||||
rng = random.Random(seed)
|
||||
values = []
|
||||
for _ in range(resamples):
|
||||
sample = [r for g in (rng.choice(names) for _ in names) for r in by_group[g]]
|
||||
value = stat(sample)
|
||||
if value is not None:
|
||||
values.append(value)
|
||||
if not values:
|
||||
return None
|
||||
values.sort()
|
||||
return values[int(0.025 * (len(values) - 1))], values[int(0.975 * (len(values) - 1))]
|
||||
|
||||
|
||||
def summary(rows: Sequence[Prediction], *, resamples: int = BOOTSTRAP, seed: int = 0) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {"candidates": len(rows), "groups": len({r.group for r in rows})}
|
||||
|
||||
def statistic(level: str, name: str) -> Callable[[Sequence[Prediction]], float | None]:
|
||||
return lambda sample: _at(sample, level)[name]
|
||||
|
||||
for level in ("block", "advisory"):
|
||||
point = _at(rows, level)
|
||||
out[level] = {
|
||||
name: {"value": value, "ci": cluster_bootstrap(rows, statistic(level, name), resamples=resamples, seed=seed)}
|
||||
for name, value in point.items()
|
||||
if value is not None
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def floor_status(rows: Sequence[Prediction]) -> str:
|
||||
pos = len({r.group for r in rows if r.label == 1})
|
||||
neg = len({r.group for r in rows if r.label == 0})
|
||||
level = min(pos, neg)
|
||||
return "per_family" if level >= PER_FAMILY_FLOOR else "pooled" if level >= POOLED_FLOOR else "insufficient data"
|
||||
|
||||
|
||||
def strata(rows: Sequence[Prediction], key: Callable[[Prediction], str], **kw: Any) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
for name in sorted({key(r) for r in rows}):
|
||||
part = [r for r in rows if key(r) == name]
|
||||
status = floor_status(part)
|
||||
out[name] = {"status": status} if status == "insufficient data" else {"status": status, **summary(part, **kw)}
|
||||
return out
|
||||
|
||||
|
||||
def explain(rationale: str, x: Mapping[str, Any]) -> list[str]:
|
||||
"""The signals the rationale names that the record holds (a name it invents is dropped)."""
|
||||
named = dict.fromkeys(m.group(0) for m in _SIGNAL.finditer(rationale))
|
||||
return [n for n in named if n in BY_NAME and n in x and x[n] is not None]
|
||||
|
||||
|
||||
def incompatible(artifact: Mapping[str, Any], record: Mapping[str, Any], signature_digest: str) -> str | None:
|
||||
"""Why a program may not decide a record (the caller falls back to today's behaviour and says so)."""
|
||||
if artifact.get("schema_version") != SCHEMA_VERSION or record.get("schema_version") != SCHEMA_VERSION:
|
||||
return "schema version differs"
|
||||
if artifact.get("feature_set_digest") != feature_set_digest() or record.get("feature_set_digest") != feature_set_digest():
|
||||
return "feature set differs"
|
||||
if (artifact.get("signature") or {}).get("digest") != signature_digest:
|
||||
return "signature differs"
|
||||
if artifact.get("profile") != record.get("profile"):
|
||||
return "profile differs"
|
||||
return None
|
||||
|
||||
|
||||
def targets(folds: Sequence[Fold], examples: Sequence[Example], *, skip: Iterable[str] = ()) -> list[tuple[Example, Fold]]:
|
||||
"""(example, fold) for every example of every usable fold's evaluated groups; ``skip`` families are never asked
|
||||
(the families below the floor: no money is spent on them)."""
|
||||
skipped = set(skip)
|
||||
return [(e, f) for f in folds if f.status == "ok" for e in examples if e.group in f.eval_groups and e.family not in skipped]
|
||||
|
||||
|
||||
def predict(
|
||||
program: Program,
|
||||
folds: Sequence[Fold],
|
||||
caller: Caller,
|
||||
candidate_of: Callable[[Any], Candidate | None],
|
||||
*,
|
||||
seed: int = 0,
|
||||
pairs: Sequence[tuple[Example, Fold]] | None = None,
|
||||
) -> list[Prediction]:
|
||||
"""Decide every example of every fold's evaluated groups (or the given ``pairs``), each under its own fold."""
|
||||
out: list[Prediction] = []
|
||||
for example, fold in pairs if pairs is not None else targets(folds, program.memory):
|
||||
candidate = candidate_of(example)
|
||||
if candidate is None:
|
||||
continue
|
||||
decision = program.decide(candidate, fold, caller, seed=seed)
|
||||
out.append(
|
||||
Prediction(
|
||||
fold.name, example.family, example.group, example.label, decision.s, decision.verdict, example.unit, example.language,
|
||||
example.source, decision.parse_failed,
|
||||
)
|
||||
) # fmt: skip
|
||||
return out
|
||||
|
||||
|
||||
def unevaluable_families(examples: Sequence[Example], *, floor: int = POOLED_FLOOR) -> list[str]:
|
||||
"""Families below the pooled floor of positive and negative repository groups: reported, never evaluated."""
|
||||
out = []
|
||||
for family in sorted({e.family for e in examples}):
|
||||
pos = len({e.group for e in examples if e.family == family and e.label == 1})
|
||||
neg = len({e.group for e in examples if e.family == family and e.label == 0})
|
||||
if min(pos, neg) < floor:
|
||||
out.append(family)
|
||||
return out
|
||||
|
||||
|
||||
def learning_curve(
|
||||
program: Program,
|
||||
pairs: Sequence[tuple[Example, Fold]],
|
||||
caller: Caller,
|
||||
candidate_of: Callable[[Any], Candidate | None],
|
||||
*,
|
||||
fractions: Sequence[float] = (0.0, 0.25, 0.5, 1.0),
|
||||
seed: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""The metric as a function of memory size: a seeded share of the memory's groups, the same fixed candidates."""
|
||||
from .compile import balanced_brier
|
||||
|
||||
groups = sorted({e.group for e in program.memory}, key=lambda g: hashlib.sha256(f"{seed}\x00curve\x00{g}".encode()).hexdigest())
|
||||
points = []
|
||||
for fraction in fractions:
|
||||
kept = set(groups[: int(len(groups) * fraction + 0.5)])
|
||||
sub = replace(program, memory=[e for e in program.memory if e.group in kept])
|
||||
preds = predict(sub, [], caller, candidate_of, seed=seed, pairs=pairs)
|
||||
brier = balanced_brier([(p.family, p.label, p.group, p.s) for p in preds])
|
||||
points.append({"memory_fraction": fraction, "memory_groups": len(kept), "candidates": len(preds), "balanced_brier": brier})
|
||||
return points
|
||||
|
||||
|
||||
def evaluate(preds: Sequence[Prediction], *, resamples: int = BOOTSTRAP, seed: int = 0, **floors: int) -> dict[str, Any]:
|
||||
"""Cross-fitted calibration, its check, nested operating points and the report (counts and intervals only)."""
|
||||
failed = sum(r.parse_failed for r in preds)
|
||||
answers = len(preds) or 1
|
||||
rows = [Row(r.fold, r.family, r.group, r.label, r.s) for r in preds]
|
||||
ps, maps = cross_fit(rows, **floors)
|
||||
calibrated = [replace(r, p=p) for r, p in zip(preds, ps, strict=True)]
|
||||
decided = [r for r in calibrated if r.p is not None]
|
||||
checked = (
|
||||
check([float(r.p) for r in decided if r.p is not None], [r.label for r in decided], [r.group for r in decided], seed=seed)
|
||||
if decided
|
||||
else None
|
||||
)
|
||||
nested, points = nested_points(calibrated)
|
||||
report: dict[str, Any] = {
|
||||
"status": "instrument failure" if failed / answers > PARSE_FAILURE_LIMIT else "ok",
|
||||
"parse_failed_share": round(failed / answers, 4),
|
||||
"calibration": checked.as_dict() if checked else None,
|
||||
"offered": offered(checked) if checked else {"block": {"offered": False, "reason": "insufficient data"}},
|
||||
"points_by_fold": {f: p.as_dict() for f, p in points.items()},
|
||||
"maps_by_fold": {f: m.as_dict() for f, m in maps.items()},
|
||||
"overall": summary([r for r in nested if r.p is not None], resamples=resamples, seed=seed),
|
||||
"by_family": strata(nested, lambda r: r.family, resamples=resamples, seed=seed),
|
||||
"by_language": strata(nested, lambda r: r.language, resamples=resamples, seed=seed),
|
||||
"undecided": sum(1 for r in nested if r.p is None),
|
||||
}
|
||||
if checked is not None and not checked.holds:
|
||||
report["overall"]["block"] = {"offered": False, "reason": "calibration"}
|
||||
return report
|
||||
|
||||
|
||||
def candidate_from_example(
|
||||
excerpt_text: Callable[[str], str | None], vectors: Mapping[str, Sequence[float]] | None = None
|
||||
) -> Callable[[Any], Candidate | None]:
|
||||
"""An evaluation candidate from a memory example: its record and stored excerpt (a 40-line example excerpt)."""
|
||||
|
||||
def make(example: Any) -> Candidate | None:
|
||||
code = excerpt_text(example.excerpt_sha)
|
||||
if not code:
|
||||
return None
|
||||
return Candidate(Target.of(example, (vectors or {}).get(example.excerpt_sha)), code, example.language, tuple(example.roles))
|
||||
|
||||
return make
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PARSE_FAILURE_LIMIT", "TARGET_PRECISION", "TARGET_RECALL", "Points", "Prediction", "candidate_from_example", "choose_points",
|
||||
"cluster_bootstrap", "evaluate", "explain", "floor_status", "incompatible", "learning_curve", "nested_points", "predict", "strata",
|
||||
"summary", "targets", "unevaluable_families",
|
||||
"wilson_lower",
|
||||
] # fmt: skip
|
||||
Reference in New Issue
Block a user