mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
7f6bbacb6e
learn/compile.toml: the pre-registered spec -- balanced Brier score of the raw score on C_val (cells by family x label, repositories capped at 5, unsure = 0.5, ties within 0.005 to fewer prompt tokens), seed, the compile split (25%, boot 60/40, compile-split-only memory), bootstrap (150 candidates, confidence >= 0.7, 4 sets of 3), search (6 proposals + baseline, 20 errors shown, 60 validation candidates), k = 1, the retrieval settings, and the $3 / 1,200-call ceiling. learn/compile.py: bootstrap few-shot keeps only right, confident rationales that cite a line and name no repository, path or CVE/GHSA id; seeded demo sets (positive, negative, any; families mixed) with other-framework alternates; one proposal call; 7 instructions scored with set 1, the best with sets 2-4. The artifact is canonical JSON stored as programs/<program_id>.json, program_id = sha256 of the artifact without its id; it records the spec sha, memory/compile-groups/folds digests and the response-cache digest, with spend priced from cached usage, so a rerun from the cache is byte-identical at $0 (tested on FileStore and MinioStore through the fake client; real MinIO gated as today). A ceiling ends the compile `unfinished` with no artifact. packageable() keeps only permissively licensed demonstrations (MIT, BSD, Apache-2.0, ISC, zlib), swapping in a permissive alternate or dropping, under a new program id. learn/calibrate.py: Platt scaling on logit(s), pure Python Newton, per family / pooled with family intercepts / insufficient data by the floors; cross-fitted (fold j's map is fitted on the other folds only); the check (5-bin reliability, slope and intercept with repository-bootstrap CIs, ECE <= 0.05), ADVISORY only when it fails. learn/evaluate.py: predictions under each fold's boundary, families below the floor never asked; operating points read from the calibrated, cross-fitted predictions of the other folds (BLOCK: Wilson lower bound >= 0.95, never on a majority unsure; ADVISORY: recall >= 0.90); repository-cluster bootstrap intervals; strata with "insufficient data"; >5% parse failures is an instrument failure; explanations keep only signals the record holds; version mismatch names the fallback reason; the memory-size learning curve. `ousast learn compile|evaluate|curve`: paid runs need --budget-usd (exit 3 at the ceiling), --replay-only runs from the cache. Nothing was spent: every test and the dry run used scripted clients. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
63 lines
3.3 KiB
Python
63 lines
3.3 KiB
Python
"""Calibration (learned-decision-engine design 4.4, task 6.5): Platt on a known miscalibration, cross-fitting (fold
|
|
j's map never saw fold j), the slope/intercept/ECE check, and the ADVISORY-only downgrade."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import random
|
|
|
|
import pytest
|
|
|
|
from openultrasast.learn.calibrate import Row, check, cross_fit, fit_map, fit_platt, logit, offered, sigmoid
|
|
|
|
|
|
def _sample(n: int, a: float, b: float, seed: int = 0) -> tuple[list[float], list[int]]:
|
|
"""Scores whose true probability is ``sigmoid(a logit(s) + b)``: a known miscalibration of ``s``."""
|
|
rng = random.Random(seed)
|
|
scores = [rng.uniform(0.02, 0.98) for _ in range(n)]
|
|
labels = [1 if rng.random() < sigmoid(a * logit(s) + b) else 0 for s in scores]
|
|
return scores, labels
|
|
|
|
|
|
def test_platt_recovers_a_known_miscalibration() -> None:
|
|
scores, labels = _sample(6000, 2.0, -0.5)
|
|
a, b = fit_platt(scores, labels)
|
|
assert a == pytest.approx(2.0, abs=0.15) and b == pytest.approx(-0.5, abs=0.15)
|
|
|
|
|
|
def _rows(n_groups: int, a: float, b: float, *, family: str = "injection", folds: int = 5, seed: int = 0) -> list[Row]:
|
|
scores, labels = _sample(n_groups * 10, a, b, seed)
|
|
return [Row(f"f{(i // 10) % folds}", family, f"g{i // 10}", y, s) for i, (s, y) in enumerate(zip(scores, labels, strict=True))]
|
|
|
|
|
|
def test_cross_fitting_never_fits_a_fold_on_itself() -> None:
|
|
rows = _rows(100, 2.0, 0.0)
|
|
ps, maps = cross_fit(rows, per_family_floor=10, pooled_floor=5)
|
|
assert all(p is not None for p in ps)
|
|
for fold, mapping in maps.items():
|
|
expected = fit_map([r for r in rows if r.fold != fold], per_family_floor=10, pooled_floor=5)
|
|
assert mapping == expected
|
|
# moving fold f0's labels changes every map except f0's own
|
|
flipped = [Row(r.fold, r.family, r.group, 1 - r.label if r.fold == "f0" else r.label, r.s) for r in rows]
|
|
_, flipped_maps = cross_fit(flipped, per_family_floor=10, pooled_floor=5)
|
|
assert flipped_maps["f0"] == maps["f0"] and flipped_maps["f1"] != maps["f1"]
|
|
|
|
|
|
def test_floors_per_family_pooled_and_insufficient() -> None:
|
|
rows = _rows(60, 1.0, 0.0, family="injection") + _rows(12, 1.0, 0.3, family="path", seed=1) + _rows(3, 1.0, 0, family="memory", seed=2)
|
|
mapping = fit_map(rows, per_family_floor=20, pooled_floor=5)
|
|
assert set(mapping.per_family) == {"injection"} and set(mapping.intercepts) == {"path"} and mapping.insufficient == ("memory",)
|
|
assert mapping.apply("memory", 0.7) is None and mapping.apply("path", 0.7) is not None
|
|
|
|
|
|
def test_the_check_holds_on_calibrated_scores_and_fails_otherwise() -> None:
|
|
scores, labels = _sample(1500, 1.0, 0.0, seed=3)
|
|
groups = [f"g{i // 10}" for i in range(len(scores))]
|
|
good = check(scores, labels, groups, resamples=40)
|
|
assert good.holds and good.ece <= 0.05 and good.slope_ci[0] <= 1 <= good.slope_ci[1]
|
|
assert offered(good) == {"block": {"offered": True, "reason": None}, "advisory": {"offered": True, "reason": None}}
|
|
overconfident = [min(0.99, max(0.01, 0.5 + 2.5 * (s - 0.5))) for s in scores]
|
|
bad = check(overconfident, labels, groups, resamples=40)
|
|
assert not bad.holds and bad.slope < 1
|
|
assert offered(bad)["block"] == {"offered": False, "reason": "calibration"} and offered(bad)["advisory"]["offered"]
|
|
assert len(good.reliability) == 5 and sum(b["n"] for b in good.reliability) == 1500
|