feat(learn): compile, calibrate, evaluate (decision-engine 6.5)

learn/compile.toml: the pre-registered spec -- balanced Brier score of the
raw score on C_val (cells by family x label, repositories capped at 5,
unsure = 0.5, ties within 0.005 to fewer prompt tokens), seed, the
compile split (25%, boot 60/40, compile-split-only memory), bootstrap
(150 candidates, confidence >= 0.7, 4 sets of 3), search (6 proposals +
baseline, 20 errors shown, 60 validation candidates), k = 1, the
retrieval settings, and the $3 / 1,200-call ceiling.

learn/compile.py: bootstrap few-shot keeps only right, confident
rationales that cite a line and name no repository, path or CVE/GHSA id;
seeded demo sets (positive, negative, any; families mixed) with
other-framework alternates; one proposal call; 7 instructions scored with
set 1, the best with sets 2-4. The artifact is canonical JSON stored as
programs/<program_id>.json, program_id = sha256 of the artifact without
its id; it records the spec sha, memory/compile-groups/folds digests and
the response-cache digest, with spend priced from cached usage, so a
rerun from the cache is byte-identical at $0 (tested on FileStore and
MinioStore through the fake client; real MinIO gated as today). A ceiling
ends the compile `unfinished` with no artifact. packageable() keeps only
permissively licensed demonstrations (MIT, BSD, Apache-2.0, ISC, zlib),
swapping in a permissive alternate or dropping, under a new program id.

learn/calibrate.py: Platt scaling on logit(s), pure Python Newton, per
family / pooled with family intercepts / insufficient data by the floors;
cross-fitted (fold j's map is fitted on the other folds only); the check
(5-bin reliability, slope and intercept with repository-bootstrap CIs,
ECE <= 0.05), ADVISORY only when it fails.

learn/evaluate.py: predictions under each fold's boundary, families below
the floor never asked; operating points read from the calibrated,
cross-fitted predictions of the other folds (BLOCK: Wilson lower bound
>= 0.95, never on a majority unsure; ADVISORY: recall >= 0.90);
repository-cluster bootstrap intervals; strata with "insufficient data";
>5% parse failures is an instrument failure; explanations keep only
signals the record holds; version mismatch names the fallback reason;
the memory-size learning curve.

`ousast learn compile|evaluate|curve`: paid runs need --budget-usd (exit
3 at the ceiling), --replay-only runs from the cache. Nothing was spent:
every test and the dry run used scripted clients.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_learn_calibrate.py

@@ -0,0 +1,62 @@
+"""Calibration (learned-decision-engine design 4.4, task 6.5): Platt on a known miscalibration, cross-fitting (fold
+j's map never saw fold j), the slope/intercept/ECE check, and the ADVISORY-only downgrade."""
+
+from __future__ import annotations
+
+import random
+
+import pytest
+
+from openultrasast.learn.calibrate import Row, check, cross_fit, fit_map, fit_platt, logit, offered, sigmoid
+
+
+def _sample(n: int, a: float, b: float, seed: int = 0) -> tuple[list[float], list[int]]:
+    """Scores whose true probability is ``sigmoid(a logit(s) + b)``: a known miscalibration of ``s``."""
+    rng = random.Random(seed)
+    scores = [rng.uniform(0.02, 0.98) for _ in range(n)]
+    labels = [1 if rng.random() < sigmoid(a * logit(s) + b) else 0 for s in scores]
+    return scores, labels
+
+
+def test_platt_recovers_a_known_miscalibration() -> None:
+    scores, labels = _sample(6000, 2.0, -0.5)
+    a, b = fit_platt(scores, labels)
+    assert a == pytest.approx(2.0, abs=0.15) and b == pytest.approx(-0.5, abs=0.15)
+
+
+def _rows(n_groups: int, a: float, b: float, *, family: str = "injection", folds: int = 5, seed: int = 0) -> list[Row]:
+    scores, labels = _sample(n_groups * 10, a, b, seed)
+    return [Row(f"f{(i // 10) % folds}", family, f"g{i // 10}", y, s) for i, (s, y) in enumerate(zip(scores, labels, strict=True))]
+
+
+def test_cross_fitting_never_fits_a_fold_on_itself() -> None:
+    rows = _rows(100, 2.0, 0.0)
+    ps, maps = cross_fit(rows, per_family_floor=10, pooled_floor=5)
+    assert all(p is not None for p in ps)
+    for fold, mapping in maps.items():
+        expected = fit_map([r for r in rows if r.fold != fold], per_family_floor=10, pooled_floor=5)
+        assert mapping == expected
+    # moving fold f0's labels changes every map except f0's own
+    flipped = [Row(r.fold, r.family, r.group, 1 - r.label if r.fold == "f0" else r.label, r.s) for r in rows]
+    _, flipped_maps = cross_fit(flipped, per_family_floor=10, pooled_floor=5)
+    assert flipped_maps["f0"] == maps["f0"] and flipped_maps["f1"] != maps["f1"]
+
+
+def test_floors_per_family_pooled_and_insufficient() -> None:
+    rows = _rows(60, 1.0, 0.0, family="injection") + _rows(12, 1.0, 0.3, family="path", seed=1) + _rows(3, 1.0, 0, family="memory", seed=2)
+    mapping = fit_map(rows, per_family_floor=20, pooled_floor=5)
+    assert set(mapping.per_family) == {"injection"} and set(mapping.intercepts) == {"path"} and mapping.insufficient == ("memory",)
+    assert mapping.apply("memory", 0.7) is None and mapping.apply("path", 0.7) is not None
+
+
+def test_the_check_holds_on_calibrated_scores_and_fails_otherwise() -> None:
+    scores, labels = _sample(1500, 1.0, 0.0, seed=3)
+    groups = [f"g{i // 10}" for i in range(len(scores))]
+    good = check(scores, labels, groups, resamples=40)
+    assert good.holds and good.ece <= 0.05 and good.slope_ci[0] <= 1 <= good.slope_ci[1]
+    assert offered(good) == {"block": {"offered": True, "reason": None}, "advisory": {"offered": True, "reason": None}}
+    overconfident = [min(0.99, max(0.01, 0.5 + 2.5 * (s - 0.5))) for s in scores]
+    bad = check(overconfident, labels, groups, resamples=40)
+    assert not bad.holds and bad.slope < 1
+    assert offered(bad)["block"] == {"offered": False, "reason": "calibration"} and offered(bad)["advisory"]["offered"]
+    assert len(good.reliability) == 5 and sum(b["n"] for b in good.reliability) == 1500

tests/test_learn_compile.py

@@ -0,0 +1,193 @@
+"""Compilation (learned-decision-engine design 4.3 and 5, task 6.5): bootstrap few-shot keeps only right, confident,
+identity-free demonstrations; instruction search on the pre-registered metric; the compile split stays out of every
+evaluation fold; a ceiling ends the compile unfinished; a rerun from the response cache reproduces the artifact
+byte for byte at $0 (``test_reproducible_from_cache``); the permissive-licence filter for packaged programs."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from pathlib import Path
+
+import pytest
+from learn_fixtures import ScriptedChat, by_id, corpus, default_answer, example
+from test_plane_memory import FakeObjects, _real_minio
+
+from openultrasast.learn import compile as compile_module
+from openultrasast.learn.compile import (
+    DEFAULT_SPEC,
+    CompileError,
+    Scored,
+    balanced_brier,
+    best,
+    canonical,
+    compile_program,
+    draw_demo_sets,
+    identity_hits,
+    load_program,
+    load_spec,
+    packageable,
+    permissive,
+    program_id,
+)
+from openultrasast.learn.folds import Fold, FoldError
+from openultrasast.learn.program import Caller, Demo
+from openultrasast.plane.budget import MeteredClient
+from openultrasast.plane.memory import FileStore, MemoryStore, MinioStore
+
+PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
+MEMORY = corpus(groups=24, per_group=4)
+CODE = {e.excerpt_sha: f"   10  def f(q):\n   11      return {'execute(q)' if e.label else 'escape(q)'}\n" for e in MEMORY}
+SPEC = load_spec(DEFAULT_SPEC)
+PROPOSALS = [f"Instruction {i}: be careful with the signals." if i == 3 else f"Instruction {i}." for i in range(1, 7)]
+
+
+def excerpt_text(sha: str) -> str | None:
+    return CODE.get(sha)
+
+
+def answer(messages: Sequence[Mapping[str, object]], temperature: float | None) -> str:
+    """Proposals on request; a better calibrated confidence under the 'careful' instruction, so the search moves."""
+    system = str(messages[0]["content"])
+    if system.startswith("Propose instructions"):
+        return json.dumps({"instructions": PROPOSALS})
+    data = json.loads(default_answer(messages, temperature))
+    if "careful" in system:
+        data["confidence"] = 0.95
+    return json.dumps(data)
+
+
+@pytest.fixture(params=["file", "fake-minio", "minio"])
+def store(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[MemoryStore]:
+    if request.param == "file":
+        yield FileStore(tmp_path / "memory")
+    elif request.param == "minio":
+        yield from _real_minio()
+    else:
+        yield MinioStore(FakeObjects("works"), "bucket", "p")
+
+
+def _caller(chat: ScriptedChat, store: MemoryStore | None = None, **budget: object) -> Caller:
+    return Caller(MeteredClient(chat, prices=PRICES, **budget), "deepseek-flash", PRICES, store=store)  # type: ignore[arg-type]
+
+
+def _compile(caller: Caller, store: MemoryStore | None = None, memory: Sequence = MEMORY) -> compile_module.CompileResult:
+    return compile_program(memory, caller, excerpt_text, spec=SPEC, store=store, created="2026-10-01", code_commit="test")
+
+
+def test_reproducible_from_cache(store: MemoryStore) -> None:
+    first_chat = ScriptedChat(answer)
+    first = _compile(_caller(first_chat, store), store)
+    assert first.status == "done" and first.calls > 0 and first.calls == len(first_chat.calls)
+    second_chat = ScriptedChat(answer)
+    second = _compile(_caller(second_chat, store), store)
+    assert second.calls == 0 and second_chat.calls == [] and second.replayed > 0
+    assert first.artifact is not None and second.artifact is not None
+    assert canonical(first.artifact) == canonical(second.artifact) and first.program_id == second.program_id
+    assert load_program(store, str(first.program_id)) == first.artifact
+    assert first.artifact["compile"]["usd"] == second.artifact["compile"]["usd"] > 0
+
+
+def test_the_search_picks_a_better_instruction_and_records_every_configuration() -> None:
+    result = _compile(_caller(ScriptedChat(answer)))
+    artifact = result.artifact
+    assert artifact is not None and artifact["instruction_source"] == "proposed:3" and "careful" in artifact["instruction"]
+    configurations = artifact["compile"]["configurations"]
+    assert len(configurations) == 7 + 3  # 7 instructions with demo set 1, then the best with sets 2-4
+    baseline = configurations[0]["score"]
+    assert min(c["score"] for c in configurations) < baseline
+    assert artifact["compile"]["spec_sha256"] == SPEC.sha256 and artifact["program_id"] == program_id(artifact)
+    assert artifact["data"]["folds_digest"] and len(artifact["demos"]) == 3
+
+
+def test_bootstrap_keeps_only_right_confident_identity_free_rationales() -> None:
+    def naming(messages: Sequence[Mapping[str, object]], temperature: float | None) -> str:
+        if str(messages[0]["content"]).startswith("Propose"):
+            return json.dumps({"instructions": []})
+        data = json.loads(default_answer(messages, temperature))
+        tail = str(messages[-1]["content"]).rsplit("Candidate:", 1)[1]
+        if "execute" in tail:
+            data["rationale"] = "As in CVE-2021-1234 and app/views.py, the value reaches execute."
+        return json.dumps(data)
+
+    def timid(messages: Sequence[Mapping[str, object]], temperature: float | None) -> str:
+        return json.dumps({**json.loads(default_answer(messages, temperature)), "confidence": 0.5})
+
+    with pytest.raises(CompileError, match="kept no demonstration"):
+        _compile(_caller(ScriptedChat(timid)))
+    result = _compile(_caller(ScriptedChat(naming)))
+    assert result.artifact is not None
+    rejected = result.artifact["compile"]["bootstrap"]["rejected"]
+    assert rejected.get("rationale names an identity", 0) > 0
+    for demo in result.artifact["demos"]:
+        assert identity_hits(demo["rationale"]) == [] and demo["label"] == 0  # every positive named an identity
+
+
+def test_identity_scan() -> None:
+    assert identity_hits("See CVE-2020-0001 here.") == ["CVE-2020-0001"]
+    assert identity_hits("Line 4 of views.py passes q.") == ["views.py"]
+    assert identity_hits("the acme helper", ("acme/webapp", "acme")) == ["acme"]
+    assert identity_hits("Line 11 passes the parameter to the query.", ("acme/webapp", "acme")) == []
+
+
+def test_the_compile_split_never_meets_an_evaluation_fold(monkeypatch: pytest.MonkeyPatch) -> None:
+    real = compile_module.outer_folds
+
+    def leaking(examples, **kw):  # type: ignore[no-untyped-def]
+        folds = real(examples, **kw)
+        return [*folds, Fold("outer", "leak", frozenset({e.group for e in examples}))]
+
+    monkeypatch.setattr(compile_module, "outer_folds", leaking)
+    with pytest.raises(FoldError):
+        _compile(_caller(ScriptedChat(answer)))
+
+
+def test_a_ceiling_ends_the_compile_unfinished_with_no_artifact(tmp_path: Path) -> None:
+    store = FileStore(tmp_path / "memory")
+    result = _compile(_caller(ScriptedChat(answer), store, budget_calls=5), store)
+    assert result.status == "unfinished" and result.artifact is None and result.program_id is None
+    assert "calls budget exhausted" in str(result.reason) and store.blob_names("programs") == []
+
+
+def test_balanced_brier_weights_cells_and_caps_repositories() -> None:
+    rows = [("a", 1, "g1", 1.0)] * 10 + [("a", 1, "g2", 0.0)] + [("a", 0, "g3", 0.0)]
+    # cell (a, 1): g1 has 10 rows of error 0 (capped to weight 0.5 each = 5), g2 one row of error 1 -> 1/6
+    assert balanced_brier(rows, repository_cap=5) == pytest.approx((1 / 6 + 0.0) / 2)
+    assert balanced_brier([]) is None
+
+
+def test_ties_go_to_fewer_prompt_tokens() -> None:
+    a = Scored(0, 0, 0.100, {}, 900)
+    b = Scored(1, 0, 0.103, {}, 500)
+    c = Scored(2, 0, 0.090, {}, 2000)
+    assert best([a, b], 0.005) == b and best([a, b, c], 0.005) == c
+
+
+def test_demo_sets_mix_labels_and_carry_framework_alternates() -> None:
+    examples = [example(f"d{i}", group=f"g{i}/r", label=i % 2, family=("injection", "path")[i % 3 == 0],
+                        frameworks=("flask",) if i < 4 else ()) for i in range(12)]  # fmt: skip
+    pool = [Demo(e.id, e.excerpt_sha, e.label, e.family, "x", 0.9, "r", (1,)) for e in examples]
+    sets = draw_demo_sets(pool, by_id(examples), sets=4, size=3, seed=1)
+    assert len(sets) == 4 and draw_demo_sets(pool, by_id(examples), sets=4, size=3, seed=1) == sets
+    index = by_id(examples)
+    for chosen in sets:
+        assert {d.label for d in chosen[:2]} == {0, 1} and len(chosen) == 3
+        for demo in chosen:
+            if demo.alternate is not None:
+                assert demo.alternate.label == demo.label
+                assert not set(index[demo.example_id].frameworks) & set(index[demo.alternate.example_id].frameworks)
+
+
+def test_only_permissively_licensed_demonstrations_are_packaged() -> None:
+    assert all(permissive(x) for x in ("MIT", "BSD-3-Clause", "Apache-2.0", "ISC", "Zlib", "mit license"))
+    assert not any(permissive(x) for x in ("GPL-3.0", "AGPL-3.0", "LGPL-2.1", "unlicensed", "", "proprietary", "MPL-2.0"))
+    mit, gpl, unknown, apache = (example(n, group=f"{n}/r", label=1, license=lic) for n, lic in
+                                 (("mit", "MIT"), ("gpl", "GPL-3.0"), ("unk", ""), ("apache", "Apache-2.0")))  # fmt: skip
+    demo = lambda e, alt=None: {"example_id": e.id, "excerpt_sha": e.excerpt_sha, "label": 1, "rationale": "r", "alternate": alt}  # noqa: E731
+    artifact = {"program_id": "x", "demos": [demo(mit), demo(gpl, demo(apache)), demo(unknown, demo(gpl))], "instruction": "i"}
+    artifact["program_id"] = program_id(artifact)
+    packaged, tally = packageable(artifact, by_id([mit, gpl, unknown, apache]))
+    assert tally == {"kept": 1, "swapped": 1, "dropped": 1}
+    assert [d["example_id"] for d in packaged["demos"]] == [mit.id, apache.id]
+    assert packaged["program_id"] == program_id(packaged) != artifact["program_id"]
+    assert packaged["packaged"]["from"] == artifact["program_id"]

tests/test_learn_decide.py

@@ -0,0 +1,136 @@
+"""Operating points and the evaluation report (learned-decision-engine design 4.4-4.5, task 6.5): points read from
+calibrated, cross-fitted predictions of the other folds; ``unsure`` never BLOCKs; explanation fields; version
+mismatch falls back; ``insufficient data``; parse failures are an instrument failure. Every client is scripted."""
+
+from __future__ import annotations
+
+import json
+import random
+
+import pytest
+from learn_fixtures import ScriptedChat, corpus
+
+from openultrasast.learn.evaluate import (
+    Prediction,
+    candidate_from_example,
+    choose_points,
+    evaluate,
+    explain,
+    incompatible,
+    learning_curve,
+    nested_points,
+    predict,
+    strata,
+    targets,
+    unevaluable_families,
+    wilson_lower,
+)
+from openultrasast.learn.folds import compile_split, outer_folds
+from openultrasast.learn.program import SIGNATURE, Caller, Program, ProgramSpec
+from openultrasast.learn.schema import SCHEMA_VERSION, feature_set_digest
+from openultrasast.plane.budget import MeteredClient
+
+
+def _preds(n: int = 300, *, folds: int = 3, seed: int = 0, unsure_every: int = 0) -> list[Prediction]:
+    rng = random.Random(seed)
+    out = []
+    for i in range(n):
+        label = int(rng.random() < 0.4)
+        p = min(0.99, max(0.01, rng.gauss(0.8 if label else 0.25, 0.12)))
+        verdict = "unsure" if unsure_every and i % unsure_every == 0 else ("vulnerable" if p > 0.5 else "not_vulnerable")
+        out.append(Prediction(f"f{i % folds}", "injection", f"g{i // 3}", label, p, verdict, "delta", "python", p=p))
+    return out
+
+
+def test_wilson_lower_bound() -> None:
+    assert wilson_lower(0, 0) == 0.0 and wilson_lower(10, 10) == pytest.approx(0.722, abs=0.001)
+    assert wilson_lower(95, 100) == pytest.approx(0.888, abs=0.001)
+
+
+def test_points_come_from_the_other_folds_only() -> None:
+    preds = _preds()
+    decided, points = nested_points(preds)
+    for fold, pts in points.items():
+        assert pts == choose_points([r for r in preds if r.fold != fold])
+    perturbed = [Prediction(**{**r.__dict__, "label": 1 - r.label}) if r.fold == "f0" else r for r in preds]
+    _, again = nested_points(perturbed)
+    assert again["f0"] == points["f0"] and again["f1"] != points["f1"]
+    assert {r.point for r in decided} <= {"BLOCK", "ADVISORY", "DROP"}
+
+
+def test_block_needs_the_wilson_bound_and_advisory_the_recall() -> None:
+    pts = choose_points(_preds(600))
+    assert pts.advisory is not None
+    rows = _preds(600)
+    positives = sum(r.label for r in rows)
+    assert sum(r.label for r in rows if (r.p or 0) >= pts.advisory) / positives >= 0.90
+    if pts.block is not None:
+        chosen = [r for r in rows if (r.p or 0) >= pts.block and r.blockable]
+        assert wilson_lower(sum(r.label for r in chosen), len(chosen)) >= 0.95
+    noisy = [Prediction("f0", "injection", f"g{i}", i % 2, 0.6, "vulnerable", p=0.6) for i in range(50)]
+    unreachable = choose_points(noisy)
+    assert unreachable.block is None and not unreachable.block_offered and unreachable.block_reason == "precision_unreachable"
+
+
+def test_a_majority_unsure_never_blocks() -> None:
+    preds = [Prediction(f"f{i % 2}", "injection", f"g{i}", 1, 0.99, "unsure", "delta", p=0.99) for i in range(40)]
+    preds += [Prediction(f"f{i % 2}", "injection", f"n{i}", 0, 0.01, "not_vulnerable", "delta", p=0.01) for i in range(40)]
+    decided, _ = nested_points(preds)
+    assert not any(r.point == "BLOCK" for r in decided)
+    assert all(r.point == "ADVISORY" for r in decided if r.label == 1)
+
+
+def test_explanations_keep_only_signals_the_record_holds() -> None:
+    x = {"qr.enabled_hits": 2, "eng.findings": 0, "eng.rung_max": None}
+    assert explain("qr.enabled_hits and eng.findings agree; eng.rung_max and made.up say otherwise", x) == [
+        "qr.enabled_hits",
+        "eng.findings",
+    ]
+
+
+def test_a_version_mismatch_falls_back_with_the_reason() -> None:
+    artifact = {"schema_version": SCHEMA_VERSION, "feature_set_digest": feature_set_digest(), "signature": {"digest": SIGNATURE.digest},
+                "profile": "static"}  # fmt: skip
+    record = {"schema_version": SCHEMA_VERSION, "feature_set_digest": feature_set_digest(), "profile": "static"}
+    assert incompatible(artifact, record, SIGNATURE.digest) is None
+    assert incompatible({**artifact, "feature_set_digest": "old"}, record, SIGNATURE.digest) == "feature set differs"
+    assert incompatible(artifact, {**record, "schema_version": 0}, SIGNATURE.digest) == "schema version differs"
+    assert incompatible({**artifact, "signature": {"digest": "x"}}, record, SIGNATURE.digest) == "signature differs"
+    assert incompatible(artifact, {**record, "profile": "plane"}, SIGNATURE.digest) == "profile differs"
+
+
+def test_a_family_below_the_floor_is_insufficient_data_never_a_number() -> None:
+    preds = _preds(90)
+    few = [Prediction("f0", "memory", "openssl/openssl", i % 2, 0.5, "unsure", p=0.5) for i in range(20)]
+    report = strata([*preds, *few], lambda r: r.family, resamples=50)
+    assert report["memory"] == {"status": "insufficient data"}
+    assert report["injection"]["status"] in ("per_family", "pooled") and "advisory" in report["injection"]
+
+
+def _code(sha: str) -> str:
+    return f"   10  def f(q):\n   11      return {'execute(q)' if int(sha, 16) % 2 else 'escape(q)'}\n"
+
+
+def test_the_evaluation_runs_end_to_end_on_scripted_answers() -> None:
+    memory = corpus(groups=30, per_group=4)
+    codes = {e.excerpt_sha: f"   10  def f(q):\n   11      return {'execute(q)' if e.label else 'escape(q)'}\n" for e in memory}
+    split = compile_split(memory)
+    folds = outer_folds(memory, exclude=split.groups)
+    pairs = targets(folds, memory, skip=unevaluable_families(memory, floor=3))
+    assert pairs and all(e.group in f.eval_groups for e, f in pairs)
+    program = Program(ProgramSpec(), memory, codes.get)
+    chat = ScriptedChat()
+    caller = Caller(MeteredClient(chat), "deepseek-flash")
+    preds = predict(program, folds, caller, candidate_from_example(codes.get), pairs=pairs)
+    assert len(preds) == len(pairs) == caller.calls + caller.replayed and caller.calls == len(chat.calls)  # identical prompts replay
+    report = evaluate(preds, resamples=50, per_family_floor=10, pooled_floor=3)
+    assert report["status"] == "ok" and report["parse_failed_share"] == 0 and report["calibration"] is not None
+    assert set(report["points_by_fold"]) == {f.name for f in folds if f.eval_groups}
+    curve = learning_curve(program, pairs[:20], caller, candidate_from_example(codes.get), fractions=(0.0, 1.0))
+    assert [p["memory_fraction"] for p in curve] == [0.0, 1.0] and curve[0]["memory_groups"] == 0
+
+
+def test_more_than_five_percent_parse_failures_is_an_instrument_failure() -> None:
+    preds = [Prediction(f"f{i % 3}", "injection", f"g{i}", i % 2, 0.5, "unsure", parse_failed=int(i % 10 == 0)) for i in range(60)]
+    assert evaluate(preds, resamples=20, per_family_floor=5, pooled_floor=2)["status"] == "instrument failure"
+    assert json.dumps(evaluate(preds[1:10], resamples=5, per_family_floor=5, pooled_floor=2), default=str)