bench(learn): injection first slice -- compiled AI classifier, out-of-repo evaluation (decision-engine 6.2, 6.6)

6.2: the 724 static examples' 616 distinct excerpts embedded with openai/text-embedding-3-small via
OpenRouter, 198,819 tokens, $0.0040 metered (ceiling $1).

6.6, family injection only, memory = every family (at most 2 contrast examples per prompt), input
profile v1: smoke fixed k = 5 (repeat prefix-cache hit share 0.96); one compile (program c1237a49,
proposed instruction 5, balanced Brier 0.128 on 13 C_val candidates vs 0.199 baseline); outer grouped
5-fold, 110 out-of-repository candidates in 29 groups: ADVISORY recall 0.90 [0.82, 0.97], precision
0.54 [0.44, 0.65], AUC 0.74; BLOCK not offered (ECE 0.064 > 0.05; no fold reached a Wilson lower bound
of 0.95); canary re-ask agreement 0.83. $0.91 metered at list price; DeepSeek balance $18.20 -> $17.90.

Instrument fixes found on the way, each with a test:
- memory build: pair sides join the record stored under their harvest unit pin (--units), and only the
  built profile's records are read (static and plane shared a key; 85 -> 724 examples);
- retrieval: stage 1 per side (family / contrast) and at most 2 contrast examples, never padded;
- program: the JSON answer contract is part of the signature, appended to proposed instructions
  (DeepSeek JSON mode refused a prompt without "json");
- compile: candidate_families for a one-family compile; compile.toml k = 5 from the smoke rule.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_learn_compile.py

@@ -6,7 +6,7 @@ byte for byte at $0 (``test_reproducible_from_cache``); the permissive-licence f
 from __future__ import annotations
 
 import json
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Mapping, Sequence
 from pathlib import Path
 
 import pytest
@@ -30,6 +30,7 @@ from openultrasast.learn.compile import (
     permissive,
     program_id,
 )
+from openultrasast.learn.examples import Example
 from openultrasast.learn.folds import Fold, FoldError
 from openultrasast.learn.program import Caller, Demo
 from openultrasast.plane.budget import MeteredClient
@@ -191,3 +192,23 @@ def test_only_permissively_licensed_demonstrations_are_packaged() -> None:
     assert [d["example_id"] for d in packaged["demos"]] == [mit.id, apache.id]
     assert packaged["program_id"] == program_id(packaged) != artifact["program_id"]
     assert packaged["packaged"]["from"] == artifact["program_id"]
+
+
+def test_a_one_family_compile_asks_only_that_family(monkeypatch: pytest.MonkeyPatch) -> None:
+    """``candidate_families`` limits the bootstrap and validation candidates; the compile memory keeps every family
+    (retrieval's contrast examples), so the artifact's data counts are those of the whole pool."""
+    asked: dict[str, set[str]] = {}
+    real = compile_module._order
+
+    def spy(items: Iterable[Example], seed: int, salt: str) -> list[Example]:
+        items = list(items)
+        asked.setdefault(salt, set()).update(e.family for e in items)
+        return real(items, seed, salt)
+
+    monkeypatch.setattr(compile_module, "_order", spy)
+    result = compile_program(MEMORY, _caller(ScriptedChat(answer)), excerpt_text, spec=SPEC, candidate_families=("injection",))
+    assert result.status == "done" and result.artifact is not None
+    assert asked["boot"] == asked["val"] == {"injection"}
+    assert result.artifact["data"]["examples"] == len(MEMORY)
+    with pytest.raises(CompileError, match="compile split too small"):
+        compile_program(MEMORY, _caller(ScriptedChat(answer)), excerpt_text, spec=SPEC, candidate_families=("memory",))

tests/test_learn_examples.py

@@ -18,6 +18,7 @@ from openultrasast.learn.examples import (
     build_examples,
     label_pin,
     load_examples,
+    pair_pins,
     record_key,
     records_by_key,
 )
@@ -155,3 +156,18 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side(
     assert count.absent_fixed_side == 1 and count.no_excerpt == 0 and build.failing() == []
     unread = build_examples([fixed], records_by_key([feature(fixed)]), lambda lab, role: None, sources=SOURCES)
     assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"]
+
+
+def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None:
+    """The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under
+    :func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found."""
+    lab = label(source="pairs", source_ref="case-9", pin="")
+    unit_pin = "a" * 40
+    stored = {**feature(lab), "pin": unit_pin}
+    units = {"h-1": {"source": "pairs", "ref": "case-9", "side": "vulnerable", "pin": unit_pin, "repo": lab["repo"]},
+             "h-2": {"source": "population-v1", "ref": "case-9", "side": "vulnerable", "pin": PIN, "repo": lab["repo"]}}  # fmt: skip
+    pins = pair_pins(units)
+    assert pins == {("case-9", "vulnerable"): unit_pin}
+    assert build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES).summary()["examples"] == 0
+    build = build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES, pins=pins)
+    assert build.summary()["examples"] == 1 and build.rows[0]["pin"] == unit_pin

tests/test_learn_program.py

@@ -15,6 +15,8 @@ from learn_fixtures import ScriptedChat, by_id, corpus, x_record
 from openultrasast.learn import program as program_module
 from openultrasast.learn.folds import outer_folds
 from openultrasast.learn.program import (
+    ANSWER_FORMAT,
+    BASELINE_INSTRUCTION,
     PARSE_FAILED,
     Answer,
     Caller,
@@ -121,6 +123,15 @@ def test_the_prefix_is_byte_identical_across_candidates() -> None:
     assert "Demonstration 1:" in a.prefix and a.messages[1] != b.messages[1]
 
 
+def test_every_instruction_carries_the_answer_contract() -> None:
+    """A proposed instruction rarely restates the JSON contract; the prefix adds it, once, so JSON mode is accepted
+    (DeepSeek refuses a prompt without the word "json") and the answer parses. The baseline prefix is unchanged."""
+    proposed, _ = program(instruction="Weigh the sanitiser before the sink.").prepare(candidate(), FOLD)
+    baseline, _ = program().prepare(candidate(), FOLD)
+    assert proposed.prefix.startswith("Weigh the sanitiser before the sink.\n") and proposed.prefix.count(ANSWER_FORMAT) == 1
+    assert baseline.prefix.count(ANSWER_FORMAT) == 1 and baseline.prefix.startswith(BASELINE_INSTRUCTION)
+
+
 def test_parsing_valid_invalid_and_out_of_range_citations() -> None:
     lines = frozenset({40, 41})
     ok = parse_answer(

tests/test_learn_retrieve.py

@@ -8,7 +8,7 @@ from dataclasses import replace
 import pytest
 from learn_fixtures import by_id, corpus, example, x_record
 
-from openultrasast.learn.folds import compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
+from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
 from openultrasast.learn.retrieve import (
     BoundaryViolation,
     Target,
@@ -102,6 +102,25 @@ def test_balance_caps_labels_groups_and_other_families() -> None:
     crowded = [example(f"c{i}", group="one/repo", label=i % 2) for i in range(10)]
     taken, state = balance(crowded, "injection", 6)
     assert len(taken) == 2 and state == "short"  # at most 2 per repository group
+    scarce = [example("own", group="own/repo", label=1)] + [
+        example(f"o{i}", group=f"o{i}/r", label=i % 2, family="path") for i in range(10)
+    ]
+    taken, state = balance(scarce, "injection", 6)
+    assert sum(e.family != "injection" for e in taken) == 2 and len(taken) == 3 and state == "short"  # never padded
+    absent = [example(f"a{i}", group=f"a{i}/r", label=i % 2, family="path") for i in range(10)]
+    taken, state = balance(absent, "injection", 6)
+    assert len(taken) == 6 and state == "ok"  # no example of the family at all: the others fill under the caps
+
+
+def test_a_minority_family_reaches_stage_two_and_contrast_stays_at_two() -> None:
+    """Stage 1 keeps the n1 nearest of the target's family beside the n1 nearest others: 50 contrast examples nearer
+    in signals cannot push the family out, and the balance rule takes at most 2 of them."""
+    others = [example(f"o{i}", group=f"o{i}/r", label=i % 2, family="path", hits=0) for i in range(50)]
+    own = [example(f"own{i}", group=f"own{i}/r", label=i % 2, hits=6) for i in range(4)]
+    fold = Fold("compile", "compile", frozenset(), phase="compile")
+    got = retrieve(_target(hits=0), [*others, *own], fold, n1=40, k=6)
+    assert sum(e.family == "injection" for e in got.examples) == 4
+    assert sum(e.family != "injection" for e in got.examples) == 2
 
 
 def test_near_duplicates_drop_in_evaluation_and_stay_in_deployment() -> None: