mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
bench(learn): injection first slice -- compiled AI classifier, out-of-repo evaluation (decision-engine 6.2, 6.6)
6.2: the 724 static examples' 616 distinct excerpts embedded with openai/text-embedding-3-small via OpenRouter, 198,819 tokens, $0.0040 metered (ceiling $1). 6.6, family injection only, memory = every family (at most 2 contrast examples per prompt), input profile v1: smoke fixed k = 5 (repeat prefix-cache hit share 0.96); one compile (program c1237a49, proposed instruction 5, balanced Brier 0.128 on 13 C_val candidates vs 0.199 baseline); outer grouped 5-fold, 110 out-of-repository candidates in 29 groups: ADVISORY recall 0.90 [0.82, 0.97], precision 0.54 [0.44, 0.65], AUC 0.74; BLOCK not offered (ECE 0.064 > 0.05; no fold reached a Wilson lower bound of 0.95); canary re-ask agreement 0.83. $0.91 metered at list price; DeepSeek balance $18.20 -> $17.90. Instrument fixes found on the way, each with a test: - memory build: pair sides join the record stored under their harvest unit pin (--units), and only the built profile's records are read (static and plane shared a key; 85 -> 724 examples); - retrieval: stage 1 per side (family / contrast) and at most 2 contrast examples, never padded; - program: the JSON answer contract is part of the signature, appended to proposed instructions (DeepSeek JSON mode refused a prompt without "json"); - compile: candidate_families for a one-family compile; compile.toml k = 5 from the smoke rule. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
68294e9db3
commit
7b17eacd44
@@ -8,7 +8,7 @@ from dataclasses import replace
|
||||
import pytest
|
||||
from learn_fixtures import by_id, corpus, example, x_record
|
||||
|
||||
from openultrasast.learn.folds import compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
|
||||
from openultrasast.learn.folds import Fold, compile_split, leave_one_framework_out, leave_one_source_out, outer_folds
|
||||
from openultrasast.learn.retrieve import (
|
||||
BoundaryViolation,
|
||||
Target,
|
||||
@@ -102,6 +102,25 @@ def test_balance_caps_labels_groups_and_other_families() -> None:
|
||||
crowded = [example(f"c{i}", group="one/repo", label=i % 2) for i in range(10)]
|
||||
taken, state = balance(crowded, "injection", 6)
|
||||
assert len(taken) == 2 and state == "short" # at most 2 per repository group
|
||||
scarce = [example("own", group="own/repo", label=1)] + [
|
||||
example(f"o{i}", group=f"o{i}/r", label=i % 2, family="path") for i in range(10)
|
||||
]
|
||||
taken, state = balance(scarce, "injection", 6)
|
||||
assert sum(e.family != "injection" for e in taken) == 2 and len(taken) == 3 and state == "short" # never padded
|
||||
absent = [example(f"a{i}", group=f"a{i}/r", label=i % 2, family="path") for i in range(10)]
|
||||
taken, state = balance(absent, "injection", 6)
|
||||
assert len(taken) == 6 and state == "ok" # no example of the family at all: the others fill under the caps
|
||||
|
||||
|
||||
def test_a_minority_family_reaches_stage_two_and_contrast_stays_at_two() -> None:
|
||||
"""Stage 1 keeps the n1 nearest of the target's family beside the n1 nearest others: 50 contrast examples nearer
|
||||
in signals cannot push the family out, and the balance rule takes at most 2 of them."""
|
||||
others = [example(f"o{i}", group=f"o{i}/r", label=i % 2, family="path", hits=0) for i in range(50)]
|
||||
own = [example(f"own{i}", group=f"own{i}/r", label=i % 2, hits=6) for i in range(4)]
|
||||
fold = Fold("compile", "compile", frozenset(), phase="compile")
|
||||
got = retrieve(_target(hits=0), [*others, *own], fold, n1=40, k=6)
|
||||
assert sum(e.family == "injection" for e in got.examples) == 4
|
||||
assert sum(e.family != "injection" for e in got.examples) == 2
|
||||
|
||||
|
||||
def test_near_duplicates_drop_in_evaluation_and_stay_in_deployment() -> None:
|
||||
|
||||
Reference in New Issue
Block a user