mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5.5
f5180a14ec
learn/retrieve.py: stage 1 is a per-instrument Gower distance over the profile's allow-listed features with explicit missing (both ran: mean feature distance; differing states 1; same non-ran state 0; null in one 1; priors dropped with priors=off), n1 = 40 with seeded tie-breaks; stage 2 re-ranks by r = lam(1 - D) + (1 - lam)cos, signals_only without the target's vector; the balance rule takes k_ret = 6 (<= k/2 per label, <= 2 per group, <= 2 other families; `short` recorded). eligible() is the only way into a prompt: not the target's group, not an evaluated group, not the held-out source or framework, and outside deployment not a near-duplicate (cos >= 0.98). assert_boundary repeats the check on rendered ids; demonstrations of a held-out framework swap to their alternates; deployment_fold excludes a scanned repository that is a corpus group; seed_memory never hands a case its own group. learn/folds.py: compile split (25% of groups per primary family, 60/40 boot/val), outer grouped 5-fold stratified by family and label mix, leave-one-source-out, leave-one-framework-out (insufficient data below 10 groups), assert_disjoint and a folds digest; all seeded. Tests: test_learn_retrieve.py (hand-computed Gower, lam 0/0.5, balance, near-duplicates, test_boundary over outer/LOSO/LOFO folds with demonstrations), test_learn_folds.py. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
77 lines
3.3 KiB
Python
77 lines
3.3 KiB
Python
"""Folds by repository group (learned-decision-engine design 4.3-4.4, task 6.3)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from learn_fixtures import corpus
|
|
|
|
from openultrasast.learn.folds import (
|
|
INSUFFICIENT,
|
|
Fold,
|
|
FoldError,
|
|
assert_disjoint,
|
|
compile_split,
|
|
folds_digest,
|
|
leave_one_framework_out,
|
|
leave_one_source_out,
|
|
outer_folds,
|
|
)
|
|
|
|
|
|
def test_compile_split_is_a_quarter_of_groups_and_disjoint_from_every_evaluation_fold() -> None:
|
|
examples = corpus(groups=20)
|
|
split = compile_split(examples, seed=1)
|
|
groups = {e.group for e in examples}
|
|
assert len(split.groups) == 6 and split.groups <= groups # 25% of each family's 10 groups, rounded: 3 + 3
|
|
assert split.boot | split.val == split.groups and not split.boot & split.val and len(split.boot) == 4
|
|
folds = [
|
|
*outer_folds(examples, seed=1, exclude=split.groups),
|
|
*leave_one_source_out(examples, exclude=split.groups),
|
|
*leave_one_framework_out(examples, exclude=split.groups, min_groups=2),
|
|
]
|
|
assert_disjoint(split, folds)
|
|
for fold in folds:
|
|
assert not fold.eval_groups & split.groups
|
|
|
|
|
|
def test_outer_folds_partition_the_remaining_groups_stratified() -> None:
|
|
examples = corpus(groups=20)
|
|
split = compile_split(examples, seed=1)
|
|
folds = outer_folds(examples, seed=1, exclude=split.groups)
|
|
assert len(folds) == 5
|
|
union = set().union(*(f.eval_groups for f in folds))
|
|
assert union == {e.group for e in examples} - split.groups and sum(len(f.eval_groups) for f in folds) == len(union)
|
|
assert max(len(f.eval_groups) for f in folds) - min(len(f.eval_groups) for f in folds) <= 1
|
|
|
|
|
|
def test_folds_are_deterministic_by_seed() -> None:
|
|
examples = corpus(groups=20)
|
|
a, b = compile_split(examples, seed=7), compile_split(examples, seed=7)
|
|
assert a == b and outer_folds(examples, seed=7) == outer_folds(examples, seed=7)
|
|
assert folds_digest(a, outer_folds(examples, seed=7)) == folds_digest(b, outer_folds(examples, seed=7))
|
|
assert any(compile_split(examples, seed=s).groups != a.groups for s in range(1, 6))
|
|
|
|
|
|
def test_leave_one_source_and_framework_out() -> None:
|
|
examples = corpus(groups=12)
|
|
sources = leave_one_source_out(examples)
|
|
assert {f.held_source for f in sources} == {"pairs", "population-v1", "population-v2"}
|
|
for fold in sources:
|
|
assert fold.eval_groups == {e.group for e in examples if e.source == fold.held_source}
|
|
frameworks = {f.held_framework: f for f in leave_one_framework_out(examples)}
|
|
assert set(frameworks) == {"flask", "django"}
|
|
assert all(f.status == INSUFFICIENT and not f.eval_groups for f in frameworks.values()) # 3 groups each < 10
|
|
enough = {f.held_framework: f for f in leave_one_framework_out(examples, min_groups=3)}
|
|
assert enough["flask"].status == "ok" and enough["flask"].eval_groups == {e.group for e in examples if "flask" in e.frameworks}
|
|
|
|
|
|
def test_a_shared_group_is_refused() -> None:
|
|
examples = corpus(groups=8)
|
|
split = compile_split(examples, seed=0)
|
|
leaked = Fold("outer", "outer-x", frozenset({next(iter(split.groups))}))
|
|
with pytest.raises(FoldError, match="compile split"):
|
|
assert_disjoint(split, [leaked])
|
|
twice = [Fold("outer", "a", frozenset({"g"})), Fold("outer", "b", frozenset({"g"}))]
|
|
with pytest.raises(FoldError, match="in a and b"):
|
|
assert_disjoint(split, twice)
|