mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
bench(learn): injection first slice -- compiled AI classifier, out-of-repo evaluation (decision-engine 6.2, 6.6)
6.2: the 724 static examples' 616 distinct excerpts embedded with openai/text-embedding-3-small via OpenRouter, 198,819 tokens, $0.0040 metered (ceiling $1). 6.6, family injection only, memory = every family (at most 2 contrast examples per prompt), input profile v1: smoke fixed k = 5 (repeat prefix-cache hit share 0.96); one compile (program c1237a49, proposed instruction 5, balanced Brier 0.128 on 13 C_val candidates vs 0.199 baseline); outer grouped 5-fold, 110 out-of-repository candidates in 29 groups: ADVISORY recall 0.90 [0.82, 0.97], precision 0.54 [0.44, 0.65], AUC 0.74; BLOCK not offered (ECE 0.064 > 0.05; no fold reached a Wilson lower bound of 0.95); canary re-ask agreement 0.83. $0.91 metered at list price; DeepSeek balance $18.20 -> $17.90. Instrument fixes found on the way, each with a test: - memory build: pair sides join the record stored under their harvest unit pin (--units), and only the built profile's records are read (static and plane shared a key; 85 -> 724 examples); - retrieval: stage 1 per side (family / contrast) and at most 2 contrast examples, never padded; - program: the JSON answer contract is part of the signature, appended to proposed instructions (DeepSeek JSON mode refused a prompt without "json"); - compile: candidate_families for a one-family compile; compile.toml k = 5 from the smoke rule. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
68294e9db3
commit
7b17eacd44
@@ -25,6 +25,7 @@ def add_commands(learn_sub: Any) -> None:
|
||||
build.add_argument("--cache", type=Path, help="clones and checkouts (default ~/.cache/openultrasast)")
|
||||
build.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
|
||||
build.add_argument("--profile", choices=("static", "plane"), default="static")
|
||||
build.add_argument("--units", type=Path, help="the harvest units.json: the pin each pair side's record is stored under")
|
||||
build.add_argument("--dry-run", action="store_true", help="count, write nothing")
|
||||
embed = memory_sub.add_parser("embed", help="embed the stored excerpts (OpenRouter, metered; cached by excerpt sha)")
|
||||
embed.add_argument("--memory", help="the store (default OUSAST_MEMORY, then results_root()/memory)")
|
||||
@@ -209,19 +210,21 @@ def _memory_embed(args: argparse.Namespace) -> int:
|
||||
def _memory_build(args: argparse.Namespace) -> int:
|
||||
"""Exit 2 on a refused label (``origin: user``, a source outside the list) or a source with >10% unread excerpts."""
|
||||
from ..plane.memory import open_store
|
||||
from .examples import ExampleBuildError, SideReader, build_examples, read_jsonl, records_by_key
|
||||
from .examples import ExampleBuildError, SideReader, build_examples, pair_pins, read_jsonl, records_by_key
|
||||
from .labels import DEFAULT_CACHE, DEFAULT_SOURCES, load_sources
|
||||
|
||||
sources = load_sources(args.sources or DEFAULT_SOURCES)
|
||||
labels = read_jsonl(args.labels)
|
||||
store = None if args.dry_run else open_store(args.memory)
|
||||
rows = [row for path in args.features for row in read_jsonl(path)]
|
||||
rows = [row for path in args.features for row in read_jsonl(path) if row.get("profile", args.profile) == args.profile]
|
||||
if store is not None:
|
||||
rows += [r.row for r in store.rows(kind="features")]
|
||||
# one profile only: the static and plane records of a candidate share a key, and the other would overwrite it
|
||||
rows += [r.row for r in store.rows(kind="features") if r.row.get("profile") == args.profile]
|
||||
reader = SideReader(args.root, args.cache or DEFAULT_CACHE, sources, labels)
|
||||
try:
|
||||
build = build_examples(
|
||||
labels, records_by_key(rows), reader, sources=sources, store=store, profile=args.profile, license_of=reader.license,
|
||||
pins=pair_pins(json.loads(args.units.read_text(encoding="utf-8"))) if args.units is not None else None,
|
||||
) # fmt: skip
|
||||
except ExampleBuildError as exc:
|
||||
print(f"learn memory build: {exc}", file=sys.stderr)
|
||||
|
||||
@@ -354,12 +354,14 @@ def compile_program(
|
||||
vectors: Mapping[str, Sequence[float]] | None = None,
|
||||
identities: Callable[[Example], Iterable[str]] | None = None,
|
||||
families: Sequence[str] = (),
|
||||
candidate_families: Sequence[str] = (),
|
||||
store: MemoryStore | None = None,
|
||||
created: str = "",
|
||||
code_commit: str = "",
|
||||
) -> CompileResult:
|
||||
"""Compile one profile's program from ``memory`` (module docstring). Deterministic given the memory, the spec and
|
||||
the response cache."""
|
||||
the response cache. ``candidate_families`` limits the bootstrap and validation candidates to those families (a
|
||||
one-family slice); retrieval still draws on every family of the compile split, for contrast examples."""
|
||||
seed = spec.seed
|
||||
pool = [e for e in memory if e.profile == profile]
|
||||
split: CompileSplit = compile_split(pool, fraction=float(spec["split"].get("compile_fraction", 0.25)),
|
||||
@@ -368,8 +370,9 @@ def compile_program(
|
||||
assert_disjoint(split, evaluation)
|
||||
compile_memory = [e for e in pool if e.group in split.groups]
|
||||
fold = Fold("compile", "compile", frozenset(), phase="compile")
|
||||
boot = _order([e for e in compile_memory if e.group in split.boot], seed, "boot")[: int(spec["bootstrap"].get("max_candidates", 150))]
|
||||
val = _order([e for e in compile_memory if e.group in split.val], seed, "val")[: int(spec["search"].get("val_candidates", 60))]
|
||||
asked = [e for e in compile_memory if not candidate_families or e.family in candidate_families]
|
||||
boot = _order([e for e in asked if e.group in split.boot], seed, "boot")[: int(spec["bootstrap"].get("max_candidates", 150))]
|
||||
val = _order([e for e in asked if e.group in split.val], seed, "val")[: int(spec["search"].get("val_candidates", 60))]
|
||||
if not boot or not val:
|
||||
raise CompileError(f"compile split too small: {len(boot)} bootstrap and {len(val)} validation candidates")
|
||||
retrieval, classify = spec["retrieval"], spec["classify"]
|
||||
|
||||
@@ -29,7 +29,9 @@ val_candidates = 60
|
||||
|
||||
[classify]
|
||||
model = "deepseek-flash"
|
||||
k = 1 # k and lam are A/B variants (design section 6), not searched here
|
||||
k = 5 # fixed by the smoke run before any compile spend (task 6.6): samples 2..5 hit the
|
||||
# provider prefix cache for 0.96 of their prompt tokens (>= 0.8), 2026-10-01. k and lam
|
||||
# remain A/B variants (design section 6), not searched here
|
||||
max_output_tokens = 300
|
||||
inputs = "v1" # the input profile: the engine instrument is withheld from prompts and retrieval (2026-10-01)
|
||||
|
||||
|
||||
@@ -22,7 +22,9 @@ Memory examples carry **no rationale** (code, signals, roles, family and label o
|
||||
``fix_function`` (a different handler of the same snapshot).
|
||||
|
||||
Pair labels have no commit pin (the corpus holds excerpts); their envelope pin is :func:`label_pin`, a stable
|
||||
40-hex digest of the pair and its side, so the vulnerable and the fixed side stay distinct rows.
|
||||
40-hex digest of the pair and its side, so the vulnerable and the fixed side stay distinct rows. The harvest keyed a
|
||||
pair side's feature record by its unit's pin instead (the blob sha1 of the side's excerpt, ``plane/harvest.py``):
|
||||
:func:`pair_pins` reads that mapping from the harvest ``units.json`` and the build uses it when given.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -62,6 +64,11 @@ def label_pin(label: Mapping[str, Any]) -> str:
|
||||
return hashlib.sha1(subject.encode("utf-8")).hexdigest() # noqa: S324 -- a name, not a security digest
|
||||
|
||||
|
||||
def pair_pins(units: Mapping[str, Mapping[str, Any]]) -> dict[tuple[str, str], str]:
|
||||
"""Harvest ``units.json`` -> ``(case reference, side) -> pin`` for the pair units (the pin their records carry)."""
|
||||
return {(str(u["ref"]), str(u["side"])): str(u["pin"]) for u in units.values() if u.get("source") == "pairs"}
|
||||
|
||||
|
||||
def record_key(repo: str, pin: str, candidate: str, family: str, unit: str) -> tuple[str, str, str, str, str]:
|
||||
return repo_key(repo), pin, candidate, family, unit
|
||||
|
||||
@@ -127,9 +134,11 @@ def build_examples(
|
||||
license_of: Callable[[Mapping[str, Any]], str] | None = None,
|
||||
function_of: Callable[[Mapping[str, Any], str], str | None] | None = None,
|
||||
bounds: Bounds = EXAMPLE,
|
||||
pins: Mapping[tuple[str, str], str] | None = None,
|
||||
) -> MemoryBuild:
|
||||
"""Example rows for the verified labels that have a feature record (``records`` by :func:`record_key`) and an
|
||||
excerpt; with ``store`` the excerpt blobs and the rows are written. Delta units append the base->head diff."""
|
||||
excerpt; with ``store`` the excerpt blobs and the rows are written. Delta units append the base->head diff.
|
||||
``pins`` (:func:`pair_pins`) gives a pair side the pin its harvest record was stored under."""
|
||||
rows = [_as_dict(label) for label in labels]
|
||||
allowed = {s.id for s in sources.sources} | {d for d, kind in DERIVED_SOURCES.items() if any(s.kind == kind for s in sources.sources)}
|
||||
for row in rows:
|
||||
@@ -145,7 +154,9 @@ def build_examples(
|
||||
continue
|
||||
count = build.sources.setdefault(str(label["source"]), SourceCount())
|
||||
count.labels += 1
|
||||
pin = label_pin(label)
|
||||
side = (str(label.get("source_ref")), str(label.get("pin_role") or "vulnerable"))
|
||||
pin = (pins or {}).get(side) if label.get("source") == "pairs" else None
|
||||
pin = pin or label_pin(label)
|
||||
candidate, family, unit = str(label["candidate"]), str(label["family"]), str(label["unit"])
|
||||
record = records.get(record_key(str(label["repo"]), pin, candidate, family, unit))
|
||||
if record is None or record.get("profile") != profile:
|
||||
@@ -307,5 +318,5 @@ def records_by_key(rows: Iterable[Mapping[str, Any]]) -> dict[tuple[str, str, st
|
||||
|
||||
__all__ = [
|
||||
"EXAMPLE_KIND", "NO_EXCERPT_LIMIT", "Example", "ExampleBuildError", "MemoryBuild", "SideReader", "build_examples",
|
||||
"label_pin", "label_set_digest", "load_examples", "read_jsonl", "record_key", "records_by_key",
|
||||
"label_pin", "label_set_digest", "load_examples", "pair_pins", "read_jsonl", "record_key", "records_by_key",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -87,16 +87,21 @@ SIGNATURE = Signature(
|
||||
)
|
||||
)
|
||||
|
||||
BASELINE_INSTRUCTION = (
|
||||
"You decide whether a function contains an exploitable security vulnerability of the given family.\n"
|
||||
"You see the function's code (numbered lines), the signals static instruments emitted for it, and the roles\n"
|
||||
"inferred for its calls. Signals are evidence, not verdicts: rules over-report, and a missing instrument is not a\n"
|
||||
"negative. Similar labelled functions from other repositories follow as examples; reason about this code.\n"
|
||||
# The answer contract is part of the signature, not of the searched instruction: a proposed instruction that leaves it
|
||||
# out still gets it (``render_prefix``), so every prompt asks for JSON (DeepSeek's JSON mode refuses a prompt that
|
||||
# never says "json") and every answer can parse.
|
||||
ANSWER_FORMAT = (
|
||||
'Answer with one JSON object: {"verdict": "vulnerable" | "not_vulnerable" | "unsure", "family": <family or\n'
|
||||
'"none">, "confidence": <probability in [0, 1] that your verdict is right>, "rationale": <at most 3 sentences>,\n'
|
||||
'"cited_lines": [<line numbers of the code that support the verdict>]}. Cite at least one line. Say "unsure"\n'
|
||||
"when the code does not decide it."
|
||||
)
|
||||
BASELINE_INSTRUCTION = (
|
||||
"You decide whether a function contains an exploitable security vulnerability of the given family.\n"
|
||||
"You see the function's code (numbered lines), the signals static instruments emitted for it, and the roles\n"
|
||||
"inferred for its calls. Signals are evidence, not verdicts: rules over-report, and a missing instrument is not a\n"
|
||||
"negative. Similar labelled functions from other repositories follow as examples; reason about this code.\n" + ANSWER_FORMAT
|
||||
)
|
||||
|
||||
|
||||
# --- rendering ---------------------------------------------------------------------------------------------------------
|
||||
@@ -232,7 +237,8 @@ ExcerptText = Callable[[str], str | None]
|
||||
|
||||
def render_prefix(spec: ProgramSpec, demos: Sequence[Demo], examples: Mapping[str, Example], excerpt_text: ExcerptText) -> str:
|
||||
"""The instruction and the demonstrations: identical for every candidate of one program and fold."""
|
||||
parts = [spec.instruction.strip()]
|
||||
instruction = spec.instruction.strip()
|
||||
parts = [instruction if ANSWER_FORMAT in instruction else f"{instruction}\n{ANSWER_FORMAT}"]
|
||||
for number, demo in enumerate(demos, 1):
|
||||
example = examples[demo.example_id]
|
||||
code = excerpt_text(demo.excerpt_sha) or ""
|
||||
@@ -481,7 +487,8 @@ class Program:
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BASELINE_INSTRUCTION", "PARSE_FAILED", "SIGNATURE", "UNSURE", "VERDICTS", "Answer", "Caller", "Candidate", "Decision", "Demo",
|
||||
"ANSWER_FORMAT", "BASELINE_INSTRUCTION", "PARSE_FAILED", "SIGNATURE", "UNSURE", "VERDICTS", "Answer", "Caller", "Candidate",
|
||||
"Decision", "Demo",
|
||||
"Program", "ProgramSpec", "Prompt", "ReplayMiss", "aggregate", "classify", "label_word", "line_numbers",
|
||||
"params_digest", "parse_answer", "render", "render_prefix", "render_roles", "render_signals", "request_key",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -11,12 +11,13 @@ profile, computed per instrument block so that missing is explicit, never impute
|
||||
- ``D`` is the uniform mean over the profile's blocks; prior features are dropped with ``priors = "off"``, and the blocks
|
||||
of instruments the input profile withholds (``inputs``, default ``v1``: the engine) are left out entirely.
|
||||
|
||||
Ties break by ``sha256(example id + seed)``; the first ``n1 = 40`` go to stage 2, a cosine re-rank of the code
|
||||
Ties break by ``sha256(example id + seed)``; the first ``n1 = 40`` of the candidate's family and the first ``n1``
|
||||
of the other families go to stage 2, a cosine re-rank of the code
|
||||
embeddings, ``r = lam (1 - D) + (1 - lam) cos``. Without the target's vector the whole retrieval is
|
||||
``signals_only``; an example without a vector scores ``cos = 1 - D`` (its signal similarity, so it keeps its
|
||||
stage-1 standing). The **balance rule** then takes ``k_ret = 6``: at most ``k/2`` per label, at most 2 per
|
||||
repository group, at most 2 from other families while the candidate's family has examples left; ``short`` when
|
||||
the pool cannot fill it.
|
||||
repository group, at most 2 contrast examples from other families whenever the pool holds the candidate's family
|
||||
(never padded with more); ``short`` when the pool cannot fill it.
|
||||
|
||||
**The boundary.** :func:`eligible` is the only way an example enters a prompt -- retrieved examples and compiled
|
||||
demonstrations alike: not the target's group, not a group the fold evaluates, not the held-out source, not the
|
||||
@@ -43,6 +44,7 @@ LAM = 0.5
|
||||
NEAR_DUPLICATE_COS = 0.98
|
||||
UNBOUNDED_CAP = 10.0
|
||||
PER_GROUP = 2
|
||||
CONTRAST = 2 # contrast examples from other families, at most, while the pool holds the target's family
|
||||
|
||||
|
||||
class BoundaryViolation(AssertionError):
|
||||
@@ -153,35 +155,26 @@ def _tie(example_id: str, seed: int) -> str:
|
||||
|
||||
|
||||
def balance(ranked: Sequence[Example], family: str, k: int = K_RET) -> tuple[list[Example], str]:
|
||||
"""Take up to ``k`` in rank order: <= k/2 per label, <= 2 per group, <= 2 other families while the target's family
|
||||
has examples left; then, if short, fill from other families under the label and group caps."""
|
||||
per_label, other_cap = k // 2, min(2, k)
|
||||
"""Take up to ``k`` in rank order: <= k/2 per label, <= 2 per group, and at most 2 contrast examples from other
|
||||
families whenever the pool holds any example of the target's family (maintainer, 2026-10-01: "allow 2 contrast").
|
||||
A pool without the target's family takes from the others under the label and group caps. ``short`` when the
|
||||
pool cannot fill ``k`` or a label minimum: what exists is taken, never padded with more contrast."""
|
||||
per_label = k // 2
|
||||
other_cap = min(CONTRAST, k) if any(e.family == family for e in ranked) else k
|
||||
taken: list[Example] = []
|
||||
labels: dict[int, int] = {}
|
||||
groups: dict[str, int] = {}
|
||||
|
||||
def fits(example: Example) -> bool:
|
||||
return labels.get(example.label, 0) < per_label and groups.get(example.group, 0) < PER_GROUP
|
||||
|
||||
def take(example: Example) -> None:
|
||||
taken.append(example)
|
||||
labels[example.label] = labels.get(example.label, 0) + 1
|
||||
groups[example.group] = groups.get(example.group, 0) + 1
|
||||
|
||||
others = 0
|
||||
for example in ranked:
|
||||
if len(taken) == k:
|
||||
break
|
||||
if example.family != family and others >= other_cap:
|
||||
continue
|
||||
if fits(example):
|
||||
if labels.get(example.label, 0) < per_label and groups.get(example.group, 0) < PER_GROUP:
|
||||
others += example.family != family
|
||||
take(example)
|
||||
for example in ranked:
|
||||
if len(taken) == k:
|
||||
break
|
||||
if example.id not in {e.id for e in taken} and fits(example):
|
||||
take(example)
|
||||
taken.append(example)
|
||||
labels[example.label] = labels.get(example.label, 0) + 1
|
||||
groups[example.group] = groups.get(example.group, 0) + 1
|
||||
pool_labels = {e.label for e in ranked}
|
||||
short = len(taken) < k or any(labels.get(lab, 0) < min(2, per_label) for lab in (0, 1) if lab in pool_labels)
|
||||
return taken, "short" if short else "ok"
|
||||
@@ -202,10 +195,14 @@ def retrieve(
|
||||
) -> Retrieval:
|
||||
"""The ``k`` balanced examples for ``target`` from the eligible part of ``pool`` (module docstring)."""
|
||||
candidates = [e for e in pool if e.profile == target.profile and eligible(e, target, fold)]
|
||||
scored = sorted(
|
||||
ranked_all = sorted(
|
||||
((gower(target.x, target.instruments, e.x, e.instruments, target.profile, priors=priors, inputs=inputs), e) for e in candidates),
|
||||
key=lambda pair: (pair[0], _tie(pair[1].id, seed)),
|
||||
)[:n1]
|
||||
)
|
||||
# stage 1 per side: the n1 nearest of the target's family and the n1 nearest contrast examples, so a family that is a
|
||||
# minority of the memory still reaches stage 2 (the balance rule then takes at most 2 contrast examples)
|
||||
own = [pair for pair in ranked_all if pair[1].family == target.family][:n1]
|
||||
scored = own + [pair for pair in ranked_all if pair[1].family != target.family][:n1]
|
||||
mode = "signals+embeddings" if target.vector is not None and vectors else "signals_only"
|
||||
near = without = 0
|
||||
reranked: list[tuple[float, str, Example]] = []
|
||||
|
||||
Reference in New Issue
Block a user