mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 17:58:18 +00:00
Marius CiepluchandClaude Opus 5.5
7b17eacd44
6.2: the 724 static examples' 616 distinct excerpts embedded with openai/text-embedding-3-small via OpenRouter, 198,819 tokens, $0.0040 metered (ceiling $1). 6.6, family injection only, memory = every family (at most 2 contrast examples per prompt), input profile v1: smoke fixed k = 5 (repeat prefix-cache hit share 0.96); one compile (program c1237a49, proposed instruction 5, balanced Brier 0.128 on 13 C_val candidates vs 0.199 baseline); outer grouped 5-fold, 110 out-of-repository candidates in 29 groups: ADVISORY recall 0.90 [0.82, 0.97], precision 0.54 [0.44, 0.65], AUC 0.74; BLOCK not offered (ECE 0.064 > 0.05; no fold reached a Wilson lower bound of 0.95); canary re-ask agreement 0.83. $0.91 metered at list price; DeepSeek balance $18.20 -> $17.90. Instrument fixes found on the way, each with a test: - memory build: pair sides join the record stored under their harvest unit pin (--units), and only the built profile's records are read (static and plane shared a key; 85 -> 724 examples); - retrieval: stage 1 per side (family / contrast) and at most 2 contrast examples, never padded; - program: the JSON answer contract is part of the signature, appended to proposed instructions (DeepSeek JSON mode refused a prompt without "json"); - compile: candidate_families for a one-family compile; compile.toml k = 5 from the smoke rule. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
48 lines
2.2 KiB
TOML
48 lines
2.2 KiB
TOML
# The pre-registered compile specification of the decision engine's program (learned-decision-engine design 4.3).
|
|
# Committed before any compile runs: the metric, the seeds, the search space and the budget are fixed here, and the
|
|
# compiled artifact records this file's sha256. Changing anything below after looking at a result is a new spec.
|
|
version = 1
|
|
seed = 0
|
|
|
|
[metric]
|
|
name = "balanced_brier_v1" # Brier score of the raw score s on C_val, each (family, label) cell weighted equally
|
|
repository_cap = 5 # each repository counts at most 5 effective rows per cell
|
|
unsure_score = 0.5 # unsure and parse_failed answers score as s = 0.5
|
|
tie_tolerance = 0.005 # within this, fewer prompt tokens wins, then the earlier configuration
|
|
secondary = ["recall_at_precision_0.90", "unsure_share"] # reported, never optimised
|
|
|
|
[split]
|
|
compile_fraction = 0.25 # of the repository groups, per primary family; never an evaluation fold
|
|
boot_fraction = 0.6 # C_boot (demonstrations) vs C_val (scoring)
|
|
retrieval = "compile_only" # during the compile, memory is the compile split only: no evaluated group shapes it
|
|
|
|
[bootstrap]
|
|
max_candidates = 150
|
|
min_confidence = 0.7
|
|
demo_sets = 4
|
|
demos_per_set = 3 # one positive, one negative, one of either; families mixed
|
|
|
|
[search]
|
|
proposals = 6 # instructions proposed in one call, scored with the hand-written baseline (7)
|
|
errors_shown = 20 # summarised bootstrap errors in the proposal prompt (signals and verdicts, no code)
|
|
val_candidates = 60
|
|
|
|
[classify]
|
|
model = "deepseek-flash"
|
|
k = 5 # fixed by the smoke run before any compile spend (task 6.6): samples 2..5 hit the
|
|
# provider prefix cache for 0.96 of their prompt tokens (>= 0.8), 2026-10-01. k and lam
|
|
# remain A/B variants (design section 6), not searched here
|
|
max_output_tokens = 300
|
|
inputs = "v1" # the input profile: the engine instrument is withheld from prompts and retrieval (2026-10-01)
|
|
|
|
[retrieval]
|
|
n1 = 40
|
|
k = 6
|
|
lam = 0.5
|
|
near_duplicate_cos = 0.98
|
|
priors = "off"
|
|
|
|
[budget]
|
|
usd = 3.0 # a compile that reaches it is `unfinished` and writes no artifact
|
|
calls = 1200
|