marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-18 12:39:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

learn(exp-003): decision-rule experiments and the precision-bound BLOCK arm

Browse Source 
kind: rule experiments score once from the response cache behind a
zero-ceiling meter (a miss is excluded and counted, never asked) and apply
each arm's BLOCK rule to the same held-out scores. Arm B chooses a per-family
raw-score threshold on the other outer folds only (lowest t with Wilson 95%
LB >= 0.95); a test fails if a fold's own labels move its threshold.
Manifest and frozen units (353 units, 146 groups, 244 paired) for exp-003.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-03 08:02:51 +02:00
co-authored by Claude Opus 5.5
parent 432fd2947e
commit dd7ebc119b
6 changed files with 1083 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openultrasast/learn/cli.py
+19 -7
View File
@@ -122,11 +122,17 @@ def _experiment(args: argparse.Namespace) -> int:
print(json.dumps({k: row[k] for k in shown}, indent=2))
return 0
if args.experiment_command == "analyse":
row = ex.check_registered(store, manifest, need_units=False)
row = ex.check_registered(store, manifest, need_units=manifest.kind == "rule")
provenance = {"code_commit": ex.committed_at_head(manifest.path)}
report = ex.analyse(manifest, ex.load_outcomes(store, manifest.id), resamples=args.resamples, provenance=provenance)
outcomes = ex.load_outcomes(store, manifest.id)
if manifest.kind == "rule":
units = ex.read_units(manifest.units_file)
report = ex.analyse_rule(manifest, outcomes, units, resamples=args.resamples, provenance=provenance)
else:
report = ex.analyse(manifest, outcomes, resamples=args.resamples, provenance=provenance)
if args.record:
store.put_row(ex.result_row(manifest, str(row["pin"]), report))
result = ex.rule_result_row if manifest.kind == "rule" else ex.result_row
store.put_row(result(manifest, str(row["pin"]), report))
_write(report, args.out or (ex.RESULT_DIR / manifest.id / "result.json" if args.record else None))
return 0
from .embeddings import EmbeddingCache
@@ -143,6 +149,16 @@ def _experiment(args: argparse.Namespace) -> int:
cache = EmbeddingCache(store)
vectors = {sha: v for sha in cache.names() if (v := cache.get(sha)) is not None} or None
parameters = _model_parameters(args.manifest_model)
def excerpt_text(sha: str) -> str | None:
data = store.get_blob("excerpts", sha)
return data.decode("utf-8") if data is not None else None
if manifest.kind == "rule": # zero cost: cache only, behind a zero-ceiling meter; no endpoint is resolved
caller, meter = ex.zero_cost_caller(_arm_model(manifest, manifest.arms["A"], store), parameters, store)
summary = ex.run_rule(store, manifest, memory, caller, meter, excerpt_text, vectors)
_write(summary.as_dict(), args.out)
return 0
callers: dict[str, Caller] = {}
for name, arm in manifest.arms.items():
if arm.replay_only:
@@ -157,10 +173,6 @@ def _experiment(args: argparse.Namespace) -> int:
client = MeteredClient(resolved[0], prices=parameters, budget_usd=arm.budget_usd)
callers[name] = Caller(client, _arm_model(manifest, arm, store), parameters, store=store)
def excerpt_text(sha: str) -> str | None:
data = store.get_blob("excerpts", sha)
return data.decode("utf-8") if data is not None else None
early = not args.no_early_stop
summary = ex.run(store, manifest, memory, callers, excerpt_text, vectors, stop_early=early, resamples=args.resamples)
except ex.ExperimentError as exc: