diff --git a/.kiro/specs/learned-decision-engine/tasks.md b/.kiro/specs/learned-decision-engine/tasks.md index 941324c..1d756d1 100644 --- a/.kiro/specs/learned-decision-engine/tasks.md +++ b/.kiro/specs/learned-decision-engine/tasks.md @@ -133,7 +133,10 @@ a model or embedding call. annotation checks / the `experiment-outcome` task are not written) and `exp-002-retrieval-ensemble` (arm A today's k = 5 temperature sampling, arm B the retrieval ensemble; primary canary agreement (target 0.90) and within-pair AUC, secondary usd per candidate, AUC, ADVISORY recall/precision; families injection and access_control on the - slice-2 candidates; ceiling $4: A $1, B $3). + slice-2 candidates; ceiling $4: A $1, B $3). Both registered in `s3://sast-memory` at fbc3220 + (`benchmarks/experiments//registration.json`). exp-002's units are `pending` too: on 2026-10-02 every example + row in the store carried null plane-only instruments (`agree`, `model_sinks`, `verify`) and `load_examples` raised + on them, so `ousast learn experiment units` could not freeze the candidates. - [ ] 7.3 exp-001 run and recorded (~$7.7 expected, ceiling $15) -- an inconclusive result is pre-registered as likely; needs the frozen units file first. - [ ] 7.4 exp-002 run and recorded **(budget go-ahead: ~$2 expected, ceiling $4)**; further program variants diff --git a/benchmarks/experiments/exp-001-known-callers/registration.json b/benchmarks/experiments/exp-001-known-callers/registration.json new file mode 100644 index 0000000..444a260 --- /dev/null +++ b/benchmarks/experiments/exp-001-known-callers/registration.json @@ -0,0 +1,11 @@ +{ + "experiment": "exp-001-known-callers", + "registered": "2026-10-02", + "store": "s3://sast-memory", + "manifest": "plane/experiments/exp-001-known-callers.yaml", + "manifest_digest": "76b54aa5ae908a4cd7797f3b4654acf03b90e5ad5d068216036e722104313929", + "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", + "units_status": "pending", + "units_note": "model-free candidate generation on the 41 development repositories is not built; the plane's experiment-outcome task and Run annotation checks are not written", + "status": "registered, not run" +} diff --git a/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json b/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json new file mode 100644 index 0000000..c454261 --- /dev/null +++ b/benchmarks/experiments/exp-002-retrieval-ensemble/registration.json @@ -0,0 +1,11 @@ +{ + "experiment": "exp-002-retrieval-ensemble", + "registered": "2026-10-02", + "store": "s3://sast-memory", + "manifest": "plane/experiments/exp-002-retrieval-ensemble.yaml", + "manifest_digest": "f5004fb8618b1f7cc7dcf298d42b4ac1b6c0c1aa72a97299c77f937bd6175148", + "pin": "fbc3220bbe1c8a4f8afa0a724fd41a51b390dd22", + "units_status": "pending", + "units_note": "`ousast learn experiment units` could not freeze the slice-2 candidates on 2026-10-02: all 904 example rows in the store carry null for the plane-only instruments (agree, model_sinks, verify) and learn.examples.load_examples (unchanged since a90e570) raises on them; freeze, commit and register again once the rows read", + "status": "registered, not run" +}