mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
merge(learn): advisory-fix harvest and slice 2 -- six families out-of-repo
Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
@@ -1227,6 +1227,15 @@ start only after the maintainer confirms their budget against the balance read j
|
||||
0.90 has precision 0.54 (prevalence 0.53); the majority `vulnerable` verdict alone has precision 0.86, recall 0.43.
|
||||
Canary re-ask agreement at temperature 0 was 0.83, below the 0.9 the adoption check requires.
|
||||
|
||||
- **Slice 2, 2026-10-02 (six families, after the advisory-fix harvest).** Two instrument defects stopped runs before any
|
||||
number: the memory build's side reader knew only the first pair-kind source (all 194 advisory sides unread), and
|
||||
compiled demonstrations skipped the near-duplicate rule that retrieval and the pre-call assertion apply (the
|
||||
assertion stopped the untrusted_destination evaluation). Both fixed with tests. A confound to read with the new
|
||||
families' numbers: every single-label repository group in their evaluation holds positives only (the pre-advisory
|
||||
corpus had no negatives there), so pooled AUC mixes "vulnerable vs its own fix" with "vulnerable vs another
|
||||
repository's fix"; the record reports the paired-group AUC beside it (0.72-0.93, lower than pooled 0.77-0.96).
|
||||
Memorisation is visible: a verify pass named the advisory's CVE id for a public fix.
|
||||
|
||||
## Maintainer decisions at design approval (2026-09-30)
|
||||
|
||||
- Design approved as written.
|
||||
|
||||
@@ -38,6 +38,10 @@ a model or embedding call.
|
||||
- Done 2026-10-01 (`benchmarks/measurements/2026-10-01-decision-engine-harvest/record.json`): verify $2.54 and roles
|
||||
$1.40 by attribution. The engine job finished the same day (682 of 686 pair sides; the 2 family-unknown pairs
|
||||
not run) and the feature records were rebuilt; dev-php pins have no engine run (`missing`).
|
||||
- Advisory-fix pairs harvested 2026-10-02 (`benchmarks/measurements/2026-10-02-decision-engine-harvest-advisory/record.json`):
|
||||
194 sides, verify a/b on the 34 untrusted_destination sides and model roles on all ($0.38 metered, ceiling $5),
|
||||
engine on the host (54 ran, 58 asked no question, 82 no language model); static memory 724 -> 904 examples. The
|
||||
leak audit flags nothing new in the static profile; the plane profile newly flags `verify.flag_b` and `ms.operation`.
|
||||
- _Requirements: 1.1_
|
||||
|
||||
- [ ] 6. Program, local memory and compilation (design sections 4-5)
|
||||
@@ -103,6 +107,12 @@ a model or embedding call.
|
||||
- [ ] 6.7 Second paid slice **(budget go-ahead: ~$7 expected, ceiling $10; needs a top-up)**
|
||||
- Leave-one-source-out (~$1.4-2.1), leave-one-framework-out where a framework reaches 10 groups ($0 today),
|
||||
learning curves over memory size (0/25/50/100%) and number of sources (~$5.2), recorded beside 6.6.
|
||||
- Partial 2026-10-02 (`benchmarks/measurements/2026-10-02-decision-engine-slice-2/record.json`): after the
|
||||
advisory-fix harvest (memory 724 -> 904), one compile per evaluable family and its outer-fold evaluation for all
|
||||
six (injection, untrusted_destination, path, output_encoding, deserialization, access_control); out-of-repository
|
||||
AUC 0.77-0.96, BLOCK offered for none (calibration holds only for output_encoding, where no fold reaches the
|
||||
Wilson bound); one memory-size point for injection (724 vs 904 examples, paired: AUC 0.76 vs 0.77). $2.88
|
||||
metered (ceiling $8). Not done: leave-one-source-out, the 0/25/50% curve points, the source-count curve.
|
||||
- _Requirements: 5.1, 5.2, 8.3_
|
||||
|
||||
- [ ] 7. Experiments (exp-001 and program variants need budget go-ahead)
|
||||
|
||||
@@ -0,0 +1,778 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"embeddings": {
|
||||
"cached": 616,
|
||||
"calls": 6,
|
||||
"embedded": 172,
|
||||
"model": "openai/text-embedding-3-small",
|
||||
"requested": 788,
|
||||
"tokens": 43397,
|
||||
"usd": 0.0008679400000000001
|
||||
},
|
||||
"engine": {
|
||||
"by_side_state": {
|
||||
"fixed:failed": 30,
|
||||
"fixed:none": 41,
|
||||
"fixed:ran": 26,
|
||||
"vuln:failed": 28,
|
||||
"vuln:none": 41,
|
||||
"vuln:ran": 28
|
||||
},
|
||||
"frozen_source_commit": "7b17eacd44fa22ba56d96da3af1491cef2331987",
|
||||
"image": [
|
||||
"sha256:e6f8dc6cc7547f7118bd058687de676e0fcf741799d1d3839b3e8648ce190fbb"
|
||||
],
|
||||
"not_ran_reasons": {
|
||||
"failed (read its input, asked no question)": 58,
|
||||
"no engine model for csharp": 18,
|
||||
"no engine model for go": 22,
|
||||
"no engine model for ruby": 22,
|
||||
"no engine model for typescript": 20
|
||||
},
|
||||
"note": "none = no engine model for the language; failed = read its input and asked no question (never zero findings)",
|
||||
"ran_with_findings": 3,
|
||||
"seconds_total": 7400,
|
||||
"sides": 194,
|
||||
"state": {
|
||||
"failed": 58,
|
||||
"none": 82,
|
||||
"ran": 54
|
||||
}
|
||||
},
|
||||
"features": {
|
||||
"absent_on_side": {},
|
||||
"dropped_stale_rows": 0,
|
||||
"instrument_states_plane": {
|
||||
"access_control:agree": {
|
||||
"none": 40
|
||||
},
|
||||
"access_control:delta": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:engine": {
|
||||
"failed": 10,
|
||||
"none": 20,
|
||||
"ran": 10
|
||||
},
|
||||
"access_control:entry_points": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:facts": {
|
||||
"not_applicable": 40
|
||||
},
|
||||
"access_control:language": {
|
||||
"ran": 40
|
||||
},
|
||||
"access_control:model_sinks": {
|
||||
"ran": 40
|
||||
},
|
||||
"access_control:quick": {
|
||||
"none": 16,
|
||||
"ran": 24
|
||||
},
|
||||
"access_control:roles": {
|
||||
"failed": 4,
|
||||
"none": 4,
|
||||
"ran": 32
|
||||
},
|
||||
"access_control:source": {
|
||||
"failed": 4,
|
||||
"none": 4,
|
||||
"ran": 32
|
||||
},
|
||||
"access_control:verify": {
|
||||
"none": 40
|
||||
},
|
||||
"deserialization:agree": {
|
||||
"none": 42
|
||||
},
|
||||
"deserialization:delta": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:engine": {
|
||||
"failed": 15,
|
||||
"none": 14,
|
||||
"ran": 13
|
||||
},
|
||||
"deserialization:entry_points": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:facts": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"deserialization:language": {
|
||||
"ran": 42
|
||||
},
|
||||
"deserialization:model_sinks": {
|
||||
"ran": 42
|
||||
},
|
||||
"deserialization:quick": {
|
||||
"none": 12,
|
||||
"ran": 30
|
||||
},
|
||||
"deserialization:roles": {
|
||||
"none": 6,
|
||||
"ran": 36
|
||||
},
|
||||
"deserialization:source": {
|
||||
"none": 6,
|
||||
"ran": 36
|
||||
},
|
||||
"deserialization:verify": {
|
||||
"none": 42
|
||||
},
|
||||
"output_encoding:agree": {
|
||||
"none": 42
|
||||
},
|
||||
"output_encoding:delta": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:engine": {
|
||||
"failed": 15,
|
||||
"none": 20,
|
||||
"ran": 7
|
||||
},
|
||||
"output_encoding:entry_points": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:facts": {
|
||||
"not_applicable": 42
|
||||
},
|
||||
"output_encoding:language": {
|
||||
"ran": 42
|
||||
},
|
||||
"output_encoding:model_sinks": {
|
||||
"ran": 42
|
||||
},
|
||||
"output_encoding:quick": {
|
||||
"none": 12,
|
||||
"ran": 30
|
||||
},
|
||||
"output_encoding:roles": {
|
||||
"failed": 2,
|
||||
"none": 4,
|
||||
"ran": 36
|
||||
},
|
||||
"output_encoding:source": {
|
||||
"failed": 2,
|
||||
"none": 4,
|
||||
"ran": 36
|
||||
},
|
||||
"output_encoding:verify": {
|
||||
"none": 42
|
||||
},
|
||||
"path:agree": {
|
||||
"none": 36
|
||||
},
|
||||
"path:delta": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:engine": {
|
||||
"failed": 6,
|
||||
"none": 16,
|
||||
"ran": 14
|
||||
},
|
||||
"path:entry_points": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:facts": {
|
||||
"not_applicable": 36
|
||||
},
|
||||
"path:language": {
|
||||
"ran": 36
|
||||
},
|
||||
"path:model_sinks": {
|
||||
"ran": 36
|
||||
},
|
||||
"path:quick": {
|
||||
"none": 14,
|
||||
"ran": 22
|
||||
},
|
||||
"path:roles": {
|
||||
"none": 2,
|
||||
"ran": 34
|
||||
},
|
||||
"path:source": {
|
||||
"none": 2,
|
||||
"ran": 34
|
||||
},
|
||||
"path:verify": {
|
||||
"none": 36
|
||||
},
|
||||
"untrusted_destination:agree": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:delta": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:engine": {
|
||||
"failed": 12,
|
||||
"none": 12,
|
||||
"ran": 10
|
||||
},
|
||||
"untrusted_destination:entry_points": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:facts": {
|
||||
"not_applicable": 34
|
||||
},
|
||||
"untrusted_destination:language": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:model_sinks": {
|
||||
"ran": 34
|
||||
},
|
||||
"untrusted_destination:quick": {
|
||||
"none": 8,
|
||||
"ran": 26
|
||||
},
|
||||
"untrusted_destination:roles": {
|
||||
"failed": 2,
|
||||
"none": 2,
|
||||
"ran": 30
|
||||
},
|
||||
"untrusted_destination:source": {
|
||||
"failed": 2,
|
||||
"none": 2,
|
||||
"ran": 30
|
||||
},
|
||||
"untrusted_destination:verify": {
|
||||
"ran": 34
|
||||
}
|
||||
},
|
||||
"records": {
|
||||
"plane:access_control": 40,
|
||||
"plane:deserialization": 42,
|
||||
"plane:output_encoding": 42,
|
||||
"plane:path": 36,
|
||||
"plane:untrusted_destination": 34,
|
||||
"static:access_control": 40,
|
||||
"static:deserialization": 42,
|
||||
"static:output_encoding": 42,
|
||||
"static:path": 36,
|
||||
"static:untrusted_destination": 34
|
||||
},
|
||||
"skipped_label_rows": {}
|
||||
},
|
||||
"host_peaks": {
|
||||
"disk_free_mb_min": 2630,
|
||||
"mem_available_mb_min": 2341
|
||||
},
|
||||
"labels": {
|
||||
"advisory_pin_rows": 194,
|
||||
"note": "regenerated 2026-10-02; identical to the 2026-10-01 snapshot apart from the `created` field",
|
||||
"rows": 6589,
|
||||
"same_counts_as": "2026-10-01-decision-engine-labels-advisory-fixes/labels.json"
|
||||
},
|
||||
"leak_audit": {
|
||||
"command": "ousast learn audit-leaks --profile {plane,static} --units <merged units.json> (threshold 0.20, min 10 pairs)",
|
||||
"pairs": {
|
||||
"access_control": 20,
|
||||
"all": 376,
|
||||
"config_secrets": 12,
|
||||
"deserialization": 21,
|
||||
"injection": 43,
|
||||
"memory": 171,
|
||||
"output_encoding": 61,
|
||||
"path": 19,
|
||||
"unknown": 2,
|
||||
"untrusted_destination": 27
|
||||
},
|
||||
"plane": {
|
||||
"flagged": [
|
||||
{
|
||||
"correct": 10,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.75,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 16,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 19,
|
||||
"scope": "path",
|
||||
"separation": 0.7368,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 8,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.6667,
|
||||
"signal": "verify.turns_mean",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 13,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.6,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "agree.final",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "verify.votes",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 14,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5185,
|
||||
"signal": "verify.flag_a",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.4815,
|
||||
"signal": "verify.site_offset",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 12,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.4074,
|
||||
"signal": "verify.flag_b",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 6,
|
||||
"direction": null,
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.3,
|
||||
"signal": "ms.operation",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 9,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 21,
|
||||
"scope": "deserialization",
|
||||
"separation": 0.2857,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2623,
|
||||
"signal": "eng.witness_steps_min",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 7,
|
||||
"direction": null,
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.2593,
|
||||
"signal": "ms.operation",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 21,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 43,
|
||||
"scope": "injection",
|
||||
"separation": 0.2558,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 10
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": null,
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "agree.final",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.flag_a",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.site_offset",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 3,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.25,
|
||||
"signal": "verify.votes",
|
||||
"wrong": 0
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.rung_max",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2131,
|
||||
"signal": "eng.findings",
|
||||
"wrong": 2
|
||||
}
|
||||
],
|
||||
"flagged_signals": [
|
||||
"agree.final",
|
||||
"eng.findings",
|
||||
"eng.rung_max",
|
||||
"eng.sink_kind",
|
||||
"eng.sink_origin",
|
||||
"eng.source_kind",
|
||||
"eng.source_origin",
|
||||
"eng.witness_steps_min",
|
||||
"facts.function_lines",
|
||||
"ms.operation",
|
||||
"verify.flag_a",
|
||||
"verify.flag_b",
|
||||
"verify.site_offset",
|
||||
"verify.turns_mean",
|
||||
"verify.votes"
|
||||
],
|
||||
"new_signal_names_vs_2026_10_01": [
|
||||
"ms.operation",
|
||||
"verify.flag_b"
|
||||
]
|
||||
},
|
||||
"reading": "The static profile (the one slice 2 runs) flags only engine signals and facts.function_lines, both withheld by input profile v1. The plane profile newly flags verify.flag_b and ms.operation (plane-only instruments, never in a static/v1 prompt); verify flags point to the vulnerable side, as a real detector would, but whether they track the label must be decided before any plane-profile program runs.",
|
||||
"static": {
|
||||
"flagged": [
|
||||
{
|
||||
"correct": 10,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 12,
|
||||
"scope": "config_secrets",
|
||||
"separation": 0.75,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 16,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 19,
|
||||
"scope": "path",
|
||||
"separation": 0.7368,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 13,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 20,
|
||||
"scope": "access_control",
|
||||
"separation": 0.6,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 27,
|
||||
"scope": "untrusted_destination",
|
||||
"separation": 0.5556,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 9,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 21,
|
||||
"scope": "deserialization",
|
||||
"separation": 0.2857,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 3
|
||||
},
|
||||
{
|
||||
"correct": 18,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2623,
|
||||
"signal": "eng.witness_steps_min",
|
||||
"wrong": 2
|
||||
},
|
||||
{
|
||||
"correct": 21,
|
||||
"direction": "higher on fixed",
|
||||
"pairs": 43,
|
||||
"scope": "injection",
|
||||
"separation": 0.2558,
|
||||
"signal": "facts.function_lines",
|
||||
"wrong": 10
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.rung_max",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.sink_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_kind",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": null,
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2295,
|
||||
"signal": "eng.source_origin",
|
||||
"wrong": 1
|
||||
},
|
||||
{
|
||||
"correct": 15,
|
||||
"direction": "higher on vulnerable",
|
||||
"pairs": 61,
|
||||
"scope": "output_encoding",
|
||||
"separation": 0.2131,
|
||||
"signal": "eng.findings",
|
||||
"wrong": 2
|
||||
}
|
||||
],
|
||||
"flagged_signals": [
|
||||
"eng.findings",
|
||||
"eng.rung_max",
|
||||
"eng.sink_kind",
|
||||
"eng.sink_origin",
|
||||
"eng.source_kind",
|
||||
"eng.source_origin",
|
||||
"eng.witness_steps_min",
|
||||
"facts.function_lines"
|
||||
],
|
||||
"outside_v1_withheld": []
|
||||
}
|
||||
},
|
||||
"memory_build": {
|
||||
"advisory_sides_not_examples": {
|
||||
"absent_fixed_side": 7,
|
||||
"no_excerpt": 7
|
||||
},
|
||||
"by_source": {
|
||||
"adjudications": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 34,
|
||||
"labels": 43,
|
||||
"no_excerpt": 9,
|
||||
"no_record": 0,
|
||||
"recorded": 43,
|
||||
"unread": 0
|
||||
},
|
||||
"dev-php": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 3,
|
||||
"labels": 3,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 0,
|
||||
"recorded": 3,
|
||||
"unread": 0
|
||||
},
|
||||
"pairs": {
|
||||
"absent_fixed_side": 7,
|
||||
"examples": 740,
|
||||
"labels": 1544,
|
||||
"no_excerpt": 7,
|
||||
"no_record": 790,
|
||||
"recorded": 754,
|
||||
"unread": 0
|
||||
},
|
||||
"population-v1": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 85,
|
||||
"labels": 170,
|
||||
"no_excerpt": 0,
|
||||
"no_record": 85,
|
||||
"recorded": 85,
|
||||
"unread": 0
|
||||
},
|
||||
"population-v2": {
|
||||
"absent_fixed_side": 0,
|
||||
"examples": 42,
|
||||
"labels": 86,
|
||||
"no_excerpt": 1,
|
||||
"no_record": 43,
|
||||
"recorded": 43,
|
||||
"unread": 0
|
||||
}
|
||||
},
|
||||
"examples": 904,
|
||||
"examples_added": 180,
|
||||
"examples_before": 724,
|
||||
"failing_reason": "adjudications: 9 of 43 anonymous functions, unchanged since 2026-10-01 (exit 2 recorded, rows kept)",
|
||||
"failing_sources": [
|
||||
"adjudications"
|
||||
],
|
||||
"instrument_fix": "SideReader read only the first pair-kind source, so all 194 advisory sides were `unread` (0 examples) on the first rebuild; fixed in learn/examples.py with a test; the rebuild then read every side",
|
||||
"profile": "static"
|
||||
},
|
||||
"note": "Decision-engine harvest of the 97 advisory-fix pairs (benchmarks/pairs/advisory-fixes, 194 sides), COUNTS ONLY. Plane Runs hadv-verify (verify a/b + agree; verify families only: untrusted_destination) and hadv-roles (model roles, every side) on ax, generated by `ousast plane harvest --catalog benchmarks/pairs/advisory-fixes/catalog.toml --ceiling 2` from the label rows of those pairs; host engine features (benchmarks/learn/engine_pairs.py, frozen source, one container, --memory 3g); feature records (benchmarks/learn/build_features.py); static memory rebuilt with --units (old + new harvest units merged). Unit lists, label rows and excerpts name functions and stay local.",
|
||||
"roles": {
|
||||
"files_classified": 194,
|
||||
"role_rows": 660,
|
||||
"sides": 194,
|
||||
"skipped_units": {
|
||||
"oversized_over_20KB_inline": 0
|
||||
},
|
||||
"tasks_done": 20
|
||||
},
|
||||
"runner_image": "localhost:5001/ousast-runner@sha256:412db0a1504af8f1b23a395af8d1dcbd152c1bb6d16594f2b733067163a31884 (not rebuilt: no change under plane/tasks since it was built)",
|
||||
"spend": {
|
||||
"ceiling_usd": 5.0,
|
||||
"deepseek_balance_usd": {
|
||||
"after_features": 17.73,
|
||||
"after_runs": 17.75,
|
||||
"before": 17.88
|
||||
},
|
||||
"embeddings_openrouter_usd": 0.000868,
|
||||
"metered_usd": {
|
||||
"pilot_included": true,
|
||||
"roles": 0.1285,
|
||||
"total": 0.3802,
|
||||
"verify": 0.2517
|
||||
},
|
||||
"smoke": "plain JSON + one tool call through verify.build_client before any Run (one reply carried a stray `<|OPENAI|>` token in JSON mode on a trivial prompt; 0/16 on a classifier-shaped probe)",
|
||||
"task_budget_sum_usd": {
|
||||
"roles": 0.669,
|
||||
"verify": 0.816
|
||||
}
|
||||
},
|
||||
"verify": {
|
||||
"flagged_by_pass_units": {
|
||||
"fixed:none": 16,
|
||||
"fixed:one": 1,
|
||||
"vulnerable:both": 10,
|
||||
"vulnerable:none": 4,
|
||||
"vulnerable:one": 3
|
||||
},
|
||||
"hunts_per_pass": 34,
|
||||
"skipped_units": {
|
||||
"oversized_over_20KB_inline": 0
|
||||
},
|
||||
"tasks_done": 102,
|
||||
"units": 34,
|
||||
"usage": {
|
||||
"completion_tokens": 66734,
|
||||
"prompt_cache_hit_tokens": 444926,
|
||||
"prompt_tokens": 802586
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -265,8 +265,10 @@ class SideReader:
|
||||
if self._pairs is None:
|
||||
from ..pairs import load_pair_catalog
|
||||
|
||||
source = next(s for s in self.sources.sources if s.kind == "pairs")
|
||||
self._pairs = {case.name: case for case in load_pair_catalog(self.root / source.files[0])}
|
||||
# every pair-kind source (the corpus catalog and e.g. advisory-fixes): labels name both as source "pairs"
|
||||
self._pairs = {}
|
||||
for source in (s for s in self.sources.sources if s.kind == "pairs"):
|
||||
self._pairs.update({case.name: case for case in load_pair_catalog(self.root / source.files[0])})
|
||||
return self._pairs
|
||||
|
||||
def license(self, label: Mapping[str, Any]) -> str:
|
||||
|
||||
@@ -475,7 +475,8 @@ class Program:
|
||||
target = candidate.target
|
||||
got = retrieve(target, self.memory, fold, vectors=self.vectors, n1=self.spec.n1, k=self.spec.k_ret, lam=self.spec.lam, seed=seed,
|
||||
priors=self.spec.priors, inputs=self.spec.inputs) # fmt: skip
|
||||
usable = [Demo.from_dict(d) for d in demonstrations_for([d.as_dict() for d in self.spec.demos], self.index, target, fold)]
|
||||
demos = demonstrations_for([d.as_dict() for d in self.spec.demos], self.index, target, fold, vectors=self.vectors)
|
||||
usable = [Demo.from_dict(d) for d in demos]
|
||||
prompt = render(self.spec, candidate, got.examples, usable, self.index, self.excerpt_text)
|
||||
assert_boundary([*prompt.example_ids, *prompt.demo_ids], self.index, target, fold, vectors=self.vectors)
|
||||
return prompt, got
|
||||
|
||||
@@ -240,19 +240,32 @@ def assert_boundary(
|
||||
|
||||
|
||||
def demonstrations_for(
|
||||
demos: Sequence[Mapping[str, Any]], examples: Mapping[str, Example], target: Target, fold: Fold
|
||||
demos: Sequence[Mapping[str, Any]],
|
||||
examples: Mapping[str, Example],
|
||||
target: Target,
|
||||
fold: Fold,
|
||||
*,
|
||||
vectors: Mapping[str, Sequence[float]] | None = None,
|
||||
) -> list[Mapping[str, Any]]:
|
||||
"""The compiled demonstrations usable for ``target``: an ineligible one is swapped for its alternate when that
|
||||
is eligible, and dropped otherwise (leave-one-framework-out on a demonstration's framework)."""
|
||||
is eligible, and dropped otherwise (leave-one-framework-out on a demonstration's framework; in evaluation, a
|
||||
demonstration whose excerpt is a near-duplicate of the target's, the same rule as retrieval and
|
||||
:func:`assert_boundary`)."""
|
||||
|
||||
def ok(example: Example | None) -> bool:
|
||||
if example is None:
|
||||
return False
|
||||
vector = (vectors or {}).get(example.excerpt_sha)
|
||||
cos = cosine(target.vector, vector) if target.vector is not None and vector is not None else None
|
||||
return eligible(example, target, fold, cos=cos)
|
||||
|
||||
out: list[Mapping[str, Any]] = []
|
||||
for demo in demos:
|
||||
example = examples.get(str(demo["example_id"]))
|
||||
if example is not None and eligible(example, target, fold):
|
||||
if ok(examples.get(str(demo["example_id"]))):
|
||||
out.append(demo)
|
||||
continue
|
||||
alternate = demo.get("alternate")
|
||||
alt = examples.get(str(alternate.get("example_id"))) if isinstance(alternate, Mapping) else None
|
||||
if isinstance(alternate, Mapping) and alt is not None and eligible(alt, target, fold):
|
||||
if isinstance(alternate, Mapping) and ok(examples.get(str(alternate.get("example_id")))):
|
||||
out.append(alternate)
|
||||
return out
|
||||
|
||||
|
||||
@@ -171,3 +171,34 @@ def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> Non
|
||||
assert build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES).summary()["examples"] == 0
|
||||
build = build_examples([lab], records_by_key([stored]), lines_of, sources=SOURCES, pins=pins)
|
||||
assert build.summary()["examples"] == 1 and build.rows[0]["pin"] == unit_pin
|
||||
|
||||
|
||||
def test_side_reader_reads_every_pair_kind_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Labels from the corpus catalog and from a second pair-kind source (advisory-fixes) are both source ``pairs``;
|
||||
the reader must find either side, not only the first catalog's (the 2026-10-02 advisory rebuild read 0 of 194)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
import openultrasast.pairs as pairs
|
||||
from openultrasast.learn.examples import SideReader
|
||||
|
||||
files = {}
|
||||
for name in ("corpus-case", "advisory-case"):
|
||||
files[name] = tmp_path / f"{name}.py"
|
||||
files[name].write_text("def handler(request):\n return 1\n")
|
||||
catalogs = {"corpus.toml": "corpus-case", "advisory.toml": "advisory-case"}
|
||||
|
||||
def fake(path: Path) -> tuple[Any, ...]:
|
||||
name = catalogs[Path(path).name]
|
||||
return (SimpleNamespace(name=name, vuln_file=files[name], fixed_file=files[name], license="MIT"),)
|
||||
|
||||
monkeypatch.setattr(pairs, "load_pair_catalog", fake)
|
||||
sources = Sources(
|
||||
(Source("pairs", "pairs", ("corpus.toml",), (), "2026-09-30", "development"),
|
||||
Source("advisory-fixes", "pairs", ("advisory.toml",), (), "2026-10-01", "development")),
|
||||
frozenset(), {}, Path("sources.toml"),
|
||||
) # fmt: skip
|
||||
reader = SideReader(tmp_path, tmp_path, sources, [])
|
||||
for name in catalogs.values():
|
||||
lab = label(source="pairs", source_ref=name, pin="")
|
||||
assert reader(lab, "vulnerable") == ["def handler(request):", " return 1"]
|
||||
assert reader.license(lab) == "MIT"
|
||||
|
||||
@@ -136,6 +136,24 @@ def test_near_duplicates_drop_in_evaluation_and_stay_in_deployment() -> None:
|
||||
assert kept.near_duplicates == 0 and twin.id in {e.id for e in kept.examples}
|
||||
|
||||
|
||||
def test_a_near_duplicate_demonstration_is_swapped_in_evaluation_and_kept_in_deployment() -> None:
|
||||
"""A compiled demonstration whose excerpt is a near-duplicate of the target's (cos >= 0.98) is ineligible in
|
||||
evaluation, like a retrieved one: swapped for its alternate, or dropped. Before 2026-10-02 only the pre-call
|
||||
assertion checked it, and the advisory-fix slice stopped there (BoundaryViolation: near duplicate)."""
|
||||
twin = example("demo-twin", group="fork/repo", label=1)
|
||||
alt = example("demo-alt", group="a/b", label=1)
|
||||
vectors = {twin.excerpt_sha: [1.0, 0.0], alt.excerpt_sha: [0.0, 1.0]}
|
||||
target = replace(_target(), vector=[1.0, 0.0])
|
||||
evaluation = replace(outer_folds([twin], k=1)[0], eval_groups=frozenset())
|
||||
demos = [{"example_id": twin.id, "alternate": {"example_id": alt.id}}]
|
||||
usable = demonstrations_for(demos, by_id([twin, alt]), target, evaluation, vectors=vectors)
|
||||
assert [d["example_id"] for d in usable] == [alt.id]
|
||||
assert_boundary([d["example_id"] for d in usable], by_id([twin, alt]), target, evaluation, vectors=vectors)
|
||||
assert demonstrations_for([{"example_id": twin.id, "alternate": None}], by_id([twin]), target, evaluation, vectors=vectors) == []
|
||||
deployed, _ = deployment_fold("https://github.com/someone/else", ["fork/repo"])
|
||||
assert [d["example_id"] for d in demonstrations_for(demos, by_id([twin, alt]), target, deployed, vectors=vectors)] == [twin.id]
|
||||
|
||||
|
||||
def test_deployment_excludes_the_scanned_repository_when_it_is_a_corpus_group() -> None:
|
||||
own = example("own", group="acme/webapp", label=1)
|
||||
fold, note = deployment_fold("https://github.com/Acme/WebApp.git", ["acme/webapp", "x/y"])
|
||||
|
||||
Reference in New Issue
Block a user