mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
Marius CiepluchandClaude Opus 5.5
9e3c83cfdc
Maintainer decision "Neutralise both for v1". 1. Phantom candidates. A pair side's candidates are now only the functions that side declares (plane/harvest.py `declares`, the excerpt builder's function_bounds); a labelled function absent from a side is listed under `absent_on_side` and never asked, and a side declaring none of its verify family's functions is no verify unit. The recorded harvest units held 58 phantom verify candidates (53 fixed, 5 vulnerable sides). build_features.py skips label rows whose side does not declare the function and drops their earlier rows (MemoryStore.drop_rows); rebuilt the pair records from the recorded signals, no model call: 126 label rows absent_on_side (117 fixed, 9 vulnerable) -- access_control 13, config_secrets 4, deserialization 20, injection 28, memory 4, output_encoding 16, path 15, untrusted_destination 26 -- so 252 feature rows dropped (1712 -> 1460); no other row changed. 2. Engine withheld. Input profiles in learn/schema.py (`v1` withholds the engine instrument, `full` withholds nothing); ProgramSpec.inputs defaults to v1 and compile.toml records it. Neither the engine's state nor its values reach the candidate, examples, demonstrations or the instruction proposer's error summary, and the Gower retrieval distance leaves the engine block out. Stored records are unchanged. design.md Risks records the decision and that the engine returns only through a pre-registered A/B on assumed-benign and benign-control negatives. 3. `ousast learn audit-leaks` (learn/leaks.py): per feature and per instrument state, separation = (pairs the signal alone puts on the right side - on the wrong side) / pairs, over all pairs and per family; flagged at >= 0.20 over >= 10 pairs. Counts under benchmarks/measurements/2026-10-01-decision-engine-leak-audit/ (before and after the rebuild). After: facts.function_lines (0.80 untrusted_destination, 0.75 config_secrets, 0.26 injection; higher on fixed), verify.turns_mean 0.67 config_secrets, eng.* 0.35-0.40 output_encoding, verify.flag_a/votes/ site_offset and agree.final 0.20-0.30 on 10-12 pair families. Before the rebuild the phantom fixed sides drove state.source/state.roles/ms.flagged to 1.0 and state.engine to 0.95 on deserialization. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
237 lines
5.9 KiB
JSON
237 lines
5.9 KiB
JSON
{
|
|
"audit": {
|
|
"after_rebuild": {
|
|
"flagged": [
|
|
{
|
|
"correct": 8,
|
|
"direction": "higher on fixed",
|
|
"pairs": 10,
|
|
"scope": "untrusted_destination",
|
|
"separation": 0.8,
|
|
"signal": "facts.function_lines",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 8,
|
|
"direction": "higher on fixed",
|
|
"pairs": 12,
|
|
"scope": "config_secrets",
|
|
"separation": 0.6667,
|
|
"signal": "verify.turns_mean",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 18,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.4,
|
|
"signal": "eng.witness_steps_min",
|
|
"wrong": 2
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.findings",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": null,
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.rung_max",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": null,
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.sink_kind",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": null,
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.sink_origin",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": null,
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.source_kind",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 15,
|
|
"direction": null,
|
|
"pairs": 40,
|
|
"scope": "output_encoding",
|
|
"separation": 0.35,
|
|
"signal": "eng.source_origin",
|
|
"wrong": 1
|
|
},
|
|
{
|
|
"correct": 3,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 10,
|
|
"scope": "untrusted_destination",
|
|
"separation": 0.3,
|
|
"signal": "verify.flag_a",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 3,
|
|
"direction": null,
|
|
"pairs": 12,
|
|
"scope": "config_secrets",
|
|
"separation": 0.25,
|
|
"signal": "agree.final",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 3,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 12,
|
|
"scope": "config_secrets",
|
|
"separation": 0.25,
|
|
"signal": "verify.site_offset",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 3,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 12,
|
|
"scope": "config_secrets",
|
|
"separation": 0.25,
|
|
"signal": "verify.votes",
|
|
"wrong": 0
|
|
},
|
|
{
|
|
"correct": 2,
|
|
"direction": "higher on vulnerable",
|
|
"pairs": 10,
|
|
"scope": "untrusted_destination",
|
|
"separation": 0.2,
|
|
"signal": "eng.completion",
|
|
"wrong": 0
|
|
}
|
|
],
|
|
"pairs": {
|
|
"all": 279,
|
|
"config_secrets": 12,
|
|
"injection": 43,
|
|
"memory": 171,
|
|
"output_encoding": 40,
|
|
"path": 1,
|
|
"unknown": 2,
|
|
"untrusted_destination": 10
|
|
}
|
|
},
|
|
"before_rebuild": {
|
|
"flagged_signals": [
|
|
"agree.final",
|
|
"eng.completion",
|
|
"eng.degraded",
|
|
"eng.findings",
|
|
"eng.rung_max",
|
|
"eng.sink_kind",
|
|
"eng.sink_origin",
|
|
"eng.source_kind",
|
|
"eng.source_origin",
|
|
"eng.witness_steps_min",
|
|
"facts.function_lines",
|
|
"ms.flagged",
|
|
"ms.operation",
|
|
"qr.enabled_hits",
|
|
"qr.max_precision_estimate",
|
|
"qr.mechanism.source_reaches_sink",
|
|
"qr.prior_hits",
|
|
"roles.sink_confidence",
|
|
"roles.source_in_function",
|
|
"state.engine",
|
|
"state.roles",
|
|
"state.source",
|
|
"verify.flag_a",
|
|
"verify.flag_b",
|
|
"verify.site_offset",
|
|
"verify.turns_mean",
|
|
"verify.votes"
|
|
],
|
|
"pairs": {
|
|
"access_control": 13,
|
|
"all": 396,
|
|
"config_secrets": 16,
|
|
"deserialization": 20,
|
|
"injection": 66,
|
|
"memory": 173,
|
|
"output_encoding": 56,
|
|
"path": 14,
|
|
"unknown": 2,
|
|
"untrusted_destination": 36
|
|
}
|
|
},
|
|
"command": "ousast learn audit-leaks --profile plane --units <harvest units.json> (threshold 0.20, min 10 pairs)",
|
|
"metric": "separation = (pairs the signal alone puts on the right side - on the wrong side) / pairs; see learn/leaks.py"
|
|
},
|
|
"created": "2026-10-01",
|
|
"label_snapshot_sha256_prefix": "18337014d8a5c8df",
|
|
"note": "Maintainer decision 'Neutralise both for v1' (learned-decision-engine). COUNTS ONLY. (1) Pair-side candidates come only from functions the side declares (plane/harvest.py `declares`, the excerpt builder's function_bounds); the harvest feature records were rebuilt with benchmarks/learn/build_features.py --kinds pairs from the recorded signals (no model call): label rows whose side does not declare the function are skipped (absent_on_side) and their earlier rows dropped from the store. (2) The v1 program withholds the engine instrument from prompts and the retrieval distance. (3) `ousast learn audit-leaks` on the plane profile of the memory store, before and after the rebuild.",
|
|
"rebuild": {
|
|
"absent_on_side_by_side": {
|
|
"fixed": 117,
|
|
"vulnerable": 9
|
|
},
|
|
"absent_on_side_label_rows_by_family": {
|
|
"access_control": 13,
|
|
"config_secrets": 4,
|
|
"deserialization": 20,
|
|
"injection": 28,
|
|
"memory": 4,
|
|
"output_encoding": 16,
|
|
"path": 15,
|
|
"untrusted_destination": 26
|
|
},
|
|
"feature_rows_after": 1460,
|
|
"feature_rows_before": 1712,
|
|
"feature_rows_changed_otherwise": 0,
|
|
"feature_rows_dropped": 252,
|
|
"phantom_verify_candidates_in_recorded_units": {
|
|
"fixed": 53,
|
|
"vulnerable": 5
|
|
},
|
|
"records_after_rebuild_pairs": {
|
|
"plane:access_control": 13,
|
|
"plane:config_secrets": 4,
|
|
"plane:deserialization": 20,
|
|
"plane:injection": 42,
|
|
"plane:memory": 342,
|
|
"plane:output_encoding": 96,
|
|
"plane:path": 13,
|
|
"plane:unknown": 4,
|
|
"plane:untrusted_destination": 26,
|
|
"static:access_control": 13,
|
|
"static:config_secrets": 4,
|
|
"static:deserialization": 20,
|
|
"static:injection": 42,
|
|
"static:memory": 342,
|
|
"static:output_encoding": 96,
|
|
"static:path": 13,
|
|
"static:unknown": 4,
|
|
"static:untrusted_destination": 26
|
|
}
|
|
}
|
|
}
|