marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(learn): fixed-side negatives only where the fixed side declares the function

Browse Source 
A pair now gives a fixed-side negative only when the fixed side's source declares the
labelled function (the excerpt builder's matcher; a name on a code line where the matcher
resolves neither side), or when a deterministic rename at a distinct fix commit replaces it
(one new function, same parameter list, changed body: provenance fixed_side_moved). Anything
else is counted under pair_fixed_side.absent_fixed_side, not labelled.

108 pair labels lose their negative: 80 seeded pointer pairs whose fixed side is a guard
helper of the same commit, and 28 Real-Vuln-Benchmark traps whose fixed side is a sibling
handler of the same snapshot (4 of those handlers are labelled positives themselves). The
memory build no longer substitutes the catalog's fix_function on the fixed side. The pair
cache is an allowed label-source tree so pointer sides are read through the guard.

Snapshot: benchmarks/measurements/2026-10-01-decision-engine-labels (counts only).

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
2026-10-01 11:03:54 +02:00
co-authored by Claude Opus 5.5
parent 7f6bbacb6e
commit 658f2d5e39
5 changed files with 711 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/measurements/2026-10-01-decision-engine-labels/labels.json
+477
View File
@@ -0,0 +1,477 @@
{
"assumed_benign_history": {
"accepted": 969,
"commits_scanned": 36000,
"rejected": {
"bulk change": 278,
"dependency": 2206,
"english": 4251,
"later fix touches its files": 3168,
"near_fix": 4911,
"no changed function": 50,
"no product source": 2368,
"other_language": 2,
"protocol pin": 55,
"reference": 77
},
"repositories": 28
},
"conditional_negatives": {
"assumed_benign": {
"by_language": {
"go": 18,
"javascript": 2136,
"php": 832,
"python": 1049,
"typescript": 636
},
"groups": 27,
"pushes": 969,
"rows": 4671
},
"benign_control": {
"by_language": {
"javascript": 7,
"php": 27,
"python": 23,
"typescript": 15
},
"groups": 24,
"pushes": 24,
"rows": 72
}
},
"created": "2026-10-01",
"excluded_unread": [
"candidates-v3-php.json",
"freeze-v3-php.json",
"population-v3-php.toml",
"protocol-v3.md",
"research-2026-09-30-php.md"
],
"floors": {
"per_family": 20,
"pooled": 10,
"unit": "repository groups with positive and with negative pin labels"
},
"group_merges": {
"advisory": 0,
"alias": 0,
"commit": 0
},
"groups": 105,
"labels_sha256": "e71815fffefcc59d6323979eb09abbbcb5b01f9e8570fe3a32a825977a8d2ba4",
"note": "First label snapshot of the learned decision engine (task 4), COUNTS ONLY: rows and repository groups per source, family, language, framework, split and unit. No candidate, path, function, commit or repository name. Built by `ousast learn labels` from src/openultrasast/learn/sources.toml (fail-closed; population v3 excluded unread). verified_pin_labels exclude conditional rows (benign_control, assumed_benign), which need a candidate signal at the tip and none at the base. `model` applies the design's floors to repository groups with positive and with negative pin labels. The full rows (labels_sha256) stay local: they name functions. Rebuilt 2026-10-01 after the fixed-side fix: a pair gives a fixed-side negative only where the fixed side's source declares the labelled function (or a deterministic rename at a distinct fix commit, provenance fixed_side_moved); otherwise it is counted under pair_fixed_side.absent_fixed_side. 108 pair labels lost their negative: 80 seeded pointer pairs whose fixed side is a guard helper of the same commit, and 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler of the same snapshot (fix_function; 4 of those handlers are labelled positives themselves). No renamed equivalent exists (fixed_side_moved = 0). Supersedes 2026-09-30 for the per-family floors.",
"pair_fixed_side": {
"absent_fixed_side": {
"by_family": {
"access_control": 13,
"config_secrets": 4,
"deserialization": 20,
"injection": 18,
"output_encoding": 16,
"path": 11,
"untrusted_destination": 26
},
"labels": 108
},
"fixed_side_moved": 0
},
"rows": 6201,
"schema": 1,
"skipped": {
"pairs: label names no function": 44,
"pairs: no fixed-side negative (fixed side is another function of the same snapshot)": 108,
"pairs: title tier (sensitivity arm)": 58,
"pairs: unscorable pair": 7,
"population: positive the fix did not change (no negative)": 22
},
"sources": [
"adjudications",
"assumed_benign",
"dev-php",
"pairs",
"population-v1",
"population-v2"
],
"verified_delta_labels": {
"access_control:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 12,
"positive_rows": 13
},
"config_secrets:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 5,
"positive_rows": 18
},
"config_secrets:repair": {
"negative_groups": 2,
"negative_rows": 12,
"positive_groups": 0,
"positive_rows": 0
},
"deserialization:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 20,
"positive_rows": 20
},
"injection:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 42,
"positive_rows": 77
},
"injection:repair": {
"negative_groups": 33,
"negative_rows": 48,
"positive_groups": 0,
"positive_rows": 0
},
"memory:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 5,
"positive_rows": 173
},
"memory:repair": {
"negative_groups": 5,
"negative_rows": 173,
"positive_groups": 0,
"positive_rows": 0
},
"output_encoding:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 15,
"positive_rows": 56
},
"output_encoding:repair": {
"negative_groups": 1,
"negative_rows": 40,
"positive_groups": 0,
"positive_rows": 0
},
"path:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 14,
"positive_rows": 14
},
"path:repair": {
"negative_groups": 3,
"negative_rows": 3,
"positive_groups": 0,
"positive_rows": 0
},
"unknown:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 2,
"positive_rows": 2
},
"unknown:repair": {
"negative_groups": 2,
"negative_rows": 2,
"positive_groups": 0,
"positive_rows": 0
},
"untrusted_destination:introduce": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 35,
"positive_rows": 45
},
"untrusted_destination:repair": {
"negative_groups": 7,
"negative_rows": 10,
"positive_groups": 0,
"positive_rows": 0
}
},
"verified_pin_labels": {
"by_family": {
"access_control": {
"model": "insufficient",
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 12,
"positive_rows": 13
},
"config_secrets": {
"model": "insufficient",
"negative_groups": 2,
"negative_rows": 13,
"positive_groups": 5,
"positive_rows": 18
},
"deserialization": {
"model": "insufficient",
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 20,
"positive_rows": 20
},
"injection": {
"model": "per_family",
"negative_groups": 33,
"negative_rows": 71,
"positive_groups": 44,
"positive_rows": 81
},
"memory": {
"model": "insufficient",
"negative_groups": 5,
"negative_rows": 173,
"positive_groups": 5,
"positive_rows": 173
},
"output_encoding": {
"model": "insufficient",
"negative_groups": 1,
"negative_rows": 40,
"positive_groups": 15,
"positive_rows": 56
},
"path": {
"model": "insufficient",
"negative_groups": 3,
"negative_rows": 3,
"positive_groups": 15,
"positive_rows": 15
},
"unknown": {
"model": "insufficient",
"negative_groups": 2,
"negative_rows": 2,
"positive_groups": 2,
"positive_rows": 2
},
"untrusted_destination": {
"model": "insufficient",
"negative_groups": 8,
"negative_rows": 26,
"positive_groups": 35,
"positive_rows": 46
}
},
"by_framework": {
"django": {
"negative_groups": 1,
"negative_rows": 2,
"positive_groups": 1,
"positive_rows": 2
},
"express": {
"negative_groups": 3,
"negative_rows": 6,
"positive_groups": 4,
"positive_rows": 12
},
"flask": {
"negative_groups": 2,
"negative_rows": 3,
"positive_groups": 2,
"positive_rows": 3
},
"httpx": {
"negative_groups": 3,
"negative_rows": 4,
"positive_groups": 3,
"positive_rows": 6
},
"lodash": {
"negative_groups": 5,
"negative_rows": 9,
"positive_groups": 6,
"positive_rows": 15
},
"node-http-clients": {
"negative_groups": 4,
"negative_rows": 8,
"positive_groups": 5,
"positive_rows": 9
},
"none": {
"negative_groups": 8,
"negative_rows": 25,
"positive_groups": 9,
"positive_rows": 36
},
"pyyaml": {
"negative_groups": 3,
"negative_rows": 5,
"positive_groups": 3,
"positive_rows": 6
},
"react": {
"negative_groups": 4,
"negative_rows": 4,
"positive_groups": 4,
"positive_rows": 5
},
"requests": {
"negative_groups": 5,
"negative_rows": 8,
"positive_groups": 5,
"positive_rows": 10
},
"sanitize-html": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 1,
"positive_rows": 1
},
"sqlalchemy": {
"negative_groups": 2,
"negative_rows": 2,
"positive_groups": 2,
"positive_rows": 3
},
"unknown": {
"negative_groups": 40,
"negative_rows": 275,
"positive_groups": 77,
"positive_rows": 346
},
"vue": {
"negative_groups": 3,
"negative_rows": 5,
"positive_groups": 4,
"positive_rows": 6
},
"wordpress": {
"negative_groups": 5,
"negative_rows": 8,
"positive_groups": 9,
"positive_rows": 13
}
},
"by_language": {
"c_cpp": {
"negative_groups": 5,
"negative_rows": 173,
"positive_groups": 5,
"positive_rows": 173
},
"java": {
"negative_groups": 1,
"negative_rows": 40,
"positive_groups": 1,
"positive_rows": 40
},
"javascript": {
"negative_groups": 25,
"negative_rows": 30,
"positive_groups": 22,
"positive_rows": 31
},
"php": {
"negative_groups": 9,
"negative_rows": 26,
"positive_groups": 13,
"positive_rows": 22
},
"python": {
"negative_groups": 10,
"negative_rows": 23,
"positive_groups": 59,
"positive_rows": 132
},
"typescript": {
"negative_groups": 7,
"negative_rows": 33,
"positive_groups": 7,
"positive_rows": 23
},
"unknown": {
"negative_groups": 1,
"negative_rows": 3,
"positive_groups": 1,
"positive_rows": 3
}
},
"by_source": {
"adjudications": {
"negative_groups": 15,
"negative_rows": 40,
"positive_groups": 3,
"positive_rows": 3
},
"dev-php": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 3,
"positive_rows": 3
},
"pairs": {
"negative_groups": 25,
"negative_rows": 235,
"positive_groups": 74,
"positive_rows": 343
},
"population-v1": {
"negative_groups": 11,
"negative_rows": 35,
"positive_groups": 11,
"positive_rows": 50
},
"population-v2": {
"negative_groups": 14,
"negative_rows": 18,
"positive_groups": 17,
"positive_rows": 25
}
},
"by_split": {
"dev-php": {
"negative_groups": 0,
"negative_rows": 0,
"positive_groups": 3,
"positive_rows": 3
},
"holdout": {
"negative_groups": 12,
"negative_rows": 135,
"positive_groups": 63,
"positive_rows": 189
},
"population-v1": {
"negative_groups": 11,
"negative_rows": 46,
"positive_groups": 11,
"positive_rows": 51
},
"population-v2": {
"negative_groups": 15,
"negative_rows": 47,
"positive_groups": 17,
"positive_rows": 27
},
"train": {
"negative_groups": 16,
"negative_rows": 100,
"positive_groups": 65,
"positive_rows": 154
}
},
"design_informed": {
"adjudications": {
"negative_groups": 13,
"negative_rows": 29,
"positive_groups": 2,
"positive_rows": 2
},
"population-v2": {
"negative_groups": 14,
"negative_rows": 18,
"positive_groups": 17,
"positive_rows": 25
}
},
"privileged_positives": 2
}
}