mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
merge(learn): recovered negatives; excerpt matcher covers more declaration forms
Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"note": "COUNTS ONLY. Where the decision engine's positives-only repository groups come from, before any change. Memory = the static example rows of the RustFS store as of the 2026-10-02 slice-2 rebuild (904 examples); labels = the 2026-10-02 regeneration (6,589 rows, identical in counts to 2026-10-01-decision-engine-labels-advisory-fixes); feature records = the 924 static records of the store; harvest units = the merged units of the corpus and advisory harvests. Every (family, group) of the memory that holds positive and no negative example was traced back: negative label row present? feature record for it? excerpt of it? The reason is the first link that is missing. A second table accounts for the pair sides that have a label but no example at all (those groups are absent from the memory on BOTH sides, so they do not show as positives-only; they are where the fixed-side negatives the matcher owes come from). No candidate, path, function, commit or repository name is recorded here; the per-group detail stays in scratch.",
|
||||
"memory_before": {
|
||||
"examples": 904,
|
||||
"profiles": {"static": 904, "plane": 0},
|
||||
"by_family": {
|
||||
"access_control": {"examples_pos": 31, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12},
|
||||
"config_secrets": {"examples_pos": 18, "examples_neg": 13, "groups_pos": 5, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 3},
|
||||
"deserialization": {"examples_pos": 39, "examples_neg": 19, "groups_pos": 39, "groups_neg": 19, "groups_both": 19, "groups_pos_only": 20},
|
||||
"injection": {"examples_pos": 75, "examples_neg": 65, "groups_pos": 38, "groups_neg": 28, "groups_both": 28, "groups_pos_only": 10},
|
||||
"memory": {"examples_pos": 171, "examples_neg": 171, "groups_pos": 5, "groups_neg": 5, "groups_both": 5, "groups_pos_only": 0},
|
||||
"output_encoding": {"examples_pos": 76, "examples_neg": 60, "groups_pos": 35, "groups_neg": 21, "groups_both": 21, "groups_pos_only": 14},
|
||||
"path": {"examples_pos": 30, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12},
|
||||
"unknown": {"examples_pos": 2, "examples_neg": 2, "groups_pos": 2, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 0},
|
||||
"untrusted_destination": {"examples_pos": 61, "examples_neg": 35, "groups_pos": 51, "groups_neg": 24, "groups_both": 24, "groups_pos_only": 27}
|
||||
},
|
||||
"groups_neg_only": 0
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"total": 98,
|
||||
"by_family": {"access_control": 12, "config_secrets": 3, "deserialization": 20, "injection": 10, "output_encoding": 14, "path": 12, "untrusted_destination": 27},
|
||||
"by_family_source_reason": [
|
||||
{"family": "access_control", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function: the label builder emitted no fixed-side negative (fixed side is another function of the same snapshot)", "groups": 12},
|
||||
{"family": "config_secrets", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 3},
|
||||
{"family": "deserialization", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 20},
|
||||
{"family": "injection", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 8},
|
||||
{"family": "injection", "source": "dev-php", "reason": "no fixed pin recorded: development recipes name a vulnerable checkout only", "groups": 2},
|
||||
{"family": "output_encoding", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 14},
|
||||
{"family": "path", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 11},
|
||||
{"family": "path", "source": "dev-php", "reason": "no fixed pin recorded (development recipe)", "groups": 1},
|
||||
{"family": "untrusted_destination", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 26},
|
||||
{"family": "untrusted_destination", "source": "population-v2", "reason": "the fix did not change the labelled function: the same code at both pins is not a negative", "groups": 1}
|
||||
],
|
||||
"by_reason": {
|
||||
"fixed side lacks the function (pairs, vibe-py: 108 labels, 80 seeded pointer pairs whose fixed side is a guard helper of the same commit + 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler)": 94,
|
||||
"no fixed pin recorded (dev-php recipes)": 3,
|
||||
"fix did not change the function (population-v2)": 1,
|
||||
"matcher cannot resolve the function on the fixed pin": 0,
|
||||
"excerpt missing": 0
|
||||
},
|
||||
"check": "all 108 fixed sides of the vibe-py pairs were scanned for a `def <labelled name>` line: 0 hold one (the name occurs only in calls); these are data gaps, counted, never invented",
|
||||
"advisory_fixes_note": "no advisory-fixes group is positives-only: every one of the 97 pairs gave a fixed-side negative label; 7 of them are absent from the memory on both sides (below)"
|
||||
},
|
||||
"pair_sides_with_a_label_but_no_example": {
|
||||
"total_sides": 32,
|
||||
"pairs": 16,
|
||||
"by_catalog_state": {
|
||||
"advisory-fixes: feature record present, excerpt missing": {"sides": 14, "pairs": 7, "language": "ruby", "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}},
|
||||
"corpus (vfc, vfc-js): no feature record (declares() false on both sides)": {"sides": 18, "pairs": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}, "by_language": {"javascript": 6, "c_cpp": 1, "perl (catalog: unknown)": 2}}
|
||||
},
|
||||
"reasons": {
|
||||
"ruby under the record's language": "build_features stores the closed feature-vocabulary language (schema.LANGUAGES; `other` for Ruby, Go, C#); the memory build hands that `other` to the excerpt matcher, which then runs the brace matcher: a Ruby `def` has no brace body, so 7 pairs (14 sides) get no excerpt; 4 further Ruby pairs (8 stored examples) matched a `{ |x| ... }` block inside the method instead of the method -- 2-line excerpts, wrong spans. The 11 Go pairs (22 examples) under `other` are brace-matched to the exact body (a correct span) and 9 C# pairs likewise.",
|
||||
"javascript object-literal method": {"pairs": 3, "form": "`name: function (...) {` and `name: function anonymous(...) {`"},
|
||||
"javascript property-assignment arrow": {"pairs": 2, "form": "`module.exports.name = (a, b) => {` and `const name = module.exports.name = async a => {` (a chained assignment)"},
|
||||
"javascript prototype named function expression": {"pairs": 1, "form": "`X.prototype.name = function Name$withDollar (req, res) {`: `$` is not a `\\w` character for the declaration pattern or the brace matcher"},
|
||||
"c parameter list longer than the brace matcher's lookahead": {"pairs": 1, "form": "`Type name(a,\\n b,\\n#ifndef X\\n c\\n#else\\n d\\n#endif\\n )\\n{`: the `{` is 9 lines down, the matcher looks 8"},
|
||||
"perl sub": {"pairs": 2, "form": "`sub name {` (no parameter list; catalog language `unknown`)"}
|
||||
},
|
||||
"recoverable_by_tooling": {"fixed_side_negatives": 16, "positives": 16, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 3, "untrusted_destination": 1, "injection": 5, "memory": 2}},
|
||||
"note": "each recovered pair adds a paired group (positive and negative), not a negative to a positives-only group; the 98 positives-only groups are data gaps and stay as they are"
|
||||
},
|
||||
"label_builder_fixed_side_rule": "a fixed-side negative only where the fixed side declares the labelled function (or a deterministic rename at a distinct fix commit, fixed_side_moved = 0); unchanged",
|
||||
"instruments": {
|
||||
"memory_rows": "store.rows(kind=example|features) dumped once to scratch, 904 + 1,848 rows read",
|
||||
"matcher_sweep": "function_bounds on both sides of every `pairs` pin label: vulnerable ok 431, fixed ok 323, fixed side named-but-undeclared 108 (vibe-py, python), both sides unmatched 9 pairs",
|
||||
"language_check": "every pair record whose language is `other`: span under the record's language vs the label's: csharp same 18 sides, go different 18 (brace body vs to-next-declaration; stored spans kept), ruby none 14 (unstored) and different 8 (stored, wrong)"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"note": "COUNTS ONLY. Labels, feature records and the static memory rebuilt after the excerpt matcher learned the declaration forms recorded in 2026-10-02-decision-engine-negatives-gap. No model or embedding call: feature records reuse the recorded engine, verify and roles outputs of the harvest Runs (instruments that never ran stay `missing`); the new and changed excerpts have no vector yet (`ousast learn memory embed`, estimated below). The memory store is the RustFS bucket (versioned: prior objects stay). Candidate, path, function, commit and repository names stay local.",
|
||||
"labels": {
|
||||
"command": "ousast learn labels --root . --out <scratch> --snapshot <scratch>",
|
||||
"rows": 6589,
|
||||
"groups": 202,
|
||||
"labels_sha256_prefix": "c17daeac522090d4",
|
||||
"same_as": "the 2026-10-02 regeneration recorded in 2026-10-02-decision-engine-harvest-advisory (identical sha): the label builder's fixed-side rule already accepted every pair the matcher now resolves (its name-on-a-code-line fallback), so no label row changed",
|
||||
"pair_fixed_side": {"absent_fixed_side_labels": 108, "fixed_side_moved": 0}
|
||||
},
|
||||
"features": {
|
||||
"command": "benchmarks/learn/build_features.py --labels <the 36 label rows of the 9 corpus pairs> --units <merged units.json> (defaults: harvest-engine, harvest-verify, harvest-roles)",
|
||||
"targets": 18,
|
||||
"records": {"static:injection": 10, "static:memory": 4, "static:path": 4, "plane:injection": 10, "plane:memory": 4, "plane:path": 4},
|
||||
"absent_on_side": {},
|
||||
"instrument_states_plane_new_records": {
|
||||
"injection:engine": {"failed": 2, "none": 2, "ran": 6}, "injection:roles": {"failed": 8, "none": 2}, "injection:verify": {"missing": 2, "ran": 8},
|
||||
"memory:engine": {"failed": 2, "none": 2}, "memory:roles": {"none": 4}, "memory:verify": {"none": 4},
|
||||
"path:engine": {"failed": 4}, "path:roles": {"failed": 4}, "path:verify": {"none": 4}
|
||||
},
|
||||
"instrument_fix": "the store's ingest index remembered these 36 rows from the 2026-10-01 build (before the declares() gate) and skipped them as a repeat, although the 17:25 build had dropped them as stale: the first rebuild stored nothing (store rows 1,848 -> 1,848, memory 919). build_features now checks the rows are present after an index hit and restores them (restored_after_drop = 36). An index hit is not evidence the rows are there.",
|
||||
"store_rows": {"before": 1848, "after": 1884, "static": 942, "plane": 942},
|
||||
"advisory_ruby_pairs": "their 14 records were already in the store (language `other`); nothing rebuilt, the excerpt was the gap"
|
||||
},
|
||||
"memory": {
|
||||
"command": "ousast learn memory build --labels <labels> --units <merged units.json> --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)",
|
||||
"examples": {"before": 904, "after": 937, "added": 33},
|
||||
"by_source_after": {"adjudications": 34, "dev-php": 3, "pairs": 772, "population-v1": 85, "population-v2": 43},
|
||||
"pairs_build_counts": {"before": {"recorded": 754, "examples": 740, "absent_fixed_side": 7, "no_excerpt": 7}, "after": {"recorded": 772, "examples": 772, "absent_fixed_side": 0, "no_excerpt": 0}},
|
||||
"added_examples": {
|
||||
"advisory-fixes ruby pairs (record present, excerpt recovered)": {"pairs": 7, "positives": 7, "negatives": 7, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}},
|
||||
"corpus pairs (record and excerpt recovered)": {"pairs": 9, "positives": 9, "negatives": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}},
|
||||
"population-v2 (a `this.name = function (` assignment now matched)": {"positives": 1, "negatives": 0, "family": "injection", "note": "its fix did not change the function, so it has no negative label: one more positives-only group"}
|
||||
},
|
||||
"changed_excerpts": {"count": 9, "why": "4 advisory Ruby pairs (8 examples) had a `{ |x| ... }` block inside the method as their excerpt; 1 population-v2 TypeScript method (1 example) had the `{}` default value of a parameter as its body; both corrected by the paren-aware matcher. Their old vectors no longer name the content."},
|
||||
"unchanged_excerpts": 895,
|
||||
"vectors": {"distinct_excerpts_after": 821, "distinct_excerpts_before": 788, "without_vector_at_most": 42, "embed_dry_run_all_usd": 0.0043, "note": "no embedding call was made; `ousast learn memory embed` for the 33 new + 9 changed excerpts is a fraction of a cent. Retrieval scores an example without a vector by its signal similarity (retrieve.py), so they take part now."},
|
||||
"per_family_before_after": {
|
||||
"access_control": {"before": {"pos": 31, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 20, "groups_pos": 32, "groups_neg": 20, "pos_only": 12}},
|
||||
"config_secrets": {"before": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}, "after": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}},
|
||||
"deserialization": {"before": {"pos": 39, "neg": 19, "groups_pos": 39, "groups_neg": 19, "pos_only": 20}, "after": {"pos": 41, "neg": 21, "groups_pos": 41, "groups_neg": 21, "pos_only": 20}},
|
||||
"injection": {"before": {"pos": 75, "neg": 65, "groups_pos": 38, "groups_neg": 28, "pos_only": 10}, "after": {"pos": 81, "neg": 70, "groups_pos": 44, "groups_neg": 33, "pos_only": 11}},
|
||||
"memory": {"before": {"pos": 171, "neg": 171, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}, "after": {"pos": 173, "neg": 173, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}},
|
||||
"output_encoding": {"before": {"pos": 76, "neg": 60, "groups_pos": 35, "groups_neg": 21, "pos_only": 14}, "after": {"pos": 77, "neg": 61, "groups_pos": 36, "groups_neg": 22, "pos_only": 14}},
|
||||
"path": {"before": {"pos": 30, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 21, "groups_pos": 33, "groups_neg": 21, "pos_only": 12}},
|
||||
"unknown": {"before": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}, "after": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}},
|
||||
"untrusted_destination": {"before": {"pos": 61, "neg": 35, "groups_pos": 51, "groups_neg": 24, "pos_only": 27}, "after": {"pos": 62, "neg": 36, "groups_pos": 52, "groups_neg": 25, "pos_only": 27}}
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"before": 98,
|
||||
"after": 99,
|
||||
"by_reason_after": {
|
||||
"fixed side lacks the function (pairs, vibe-py: another function of the same snapshot; 108 labels)": 94,
|
||||
"no fixed pin recorded (dev-php recipes)": 3,
|
||||
"fix did not change the function (population-v2)": 2,
|
||||
"matcher cannot resolve the function on the fixed pin": 0,
|
||||
"excerpt missing": 0
|
||||
},
|
||||
"reading": "every recovered pair adds a paired group (both sides); the positives-only groups are data gaps that the rule `a fixed-side negative only where the fixed side declares the function` keeps counted, never invented. The one new positives-only group is a population positive that was invisible before (no excerpt) and whose fix did not change its function."
|
||||
}
|
||||
},
|
||||
"leak_audit": {
|
||||
"command": "ousast learn audit-leaks --profile {static,plane} --units <merged units.json> (threshold 0.20, min 10 pairs)",
|
||||
"pairs": {"before": {"all": 376, "injection": 43, "memory": 171, "path": 19}, "after": {"all": 385, "injection": 48, "memory": 173, "path": 21}, "unchanged": ["access_control 20", "config_secrets 12", "deserialization 21", "output_encoding 61", "unknown 2", "untrusted_destination 27"]},
|
||||
"rows_read": {"before": 1848, "after": 1884},
|
||||
"skipped": {"one side only": 172},
|
||||
"static": {"new_flags": [], "gone_flags": [], "changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]}, "flagged": 13},
|
||||
"plane": {
|
||||
"new_flags": [
|
||||
{"signal": "agree.final", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19},
|
||||
{"signal": "verify.votes", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19, "direction": "higher on vulnerable"}
|
||||
],
|
||||
"gone_flags": [],
|
||||
"changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]},
|
||||
"flagged": {"before": 25, "after": 27},
|
||||
"reading": "the five injection pairs the matcher recovered carry verify verdicts (8 of 10 sides ran) that separate their sides; agree.final and verify.votes were already flagged for untrusted_destination and config_secrets and are the verifier doing its job, not a label leak. The v1 input profile is unchanged (it withholds engine, facts.function_lines, verify.flag_b and ms.operation); agree.final and verify.votes stay in, reported here for the profile's owner."
|
||||
}
|
||||
},
|
||||
"instruments": {
|
||||
"store_dumps": "store.rows(kind=example|features) read once before and once after (904/1,848 then 937/1,884 rows)",
|
||||
"matcher_sweep_after": "function_bounds on both sides of every pairs pin label: vulnerable ok 440 (was 431), fixed ok 332 (was 323), the 108 vibe-py fixed sides still undeclared",
|
||||
"dry_build_check": "a dry build_examples over the stored records before any write: 15 new rows, 9 changed shas, 0 gone -- as expected; the live build then matched it plus the 18 restored corpus records"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user