mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
merge(learn): recovered negatives; excerpt matcher covers more declaration forms
Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
Marius Ciepluch
@@ -275,6 +275,7 @@ def main() -> int:
|
||||
skipped: Counter[str] = Counter()
|
||||
absent: Counter[str] = Counter() # family -> labelled pair candidates their side does not declare
|
||||
dropped = 0
|
||||
reingested = 0 # rows the index remembered from an earlier build but a later one had dropped
|
||||
args.scratch.mkdir(parents=True, exist_ok=True)
|
||||
chosen = [(k, r) for k, r in sorted(targets.items()) if k[0] in args.kinds.split(",")]
|
||||
for number, ((kind, ref, role_or_pin), rows) in enumerate(chosen):
|
||||
@@ -357,15 +358,24 @@ def main() -> int:
|
||||
for inst, info in rec["instruments"].items():
|
||||
state = "missing" if info.get("version") == "missing:not-run" else info["state"]
|
||||
coverage[f"{family}:{inst}"][state] += 1
|
||||
store.ingest_rows("harvest", f"features:{owner}", out_rows)
|
||||
ingested = store.ingest_rows("harvest", f"features:{owner}", out_rows)
|
||||
built = {r["id"] for r in out_rows}
|
||||
stored = store.rows(repo=memory.repo_key(repo), pin=pin, kind="features", where={"run": "harvest", "task": owner})
|
||||
present = {r.row["id"] for r in stored}
|
||||
restored = 0
|
||||
if ingested.skipped and not built <= present:
|
||||
# the index remembers these very rows from an earlier build, but a later build dropped them as stale
|
||||
# (the side did not declare the function then); an index hit is not evidence the rows are there
|
||||
store.put_rows(out_rows)
|
||||
restored = len(built - present)
|
||||
reingested += restored
|
||||
stale = [r.row["id"] for r in stored if r.row["id"] not in built]
|
||||
dropped += store.drop_rows(memory.repo_key(repo), pin, stale)
|
||||
print(json.dumps({"target": owner, "rows": len(out_rows), "dropped": len(stale)}), flush=True)
|
||||
print(json.dumps({"target": owner, "rows": len(out_rows), "dropped": len(stale), "restored": restored}), flush=True)
|
||||
counts = {
|
||||
"absent_on_side": dict(sorted(absent.items())),
|
||||
"dropped_stale_rows": dropped,
|
||||
"restored_after_drop": reingested,
|
||||
"records": {f"{p}:{f}": n for (p, f), n in sorted(tally.items())},
|
||||
"instrument_states_plane": {k: dict(sorted(v.items())) for k, v in sorted(coverage.items())},
|
||||
"skipped_label_rows": dict(skipped),
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"note": "COUNTS ONLY. Where the decision engine's positives-only repository groups come from, before any change. Memory = the static example rows of the RustFS store as of the 2026-10-02 slice-2 rebuild (904 examples); labels = the 2026-10-02 regeneration (6,589 rows, identical in counts to 2026-10-01-decision-engine-labels-advisory-fixes); feature records = the 924 static records of the store; harvest units = the merged units of the corpus and advisory harvests. Every (family, group) of the memory that holds positive and no negative example was traced back: negative label row present? feature record for it? excerpt of it? The reason is the first link that is missing. A second table accounts for the pair sides that have a label but no example at all (those groups are absent from the memory on BOTH sides, so they do not show as positives-only; they are where the fixed-side negatives the matcher owes come from). No candidate, path, function, commit or repository name is recorded here; the per-group detail stays in scratch.",
|
||||
"memory_before": {
|
||||
"examples": 904,
|
||||
"profiles": {"static": 904, "plane": 0},
|
||||
"by_family": {
|
||||
"access_control": {"examples_pos": 31, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12},
|
||||
"config_secrets": {"examples_pos": 18, "examples_neg": 13, "groups_pos": 5, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 3},
|
||||
"deserialization": {"examples_pos": 39, "examples_neg": 19, "groups_pos": 39, "groups_neg": 19, "groups_both": 19, "groups_pos_only": 20},
|
||||
"injection": {"examples_pos": 75, "examples_neg": 65, "groups_pos": 38, "groups_neg": 28, "groups_both": 28, "groups_pos_only": 10},
|
||||
"memory": {"examples_pos": 171, "examples_neg": 171, "groups_pos": 5, "groups_neg": 5, "groups_both": 5, "groups_pos_only": 0},
|
||||
"output_encoding": {"examples_pos": 76, "examples_neg": 60, "groups_pos": 35, "groups_neg": 21, "groups_both": 21, "groups_pos_only": 14},
|
||||
"path": {"examples_pos": 30, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12},
|
||||
"unknown": {"examples_pos": 2, "examples_neg": 2, "groups_pos": 2, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 0},
|
||||
"untrusted_destination": {"examples_pos": 61, "examples_neg": 35, "groups_pos": 51, "groups_neg": 24, "groups_both": 24, "groups_pos_only": 27}
|
||||
},
|
||||
"groups_neg_only": 0
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"total": 98,
|
||||
"by_family": {"access_control": 12, "config_secrets": 3, "deserialization": 20, "injection": 10, "output_encoding": 14, "path": 12, "untrusted_destination": 27},
|
||||
"by_family_source_reason": [
|
||||
{"family": "access_control", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function: the label builder emitted no fixed-side negative (fixed side is another function of the same snapshot)", "groups": 12},
|
||||
{"family": "config_secrets", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 3},
|
||||
{"family": "deserialization", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 20},
|
||||
{"family": "injection", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 8},
|
||||
{"family": "injection", "source": "dev-php", "reason": "no fixed pin recorded: development recipes name a vulnerable checkout only", "groups": 2},
|
||||
{"family": "output_encoding", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 14},
|
||||
{"family": "path", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 11},
|
||||
{"family": "path", "source": "dev-php", "reason": "no fixed pin recorded (development recipe)", "groups": 1},
|
||||
{"family": "untrusted_destination", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 26},
|
||||
{"family": "untrusted_destination", "source": "population-v2", "reason": "the fix did not change the labelled function: the same code at both pins is not a negative", "groups": 1}
|
||||
],
|
||||
"by_reason": {
|
||||
"fixed side lacks the function (pairs, vibe-py: 108 labels, 80 seeded pointer pairs whose fixed side is a guard helper of the same commit + 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler)": 94,
|
||||
"no fixed pin recorded (dev-php recipes)": 3,
|
||||
"fix did not change the function (population-v2)": 1,
|
||||
"matcher cannot resolve the function on the fixed pin": 0,
|
||||
"excerpt missing": 0
|
||||
},
|
||||
"check": "all 108 fixed sides of the vibe-py pairs were scanned for a `def <labelled name>` line: 0 hold one (the name occurs only in calls); these are data gaps, counted, never invented",
|
||||
"advisory_fixes_note": "no advisory-fixes group is positives-only: every one of the 97 pairs gave a fixed-side negative label; 7 of them are absent from the memory on both sides (below)"
|
||||
},
|
||||
"pair_sides_with_a_label_but_no_example": {
|
||||
"total_sides": 32,
|
||||
"pairs": 16,
|
||||
"by_catalog_state": {
|
||||
"advisory-fixes: feature record present, excerpt missing": {"sides": 14, "pairs": 7, "language": "ruby", "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}},
|
||||
"corpus (vfc, vfc-js): no feature record (declares() false on both sides)": {"sides": 18, "pairs": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}, "by_language": {"javascript": 6, "c_cpp": 1, "perl (catalog: unknown)": 2}}
|
||||
},
|
||||
"reasons": {
|
||||
"ruby under the record's language": "build_features stores the closed feature-vocabulary language (schema.LANGUAGES; `other` for Ruby, Go, C#); the memory build hands that `other` to the excerpt matcher, which then runs the brace matcher: a Ruby `def` has no brace body, so 7 pairs (14 sides) get no excerpt; 4 further Ruby pairs (8 stored examples) matched a `{ |x| ... }` block inside the method instead of the method -- 2-line excerpts, wrong spans. The 11 Go pairs (22 examples) under `other` are brace-matched to the exact body (a correct span) and 9 C# pairs likewise.",
|
||||
"javascript object-literal method": {"pairs": 3, "form": "`name: function (...) {` and `name: function anonymous(...) {`"},
|
||||
"javascript property-assignment arrow": {"pairs": 2, "form": "`module.exports.name = (a, b) => {` and `const name = module.exports.name = async a => {` (a chained assignment)"},
|
||||
"javascript prototype named function expression": {"pairs": 1, "form": "`X.prototype.name = function Name$withDollar (req, res) {`: `$` is not a `\\w` character for the declaration pattern or the brace matcher"},
|
||||
"c parameter list longer than the brace matcher's lookahead": {"pairs": 1, "form": "`Type name(a,\\n b,\\n#ifndef X\\n c\\n#else\\n d\\n#endif\\n )\\n{`: the `{` is 9 lines down, the matcher looks 8"},
|
||||
"perl sub": {"pairs": 2, "form": "`sub name {` (no parameter list; catalog language `unknown`)"}
|
||||
},
|
||||
"recoverable_by_tooling": {"fixed_side_negatives": 16, "positives": 16, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 3, "untrusted_destination": 1, "injection": 5, "memory": 2}},
|
||||
"note": "each recovered pair adds a paired group (positive and negative), not a negative to a positives-only group; the 98 positives-only groups are data gaps and stay as they are"
|
||||
},
|
||||
"label_builder_fixed_side_rule": "a fixed-side negative only where the fixed side declares the labelled function (or a deterministic rename at a distinct fix commit, fixed_side_moved = 0); unchanged",
|
||||
"instruments": {
|
||||
"memory_rows": "store.rows(kind=example|features) dumped once to scratch, 904 + 1,848 rows read",
|
||||
"matcher_sweep": "function_bounds on both sides of every `pairs` pin label: vulnerable ok 431, fixed ok 323, fixed side named-but-undeclared 108 (vibe-py, python), both sides unmatched 9 pairs",
|
||||
"language_check": "every pair record whose language is `other`: span under the record's language vs the label's: csharp same 18 sides, go different 18 (brace body vs to-next-declaration; stored spans kept), ruby none 14 (unstored) and different 8 (stored, wrong)"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"created": "2026-10-02",
|
||||
"note": "COUNTS ONLY. Labels, feature records and the static memory rebuilt after the excerpt matcher learned the declaration forms recorded in 2026-10-02-decision-engine-negatives-gap. No model or embedding call: feature records reuse the recorded engine, verify and roles outputs of the harvest Runs (instruments that never ran stay `missing`); the new and changed excerpts have no vector yet (`ousast learn memory embed`, estimated below). The memory store is the RustFS bucket (versioned: prior objects stay). Candidate, path, function, commit and repository names stay local.",
|
||||
"labels": {
|
||||
"command": "ousast learn labels --root . --out <scratch> --snapshot <scratch>",
|
||||
"rows": 6589,
|
||||
"groups": 202,
|
||||
"labels_sha256_prefix": "c17daeac522090d4",
|
||||
"same_as": "the 2026-10-02 regeneration recorded in 2026-10-02-decision-engine-harvest-advisory (identical sha): the label builder's fixed-side rule already accepted every pair the matcher now resolves (its name-on-a-code-line fallback), so no label row changed",
|
||||
"pair_fixed_side": {"absent_fixed_side_labels": 108, "fixed_side_moved": 0}
|
||||
},
|
||||
"features": {
|
||||
"command": "benchmarks/learn/build_features.py --labels <the 36 label rows of the 9 corpus pairs> --units <merged units.json> (defaults: harvest-engine, harvest-verify, harvest-roles)",
|
||||
"targets": 18,
|
||||
"records": {"static:injection": 10, "static:memory": 4, "static:path": 4, "plane:injection": 10, "plane:memory": 4, "plane:path": 4},
|
||||
"absent_on_side": {},
|
||||
"instrument_states_plane_new_records": {
|
||||
"injection:engine": {"failed": 2, "none": 2, "ran": 6}, "injection:roles": {"failed": 8, "none": 2}, "injection:verify": {"missing": 2, "ran": 8},
|
||||
"memory:engine": {"failed": 2, "none": 2}, "memory:roles": {"none": 4}, "memory:verify": {"none": 4},
|
||||
"path:engine": {"failed": 4}, "path:roles": {"failed": 4}, "path:verify": {"none": 4}
|
||||
},
|
||||
"instrument_fix": "the store's ingest index remembered these 36 rows from the 2026-10-01 build (before the declares() gate) and skipped them as a repeat, although the 17:25 build had dropped them as stale: the first rebuild stored nothing (store rows 1,848 -> 1,848, memory 919). build_features now checks the rows are present after an index hit and restores them (restored_after_drop = 36). An index hit is not evidence the rows are there.",
|
||||
"store_rows": {"before": 1848, "after": 1884, "static": 942, "plane": 942},
|
||||
"advisory_ruby_pairs": "their 14 records were already in the store (language `other`); nothing rebuilt, the excerpt was the gap"
|
||||
},
|
||||
"memory": {
|
||||
"command": "ousast learn memory build --labels <labels> --units <merged units.json> --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)",
|
||||
"examples": {"before": 904, "after": 937, "added": 33},
|
||||
"by_source_after": {"adjudications": 34, "dev-php": 3, "pairs": 772, "population-v1": 85, "population-v2": 43},
|
||||
"pairs_build_counts": {"before": {"recorded": 754, "examples": 740, "absent_fixed_side": 7, "no_excerpt": 7}, "after": {"recorded": 772, "examples": 772, "absent_fixed_side": 0, "no_excerpt": 0}},
|
||||
"added_examples": {
|
||||
"advisory-fixes ruby pairs (record present, excerpt recovered)": {"pairs": 7, "positives": 7, "negatives": 7, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}},
|
||||
"corpus pairs (record and excerpt recovered)": {"pairs": 9, "positives": 9, "negatives": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}},
|
||||
"population-v2 (a `this.name = function (` assignment now matched)": {"positives": 1, "negatives": 0, "family": "injection", "note": "its fix did not change the function, so it has no negative label: one more positives-only group"}
|
||||
},
|
||||
"changed_excerpts": {"count": 9, "why": "4 advisory Ruby pairs (8 examples) had a `{ |x| ... }` block inside the method as their excerpt; 1 population-v2 TypeScript method (1 example) had the `{}` default value of a parameter as its body; both corrected by the paren-aware matcher. Their old vectors no longer name the content."},
|
||||
"unchanged_excerpts": 895,
|
||||
"vectors": {"distinct_excerpts_after": 821, "distinct_excerpts_before": 788, "without_vector_at_most": 42, "embed_dry_run_all_usd": 0.0043, "note": "no embedding call was made; `ousast learn memory embed` for the 33 new + 9 changed excerpts is a fraction of a cent. Retrieval scores an example without a vector by its signal similarity (retrieve.py), so they take part now."},
|
||||
"per_family_before_after": {
|
||||
"access_control": {"before": {"pos": 31, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 20, "groups_pos": 32, "groups_neg": 20, "pos_only": 12}},
|
||||
"config_secrets": {"before": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}, "after": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}},
|
||||
"deserialization": {"before": {"pos": 39, "neg": 19, "groups_pos": 39, "groups_neg": 19, "pos_only": 20}, "after": {"pos": 41, "neg": 21, "groups_pos": 41, "groups_neg": 21, "pos_only": 20}},
|
||||
"injection": {"before": {"pos": 75, "neg": 65, "groups_pos": 38, "groups_neg": 28, "pos_only": 10}, "after": {"pos": 81, "neg": 70, "groups_pos": 44, "groups_neg": 33, "pos_only": 11}},
|
||||
"memory": {"before": {"pos": 171, "neg": 171, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}, "after": {"pos": 173, "neg": 173, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}},
|
||||
"output_encoding": {"before": {"pos": 76, "neg": 60, "groups_pos": 35, "groups_neg": 21, "pos_only": 14}, "after": {"pos": 77, "neg": 61, "groups_pos": 36, "groups_neg": 22, "pos_only": 14}},
|
||||
"path": {"before": {"pos": 30, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 21, "groups_pos": 33, "groups_neg": 21, "pos_only": 12}},
|
||||
"unknown": {"before": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}, "after": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}},
|
||||
"untrusted_destination": {"before": {"pos": 61, "neg": 35, "groups_pos": 51, "groups_neg": 24, "pos_only": 27}, "after": {"pos": 62, "neg": 36, "groups_pos": 52, "groups_neg": 25, "pos_only": 27}}
|
||||
},
|
||||
"positives_only_groups": {
|
||||
"before": 98,
|
||||
"after": 99,
|
||||
"by_reason_after": {
|
||||
"fixed side lacks the function (pairs, vibe-py: another function of the same snapshot; 108 labels)": 94,
|
||||
"no fixed pin recorded (dev-php recipes)": 3,
|
||||
"fix did not change the function (population-v2)": 2,
|
||||
"matcher cannot resolve the function on the fixed pin": 0,
|
||||
"excerpt missing": 0
|
||||
},
|
||||
"reading": "every recovered pair adds a paired group (both sides); the positives-only groups are data gaps that the rule `a fixed-side negative only where the fixed side declares the function` keeps counted, never invented. The one new positives-only group is a population positive that was invisible before (no excerpt) and whose fix did not change its function."
|
||||
}
|
||||
},
|
||||
"leak_audit": {
|
||||
"command": "ousast learn audit-leaks --profile {static,plane} --units <merged units.json> (threshold 0.20, min 10 pairs)",
|
||||
"pairs": {"before": {"all": 376, "injection": 43, "memory": 171, "path": 19}, "after": {"all": 385, "injection": 48, "memory": 173, "path": 21}, "unchanged": ["access_control 20", "config_secrets 12", "deserialization 21", "output_encoding 61", "unknown 2", "untrusted_destination 27"]},
|
||||
"rows_read": {"before": 1848, "after": 1884},
|
||||
"skipped": {"one side only": 172},
|
||||
"static": {"new_flags": [], "gone_flags": [], "changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]}, "flagged": 13},
|
||||
"plane": {
|
||||
"new_flags": [
|
||||
{"signal": "agree.final", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19},
|
||||
{"signal": "verify.votes", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19, "direction": "higher on vulnerable"}
|
||||
],
|
||||
"gone_flags": [],
|
||||
"changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]},
|
||||
"flagged": {"before": 25, "after": 27},
|
||||
"reading": "the five injection pairs the matcher recovered carry verify verdicts (8 of 10 sides ran) that separate their sides; agree.final and verify.votes were already flagged for untrusted_destination and config_secrets and are the verifier doing its job, not a label leak. The v1 input profile is unchanged (it withholds engine, facts.function_lines, verify.flag_b and ms.operation); agree.final and verify.votes stay in, reported here for the profile's owner."
|
||||
}
|
||||
},
|
||||
"instruments": {
|
||||
"store_dumps": "store.rows(kind=example|features) read once before and once after (904/1,848 then 937/1,884 rows)",
|
||||
"matcher_sweep_after": "function_bounds on both sides of every pairs pin label: vulnerable ok 440 (was 431), fixed ok 332 (was 323), the 108 vibe-py fixed sides still undeclared",
|
||||
"dry_build_check": "a dry build_examples over the stored records before any write: 15 new rows, 9 changed shas, 0 gone -- as expected; the live build then matched it plus the 18 restored corpus records"
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,17 @@ def _counter_role(label: Mapping[str, Any]) -> str:
|
||||
return "vulnerable" if label.get("pin_role") == "fixed" else "fixed"
|
||||
|
||||
|
||||
def _excerpt_language(record: Mapping[str, Any], label: Mapping[str, Any], lines: Sequence[str] | None, function: str) -> str:
|
||||
"""The language the excerpt matcher runs under: the record's (the closed feature vocabulary, ``other`` for Go,
|
||||
Ruby, C#, Perl, ...), and the label's -- the catalog's or the clone's -- where the record's resolves nothing
|
||||
(a Ruby ``def`` under ``other`` has no brace body). The record keeps its own language field."""
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
declared = str(label.get("language") or "")
|
||||
if lines and declared and declared != language and function_bounds(lines, language, function) is None:
|
||||
return declared
|
||||
return language
|
||||
|
||||
|
||||
def build_examples(
|
||||
labels: Iterable[Label | Mapping[str, Any]],
|
||||
records: Mapping[tuple[str, str, str, str, str], Mapping[str, Any]],
|
||||
@@ -165,16 +176,17 @@ def build_examples(
|
||||
count.recorded += 1
|
||||
path, _, function = candidate.partition("::")
|
||||
function = (function_of(label, str(label.get("pin_role") or "vulnerable")) if function_of else None) or function
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
identities = identities_of(str(label.get("repo") or ""), path, [str(label.get("pin") or "")])
|
||||
lines = lines_of(label, str(label.get("pin_role") or "vulnerable"))
|
||||
language = str(record.get("language") or label.get("language") or "other")
|
||||
matcher = _excerpt_language(record, label, lines, function)
|
||||
diff = None
|
||||
if unit == "delta" and lines:
|
||||
base = lines_of(label, _counter_role(label))
|
||||
head_span, base_span = function_bounds(lines, language, function), function_bounds(base or (), language, function)
|
||||
head_span, base_span = function_bounds(lines, matcher, function), function_bounds(base or (), matcher, function)
|
||||
if base and head_span and base_span:
|
||||
diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], language, identities=identities)
|
||||
shown = excerpt(lines, language, function, bounds=bounds, identities=identities, diff=diff)
|
||||
diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], matcher, identities=identities)
|
||||
shown = excerpt(lines, matcher, function, bounds=bounds, identities=identities, diff=diff)
|
||||
if shown is None:
|
||||
if not lines:
|
||||
count.unread += 1
|
||||
|
||||
@@ -14,6 +14,14 @@ otherwise tell the model the label. Both become ``<redacted>``. A delta unit add
|
||||
|
||||
The text is normalised (trailing blanks dropped, ``\\n`` endings, one final newline) before it is hashed, so the sha
|
||||
names the content an embedding and a response cache key are of.
|
||||
|
||||
Where a function is declared (:func:`function_bounds`): the repo-facts declaration pattern of the language first (its
|
||||
span runs to the next declaration), else the brace matcher, which knows ``name(...) {`` with any return type,
|
||||
qualifiers, generics or a parameter list spread over many lines (a prototype's ``;`` is not a body), object-literal
|
||||
methods ``name: function (`` / ``name: (a) =>``, assignments ``x.y.name = function (`` / ``= (a) =>`` / chained
|
||||
``const name = m.name = async a =>``, Perl ``sub name {`` and C# expression bodies. A file given as ``other`` or
|
||||
``unknown`` is sniffed for Perl and Ruby (:func:`resolve_language`), whose ``def``/``sub`` forms a brace matcher
|
||||
would miss or mistake for a block.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -180,47 +188,123 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) -
|
||||
# --- spans and excerpts -----------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
|
||||
line naming it before ``(`` whose ``{`` comes before any ``;``, to the matching ``}``. Comments are blanked
|
||||
before braces are counted."""
|
||||
name = re.split(r"::|\.|->", function)[-1]
|
||||
if not re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
return None
|
||||
call = re.compile(r"(?<![\w.>$])" + re.escape(name) + r"\s*\(")
|
||||
_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"})
|
||||
_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)")
|
||||
_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)")
|
||||
_RUBY_END = re.compile(r"^\s*end\s*$")
|
||||
_BODY_SCAN_LINES = 40 # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be
|
||||
_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b")
|
||||
_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)" # function (...) / (...) => / a =>
|
||||
|
||||
|
||||
def resolve_language(lines: Sequence[str], language: str) -> str:
|
||||
"""The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby,
|
||||
C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose
|
||||
declaration forms a brace matcher never finds and whose ``#`` comments must be redacted."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
if language in _UNKNOWN_LANGUAGES and lines:
|
||||
if any(_PERL_LINE.match(line) for line in lines):
|
||||
return "perl"
|
||||
if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines):
|
||||
return "ruby"
|
||||
return language
|
||||
|
||||
|
||||
def _blank_comments(lines: Sequence[str], language: str) -> list[str]:
|
||||
style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c"
|
||||
code: list[str] = []
|
||||
state: str | None = None
|
||||
for line in lines:
|
||||
ranges, state = _comment_ranges(line, state, language if language in SLASH_COMMENTS else "c")
|
||||
ranges, state = _comment_ranges(line, state, style)
|
||||
for start, end in reversed(ranges):
|
||||
line = line[:start] + " " * (end - start) + line[end:]
|
||||
code.append(line)
|
||||
for i, line in enumerate(code):
|
||||
found = call.search(line)
|
||||
if found is None or re.match(r"\s*(return|if|while|for|switch|else)\b", line):
|
||||
continue
|
||||
tail = "\n".join([line[found.end() :], *code[i + 1 : i + 8]])
|
||||
brace, semi = tail.find("{"), tail.find(";")
|
||||
if brace < 0 or (0 <= semi < brace):
|
||||
continue
|
||||
depth, opened = 0, False
|
||||
for j in range(i, len(code)):
|
||||
for c in line[found.end() :] if j == i else code[j]:
|
||||
return code
|
||||
|
||||
|
||||
def _declaration_heads(name: str) -> re.Pattern[str]:
|
||||
"""Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told
|
||||
apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` /
|
||||
``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>``
|
||||
(any chain). ``$`` is an identifier character (``Glance$serveRequest``)."""
|
||||
quoted = re.escape(name)
|
||||
return re.compile(
|
||||
rf"(?<![\w.>$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()"
|
||||
rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])"
|
||||
rf"|(?<![\w$]){quoted}\s*:\s*{_FUNCTION_EXPR}"
|
||||
rf"|(?<![\w$]){quoted}\s*=\s*(?:[\w$.\[\]'\"]+\s*=\s*)*{_FUNCTION_EXPR}"
|
||||
)
|
||||
|
||||
|
||||
def _body_start(code: Sequence[str], line: int, column: int) -> tuple[str, int, int] | None:
|
||||
"""Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0,
|
||||
``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a
|
||||
prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within
|
||||
:data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``."""
|
||||
depth, arrow = 0, False
|
||||
for j in range(line, min(line + _BODY_SCAN_LINES, len(code))):
|
||||
text = code[j]
|
||||
k = column if j == line else 0
|
||||
while k < len(text):
|
||||
c = text[k]
|
||||
if c == "(":
|
||||
depth += 1
|
||||
elif c == ")":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return None
|
||||
elif depth == 0:
|
||||
if c == "{":
|
||||
depth, opened = depth + 1, True
|
||||
elif c == "}" and opened:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i, j + 1
|
||||
return i, len(code)
|
||||
return ("brace", j, k)
|
||||
if arrow and c in ";,":
|
||||
return ("expression", j, k)
|
||||
if c in "};": # a statement ends (a call, a prototype) before any body opened
|
||||
return None
|
||||
if text.startswith("=>", k):
|
||||
arrow = True
|
||||
k += 1
|
||||
k += 1
|
||||
return None
|
||||
|
||||
|
||||
def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first
|
||||
declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the
|
||||
end of an expression body). Comments are blanked before anything is matched or counted."""
|
||||
name = re.split(r"::|\.|->", function)[-1]
|
||||
if not re.fullmatch(r"[A-Za-z_$][\w$]*", name):
|
||||
return None
|
||||
heads = _declaration_heads(name)
|
||||
code = _blank_comments(lines, language)
|
||||
for i, line in enumerate(code):
|
||||
if _STATEMENT_HEAD.match(line):
|
||||
continue
|
||||
for found in heads.finditer(line):
|
||||
body = _body_start(code, i, found.end())
|
||||
if body is None:
|
||||
continue
|
||||
kind, j, k = body
|
||||
if kind == "expression":
|
||||
return i, j + 1
|
||||
depth = 0
|
||||
for row in range(j, len(code)):
|
||||
for c in code[row][k:] if row == j else code[row]:
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i, row + 1
|
||||
return i, len(code)
|
||||
return None
|
||||
|
||||
|
||||
def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
"""0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ``<global>``), or
|
||||
``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, ...) and
|
||||
brace languages whose pattern misses the name use the brace matcher."""
|
||||
language = LANGUAGE_ALIASES.get(language, language)
|
||||
``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl,
|
||||
...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed
|
||||
(:func:`resolve_language`)."""
|
||||
language = resolve_language(lines, language)
|
||||
if function == GLOBAL:
|
||||
return (0, len(lines)) if lines else None
|
||||
pattern = DECLARATION.get(language)
|
||||
@@ -267,6 +351,7 @@ def excerpt(
|
||||
:func:`delta_diff`) is appended under a ``diff:`` line."""
|
||||
if not lines:
|
||||
return None
|
||||
language = resolve_language(lines, language)
|
||||
span = function_bounds(lines, language, function)
|
||||
if span is None:
|
||||
return None
|
||||
@@ -300,6 +385,7 @@ def delta_diff(
|
||||
) -> str:
|
||||
"""The base->head unified diff of a function (one line of context, no file headers), redacted, at most
|
||||
``max_lines`` lines."""
|
||||
language = resolve_language(head, language)
|
||||
diff = [line for line in difflib.unified_diff(list(base), list(head), lineterm="", n=1)][2:]
|
||||
out: list[str] = []
|
||||
for line in diff:
|
||||
@@ -314,5 +400,5 @@ def delta_diff(
|
||||
|
||||
__all__ = [
|
||||
"CANDIDATE", "DIFF_LINES", "EXAMPLE", "REDACTED", "Bounds", "Excerpt", "delta_diff", "excerpt", "excerpt_sha",
|
||||
"function_bounds", "identities_of", "normalise", "redact",
|
||||
"function_bounds", "identities_of", "normalise", "redact", "resolve_language",
|
||||
] # fmt: skip
|
||||
|
||||
@@ -158,6 +158,16 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side(
|
||||
assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"]
|
||||
|
||||
|
||||
def test_the_excerpt_matcher_falls_back_to_the_labels_language_when_the_records_resolves_nothing() -> None:
|
||||
"""A feature record carries the closed vocabulary's language (``other`` for Ruby, Go, C#); a Python ``def`` under
|
||||
``other`` has no brace body, so the matcher gets the label's language instead. The record's field is kept."""
|
||||
lab = label()
|
||||
other = {**feature(lab), "language": "other"}
|
||||
build = build_examples([lab], records_by_key([other]), lambda lab, role: CODE[role], sources=SOURCES)
|
||||
assert build.summary()["examples"] == 1 and build.rows[0]["language"] == "other" # the row keeps the vocabulary language
|
||||
assert build.sources["population-v1"].no_excerpt == 0
|
||||
|
||||
|
||||
def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None:
|
||||
"""The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under
|
||||
:func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found."""
|
||||
|
||||
@@ -136,3 +136,224 @@ def test_brace_languages_without_a_declaration_pattern() -> None:
|
||||
assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text
|
||||
assert excerpt(c, "cpp", "Parser::parse_header") is not None
|
||||
assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None # a prototype is not a body
|
||||
|
||||
|
||||
# --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) -------
|
||||
|
||||
|
||||
def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None:
|
||||
found = excerpt(code, language, function)
|
||||
return None if found is None else (found.first, found.last)
|
||||
|
||||
|
||||
def test_js_object_literal_methods() -> None:
|
||||
js = [
|
||||
"module.exports = {",
|
||||
" getInfo: function () {",
|
||||
" return this.exec('pdfinfo ' + this.path)",
|
||||
" },",
|
||||
" ps: function(pid, options, done) {",
|
||||
" var cmd = 'ps -o pcpu,rss -p ' + pid",
|
||||
" exec(cmd, done)",
|
||||
" },",
|
||||
" performAction: function anonymous(yytext, yyleng) {",
|
||||
" return eval(yytext)",
|
||||
" },",
|
||||
" short: (a) => a + 1,",
|
||||
" use: function () { return this.getInfo().then(function (info) { return info }) },",
|
||||
"}",
|
||||
]
|
||||
assert _span(js, "javascript", "getInfo") == (2, 4)
|
||||
assert _span(js, "javascript", "ps") == (5, 8)
|
||||
assert _span(js, "javascript", "performAction") == (9, 11) # the key, not the expression's own name
|
||||
assert _span(js, "javascript", "short") == (12, 12) # an expression body ends at its comma
|
||||
assert _span(js, "javascript", "then") is None # a call inside a body is not a declaration
|
||||
|
||||
|
||||
def test_js_property_assignment_functions_and_arrows() -> None:
|
||||
js = [
|
||||
"module.exports.ConfigureFilePath = (Options, FilePath) => {",
|
||||
" return path.join(Options.root, FilePath)",
|
||||
"}",
|
||||
"const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {",
|
||||
" return exec(`lsof -i :${port}`)",
|
||||
"}",
|
||||
"Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {",
|
||||
" fs.createReadStream(req.url).pipe(res)",
|
||||
"}",
|
||||
"this.getDaqValue = function (tagid, fromts, tots) {",
|
||||
" return conn.query('SELECT ' + tagid)",
|
||||
"}",
|
||||
"const total = sum(prices)",
|
||||
]
|
||||
assert _span(js, "javascript", "ConfigureFilePath") == (1, 3)
|
||||
assert _span(js, "javascript", "listProcessesOnPort") == (4, 6)
|
||||
assert _span(js, "javascript", "Glance$serveRequest") == (7, 9) # `$` is an identifier character
|
||||
assert _span(js, "javascript", "serveRequest") == (7, 9)
|
||||
assert _span(js, "javascript", "getDaqValue") == (10, 12)
|
||||
assert _span(js, "javascript", "sum") is None # an assignment of a call result declares nothing
|
||||
|
||||
|
||||
def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None:
|
||||
ts = [
|
||||
"export class Client {",
|
||||
" private retries = 3",
|
||||
" private async fetchRetry(",
|
||||
" url: string,",
|
||||
" options: any = {},",
|
||||
" ): Promise<Response> {",
|
||||
" return fetch(url, options)",
|
||||
" }",
|
||||
" handle = async (req: Request): Promise<void> => {",
|
||||
" await this.fetchRetry(req.url)",
|
||||
" }",
|
||||
"}",
|
||||
]
|
||||
assert _span(ts, "typescript", "fetchRetry") == (3, 8) # not the `{}` default value inside the parameter list
|
||||
assert _span(ts, "typescript", "handle") == (9, 11)
|
||||
|
||||
|
||||
def test_perl_sub_under_an_unknown_language() -> None:
|
||||
perl = [
|
||||
"#!/usr/bin/perl",
|
||||
"use strict;",
|
||||
"sub link_hash_cert;", # a forward declaration has no body
|
||||
"# CVE-2022-1292: shell injection fixed",
|
||||
"sub link_hash_cert {",
|
||||
" my $fname = $_[0];",
|
||||
" system(\"openssl x509 -in '$fname'\");",
|
||||
"}",
|
||||
"sub hash_dir { opendir(my $dh, $_[0]) or die; }",
|
||||
]
|
||||
assert _span(perl, "unknown", "link_hash_cert") == (5, 8)
|
||||
assert _span(perl, "other", "hash_dir") == (9, 9)
|
||||
shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,))
|
||||
assert shown is not None and "CVE-2022" not in shown.text # `#` comments are redacted once the language is known
|
||||
|
||||
|
||||
def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None:
|
||||
c = [
|
||||
"CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);",
|
||||
"",
|
||||
"CURLcode Curl_add_custom_headers(struct Curl_easy *data,",
|
||||
" bool is_connect,",
|
||||
"#ifndef USE_HYPER",
|
||||
" struct dynbuf *req",
|
||||
"#else",
|
||||
" void *req",
|
||||
"#endif",
|
||||
" )",
|
||||
"{",
|
||||
" return CURLE_OK;",
|
||||
"}",
|
||||
"std::string Parser::header(int index) const noexcept {",
|
||||
" return headers[index];",
|
||||
"}",
|
||||
"Parser::Parser(int n) : size_(n), data_(nullptr) {",
|
||||
" data_ = alloc(n);",
|
||||
"}",
|
||||
"int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }",
|
||||
]
|
||||
assert _span(c, "c", "Curl_add_custom_headers") == (3, 13) # the prototype is skipped, the body is 9 lines down
|
||||
assert _span(c, "cpp", "Parser::header") == (14, 16)
|
||||
assert _span(c, "cpp", "Parser") == (17, 19) # an initializer list's commas are not statement ends
|
||||
assert _span(c, "c", "check") is None # `if (check(1)) {` is a call: its `)` closes an outer parenthesis
|
||||
|
||||
|
||||
def test_php_methods_with_modifiers_and_return_types() -> None:
|
||||
php = [
|
||||
"<?php",
|
||||
"class Export {",
|
||||
" public static function &byRef(array $rows): ?array",
|
||||
" {",
|
||||
" return $rows;",
|
||||
" }",
|
||||
" final protected function render(string $name): void { echo $name; }",
|
||||
"}",
|
||||
]
|
||||
assert _span(php, "php", "byRef") == (3, 6)
|
||||
assert _span(php, "php", "render") == (7, 8) # a pattern language's span runs to the next declaration or the end
|
||||
|
||||
|
||||
def test_go_receivers_and_rust_generics() -> None:
|
||||
go = [
|
||||
"package main",
|
||||
"",
|
||||
"func (s *Server) handle(w http.ResponseWriter, r *http.Request) {",
|
||||
"\tio.Copy(w, r.Body)",
|
||||
"}",
|
||||
"",
|
||||
"func main() {}",
|
||||
]
|
||||
assert _span(go, "go", "handle") == (3, 6) # the declaration pattern: to the next `func`
|
||||
assert _span(go, "other", "handle") == (3, 5) # the feature vocabulary's bucket for Go: brace-matched to the body
|
||||
rust = ["pub fn parse<T: Read>(input: &mut T) -> Result<(), Error> {", " Ok(())", "}"]
|
||||
assert _span(rust, "rust", "parse") == (1, 3)
|
||||
|
||||
|
||||
def test_ruby_defs_under_the_other_language_are_sniffed() -> None:
|
||||
ruby = [
|
||||
"module Kredis",
|
||||
" class Type",
|
||||
" def self.cast_value(value)",
|
||||
" YAML.load(value)",
|
||||
" end",
|
||||
"",
|
||||
" def fetch_file(path)",
|
||||
" files.each { |f| return f if f.name == path }",
|
||||
" nil",
|
||||
" end",
|
||||
" end",
|
||||
"end",
|
||||
]
|
||||
assert _span(ruby, "ruby", "cast_value") == (3, 6)
|
||||
assert _span(ruby, "other", "cast_value") == (3, 6) # not None: the file is sniffed as Ruby
|
||||
assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12) # to the end: no later `def`
|
||||
assert _span(ruby, "c", "fetch_file") == (7, 8) # what a brace matcher takes under a named brace language: the block
|
||||
|
||||
|
||||
def test_csharp_attributes_modifiers_and_expression_bodies() -> None:
|
||||
cs = [
|
||||
"public class UploadController : Controller",
|
||||
"{",
|
||||
" [HttpPost]",
|
||||
" public async Task<IActionResult> Upload(IFormFile file)",
|
||||
" {",
|
||||
" var path = Path.Combine(root, file.FileName);",
|
||||
" return Ok(path);",
|
||||
" }",
|
||||
" protected override string Name(int index) => names[index];",
|
||||
" public abstract int Size(string key);",
|
||||
"}",
|
||||
]
|
||||
assert _span(cs, "csharp", "Upload") == (4, 8)
|
||||
assert _span(cs, "csharp", "Name") == (9, 9) # an expression-bodied member ends at its `;`
|
||||
assert _span(cs, "csharp", "Size") is None # abstract: no body
|
||||
|
||||
|
||||
def test_java_and_statement_heads_are_not_declarations() -> None:
|
||||
java = [
|
||||
"public class Handler {",
|
||||
" @Override",
|
||||
" public synchronized <T> List<T> load(String name, Class<T> type)",
|
||||
" throws IOException, SQLException {",
|
||||
" return run(name, type);",
|
||||
" }",
|
||||
" void other() {",
|
||||
' while (load("x", null) != null) { break; }',
|
||||
" }",
|
||||
"}",
|
||||
]
|
||||
assert _span(java, "java", "load") == (3, 6) # `throws A, B {`: the comma is not a statement end
|
||||
assert _span(java, "java", "run") is None
|
||||
assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4) # pattern and brace matcher agree
|
||||
|
||||
|
||||
def test_resolve_language_sniffs_only_unknown_languages() -> None:
|
||||
from openultrasast.learn.excerpt import resolve_language
|
||||
|
||||
assert resolve_language(["sub x {", "}"], "unknown") == "perl"
|
||||
assert resolve_language(["def x", "end"], "other") == "ruby"
|
||||
assert resolve_language(["def x(a):", " return a"], "other") == "other" # Python has no `end` lines
|
||||
assert resolve_language(["sub x {", "}"], "python") == "python" # a named language is never second-guessed
|
||||
assert resolve_language([], "unknown") == "unknown"
|
||||
|
||||
Reference in New Issue
Block a user