diff --git a/benchmarks/learn/build_features.py b/benchmarks/learn/build_features.py index 882fc0f..001256d 100644 --- a/benchmarks/learn/build_features.py +++ b/benchmarks/learn/build_features.py @@ -275,6 +275,7 @@ def main() -> int: skipped: Counter[str] = Counter() absent: Counter[str] = Counter() # family -> labelled pair candidates their side does not declare dropped = 0 + reingested = 0 # rows the index remembered from an earlier build but a later one had dropped args.scratch.mkdir(parents=True, exist_ok=True) chosen = [(k, r) for k, r in sorted(targets.items()) if k[0] in args.kinds.split(",")] for number, ((kind, ref, role_or_pin), rows) in enumerate(chosen): @@ -357,15 +358,24 @@ def main() -> int: for inst, info in rec["instruments"].items(): state = "missing" if info.get("version") == "missing:not-run" else info["state"] coverage[f"{family}:{inst}"][state] += 1 - store.ingest_rows("harvest", f"features:{owner}", out_rows) + ingested = store.ingest_rows("harvest", f"features:{owner}", out_rows) built = {r["id"] for r in out_rows} stored = store.rows(repo=memory.repo_key(repo), pin=pin, kind="features", where={"run": "harvest", "task": owner}) + present = {r.row["id"] for r in stored} + restored = 0 + if ingested.skipped and not built <= present: + # the index remembers these very rows from an earlier build, but a later build dropped them as stale + # (the side did not declare the function then); an index hit is not evidence the rows are there + store.put_rows(out_rows) + restored = len(built - present) + reingested += restored stale = [r.row["id"] for r in stored if r.row["id"] not in built] dropped += store.drop_rows(memory.repo_key(repo), pin, stale) - print(json.dumps({"target": owner, "rows": len(out_rows), "dropped": len(stale)}), flush=True) + print(json.dumps({"target": owner, "rows": len(out_rows), "dropped": len(stale), "restored": restored}), flush=True) counts = { "absent_on_side": dict(sorted(absent.items())), "dropped_stale_rows": dropped, + "restored_after_drop": reingested, "records": {f"{p}:{f}": n for (p, f), n in sorted(tally.items())}, "instrument_states_plane": {k: dict(sorted(v.items())) for k, v in sorted(coverage.items())}, "skipped_label_rows": dict(skipped), diff --git a/benchmarks/measurements/2026-10-02-decision-engine-negatives-gap/record.json b/benchmarks/measurements/2026-10-02-decision-engine-negatives-gap/record.json new file mode 100644 index 0000000..84162ec --- /dev/null +++ b/benchmarks/measurements/2026-10-02-decision-engine-negatives-gap/record.json @@ -0,0 +1,69 @@ +{ + "created": "2026-10-02", + "note": "COUNTS ONLY. Where the decision engine's positives-only repository groups come from, before any change. Memory = the static example rows of the RustFS store as of the 2026-10-02 slice-2 rebuild (904 examples); labels = the 2026-10-02 regeneration (6,589 rows, identical in counts to 2026-10-01-decision-engine-labels-advisory-fixes); feature records = the 924 static records of the store; harvest units = the merged units of the corpus and advisory harvests. Every (family, group) of the memory that holds positive and no negative example was traced back: negative label row present? feature record for it? excerpt of it? The reason is the first link that is missing. A second table accounts for the pair sides that have a label but no example at all (those groups are absent from the memory on BOTH sides, so they do not show as positives-only; they are where the fixed-side negatives the matcher owes come from). No candidate, path, function, commit or repository name is recorded here; the per-group detail stays in scratch.", + "memory_before": { + "examples": 904, + "profiles": {"static": 904, "plane": 0}, + "by_family": { + "access_control": {"examples_pos": 31, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12}, + "config_secrets": {"examples_pos": 18, "examples_neg": 13, "groups_pos": 5, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 3}, + "deserialization": {"examples_pos": 39, "examples_neg": 19, "groups_pos": 39, "groups_neg": 19, "groups_both": 19, "groups_pos_only": 20}, + "injection": {"examples_pos": 75, "examples_neg": 65, "groups_pos": 38, "groups_neg": 28, "groups_both": 28, "groups_pos_only": 10}, + "memory": {"examples_pos": 171, "examples_neg": 171, "groups_pos": 5, "groups_neg": 5, "groups_both": 5, "groups_pos_only": 0}, + "output_encoding": {"examples_pos": 76, "examples_neg": 60, "groups_pos": 35, "groups_neg": 21, "groups_both": 21, "groups_pos_only": 14}, + "path": {"examples_pos": 30, "examples_neg": 18, "groups_pos": 30, "groups_neg": 18, "groups_both": 18, "groups_pos_only": 12}, + "unknown": {"examples_pos": 2, "examples_neg": 2, "groups_pos": 2, "groups_neg": 2, "groups_both": 2, "groups_pos_only": 0}, + "untrusted_destination": {"examples_pos": 61, "examples_neg": 35, "groups_pos": 51, "groups_neg": 24, "groups_both": 24, "groups_pos_only": 27} + }, + "groups_neg_only": 0 + }, + "positives_only_groups": { + "total": 98, + "by_family": {"access_control": 12, "config_secrets": 3, "deserialization": 20, "injection": 10, "output_encoding": 14, "path": 12, "untrusted_destination": 27}, + "by_family_source_reason": [ + {"family": "access_control", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function: the label builder emitted no fixed-side negative (fixed side is another function of the same snapshot)", "groups": 12}, + {"family": "config_secrets", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 3}, + {"family": "deserialization", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 20}, + {"family": "injection", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 8}, + {"family": "injection", "source": "dev-php", "reason": "no fixed pin recorded: development recipes name a vulnerable checkout only", "groups": 2}, + {"family": "output_encoding", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 14}, + {"family": "path", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 11}, + {"family": "path", "source": "dev-php", "reason": "no fixed pin recorded (development recipe)", "groups": 1}, + {"family": "untrusted_destination", "source": "pairs", "slice": "vibe-py", "reason": "fixed side lacks the function (same as above)", "groups": 26}, + {"family": "untrusted_destination", "source": "population-v2", "reason": "the fix did not change the labelled function: the same code at both pins is not a negative", "groups": 1} + ], + "by_reason": { + "fixed side lacks the function (pairs, vibe-py: 108 labels, 80 seeded pointer pairs whose fixed side is a guard helper of the same commit + 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler)": 94, + "no fixed pin recorded (dev-php recipes)": 3, + "fix did not change the function (population-v2)": 1, + "matcher cannot resolve the function on the fixed pin": 0, + "excerpt missing": 0 + }, + "check": "all 108 fixed sides of the vibe-py pairs were scanned for a `def ` line: 0 hold one (the name occurs only in calls); these are data gaps, counted, never invented", + "advisory_fixes_note": "no advisory-fixes group is positives-only: every one of the 97 pairs gave a fixed-side negative label; 7 of them are absent from the memory on both sides (below)" + }, + "pair_sides_with_a_label_but_no_example": { + "total_sides": 32, + "pairs": 16, + "by_catalog_state": { + "advisory-fixes: feature record present, excerpt missing": {"sides": 14, "pairs": 7, "language": "ruby", "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}}, + "corpus (vfc, vfc-js): no feature record (declares() false on both sides)": {"sides": 18, "pairs": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}, "by_language": {"javascript": 6, "c_cpp": 1, "perl (catalog: unknown)": 2}} + }, + "reasons": { + "ruby under the record's language": "build_features stores the closed feature-vocabulary language (schema.LANGUAGES; `other` for Ruby, Go, C#); the memory build hands that `other` to the excerpt matcher, which then runs the brace matcher: a Ruby `def` has no brace body, so 7 pairs (14 sides) get no excerpt; 4 further Ruby pairs (8 stored examples) matched a `{ |x| ... }` block inside the method instead of the method -- 2-line excerpts, wrong spans. The 11 Go pairs (22 examples) under `other` are brace-matched to the exact body (a correct span) and 9 C# pairs likewise.", + "javascript object-literal method": {"pairs": 3, "form": "`name: function (...) {` and `name: function anonymous(...) {`"}, + "javascript property-assignment arrow": {"pairs": 2, "form": "`module.exports.name = (a, b) => {` and `const name = module.exports.name = async a => {` (a chained assignment)"}, + "javascript prototype named function expression": {"pairs": 1, "form": "`X.prototype.name = function Name$withDollar (req, res) {`: `$` is not a `\\w` character for the declaration pattern or the brace matcher"}, + "c parameter list longer than the brace matcher's lookahead": {"pairs": 1, "form": "`Type name(a,\\n b,\\n#ifndef X\\n c\\n#else\\n d\\n#endif\\n )\\n{`: the `{` is 9 lines down, the matcher looks 8"}, + "perl sub": {"pairs": 2, "form": "`sub name {` (no parameter list; catalog language `unknown`)"} + }, + "recoverable_by_tooling": {"fixed_side_negatives": 16, "positives": 16, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 3, "untrusted_destination": 1, "injection": 5, "memory": 2}}, + "note": "each recovered pair adds a paired group (positive and negative), not a negative to a positives-only group; the 98 positives-only groups are data gaps and stay as they are" + }, + "label_builder_fixed_side_rule": "a fixed-side negative only where the fixed side declares the labelled function (or a deterministic rename at a distinct fix commit, fixed_side_moved = 0); unchanged", + "instruments": { + "memory_rows": "store.rows(kind=example|features) dumped once to scratch, 904 + 1,848 rows read", + "matcher_sweep": "function_bounds on both sides of every `pairs` pin label: vulnerable ok 431, fixed ok 323, fixed side named-but-undeclared 108 (vibe-py, python), both sides unmatched 9 pairs", + "language_check": "every pair record whose language is `other`: span under the record's language vs the label's: csharp same 18 sides, go different 18 (brace body vs to-next-declaration; stored spans kept), ruby none 14 (unstored) and different 8 (stored, wrong)" + } +} diff --git a/benchmarks/measurements/2026-10-02-decision-engine-negatives-rebuilt/record.json b/benchmarks/measurements/2026-10-02-decision-engine-negatives-rebuilt/record.json new file mode 100644 index 0000000..ec0a3e2 --- /dev/null +++ b/benchmarks/measurements/2026-10-02-decision-engine-negatives-rebuilt/record.json @@ -0,0 +1,85 @@ +{ + "created": "2026-10-02", + "note": "COUNTS ONLY. Labels, feature records and the static memory rebuilt after the excerpt matcher learned the declaration forms recorded in 2026-10-02-decision-engine-negatives-gap. No model or embedding call: feature records reuse the recorded engine, verify and roles outputs of the harvest Runs (instruments that never ran stay `missing`); the new and changed excerpts have no vector yet (`ousast learn memory embed`, estimated below). The memory store is the RustFS bucket (versioned: prior objects stay). Candidate, path, function, commit and repository names stay local.", + "labels": { + "command": "ousast learn labels --root . --out --snapshot ", + "rows": 6589, + "groups": 202, + "labels_sha256_prefix": "c17daeac522090d4", + "same_as": "the 2026-10-02 regeneration recorded in 2026-10-02-decision-engine-harvest-advisory (identical sha): the label builder's fixed-side rule already accepted every pair the matcher now resolves (its name-on-a-code-line fallback), so no label row changed", + "pair_fixed_side": {"absent_fixed_side_labels": 108, "fixed_side_moved": 0} + }, + "features": { + "command": "benchmarks/learn/build_features.py --labels --units (defaults: harvest-engine, harvest-verify, harvest-roles)", + "targets": 18, + "records": {"static:injection": 10, "static:memory": 4, "static:path": 4, "plane:injection": 10, "plane:memory": 4, "plane:path": 4}, + "absent_on_side": {}, + "instrument_states_plane_new_records": { + "injection:engine": {"failed": 2, "none": 2, "ran": 6}, "injection:roles": {"failed": 8, "none": 2}, "injection:verify": {"missing": 2, "ran": 8}, + "memory:engine": {"failed": 2, "none": 2}, "memory:roles": {"none": 4}, "memory:verify": {"none": 4}, + "path:engine": {"failed": 4}, "path:roles": {"failed": 4}, "path:verify": {"none": 4} + }, + "instrument_fix": "the store's ingest index remembered these 36 rows from the 2026-10-01 build (before the declares() gate) and skipped them as a repeat, although the 17:25 build had dropped them as stale: the first rebuild stored nothing (store rows 1,848 -> 1,848, memory 919). build_features now checks the rows are present after an index hit and restores them (restored_after_drop = 36). An index hit is not evidence the rows are there.", + "store_rows": {"before": 1848, "after": 1884, "static": 942, "plane": 942}, + "advisory_ruby_pairs": "their 14 records were already in the store (language `other`); nothing rebuilt, the excerpt was the gap" + }, + "memory": { + "command": "ousast learn memory build --labels --units --profile static --root . (exit 2: adjudications 9 of 43 anonymous functions, unchanged since 2026-10-01; rows kept)", + "examples": {"before": 904, "after": 937, "added": 33}, + "by_source_after": {"adjudications": 34, "dev-php": 3, "pairs": 772, "population-v1": 85, "population-v2": 43}, + "pairs_build_counts": {"before": {"recorded": 754, "examples": 740, "absent_fixed_side": 7, "no_excerpt": 7}, "after": {"recorded": 772, "examples": 772, "absent_fixed_side": 0, "no_excerpt": 0}}, + "added_examples": { + "advisory-fixes ruby pairs (record present, excerpt recovered)": {"pairs": 7, "positives": 7, "negatives": 7, "by_family": {"access_control": 2, "deserialization": 2, "output_encoding": 1, "path": 1, "untrusted_destination": 1}}, + "corpus pairs (record and excerpt recovered)": {"pairs": 9, "positives": 9, "negatives": 9, "by_family": {"injection": 5, "path": 2, "memory": 2}}, + "population-v2 (a `this.name = function (` assignment now matched)": {"positives": 1, "negatives": 0, "family": "injection", "note": "its fix did not change the function, so it has no negative label: one more positives-only group"} + }, + "changed_excerpts": {"count": 9, "why": "4 advisory Ruby pairs (8 examples) had a `{ |x| ... }` block inside the method as their excerpt; 1 population-v2 TypeScript method (1 example) had the `{}` default value of a parameter as its body; both corrected by the paren-aware matcher. Their old vectors no longer name the content."}, + "unchanged_excerpts": 895, + "vectors": {"distinct_excerpts_after": 821, "distinct_excerpts_before": 788, "without_vector_at_most": 42, "embed_dry_run_all_usd": 0.0043, "note": "no embedding call was made; `ousast learn memory embed` for the 33 new + 9 changed excerpts is a fraction of a cent. Retrieval scores an example without a vector by its signal similarity (retrieve.py), so they take part now."}, + "per_family_before_after": { + "access_control": {"before": {"pos": 31, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 20, "groups_pos": 32, "groups_neg": 20, "pos_only": 12}}, + "config_secrets": {"before": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}, "after": {"pos": 18, "neg": 13, "groups_pos": 5, "groups_neg": 2, "pos_only": 3}}, + "deserialization": {"before": {"pos": 39, "neg": 19, "groups_pos": 39, "groups_neg": 19, "pos_only": 20}, "after": {"pos": 41, "neg": 21, "groups_pos": 41, "groups_neg": 21, "pos_only": 20}}, + "injection": {"before": {"pos": 75, "neg": 65, "groups_pos": 38, "groups_neg": 28, "pos_only": 10}, "after": {"pos": 81, "neg": 70, "groups_pos": 44, "groups_neg": 33, "pos_only": 11}}, + "memory": {"before": {"pos": 171, "neg": 171, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}, "after": {"pos": 173, "neg": 173, "groups_pos": 5, "groups_neg": 5, "pos_only": 0}}, + "output_encoding": {"before": {"pos": 76, "neg": 60, "groups_pos": 35, "groups_neg": 21, "pos_only": 14}, "after": {"pos": 77, "neg": 61, "groups_pos": 36, "groups_neg": 22, "pos_only": 14}}, + "path": {"before": {"pos": 30, "neg": 18, "groups_pos": 30, "groups_neg": 18, "pos_only": 12}, "after": {"pos": 33, "neg": 21, "groups_pos": 33, "groups_neg": 21, "pos_only": 12}}, + "unknown": {"before": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}, "after": {"pos": 2, "neg": 2, "groups_pos": 2, "groups_neg": 2, "pos_only": 0}}, + "untrusted_destination": {"before": {"pos": 61, "neg": 35, "groups_pos": 51, "groups_neg": 24, "pos_only": 27}, "after": {"pos": 62, "neg": 36, "groups_pos": 52, "groups_neg": 25, "pos_only": 27}} + }, + "positives_only_groups": { + "before": 98, + "after": 99, + "by_reason_after": { + "fixed side lacks the function (pairs, vibe-py: another function of the same snapshot; 108 labels)": 94, + "no fixed pin recorded (dev-php recipes)": 3, + "fix did not change the function (population-v2)": 2, + "matcher cannot resolve the function on the fixed pin": 0, + "excerpt missing": 0 + }, + "reading": "every recovered pair adds a paired group (both sides); the positives-only groups are data gaps that the rule `a fixed-side negative only where the fixed side declares the function` keeps counted, never invented. The one new positives-only group is a population positive that was invisible before (no excerpt) and whose fix did not change its function." + } + }, + "leak_audit": { + "command": "ousast learn audit-leaks --profile {static,plane} --units (threshold 0.20, min 10 pairs)", + "pairs": {"before": {"all": 376, "injection": 43, "memory": 171, "path": 19}, "after": {"all": 385, "injection": 48, "memory": 173, "path": 21}, "unchanged": ["access_control 20", "config_secrets 12", "deserialization 21", "output_encoding 61", "unknown 2", "untrusted_destination 27"]}, + "rows_read": {"before": 1848, "after": 1884}, + "skipped": {"one side only": 172}, + "static": {"new_flags": [], "gone_flags": [], "changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]}, "flagged": 13}, + "plane": { + "new_flags": [ + {"signal": "agree.final", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19}, + {"signal": "verify.votes", "scope": "injection", "separation": 0.2292, "pairs": 48, "correct": 15, "wrong": 4, "differ": 19, "direction": "higher on vulnerable"} + ], + "gone_flags": [], + "changed": {"facts.function_lines/path": [0.7368, 0.6667], "facts.function_lines/injection": [0.2558, 0.2292]}, + "flagged": {"before": 25, "after": 27}, + "reading": "the five injection pairs the matcher recovered carry verify verdicts (8 of 10 sides ran) that separate their sides; agree.final and verify.votes were already flagged for untrusted_destination and config_secrets and are the verifier doing its job, not a label leak. The v1 input profile is unchanged (it withholds engine, facts.function_lines, verify.flag_b and ms.operation); agree.final and verify.votes stay in, reported here for the profile's owner." + } + }, + "instruments": { + "store_dumps": "store.rows(kind=example|features) read once before and once after (904/1,848 then 937/1,884 rows)", + "matcher_sweep_after": "function_bounds on both sides of every pairs pin label: vulnerable ok 440 (was 431), fixed ok 332 (was 323), the 108 vibe-py fixed sides still undeclared", + "dry_build_check": "a dry build_examples over the stored records before any write: 15 new rows, 9 changed shas, 0 gone -- as expected; the live build then matched it plus the 18 restored corpus records" + } +} diff --git a/src/openultrasast/learn/examples.py b/src/openultrasast/learn/examples.py index 4964d85..f9378d0 100644 --- a/src/openultrasast/learn/examples.py +++ b/src/openultrasast/learn/examples.py @@ -122,6 +122,17 @@ def _counter_role(label: Mapping[str, Any]) -> str: return "vulnerable" if label.get("pin_role") == "fixed" else "fixed" +def _excerpt_language(record: Mapping[str, Any], label: Mapping[str, Any], lines: Sequence[str] | None, function: str) -> str: + """The language the excerpt matcher runs under: the record's (the closed feature vocabulary, ``other`` for Go, + Ruby, C#, Perl, ...), and the label's -- the catalog's or the clone's -- where the record's resolves nothing + (a Ruby ``def`` under ``other`` has no brace body). The record keeps its own language field.""" + language = str(record.get("language") or label.get("language") or "other") + declared = str(label.get("language") or "") + if lines and declared and declared != language and function_bounds(lines, language, function) is None: + return declared + return language + + def build_examples( labels: Iterable[Label | Mapping[str, Any]], records: Mapping[tuple[str, str, str, str, str], Mapping[str, Any]], @@ -165,16 +176,17 @@ def build_examples( count.recorded += 1 path, _, function = candidate.partition("::") function = (function_of(label, str(label.get("pin_role") or "vulnerable")) if function_of else None) or function - language = str(record.get("language") or label.get("language") or "other") identities = identities_of(str(label.get("repo") or ""), path, [str(label.get("pin") or "")]) lines = lines_of(label, str(label.get("pin_role") or "vulnerable")) + language = str(record.get("language") or label.get("language") or "other") + matcher = _excerpt_language(record, label, lines, function) diff = None if unit == "delta" and lines: base = lines_of(label, _counter_role(label)) - head_span, base_span = function_bounds(lines, language, function), function_bounds(base or (), language, function) + head_span, base_span = function_bounds(lines, matcher, function), function_bounds(base or (), matcher, function) if base and head_span and base_span: - diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], language, identities=identities) - shown = excerpt(lines, language, function, bounds=bounds, identities=identities, diff=diff) + diff = delta_diff(base[slice(*base_span)], lines[slice(*head_span)], matcher, identities=identities) + shown = excerpt(lines, matcher, function, bounds=bounds, identities=identities, diff=diff) if shown is None: if not lines: count.unread += 1 diff --git a/src/openultrasast/learn/excerpt.py b/src/openultrasast/learn/excerpt.py index ef9eb94..d788a15 100644 --- a/src/openultrasast/learn/excerpt.py +++ b/src/openultrasast/learn/excerpt.py @@ -14,6 +14,14 @@ otherwise tell the model the label. Both become ````. A delta unit add The text is normalised (trailing blanks dropped, ``\\n`` endings, one final newline) before it is hashed, so the sha names the content an embedding and a response cache key are of. + +Where a function is declared (:func:`function_bounds`): the repo-facts declaration pattern of the language first (its +span runs to the next declaration), else the brace matcher, which knows ``name(...) {`` with any return type, +qualifiers, generics or a parameter list spread over many lines (a prototype's ``;`` is not a body), object-literal +methods ``name: function (`` / ``name: (a) =>``, assignments ``x.y.name = function (`` / ``= (a) =>`` / chained +``const name = m.name = async a =>``, Perl ``sub name {`` and C# expression bodies. A file given as ``other`` or +``unknown`` is sniffed for Perl and Ruby (:func:`resolve_language`), whose ``def``/``sub`` forms a brace matcher +would miss or mistake for a block. """ from __future__ import annotations @@ -180,47 +188,123 @@ def identities_of(repo: str = "", path: str = "", commits: Iterable[str] = ()) - # --- spans and excerpts ----------------------------------------------------------------------------------------------- -def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: - """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first - line naming it before ``(`` whose ``{`` comes before any ``;``, to the matching ``}``. Comments are blanked - before braces are counted.""" - name = re.split(r"::|\.|->", function)[-1] - if not re.fullmatch(r"[A-Za-z_]\w*", name): - return None - call = re.compile(r"(?$])" + re.escape(name) + r"\s*\(") +_UNKNOWN_LANGUAGES = frozenset({"", "other", "unknown"}) +_PERL_LINE = re.compile(r"^\s*(?:sub\s+\w+\s*[{(;]|use\s+strict\s*;|my\s+[$@%]\w+)") +_RUBY_DEF = re.compile(r"^\s*def\s+(?:self\.)?\w+[?!=]?(?:\s*\(|\s|$)") +_RUBY_END = re.compile(r"^\s*end\s*$") +_BODY_SCAN_LINES = 40 # how far past a declaration head the body's ``{`` (or a prototype's ``;``) may be +_STATEMENT_HEAD = re.compile(r"\s*(return|if|while|for|switch|else|case|throw|await|yield)\b") +_FUNCTION_EXPR = r"(?:async\s+)?(?:function\b\s*\*?\s*[\w$]*\s*)?(?=\(|[\w$]+\s*=>)" # function (...) / (...) => / a => + + +def resolve_language(lines: Sequence[str], language: str) -> str: + """The alias-normalised language; a file given as unknown (``other``, the feature vocabulary's bucket for Go, Ruby, + C#, Perl, ...) is sniffed for Perl (``sub name {``, ``use strict;``) and Ruby (``def`` ... ``end``), whose + declaration forms a brace matcher never finds and whose ``#`` comments must be redacted.""" + language = LANGUAGE_ALIASES.get(language, language) + if language in _UNKNOWN_LANGUAGES and lines: + if any(_PERL_LINE.match(line) for line in lines): + return "perl" + if any(_RUBY_DEF.match(line) for line in lines) and any(_RUBY_END.match(line) for line in lines): + return "ruby" + return language + + +def _blank_comments(lines: Sequence[str], language: str) -> list[str]: + style = language if language in SLASH_COMMENTS or language in HASH_COMMENTS else "c" code: list[str] = [] state: str | None = None for line in lines: - ranges, state = _comment_ranges(line, state, language if language in SLASH_COMMENTS else "c") + ranges, state = _comment_ranges(line, state, style) for start, end in reversed(ranges): line = line[:start] + " " * (end - start) + line[end:] code.append(line) - for i, line in enumerate(code): - found = call.search(line) - if found is None or re.match(r"\s*(return|if|while|for|switch|else)\b", line): - continue - tail = "\n".join([line[found.end() :], *code[i + 1 : i + 8]]) - brace, semi = tail.find("{"), tail.find(";") - if brace < 0 or (0 <= semi < brace): - continue - depth, opened = 0, False - for j in range(i, len(code)): - for c in line[found.end() :] if j == i else code[j]: + return code + + +def _declaration_heads(name: str) -> re.Pattern[str]: + """Where a brace language may declare ``name`` (the body follows): ``name(`` (a definition -- or a call, told + apart by :func:`_body_start`); Perl ``sub name {``; an object-literal method ``name: function (`` / + ``name: (a) =>``; an assignment ``x.y.name = function (`` / ``= (a) =>`` / ``const name = m.name = async a =>`` + (any chain). ``$`` is an identifier character (``Glance$serveRequest``).""" + quoted = re.escape(name) + return re.compile( + rf"(?$]){quoted}\s*(?:<[^<>(){{}};]*>\s*)?(?=\()" + rf"|^\s*sub\s+{quoted}\s*(?=[{{(;])" + rf"|(? tuple[str, int, int] | None: + """Scan from a declaration head for its body: ``("brace", line, column)`` of the ``{`` at parenthesis depth 0, + ``("expression", line, column)`` of the ``;`` or ``,`` ending an arrow function's expression body, ``None`` for a + prototype (``;`` before any body), a call (a ``)`` closing an outer parenthesis, a ``}``) or no body within + :data:`_BODY_SCAN_LINES`. Parentheses are tracked so ``f(a, function () {})`` is not a definition of ``f``.""" + depth, arrow = 0, False + for j in range(line, min(line + _BODY_SCAN_LINES, len(code))): + text = code[j] + k = column if j == line else 0 + while k < len(text): + c = text[k] + if c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth < 0: + return None + elif depth == 0: if c == "{": - depth, opened = depth + 1, True - elif c == "}" and opened: - depth -= 1 - if depth == 0: - return i, j + 1 - return i, len(code) + return ("brace", j, k) + if arrow and c in ";,": + return ("expression", j, k) + if c in "};": # a statement ends (a call, a prototype) before any body opened + return None + if text.startswith("=>", k): + arrow = True + k += 1 + k += 1 + return None + + +def _brace_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: + """A brace language's definition of ``function`` (``Class::name`` and ``a.b`` reduced to the last name): the first + declaration head (:func:`_declaration_heads`) whose body :func:`_body_start` finds, to the matching ``}`` (or the + end of an expression body). Comments are blanked before anything is matched or counted.""" + name = re.split(r"::|\.|->", function)[-1] + if not re.fullmatch(r"[A-Za-z_$][\w$]*", name): + return None + heads = _declaration_heads(name) + code = _blank_comments(lines, language) + for i, line in enumerate(code): + if _STATEMENT_HEAD.match(line): + continue + for found in heads.finditer(line): + body = _body_start(code, i, found.end()) + if body is None: + continue + kind, j, k = body + if kind == "expression": + return i, j + 1 + depth = 0 + for row in range(j, len(code)): + for c in code[row][k:] if row == j else code[row]: + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return i, row + 1 + return i, len(code) return None def function_bounds(lines: Sequence[str], language: str, function: str) -> tuple[int, int] | None: """0-based ``[start, end)`` of the first declaration of ``function`` (the whole file for ````), or - ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, ...) and - brace languages whose pattern misses the name use the brace matcher.""" - language = LANGUAGE_ALIASES.get(language, language) + ``None`` when the file does not declare it. Languages without a repo-facts declaration pattern (C, C++, C#, Perl, + ...) and brace languages whose pattern misses the name use the brace matcher; an unknown language is sniffed + (:func:`resolve_language`).""" + language = resolve_language(lines, language) if function == GLOBAL: return (0, len(lines)) if lines else None pattern = DECLARATION.get(language) @@ -267,6 +351,7 @@ def excerpt( :func:`delta_diff`) is appended under a ``diff:`` line.""" if not lines: return None + language = resolve_language(lines, language) span = function_bounds(lines, language, function) if span is None: return None @@ -300,6 +385,7 @@ def delta_diff( ) -> str: """The base->head unified diff of a function (one line of context, no file headers), redacted, at most ``max_lines`` lines.""" + language = resolve_language(head, language) diff = [line for line in difflib.unified_diff(list(base), list(head), lineterm="", n=1)][2:] out: list[str] = [] for line in diff: @@ -314,5 +400,5 @@ def delta_diff( __all__ = [ "CANDIDATE", "DIFF_LINES", "EXAMPLE", "REDACTED", "Bounds", "Excerpt", "delta_diff", "excerpt", "excerpt_sha", - "function_bounds", "identities_of", "normalise", "redact", + "function_bounds", "identities_of", "normalise", "redact", "resolve_language", ] # fmt: skip diff --git a/tests/test_learn_examples.py b/tests/test_learn_examples.py index b9b7319..f3495dd 100644 --- a/tests/test_learn_examples.py +++ b/tests/test_learn_examples.py @@ -158,6 +158,16 @@ def test_a_fixed_side_without_the_function_is_a_label_defect_not_an_unread_side( assert unread.sources["population-v1"].unread == 1 and unread.failing() == ["population-v1"] +def test_the_excerpt_matcher_falls_back_to_the_labels_language_when_the_records_resolves_nothing() -> None: + """A feature record carries the closed vocabulary's language (``other`` for Ruby, Go, C#); a Python ``def`` under + ``other`` has no brace body, so the matcher gets the label's language instead. The record's field is kept.""" + lab = label() + other = {**feature(lab), "language": "other"} + build = build_examples([lab], records_by_key([other]), lambda lab, role: CODE[role], sources=SOURCES) + assert build.summary()["examples"] == 1 and build.rows[0]["language"] == "other" # the row keeps the vocabulary language + assert build.sources["population-v1"].no_excerpt == 0 + + def test_pair_sides_join_the_record_stored_under_their_harvest_unit_pin() -> None: """The harvest stores a pair side's record under its unit pin (the blob sha1 of the side's excerpt), not under :func:`label_pin`; ``pins`` from the harvest ``units.json`` joins them, and without it the record is not found.""" diff --git a/tests/test_learn_excerpt.py b/tests/test_learn_excerpt.py index df9dbdf..be5c1be 100644 --- a/tests/test_learn_excerpt.py +++ b/tests/test_learn_excerpt.py @@ -136,3 +136,224 @@ def test_brace_languages_without_a_declaration_pattern() -> None: assert shown is not None and shown.first == 3 and shown.last == 7 and "other(void)" not in shown.text assert excerpt(c, "cpp", "Parser::parse_header") is not None assert excerpt(["int f(void);", "int g(void) { return f(); }"], "c", "f") is None # a prototype is not a body + + +# --- declaration forms the brace matcher must know (each form once; a call or prototype is never a declaration) ------- + + +def _span(code: list[str], language: str, function: str) -> tuple[int, int] | None: + found = excerpt(code, language, function) + return None if found is None else (found.first, found.last) + + +def test_js_object_literal_methods() -> None: + js = [ + "module.exports = {", + " getInfo: function () {", + " return this.exec('pdfinfo ' + this.path)", + " },", + " ps: function(pid, options, done) {", + " var cmd = 'ps -o pcpu,rss -p ' + pid", + " exec(cmd, done)", + " },", + " performAction: function anonymous(yytext, yyleng) {", + " return eval(yytext)", + " },", + " short: (a) => a + 1,", + " use: function () { return this.getInfo().then(function (info) { return info }) },", + "}", + ] + assert _span(js, "javascript", "getInfo") == (2, 4) + assert _span(js, "javascript", "ps") == (5, 8) + assert _span(js, "javascript", "performAction") == (9, 11) # the key, not the expression's own name + assert _span(js, "javascript", "short") == (12, 12) # an expression body ends at its comma + assert _span(js, "javascript", "then") is None # a call inside a body is not a declaration + + +def test_js_property_assignment_functions_and_arrows() -> None: + js = [ + "module.exports.ConfigureFilePath = (Options, FilePath) => {", + " return path.join(Options.root, FilePath)", + "}", + "const listProcessesOnPort = module.exports.listProcessesOnPort = async port => {", + " return exec(`lsof -i :${port}`)", + "}", + "Glance.prototype.serveRequest = function Glance$serveRequest (req, res) {", + " fs.createReadStream(req.url).pipe(res)", + "}", + "this.getDaqValue = function (tagid, fromts, tots) {", + " return conn.query('SELECT ' + tagid)", + "}", + "const total = sum(prices)", + ] + assert _span(js, "javascript", "ConfigureFilePath") == (1, 3) + assert _span(js, "javascript", "listProcessesOnPort") == (4, 6) + assert _span(js, "javascript", "Glance$serveRequest") == (7, 9) # `$` is an identifier character + assert _span(js, "javascript", "serveRequest") == (7, 9) + assert _span(js, "javascript", "getDaqValue") == (10, 12) + assert _span(js, "javascript", "sum") is None # an assignment of a call result declares nothing + + +def test_ts_class_methods_with_modifiers_and_multiline_parameters() -> None: + ts = [ + "export class Client {", + " private retries = 3", + " private async fetchRetry(", + " url: string,", + " options: any = {},", + " ): Promise {", + " return fetch(url, options)", + " }", + " handle = async (req: Request): Promise => {", + " await this.fetchRetry(req.url)", + " }", + "}", + ] + assert _span(ts, "typescript", "fetchRetry") == (3, 8) # not the `{}` default value inside the parameter list + assert _span(ts, "typescript", "handle") == (9, 11) + + +def test_perl_sub_under_an_unknown_language() -> None: + perl = [ + "#!/usr/bin/perl", + "use strict;", + "sub link_hash_cert;", # a forward declaration has no body + "# CVE-2022-1292: shell injection fixed", + "sub link_hash_cert {", + " my $fname = $_[0];", + " system(\"openssl x509 -in '$fname'\");", + "}", + "sub hash_dir { opendir(my $dh, $_[0]) or die; }", + ] + assert _span(perl, "unknown", "link_hash_cert") == (5, 8) + assert _span(perl, "other", "hash_dir") == (9, 9) + shown = excerpt(perl, "unknown", "link_hash_cert", focus=(4,)) + assert shown is not None and "CVE-2022" not in shown.text # `#` comments are redacted once the language is known + + +def test_c_declaration_with_a_long_parameter_list_and_qualifiers() -> None: + c = [ + "CURLcode Curl_add_custom_headers(struct Curl_easy *data, bool is_connect, void *req);", + "", + "CURLcode Curl_add_custom_headers(struct Curl_easy *data,", + " bool is_connect,", + "#ifndef USE_HYPER", + " struct dynbuf *req", + "#else", + " void *req", + "#endif", + " )", + "{", + " return CURLE_OK;", + "}", + "std::string Parser::header(int index) const noexcept {", + " return headers[index];", + "}", + "Parser::Parser(int n) : size_(n), data_(nullptr) {", + " data_ = alloc(n);", + "}", + "int main(void) { if (check(1)) { return 1; } else if (header(2)) { return 2; } return 0; }", + ] + assert _span(c, "c", "Curl_add_custom_headers") == (3, 13) # the prototype is skipped, the body is 9 lines down + assert _span(c, "cpp", "Parser::header") == (14, 16) + assert _span(c, "cpp", "Parser") == (17, 19) # an initializer list's commas are not statement ends + assert _span(c, "c", "check") is None # `if (check(1)) {` is a call: its `)` closes an outer parenthesis + + +def test_php_methods_with_modifiers_and_return_types() -> None: + php = [ + " None: + go = [ + "package main", + "", + "func (s *Server) handle(w http.ResponseWriter, r *http.Request) {", + "\tio.Copy(w, r.Body)", + "}", + "", + "func main() {}", + ] + assert _span(go, "go", "handle") == (3, 6) # the declaration pattern: to the next `func` + assert _span(go, "other", "handle") == (3, 5) # the feature vocabulary's bucket for Go: brace-matched to the body + rust = ["pub fn parse(input: &mut T) -> Result<(), Error> {", " Ok(())", "}"] + assert _span(rust, "rust", "parse") == (1, 3) + + +def test_ruby_defs_under_the_other_language_are_sniffed() -> None: + ruby = [ + "module Kredis", + " class Type", + " def self.cast_value(value)", + " YAML.load(value)", + " end", + "", + " def fetch_file(path)", + " files.each { |f| return f if f.name == path }", + " nil", + " end", + " end", + "end", + ] + assert _span(ruby, "ruby", "cast_value") == (3, 6) + assert _span(ruby, "other", "cast_value") == (3, 6) # not None: the file is sniffed as Ruby + assert _span(ruby, "other", "fetch_file") == _span(ruby, "ruby", "fetch_file") == (7, 12) # to the end: no later `def` + assert _span(ruby, "c", "fetch_file") == (7, 8) # what a brace matcher takes under a named brace language: the block + + +def test_csharp_attributes_modifiers_and_expression_bodies() -> None: + cs = [ + "public class UploadController : Controller", + "{", + " [HttpPost]", + " public async Task Upload(IFormFile file)", + " {", + " var path = Path.Combine(root, file.FileName);", + " return Ok(path);", + " }", + " protected override string Name(int index) => names[index];", + " public abstract int Size(string key);", + "}", + ] + assert _span(cs, "csharp", "Upload") == (4, 8) + assert _span(cs, "csharp", "Name") == (9, 9) # an expression-bodied member ends at its `;` + assert _span(cs, "csharp", "Size") is None # abstract: no body + + +def test_java_and_statement_heads_are_not_declarations() -> None: + java = [ + "public class Handler {", + " @Override", + " public synchronized List load(String name, Class type)", + " throws IOException, SQLException {", + " return run(name, type);", + " }", + " void other() {", + ' while (load("x", null) != null) { break; }', + " }", + "}", + ] + assert _span(java, "java", "load") == (3, 6) # `throws A, B {`: the comma is not a statement end + assert _span(java, "java", "run") is None + assert _span(java[2:], "java", "load") == _span(java[2:], "c", "load") == (1, 4) # pattern and brace matcher agree + + +def test_resolve_language_sniffs_only_unknown_languages() -> None: + from openultrasast.learn.excerpt import resolve_language + + assert resolve_language(["sub x {", "}"], "unknown") == "perl" + assert resolve_language(["def x", "end"], "other") == "ruby" + assert resolve_language(["def x(a):", " return a"], "other") == "other" # Python has no `end` lines + assert resolve_language(["sub x {", "}"], "python") == "python" # a named language is never second-guessed + assert resolve_language([], "unknown") == "unknown"