From 6b85a60ff195f64ff8877a7c0562a79ee01c3bb2 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:36:32 +0200 Subject: [PATCH] perf(push): find the Python-side scale walls all at once, and remove them The loop the maintainer called out: four scale defects in the PHP transfer replay, found one per two-hour Joern run, all Python bookkeeping between engine calls. benchmarks/push/python_overhead.py ends that: the real replay() on the real PMPro pins, with a JoernBackend whose graphs are stubs and whose queries answer instantly with rows recorded from the real engine. Engine time is zero, so everything measured is ours, in about a minute, under the sampling profiler. Its first pass named two more walls: - Region discovery re-parsed the ruleset for every region: 89,127 TOML parses, 275 s of a 339 s discovery, twice per push. The fact, obligation and taxonomy loaders now keep the parsed (immutable) result keyed by each file's name, size and mtime -- an edit in place or an added file reloads. Discovery 339 s -> 7 s; output digest identical on PMPro. - affected_context decoded every span's and every line anchor's path for every context row -- a changed file has an anchor per unchanged line -- 78% of a stubbed push. Decoded once per call, anchors indexed by line and bisected, original order kept. A randomised control compares it against the row-by-row matching over 200 contexts (>100 attachments). Result: the whole stubbed PMPro push, 27,542 answered requests and 2,688 outcomes, runs in 58.5 s of Python. What is left is the engine. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo --- benchmarks/push/python_overhead.py | 129 ++++++++++++++++++ src/openultrasast/model/regions.py | 45 ++++-- src/openultrasast/model/taxonomy.py | 20 ++- src/openultrasast/semantic/facts.py | 43 +++++- .../semantic/obligations/facts.py | 18 ++- tests/test_fact_memo.py | 33 +++++ tests/test_model_change_context.py | 65 +++++++++ 7 files changed, 338 insertions(+), 15 deletions(-) create mode 100644 benchmarks/push/python_overhead.py create mode 100644 tests/test_fact_memo.py diff --git a/benchmarks/push/python_overhead.py b/benchmarks/push/python_overhead.py new file mode 100644 index 0000000..2515dbb --- /dev/null +++ b/benchmarks/push/python_overhead.py @@ -0,0 +1,129 @@ +"""The push path at full repository scale with an engine that answers instantly: what does the PYTHON cost? + +Named by the 2026-09-23 PHP transfer replay, which met four scale defects one two-hour run at a time -- a cache +that rescanned itself on every write, a relationship list searched linearly, a fsync per cache entry, and the +ruleset re-parsed per region (89,127 TOML parses in one discovery). Every one was Python bookkeeping around the +engine, every one grew with the repository, and none could show in a unit test or on a small subject. + +This runs the real `replay()` -- snapshots, discovery, partitions, evidence, change context, arbitration, +comparison, reporting, the reuse cache -- on a real checkout, with a `JoernBackend` whose graphs are stubs and +whose queries answer at once with rows recorded from the real engine on the same repository. Engine time is +zero by construction, so everything left is ours. It runs in minutes, with the sampling profiler on, and it +fails loudly if the instrument did not actually exercise the path (no questions, no rows, no stages). + +Usage: + python benchmarks/push/python_overhead.py --repository --base --head \\ + --rows [--rows ...] --out [--deadline 3600] +""" + +from __future__ import annotations + +import argparse +import json +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from openultrasast import profiling +from openultrasast.config import PushConfig +from openultrasast.cpg.backend import TIMING, CpgResult, JoernBackend +from openultrasast.push.runner import replay + +# What each partition's graph holds, so the stub census names exactly the files a real frontend would read. +EXTENSIONS = {"php": (".php",), "javascript": (".js", ".jsx", ".mjs", ".cjs"), "typescript": (".ts", ".tsx"), "python": (".py",)} + + +class StubEngine(JoernBackend): + """A backend whose engine is a table lookup: real row shapes, zero engine time.""" + + def __init__(self, templates: list[list[object]], **kwargs: Any) -> None: + super().__init__(**kwargs) + self.templates = templates + self.asked = 0 + self.graphs = 0 + + def build(self, root: Path, *, language: str = "", exclude: Any = (), execution_budget: Any = None) -> CpgResult | None: + extensions = EXTENSIONS.get(language.lower(), ()) + files = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.is_file() and p.suffix.lower() in extensions) + graph = Path(__import__("tempfile").mkdtemp(prefix="ousast-stub-")) / "cpg.bin" + graph.write_bytes(("stub graph " + language + " " + str(len(files))).encode()) + self.graphs += 1 + census = { + "files": str(len(files)), + "methods": str(max(1, 10 * len(files))), + "calls": "1", + "file_names": files, + "overlays": "dataflowOss", + } + + def run(kind: str, params: Mapping[str, object]) -> object: + if kind == "census": + return dict(census) + answer = batch(kind, {"one": params}) + return answer.get("one") if answer else None + + def batch(kind: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]: + answers: dict[str, list[object]] = {"__census__": [dict(census)]} + timing = [] + for rid in requests: + self.asked += 1 + rows = self.templates[self.asked % len(self.templates)] if kind == "taint" else [{"kind": "context_summary"}] + answers[rid] = list(rows) + timing.append({"id": rid, "ms": 1.0}) + answers[TIMING] = timing # type: ignore[assignment] + return answers + + return CpgResult(cpg_path=graph, run=run, run_batch=batch, run_batch_once=batch, execution_diagnostics=lambda: ()) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--rows", type=Path, action="append", required=True, help="recorded engine rows, {id: rows}") + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--deadline", type=float, default=3600.0) + parser.add_argument("--max-regions", type=int, default=500) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + templates = [rows for path in args.rows for rows in json.loads(path.read_text()).values() if isinstance(rows, list)] + if not templates: + raise SystemExit("no recorded rows: the stub would answer nothing and measure nothing") + print(f"{len(templates)} recorded answers from {len(args.rows)} file(s)", flush=True) + profiling.start(str(args.out / "profile.txt"), flush=10.0) + engine = StubEngine(templates) + started = time.monotonic() + delivery = replay( + args.repository, + base=args.base, + head=args.head, + artifact=args.out / "artifact.json", + config=PushConfig(deadline_seconds=args.deadline, cancellation_allowance_seconds=2.0), + backend=engine, + max_regions=args.max_regions, + cache_dir=args.out / "cache", + ) + elapsed = time.monotonic() - started + artifact = json.loads((args.out / "artifact.json").read_text()) if (args.out / "artifact.json").is_file() else {} + timings = {k: round(v, 1) for k, v in artifact.get("timings", {}).items() if k.endswith("_seconds")} + questions = sum(len(s["scan"]["question_outcomes"]) for s in artifact.get("scans", [])) + summary = { + "elapsed": round(elapsed, 1), + "graphs": engine.graphs, + "requests_answered": engine.asked, + "questions": questions, + "timings": timings, + } + (args.out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary), flush=True) + print(delivery.terminal if hasattr(delivery, "terminal") else "", flush=True) + if not engine.asked or not questions: + raise SystemExit("the instrument did not exercise the push path: nothing was asked or no question was recorded") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/model/regions.py b/src/openultrasast/model/regions.py index 9200cfe..0cf5570 100644 --- a/src/openultrasast/model/regions.py +++ b/src/openultrasast/model/regions.py @@ -18,6 +18,7 @@ Two restraints are deliberate: from __future__ import annotations +import bisect import time from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace @@ -194,6 +195,15 @@ AMBIGUOUS_CORRESPONDENCE = "ambiguous_line_correspondence" REACHABILITY_BOUNDED = "dynamic_external_or_depth_context_unresolved" +def _anchored_lines(anchors: list[tuple[int, int]] | None, start: int, end: int) -> list[int]: + """Anchor lines within ``[start, end]``, in the order the change context listed them.""" + if not anchors: + return [] + low = bisect.bisect_left(anchors, (start, -1)) + high = bisect.bisect_right(anchors, (end, float("inf"))) + return [line for line, _ in sorted(anchors[low:high], key=lambda item: item[1])] + + def _context_path(raw: str, root: Path, resolved_root: Path) -> str | None: """A context location's path relative to the root, or ``None`` when it is not a real file under it.""" try: @@ -272,6 +282,20 @@ def affected_context( inherited_gap = any(not gap.endswith(":" + AMBIGUOUS_CORRESPONDENCE) for gap in gaps) renamed = {context.decode_path(r.base_path): context.decode_path(r.head_path) for r in context.renames} checked: dict[str, bool] = {} + # Decoded ONCE per call. Every row used to decode every span's and every line anchor's path -- a changed file + # has one anchor per unchanged line -- and with the engine stubbed out that was 78% of a whole push. Anchors + # are indexed by line so a row finds its own with two bisections, in their original order. + attachable = changed | set(renamed.values()) + spans_by_path: dict[str, list[tuple[str, str, int, int]]] = {} + for span in context.spans: + old_path = context.decode_path(span.path) + match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path + spans_by_path.setdefault(match_path, []).append((span.side, old_path, span.start_line, span.end_line)) + anchors_by_path: dict[str, list[tuple[int, int]]] = {} + for index, anchor in enumerate(context.line_correspondences): + anchors_by_path.setdefault(context.decode_path(anchor.head_path), []).append((anchor.head_start_line, index)) + for anchors in anchors_by_path.values(): + anchors.sort() # Resolved once per distinct path: the same few changed files recur in every question's rows, and resolving # them again per row was half of what `affected_context` cost once its quadratic was gone. resolved: dict[str, str | None] = {} @@ -319,18 +343,15 @@ def affected_context( except (ValueError, TypeError, OSError): local_gaps.append("context_location_unavailable") continue - evidence = [] - for span in context.spans: - old_path = context.decode_path(span.path) - match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path - if match_path != path_text: - continue - if span.side == "base" or (span.start_line <= end and span.end_line >= start): - evidence.append(f"{span.side}_span:{old_path}:{span.start_line}-{span.end_line}") - for anchor in context.line_correspondences: - if context.decode_path(anchor.head_path) == path_text and start <= anchor.head_start_line <= end: - evidence.append(f"unchanged_line:{anchor.head_start_line};lexical_only") - if not evidence or path_text not in changed | set(renamed.values()): + if path_text not in attachable: + continue + evidence = [ + f"{side}_span:{old_path}:{span_start}-{span_end}" + for side, old_path, span_start, span_end in spans_by_path.get(path_text, ()) + if side == "base" or (span_start <= end and span_end >= start) + ] + evidence += [f"unchanged_line:{line};lexical_only" for line in _anchored_lines(anchors_by_path.get(path_text), start, end)] + if not evidence: continue function = str(row.get("function", "")) or None source = QuestionIdentity(identity.unit, identity.language, path_text, function, identity.family) diff --git a/src/openultrasast/model/taxonomy.py b/src/openultrasast/model/taxonomy.py index 1c1c324..0c87bce 100644 --- a/src/openultrasast/model/taxonomy.py +++ b/src/openultrasast/model/taxonomy.py @@ -90,7 +90,7 @@ class FamilyTaxonomy: return "lateral" -def load_families(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy: +def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy: """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file); a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``.""" source = path if path is not None else DEFAULT_FAMILIES_PATH @@ -192,3 +192,21 @@ __all__ = [ "Relation", "load_families", ] + + +# Parsed once per file state; see `semantic.facts.load_facts` for why and for the key. +_FAMILIES_MEMO: dict[tuple[object, ...], FamilyTaxonomy] = {} + + +def load_families(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy: + """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file); + a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``.""" + source = path if path is not None else DEFAULT_FAMILIES_PATH + try: + info = source.stat() + except OSError: + return _load_families_uncached(path, require_all=require_all) + key = (str(source.resolve()), info.st_size, info.st_mtime_ns, require_all) + if key not in _FAMILIES_MEMO: + _FAMILIES_MEMO[key] = _load_families_uncached(path, require_all=require_all) + return _FAMILIES_MEMO[key] diff --git a/src/openultrasast/semantic/facts.py b/src/openultrasast/semantic/facts.py index 694ac0e..5ff534e 100644 --- a/src/openultrasast/semantic/facts.py +++ b/src/openultrasast/semantic/facts.py @@ -116,7 +116,7 @@ class SemanticFacts: ) -def load_facts(directory: Path | None = None) -> SemanticFacts: +def _load_facts_uncached(directory: Path | None = None) -> SemanticFacts: root = directory if directory is not None else DEFAULT_FACTS_DIR if not root.is_dir(): raise FactLoadError(f"semantic fact directory missing: {root}") @@ -271,3 +271,44 @@ def _sanitizers(value: object, language: str, path: Path) -> list[SanitizerFact] ) ) return rows + + +# Parsed once per ruleset STATE, not once per call. Region discovery asks for the family specs of every region, +# and each spec call loaded the facts again: 89,127 TOML parses and 275 s of a 339 s discovery on a WordPress +# plugin, paid twice per push. The key is every fact file's name, size and modification time, so an edited +# ruleset -- a test's fixture, a maintainer's change -- is read again, and the result is immutable, so sharing +# it is safe. A load that fails is not remembered. +_FACTS_MEMO: dict[tuple[object, ...], SemanticFacts] = {} + + +_LISTED: dict[tuple[str, str], tuple[int, tuple[Path, ...]]] = {} + + +def _ruleset_state(root: Path, pattern: str) -> tuple[object, ...]: + """Every matching file's name, size and modification time -- cheaply, because every question asks. + + The directory is listed again only when its own stamp moves (a file added, removed or renamed); an edit in + place moves the file's stamp, which is read every time. Globbing and resolving on every call was 7% of a + whole push with the engine stubbed out. + """ + stamp = root.stat().st_mtime_ns + listed = _LISTED.get((str(root), pattern)) + if listed is None or listed[0] != stamp: + listed = (stamp, tuple(sorted(root.glob(pattern)))) + _LISTED[(str(root), pattern)] = listed + return (str(root), stamp, *((path.name, *_stat_of(path)) for path in listed[1])) + + +def _stat_of(path: Path) -> tuple[int, int]: + info = path.stat() + return info.st_size, info.st_mtime_ns + + +def load_facts(directory: Path | None = None) -> SemanticFacts: + root = directory if directory is not None else DEFAULT_FACTS_DIR + if not root.is_dir(): + return _load_facts_uncached(directory) + key = _ruleset_state(root, "*.toml") + if key not in _FACTS_MEMO: + _FACTS_MEMO[key] = _load_facts_uncached(directory) + return _FACTS_MEMO[key] diff --git a/src/openultrasast/semantic/obligations/facts.py b/src/openultrasast/semantic/obligations/facts.py index 17a1920..4762db8 100644 --- a/src/openultrasast/semantic/obligations/facts.py +++ b/src/openultrasast/semantic/obligations/facts.py @@ -11,6 +11,8 @@ import tomllib from dataclasses import dataclass from pathlib import Path +from ..facts import _ruleset_state + OPERATION_KINDS = ("protected_read", "protected_write", "privileged_action", "security_setting") DISCHARGER_KINDS = ("path_guard", "identity_constraint", "ownership_check", "non_permissive_value", "validated_input") PROVENANCE_KINDS = ("authenticated_context", "request_input", "constant", "unknown") @@ -73,7 +75,7 @@ class ObligationFacts: ) -def load_obligation_facts(directory: Path | None = None) -> ObligationFacts: +def _load_obligation_facts_uncached(directory: Path | None = None) -> ObligationFacts: root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR if not root.is_dir(): raise ObligationFactsError(f"obligation facts directory missing: {root}") @@ -160,3 +162,17 @@ def _strings(value: object, field: str, path: Path) -> tuple[str, ...]: if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value): raise ObligationFactsError(f"{path} has invalid {field} list") return tuple(str(item) for item in value) + + +# Parsed once per ruleset state; see `semantic.facts.load_facts` for the measurement and the key. +_OBLIGATION_MEMO: dict[tuple[object, ...], ObligationFacts] = {} + + +def load_obligation_facts(directory: Path | None = None) -> ObligationFacts: + root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR + if not root.is_dir(): + return _load_obligation_facts_uncached(directory) + key = _ruleset_state(root, "*.toml") + if key not in _OBLIGATION_MEMO: + _OBLIGATION_MEMO[key] = _load_obligation_facts_uncached(directory) + return _OBLIGATION_MEMO[key] diff --git a/tests/test_fact_memo.py b/tests/test_fact_memo.py new file mode 100644 index 0000000..c5cf3dd --- /dev/null +++ b/tests/test_fact_memo.py @@ -0,0 +1,33 @@ +"""The parsed ruleset is shared across calls, and never outlives a change to the files it came from. + +Region discovery asked for the facts once per region and re-parsed the TOML every time: 89,127 parses and +275 s of a 339 s discovery on a WordPress plugin, twice per push. +""" + +import shutil + +from openultrasast.semantic.facts import DEFAULT_FACTS_DIR, load_facts +from openultrasast.semantic.obligations.facts import DEFAULT_OBLIGATION_FACTS_DIR, load_obligation_facts + + +def test_facts_are_parsed_once_and_reread_when_a_file_changes(tmp_path): + root = tmp_path / "semantic" + shutil.copytree(DEFAULT_FACTS_DIR, root) + first = load_facts(root) + assert load_facts(root) is first, "an unchanged ruleset must not be parsed again" + python = root / "python.toml" + python.write_text(python.read_text() + '\n[[sink]]\nid = "memo-probe"\ncwe = "CWE-78"\ncalls = ["memo_probe_call"]\n') + edited = load_facts(root) + assert edited is not first and any("memo_probe_call" in sink.calls for sink in edited.sinks), "an edit in place was not seen" + (root / "zz_added.toml").write_text('language = "python"\n[[sink]]\nid = "added"\ncwe = "CWE-78"\ncalls = ["added_call"]\n') + added = load_facts(root) + assert any("added_call" in sink.calls for sink in added.sinks), "a new fact file was not seen" + + +def test_obligation_facts_are_parsed_once_and_reread_when_a_file_changes(tmp_path): + root = tmp_path / "obligations" + shutil.copytree(DEFAULT_OBLIGATION_FACTS_DIR, root) + first = load_obligation_facts(root) + assert load_obligation_facts(root) is first + (root / "python.toml").write_text((root / "python.toml").read_text().replace('version = "', 'version = "memo-', 1)) + assert load_obligation_facts(root) is not first diff --git a/tests/test_model_change_context.py b/tests/test_model_change_context.py index ea9b03c..6e083b1 100644 --- a/tests/test_model_change_context.py +++ b/tests/test_model_change_context.py @@ -472,3 +472,68 @@ def test_attaching_relationships_is_linear_in_what_it_attaches(tmp_path, monkeyp enriched, _ = affected_context(context(), questions, rows, tmp_path) assert len(enriched.relationships) == 4000 assert len(compared) < 4000, f"{len(compared)} comparisons for 4,000 relationships" + + +def _reference_evidence(context_value, path_text, start, end): + """The pre-index matching, verbatim in effect: every span and anchor decoded and tested per row.""" + renamed = {context_value.decode_path(r.base_path): context_value.decode_path(r.head_path) for r in context_value.renames} + evidence = [] + for span in context_value.spans: + old_path = context_value.decode_path(span.path) + match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path + if match_path != path_text: + continue + if span.side == "base" or (span.start_line <= end and span.end_line >= start): + evidence.append(f"{span.side}_span:{old_path}:{span.start_line}-{span.end_line}") + for anchor in context_value.line_correspondences: + if context_value.decode_path(anchor.head_path) == path_text and start <= anchor.head_start_line <= end: + evidence.append(f"unchanged_line:{anchor.head_start_line};lexical_only") + return evidence + + +def test_indexed_matching_attaches_exactly_what_the_row_by_row_matching_did(tmp_path): + """Randomised: renames, base and head spans, anchors out of order and in several files.""" + import random + + from openultrasast.model.contracts import PathRename, QuestionIdentity + from openultrasast.model.regions import affected_context + + rng = random.Random(20260923) + names = ["a.php", "b.php", "c.php", "old.php"] + for name in ["a.php", "b.php", "c.php"]: + (tmp_path / name).write_text("x\n" * 200) + attached = 0 + for _ in range(200): + spans = tuple( + ChangedSpan(rng.choice(names).encode().hex(), s, s + rng.randint(0, 20), rng.choice(["base", "head"])) + for s in (rng.randint(1, 180) for _ in range(rng.randint(0, 6))) + ) + anchors = tuple( + LineCorrespondence( + rng.choice(names).encode().hex(), rng.choice(names).encode().hex(), base, base + rng.randint(0, 3), rng.randint(1, 200) + ) + for base in (rng.randint(1, 190) for _ in range(rng.randint(0, 40))) + ) + renames = (PathRename(b"old.php".hex(), b"c.php".hex()),) if rng.random() < 0.5 else () + changed = tuple(n.encode().hex() for n in rng.sample(["a.php", "b.php", "c.php"], rng.randint(1, 3))) + ctx = ChangeContext("base", "head", changed, (), spans, renames, (), (), (), "filesystem-bytes-hex", anchors) + question = QuestionIdentity("unit", "php", "q.php", "run", "injection") + rows = [{"kind": "context_summary"}] + for _ in range(rng.randint(1, 12)): + s = rng.randint(1, 190) + rows.append( + {**row(rng.choice(["a.php", "b.php", "c.php"]), function=f"f{s}"), "startLine": s, "endLine": s + rng.randint(0, 30)} + ) + enriched, _ = affected_context(ctx, [question], {question: rows}, tmp_path) + attachable = {ctx.decode_path(p) for p in ctx.changed_paths} | {ctx.decode_path(r.head_path) for r in ctx.renames} + expected = [] + for r in rows[1:]: + evidence = _reference_evidence(ctx, r["path"], r["startLine"], r["endLine"]) + if evidence and r["path"] in attachable: + key = (r["path"], r["function"], tuple(evidence)) + if key not in expected: + expected.append(key) + got = [(rel.source.path, rel.source.function, rel.evidence) for rel in enriched.relationships] + assert got == expected + attached += len(got) + assert attached > 100, f"only {attached} relationships across 200 cases: the comparison would be vacuous"