perf(push): find the Python-side scale walls all at once, and remove them

The loop the maintainer called out: four scale defects in the PHP transfer
replay, found one per two-hour Joern run, all Python bookkeeping between
engine calls. benchmarks/push/python_overhead.py ends that: the real
replay() on the real PMPro pins, with a JoernBackend whose graphs are stubs
and whose queries answer instantly with rows recorded from the real engine.
Engine time is zero, so everything measured is ours, in about a minute,
under the sampling profiler. Its first pass named two more walls:

- Region discovery re-parsed the ruleset for every region: 89,127 TOML
  parses, 275 s of a 339 s discovery, twice per push. The fact, obligation
  and taxonomy loaders now keep the parsed (immutable) result keyed by each
  file's name, size and mtime -- an edit in place or an added file reloads.
  Discovery 339 s -> 7 s; output digest identical on PMPro.
- affected_context decoded every span's and every line anchor's path for
  every context row -- a changed file has an anchor per unchanged line --
  78% of a stubbed push. Decoded once per call, anchors indexed by line and
  bisected, original order kept. A randomised control compares it against
  the row-by-row matching over 200 contexts (>100 attachments).

Result: the whole stubbed PMPro push, 27,542 answered requests and 2,688
outcomes, runs in 58.5 s of Python. What is left is the engine.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

benchmarks/push/python_overhead.py

@@ -0,0 +1,129 @@
+"""The push path at full repository scale with an engine that answers instantly: what does the PYTHON cost?
+
+Named by the 2026-09-23 PHP transfer replay, which met four scale defects one two-hour run at a time -- a cache
+that rescanned itself on every write, a relationship list searched linearly, a fsync per cache entry, and the
+ruleset re-parsed per region (89,127 TOML parses in one discovery). Every one was Python bookkeeping around the
+engine, every one grew with the repository, and none could show in a unit test or on a small subject.
+
+This runs the real `replay()` -- snapshots, discovery, partitions, evidence, change context, arbitration,
+comparison, reporting, the reuse cache -- on a real checkout, with a `JoernBackend` whose graphs are stubs and
+whose queries answer at once with rows recorded from the real engine on the same repository. Engine time is
+zero by construction, so everything left is ours. It runs in minutes, with the sampling profiler on, and it
+fails loudly if the instrument did not actually exercise the path (no questions, no rows, no stages).
+
+Usage:
+    python benchmarks/push/python_overhead.py --repository <objects.git> --base <oid> --head <oid> \\
+        --rows <recorded rows json> [--rows ...] --out <dir> [--deadline 3600]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from openultrasast import profiling
+from openultrasast.config import PushConfig
+from openultrasast.cpg.backend import TIMING, CpgResult, JoernBackend
+from openultrasast.push.runner import replay
+
+# What each partition's graph holds, so the stub census names exactly the files a real frontend would read.
+EXTENSIONS = {"php": (".php",), "javascript": (".js", ".jsx", ".mjs", ".cjs"), "typescript": (".ts", ".tsx"), "python": (".py",)}
+
+
+class StubEngine(JoernBackend):
+    """A backend whose engine is a table lookup: real row shapes, zero engine time."""
+
+    def __init__(self, templates: list[list[object]], **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self.templates = templates
+        self.asked = 0
+        self.graphs = 0
+
+    def build(self, root: Path, *, language: str = "", exclude: Any = (), execution_budget: Any = None) -> CpgResult | None:
+        extensions = EXTENSIONS.get(language.lower(), ())
+        files = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.is_file() and p.suffix.lower() in extensions)
+        graph = Path(__import__("tempfile").mkdtemp(prefix="ousast-stub-")) / "cpg.bin"
+        graph.write_bytes(("stub graph " + language + " " + str(len(files))).encode())
+        self.graphs += 1
+        census = {
+            "files": str(len(files)),
+            "methods": str(max(1, 10 * len(files))),
+            "calls": "1",
+            "file_names": files,
+            "overlays": "dataflowOss",
+        }
+
+        def run(kind: str, params: Mapping[str, object]) -> object:
+            if kind == "census":
+                return dict(census)
+            answer = batch(kind, {"one": params})
+            return answer.get("one") if answer else None
+
+        def batch(kind: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]]:
+            answers: dict[str, list[object]] = {"__census__": [dict(census)]}
+            timing = []
+            for rid in requests:
+                self.asked += 1
+                rows = self.templates[self.asked % len(self.templates)] if kind == "taint" else [{"kind": "context_summary"}]
+                answers[rid] = list(rows)
+                timing.append({"id": rid, "ms": 1.0})
+            answers[TIMING] = timing  # type: ignore[assignment]
+            return answers
+
+        return CpgResult(cpg_path=graph, run=run, run_batch=batch, run_batch_once=batch, execution_diagnostics=lambda: ())
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repository", type=Path, required=True)
+    parser.add_argument("--base", required=True)
+    parser.add_argument("--head", required=True)
+    parser.add_argument("--rows", type=Path, action="append", required=True, help="recorded engine rows, {id: rows}")
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--deadline", type=float, default=3600.0)
+    parser.add_argument("--max-regions", type=int, default=500)
+    args = parser.parse_args()
+    args.out.mkdir(parents=True, exist_ok=True)
+
+    templates = [rows for path in args.rows for rows in json.loads(path.read_text()).values() if isinstance(rows, list)]
+    if not templates:
+        raise SystemExit("no recorded rows: the stub would answer nothing and measure nothing")
+    print(f"{len(templates)} recorded answers from {len(args.rows)} file(s)", flush=True)
+    profiling.start(str(args.out / "profile.txt"), flush=10.0)
+    engine = StubEngine(templates)
+    started = time.monotonic()
+    delivery = replay(
+        args.repository,
+        base=args.base,
+        head=args.head,
+        artifact=args.out / "artifact.json",
+        config=PushConfig(deadline_seconds=args.deadline, cancellation_allowance_seconds=2.0),
+        backend=engine,
+        max_regions=args.max_regions,
+        cache_dir=args.out / "cache",
+    )
+    elapsed = time.monotonic() - started
+    artifact = json.loads((args.out / "artifact.json").read_text()) if (args.out / "artifact.json").is_file() else {}
+    timings = {k: round(v, 1) for k, v in artifact.get("timings", {}).items() if k.endswith("_seconds")}
+    questions = sum(len(s["scan"]["question_outcomes"]) for s in artifact.get("scans", []))
+    summary = {
+        "elapsed": round(elapsed, 1),
+        "graphs": engine.graphs,
+        "requests_answered": engine.asked,
+        "questions": questions,
+        "timings": timings,
+    }
+    (args.out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
+    print(json.dumps(summary), flush=True)
+    print(delivery.terminal if hasattr(delivery, "terminal") else "", flush=True)
+    if not engine.asked or not questions:
+        raise SystemExit("the instrument did not exercise the push path: nothing was asked or no question was recorded")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/openultrasast/model/regions.py

@@ -18,6 +18,7 @@ Two restraints are deliberate:
 
 from __future__ import annotations
 
+import bisect
 import time
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, replace
@@ -194,6 +195,15 @@ AMBIGUOUS_CORRESPONDENCE = "ambiguous_line_correspondence"
 REACHABILITY_BOUNDED = "dynamic_external_or_depth_context_unresolved"
 
 
+def _anchored_lines(anchors: list[tuple[int, int]] | None, start: int, end: int) -> list[int]:
+    """Anchor lines within ``[start, end]``, in the order the change context listed them."""
+    if not anchors:
+        return []
+    low = bisect.bisect_left(anchors, (start, -1))
+    high = bisect.bisect_right(anchors, (end, float("inf")))
+    return [line for line, _ in sorted(anchors[low:high], key=lambda item: item[1])]
+
+
 def _context_path(raw: str, root: Path, resolved_root: Path) -> str | None:
     """A context location's path relative to the root, or ``None`` when it is not a real file under it."""
     try:
@@ -272,6 +282,20 @@ def affected_context(
     inherited_gap = any(not gap.endswith(":" + AMBIGUOUS_CORRESPONDENCE) for gap in gaps)
     renamed = {context.decode_path(r.base_path): context.decode_path(r.head_path) for r in context.renames}
     checked: dict[str, bool] = {}
+    # Decoded ONCE per call. Every row used to decode every span's and every line anchor's path -- a changed file
+    # has one anchor per unchanged line -- and with the engine stubbed out that was 78% of a whole push. Anchors
+    # are indexed by line so a row finds its own with two bisections, in their original order.
+    attachable = changed | set(renamed.values())
+    spans_by_path: dict[str, list[tuple[str, str, int, int]]] = {}
+    for span in context.spans:
+        old_path = context.decode_path(span.path)
+        match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path
+        spans_by_path.setdefault(match_path, []).append((span.side, old_path, span.start_line, span.end_line))
+    anchors_by_path: dict[str, list[tuple[int, int]]] = {}
+    for index, anchor in enumerate(context.line_correspondences):
+        anchors_by_path.setdefault(context.decode_path(anchor.head_path), []).append((anchor.head_start_line, index))
+    for anchors in anchors_by_path.values():
+        anchors.sort()
     # Resolved once per distinct path: the same few changed files recur in every question's rows, and resolving
     # them again per row was half of what `affected_context` cost once its quadratic was gone.
     resolved: dict[str, str | None] = {}
@@ -319,18 +343,15 @@ def affected_context(
             except (ValueError, TypeError, OSError):
                 local_gaps.append("context_location_unavailable")
                 continue
-            evidence = []
-            for span in context.spans:
-                old_path = context.decode_path(span.path)
-                match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path
-                if match_path != path_text:
-                    continue
-                if span.side == "base" or (span.start_line <= end and span.end_line >= start):
-                    evidence.append(f"{span.side}_span:{old_path}:{span.start_line}-{span.end_line}")
-            for anchor in context.line_correspondences:
-                if context.decode_path(anchor.head_path) == path_text and start <= anchor.head_start_line <= end:
-                    evidence.append(f"unchanged_line:{anchor.head_start_line};lexical_only")
-            if not evidence or path_text not in changed | set(renamed.values()):
+            if path_text not in attachable:
+                continue
+            evidence = [
+                f"{side}_span:{old_path}:{span_start}-{span_end}"
+                for side, old_path, span_start, span_end in spans_by_path.get(path_text, ())
+                if side == "base" or (span_start <= end and span_end >= start)
+            ]
+            evidence += [f"unchanged_line:{line};lexical_only" for line in _anchored_lines(anchors_by_path.get(path_text), start, end)]
+            if not evidence:
                 continue
             function = str(row.get("function", "")) or None
             source = QuestionIdentity(identity.unit, identity.language, path_text, function, identity.family)

src/openultrasast/model/taxonomy.py

@@ -90,7 +90,7 @@ class FamilyTaxonomy:
         return "lateral"
 
 
-def load_families(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
+def _load_families_uncached(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
     """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
     a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
     source = path if path is not None else DEFAULT_FAMILIES_PATH
@@ -192,3 +192,21 @@ __all__ = [
     "Relation",
     "load_families",
 ]
+
+
+# Parsed once per file state; see `semantic.facts.load_facts` for why and for the key.
+_FAMILIES_MEMO: dict[tuple[object, ...], FamilyTaxonomy] = {}
+
+
+def load_families(path: Path | None = None, *, require_all: bool = True) -> FamilyTaxonomy:
+    """Load the taxonomy. ``require_all`` demands the full closed set in order (the shipped file);
+    a scoped fixture may declare a subset, but never a family outside ``FAMILY_IDS``."""
+    source = path if path is not None else DEFAULT_FAMILIES_PATH
+    try:
+        info = source.stat()
+    except OSError:
+        return _load_families_uncached(path, require_all=require_all)
+    key = (str(source.resolve()), info.st_size, info.st_mtime_ns, require_all)
+    if key not in _FAMILIES_MEMO:
+        _FAMILIES_MEMO[key] = _load_families_uncached(path, require_all=require_all)
+    return _FAMILIES_MEMO[key]

src/openultrasast/semantic/facts.py

@@ -116,7 +116,7 @@ class SemanticFacts:
         )
 
 
-def load_facts(directory: Path | None = None) -> SemanticFacts:
+def _load_facts_uncached(directory: Path | None = None) -> SemanticFacts:
     root = directory if directory is not None else DEFAULT_FACTS_DIR
     if not root.is_dir():
         raise FactLoadError(f"semantic fact directory missing: {root}")
@@ -271,3 +271,44 @@ def _sanitizers(value: object, language: str, path: Path) -> list[SanitizerFact]
             )
         )
     return rows
+
+
+# Parsed once per ruleset STATE, not once per call. Region discovery asks for the family specs of every region,
+# and each spec call loaded the facts again: 89,127 TOML parses and 275 s of a 339 s discovery on a WordPress
+# plugin, paid twice per push. The key is every fact file's name, size and modification time, so an edited
+# ruleset -- a test's fixture, a maintainer's change -- is read again, and the result is immutable, so sharing
+# it is safe. A load that fails is not remembered.
+_FACTS_MEMO: dict[tuple[object, ...], SemanticFacts] = {}
+
+
+_LISTED: dict[tuple[str, str], tuple[int, tuple[Path, ...]]] = {}
+
+
+def _ruleset_state(root: Path, pattern: str) -> tuple[object, ...]:
+    """Every matching file's name, size and modification time -- cheaply, because every question asks.
+
+    The directory is listed again only when its own stamp moves (a file added, removed or renamed); an edit in
+    place moves the file's stamp, which is read every time. Globbing and resolving on every call was 7% of a
+    whole push with the engine stubbed out.
+    """
+    stamp = root.stat().st_mtime_ns
+    listed = _LISTED.get((str(root), pattern))
+    if listed is None or listed[0] != stamp:
+        listed = (stamp, tuple(sorted(root.glob(pattern))))
+        _LISTED[(str(root), pattern)] = listed
+    return (str(root), stamp, *((path.name, *_stat_of(path)) for path in listed[1]))
+
+
+def _stat_of(path: Path) -> tuple[int, int]:
+    info = path.stat()
+    return info.st_size, info.st_mtime_ns
+
+
+def load_facts(directory: Path | None = None) -> SemanticFacts:
+    root = directory if directory is not None else DEFAULT_FACTS_DIR
+    if not root.is_dir():
+        return _load_facts_uncached(directory)
+    key = _ruleset_state(root, "*.toml")
+    if key not in _FACTS_MEMO:
+        _FACTS_MEMO[key] = _load_facts_uncached(directory)
+    return _FACTS_MEMO[key]

src/openultrasast/semantic/obligations/facts.py

@@ -11,6 +11,8 @@ import tomllib
 from dataclasses import dataclass
 from pathlib import Path
 
+from ..facts import _ruleset_state
+
 OPERATION_KINDS = ("protected_read", "protected_write", "privileged_action", "security_setting")
 DISCHARGER_KINDS = ("path_guard", "identity_constraint", "ownership_check", "non_permissive_value", "validated_input")
 PROVENANCE_KINDS = ("authenticated_context", "request_input", "constant", "unknown")
@@ -73,7 +75,7 @@ class ObligationFacts:
         )
 
 
-def load_obligation_facts(directory: Path | None = None) -> ObligationFacts:
+def _load_obligation_facts_uncached(directory: Path | None = None) -> ObligationFacts:
     root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR
     if not root.is_dir():
         raise ObligationFactsError(f"obligation facts directory missing: {root}")
@@ -160,3 +162,17 @@ def _strings(value: object, field: str, path: Path) -> tuple[str, ...]:
     if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
         raise ObligationFactsError(f"{path} has invalid {field} list")
     return tuple(str(item) for item in value)
+
+
+# Parsed once per ruleset state; see `semantic.facts.load_facts` for the measurement and the key.
+_OBLIGATION_MEMO: dict[tuple[object, ...], ObligationFacts] = {}
+
+
+def load_obligation_facts(directory: Path | None = None) -> ObligationFacts:
+    root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR
+    if not root.is_dir():
+        return _load_obligation_facts_uncached(directory)
+    key = _ruleset_state(root, "*.toml")
+    if key not in _OBLIGATION_MEMO:
+        _OBLIGATION_MEMO[key] = _load_obligation_facts_uncached(directory)
+    return _OBLIGATION_MEMO[key]

tests/test_fact_memo.py

@@ -0,0 +1,33 @@
+"""The parsed ruleset is shared across calls, and never outlives a change to the files it came from.
+
+Region discovery asked for the facts once per region and re-parsed the TOML every time: 89,127 parses and
+275 s of a 339 s discovery on a WordPress plugin, twice per push.
+"""
+
+import shutil
+
+from openultrasast.semantic.facts import DEFAULT_FACTS_DIR, load_facts
+from openultrasast.semantic.obligations.facts import DEFAULT_OBLIGATION_FACTS_DIR, load_obligation_facts
+
+
+def test_facts_are_parsed_once_and_reread_when_a_file_changes(tmp_path):
+    root = tmp_path / "semantic"
+    shutil.copytree(DEFAULT_FACTS_DIR, root)
+    first = load_facts(root)
+    assert load_facts(root) is first, "an unchanged ruleset must not be parsed again"
+    python = root / "python.toml"
+    python.write_text(python.read_text() + '\n[[sink]]\nid = "memo-probe"\ncwe = "CWE-78"\ncalls = ["memo_probe_call"]\n')
+    edited = load_facts(root)
+    assert edited is not first and any("memo_probe_call" in sink.calls for sink in edited.sinks), "an edit in place was not seen"
+    (root / "zz_added.toml").write_text('language = "python"\n[[sink]]\nid = "added"\ncwe = "CWE-78"\ncalls = ["added_call"]\n')
+    added = load_facts(root)
+    assert any("added_call" in sink.calls for sink in added.sinks), "a new fact file was not seen"
+
+
+def test_obligation_facts_are_parsed_once_and_reread_when_a_file_changes(tmp_path):
+    root = tmp_path / "obligations"
+    shutil.copytree(DEFAULT_OBLIGATION_FACTS_DIR, root)
+    first = load_obligation_facts(root)
+    assert load_obligation_facts(root) is first
+    (root / "python.toml").write_text((root / "python.toml").read_text().replace('version = "', 'version = "memo-', 1))
+    assert load_obligation_facts(root) is not first

tests/test_model_change_context.py

@@ -472,3 +472,68 @@ def test_attaching_relationships_is_linear_in_what_it_attaches(tmp_path, monkeyp
     enriched, _ = affected_context(context(), questions, rows, tmp_path)
     assert len(enriched.relationships) == 4000
     assert len(compared) < 4000, f"{len(compared)} comparisons for 4,000 relationships"
+
+
+def _reference_evidence(context_value, path_text, start, end):
+    """The pre-index matching, verbatim in effect: every span and anchor decoded and tested per row."""
+    renamed = {context_value.decode_path(r.base_path): context_value.decode_path(r.head_path) for r in context_value.renames}
+    evidence = []
+    for span in context_value.spans:
+        old_path = context_value.decode_path(span.path)
+        match_path = renamed.get(old_path, old_path) if span.side == "base" else old_path
+        if match_path != path_text:
+            continue
+        if span.side == "base" or (span.start_line <= end and span.end_line >= start):
+            evidence.append(f"{span.side}_span:{old_path}:{span.start_line}-{span.end_line}")
+    for anchor in context_value.line_correspondences:
+        if context_value.decode_path(anchor.head_path) == path_text and start <= anchor.head_start_line <= end:
+            evidence.append(f"unchanged_line:{anchor.head_start_line};lexical_only")
+    return evidence
+
+
+def test_indexed_matching_attaches_exactly_what_the_row_by_row_matching_did(tmp_path):
+    """Randomised: renames, base and head spans, anchors out of order and in several files."""
+    import random
+
+    from openultrasast.model.contracts import PathRename, QuestionIdentity
+    from openultrasast.model.regions import affected_context
+
+    rng = random.Random(20260923)
+    names = ["a.php", "b.php", "c.php", "old.php"]
+    for name in ["a.php", "b.php", "c.php"]:
+        (tmp_path / name).write_text("x\n" * 200)
+    attached = 0
+    for _ in range(200):
+        spans = tuple(
+            ChangedSpan(rng.choice(names).encode().hex(), s, s + rng.randint(0, 20), rng.choice(["base", "head"]))
+            for s in (rng.randint(1, 180) for _ in range(rng.randint(0, 6)))
+        )
+        anchors = tuple(
+            LineCorrespondence(
+                rng.choice(names).encode().hex(), rng.choice(names).encode().hex(), base, base + rng.randint(0, 3), rng.randint(1, 200)
+            )
+            for base in (rng.randint(1, 190) for _ in range(rng.randint(0, 40)))
+        )
+        renames = (PathRename(b"old.php".hex(), b"c.php".hex()),) if rng.random() < 0.5 else ()
+        changed = tuple(n.encode().hex() for n in rng.sample(["a.php", "b.php", "c.php"], rng.randint(1, 3)))
+        ctx = ChangeContext("base", "head", changed, (), spans, renames, (), (), (), "filesystem-bytes-hex", anchors)
+        question = QuestionIdentity("unit", "php", "q.php", "run", "injection")
+        rows = [{"kind": "context_summary"}]
+        for _ in range(rng.randint(1, 12)):
+            s = rng.randint(1, 190)
+            rows.append(
+                {**row(rng.choice(["a.php", "b.php", "c.php"]), function=f"f{s}"), "startLine": s, "endLine": s + rng.randint(0, 30)}
+            )
+        enriched, _ = affected_context(ctx, [question], {question: rows}, tmp_path)
+        attachable = {ctx.decode_path(p) for p in ctx.changed_paths} | {ctx.decode_path(r.head_path) for r in ctx.renames}
+        expected = []
+        for r in rows[1:]:
+            evidence = _reference_evidence(ctx, r["path"], r["startLine"], r["endLine"])
+            if evidence and r["path"] in attachable:
+                key = (r["path"], r["function"], tuple(evidence))
+                if key not in expected:
+                    expected.append(key)
+        got = [(rel.source.path, rel.source.function, rel.evidence) for rel in enriched.relationships]
+        assert got == expected
+        attached += len(got)
+    assert attached > 100, f"only {attached} relationships across 200 cases: the comparison would be vacuous"