mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
perf(push): find the Python-side scale walls all at once, and remove them
The loop the maintainer called out: four scale defects in the PHP transfer replay, found one per two-hour Joern run, all Python bookkeeping between engine calls. benchmarks/push/python_overhead.py ends that: the real replay() on the real PMPro pins, with a JoernBackend whose graphs are stubs and whose queries answer instantly with rows recorded from the real engine. Engine time is zero, so everything measured is ours, in about a minute, under the sampling profiler. Its first pass named two more walls: - Region discovery re-parsed the ruleset for every region: 89,127 TOML parses, 275 s of a 339 s discovery, twice per push. The fact, obligation and taxonomy loaders now keep the parsed (immutable) result keyed by each file's name, size and mtime -- an edit in place or an added file reloads. Discovery 339 s -> 7 s; output digest identical on PMPro. - affected_context decoded every span's and every line anchor's path for every context row -- a changed file has an anchor per unchanged line -- 78% of a stubbed push. Decoded once per call, anchors indexed by line and bisected, original order kept. A randomised control compares it against the row-by-row matching over 200 contexts (>100 attachments). Result: the whole stubbed PMPro push, 27,542 answered requests and 2,688 outcomes, runs in 58.5 s of Python. What is left is the engine. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c629fea89a
commit
6b85a60ff1
@@ -11,6 +11,8 @@ import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..facts import _ruleset_state
|
||||
|
||||
OPERATION_KINDS = ("protected_read", "protected_write", "privileged_action", "security_setting")
|
||||
DISCHARGER_KINDS = ("path_guard", "identity_constraint", "ownership_check", "non_permissive_value", "validated_input")
|
||||
PROVENANCE_KINDS = ("authenticated_context", "request_input", "constant", "unknown")
|
||||
@@ -73,7 +75,7 @@ class ObligationFacts:
|
||||
)
|
||||
|
||||
|
||||
def load_obligation_facts(directory: Path | None = None) -> ObligationFacts:
|
||||
def _load_obligation_facts_uncached(directory: Path | None = None) -> ObligationFacts:
|
||||
root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR
|
||||
if not root.is_dir():
|
||||
raise ObligationFactsError(f"obligation facts directory missing: {root}")
|
||||
@@ -160,3 +162,17 @@ def _strings(value: object, field: str, path: Path) -> tuple[str, ...]:
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
|
||||
raise ObligationFactsError(f"{path} has invalid {field} list")
|
||||
return tuple(str(item) for item in value)
|
||||
|
||||
|
||||
# Parsed once per ruleset state; see `semantic.facts.load_facts` for the measurement and the key.
|
||||
_OBLIGATION_MEMO: dict[tuple[object, ...], ObligationFacts] = {}
|
||||
|
||||
|
||||
def load_obligation_facts(directory: Path | None = None) -> ObligationFacts:
|
||||
root = directory if directory is not None else DEFAULT_OBLIGATION_FACTS_DIR
|
||||
if not root.is_dir():
|
||||
return _load_obligation_facts_uncached(directory)
|
||||
key = _ruleset_state(root, "*.toml")
|
||||
if key not in _OBLIGATION_MEMO:
|
||||
_OBLIGATION_MEMO[key] = _load_obligation_facts_uncached(directory)
|
||||
return _OBLIGATION_MEMO[key]
|
||||
|
||||
Reference in New Issue
Block a user