mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(scan): attach change relationships in linear time
The third wall under the 13.3 replay, found by the streamed case log: the evidence and context passes finished at 14:53, then nothing -- no engine call, no log line -- for 82 minutes, until the case was killed 60 s past its 7,200 s deadline. `affected_context` tested membership with `relation not in relationships` on a list: 7,998,000 comparisons for 4,000 relationships (profiled, 83% of the time), and a push over a WordPress plugin attaches tens of thousands. The deadline is checked once per question, so a single late question could run for minutes, which is also why the process overran its cancellation allowance. Membership now goes through a set beside the ordered list, and each distinct context path is resolved once instead of once per row. Measured: 4,000 relationships 7.4 s -> 0.44 s; 400,000 in 16 s. The control counts comparisons rather than timing them, and fails on the old code with 7,998,000. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
982cb14115
commit
e7e2941e37
@@ -194,6 +194,17 @@ AMBIGUOUS_CORRESPONDENCE = "ambiguous_line_correspondence"
|
||||
REACHABILITY_BOUNDED = "dynamic_external_or_depth_context_unresolved"
|
||||
|
||||
|
||||
def _context_path(raw: str, root: Path, resolved_root: Path) -> str | None:
|
||||
"""A context location's path relative to the root, or ``None`` when it is not a real file under it."""
|
||||
try:
|
||||
path = Path(raw)
|
||||
absolute = path if path.is_absolute() else root / path
|
||||
text = absolute.resolve().relative_to(resolved_root).as_posix()
|
||||
return text if absolute.is_file() else None
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _elsewhere_unusable(row: Mapping[str, object], root: Path, checked: dict[str, bool]) -> bool:
|
||||
"""Whether a summarised set of locations holds one the itemised check would have rejected.
|
||||
|
||||
@@ -239,6 +250,11 @@ def affected_context(
|
||||
"""
|
||||
|
||||
relationships = list(context.relationships)
|
||||
# Membership through a set, order through the list. `relation not in relationships` on the list alone was
|
||||
# quadratic -- 8 million comparisons for 4,000 relationships, 7 s -- and a push over a WordPress plugin
|
||||
# produces tens of thousands, which ran 82 minutes past the evidence pass and overran the deadline, because
|
||||
# the deadline is checked once per question and a single question could take minutes.
|
||||
known = set(relationships)
|
||||
gaps = list(context.unresolved_boundaries)
|
||||
affected_gaps: set[QuestionIdentity] = set()
|
||||
changed = {context.decode_path(p) for p in context.changed_paths}
|
||||
@@ -256,6 +272,10 @@ def affected_context(
|
||||
inherited_gap = any(not gap.endswith(":" + AMBIGUOUS_CORRESPONDENCE) for gap in gaps)
|
||||
renamed = {context.decode_path(r.base_path): context.decode_path(r.head_path) for r in context.renames}
|
||||
checked: dict[str, bool] = {}
|
||||
# Resolved once per distinct path: the same few changed files recur in every question's rows, and resolving
|
||||
# them again per row was half of what `affected_context` cost once its quadratic was gone.
|
||||
resolved: dict[str, str | None] = {}
|
||||
resolved_root = root.resolve()
|
||||
for identity in questions:
|
||||
if execution_budget is not None and time.monotonic() >= execution_budget.deadline_monotonic:
|
||||
gaps.append("change_context_deadline_exhausted")
|
||||
@@ -286,11 +306,12 @@ def affected_context(
|
||||
continue
|
||||
if row.get("kind") != "context_method":
|
||||
continue
|
||||
path = Path(str(row.get("path", "")))
|
||||
raw_path = str(row.get("path", ""))
|
||||
if raw_path not in resolved:
|
||||
resolved[raw_path] = _context_path(raw_path, root, resolved_root)
|
||||
path_text = resolved[raw_path]
|
||||
try:
|
||||
absolute = path if path.is_absolute() else root / path
|
||||
path_text = absolute.resolve().relative_to(root.resolve()).as_posix()
|
||||
if not absolute.is_file():
|
||||
if path_text is None:
|
||||
raise ValueError("missing context source")
|
||||
start, end = int(str(row.get("startLine", 0))), int(str(row.get("endLine", 0)))
|
||||
if start < 1 or end < start:
|
||||
@@ -314,7 +335,8 @@ def affected_context(
|
||||
function = str(row.get("function", "")) or None
|
||||
source = QuestionIdentity(identity.unit, identity.language, path_text, function, identity.family)
|
||||
relation = AffectedRelationship(source, identity, str(row.get("relationship", "method")), tuple(evidence))
|
||||
if relation not in relationships:
|
||||
if relation not in known:
|
||||
known.add(relation)
|
||||
relationships.append(relation)
|
||||
if local_gaps or inherited_gap:
|
||||
affected_gaps.add(identity)
|
||||
|
||||
@@ -451,3 +451,24 @@ def test_the_evidence_pass_names_the_changed_files_to_the_engine(tmp_path):
|
||||
assert asked and all("contextFilter" not in r for r in asked)
|
||||
assert scan._changed_files(context("x\ny.php")) is None
|
||||
assert scan._changed_files(context("source.php")) == ("source.php",)
|
||||
|
||||
|
||||
def test_attaching_relationships_is_linear_in_what_it_attaches(tmp_path, monkeypatch):
|
||||
"""Membership went through a list: 8 million comparisons for 4,000 relationships, and a push over a
|
||||
WordPress plugin ran 82 minutes past its evidence pass and overran the deadline. Counted, not timed, so the
|
||||
control cannot flake under load."""
|
||||
from openultrasast.model.contracts import AffectedRelationship, QuestionIdentity
|
||||
from openultrasast.model.regions import affected_context
|
||||
|
||||
(tmp_path / "source.php").write_text("x\n" * 400)
|
||||
compared = []
|
||||
original = AffectedRelationship.__eq__
|
||||
monkeypatch.setattr(AffectedRelationship, "__eq__", lambda self, other: compared.append(1) or original(self, other))
|
||||
questions = [QuestionIdentity("unit", "php", f"f{i}.php", "run", "injection") for i in range(200)]
|
||||
rows = {
|
||||
q: [{"kind": "context_summary"}] + [{**row("source.php", function=f"m{j}"), "startLine": 1, "endLine": 400} for j in range(20)]
|
||||
for q in questions
|
||||
}
|
||||
enriched, _ = affected_context(context(), questions, rows, tmp_path)
|
||||
assert len(enriched.relationships) == 4000
|
||||
assert len(compared) < 4000, f"{len(compared)} comparisons for 4,000 relationships"
|
||||
|
||||
Reference in New Issue
Block a user