From f36165a4ed569d6a9991e3e7caceca42ee330db4 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:31:54 +0200 Subject: [PATCH] wip(g2): park change-scoped attempt + full diagnosis (NOT the fix) Scope+bound (change-first ordering, adaptive budget-fitted portioning, _context_pass scoped to changed files, evidence_pass_bounded degradation) is preserved here but does NOT unblock analysis: query_seconds stayed 0 across every live re-verify. Uncontended measurement proved why -- the evidence pass is ~150s fixed (first-query graph-load + census) + ~93ms/pair, so ~200s fixed overhead (with the 48s build) precedes any analysis and a 300s budget leaves ~100s. Pair-count scoping only trims the ~86s per-pair tail. The warm-session env (OUSAST_ENGINE_SESSION=1) times out at startup under contention and, even working, spares only subsequent batches -- not the first evidence query's load. Parked per decision to finish the G0 baseline first; resume on a clean host. Branch unmerged. 95 model tests pass; no push-suite regression. Co-Authored-By: Claude Opus 4.8 --- .../engine-change-scoped-analysis/brief.md | 39 ++++++++++++++ src/openultrasast/model/scan.py | 53 +++++++++++++++---- tests/test_model_change_context.py | 9 ++-- 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/.kiro/specs/engine-change-scoped-analysis/brief.md b/.kiro/specs/engine-change-scoped-analysis/brief.md index 2b97202..bff1d59 100644 --- a/.kiro/specs/engine-change-scoped-analysis/brief.md +++ b/.kiro/specs/engine-change-scoped-analysis/brief.md @@ -83,6 +83,45 @@ detection is being lost, not in Joern's capability. pass already returns, tier those in a second small pass — exact reach-into-unchanged, more code. - (iii) **Per-pair speedup:** attack the ~235 ms/pair (warm server / batching) rather than the pair count — helps every scan but is a deeper engine change and does not by itself bound a huge repo. + + **CORRECTION (2026-10-11, after implementing (i) + four live re-verifies — the cost model was wrong).** + Scope + bound was built and tested (change-first ordering, adaptive budget-fitted portioning, `_context_pass` + scoped to changed files, honest `evidence_pass_bounded` degradation; branch `engine-change-scoped-evidence-pass`). + It correctly cut the tiered pair count — but `query_seconds` stayed **0** every time. The live numbers show why: + + | pairs tiered | run_batch calls | evidence s | build s | + |---|---|---|---| + | 1015 | 1 | 238 | 52 | + | 512 | 1 | 219 | 68 | + | 64 | 2 | 220 | 68 | + | 4 (probe) | 1 | 160 | 127 | + + Evidence time is **flat regardless of pair count** → the cost is **per-*call* fixed** (~150–220 s: loading the CPG + into the query engine + the census the evidenceOnly batch computes), ~85 ms/pair on top. Pair-count scoping + therefore cannot free budget, and portioning re-pays the fixed cost per call (so (i) can be counterproductive). + Build is also large and swings 52→127 s — two engines thrashing the 7 GB host (the record-07 baseline shares it). + The real levers are now (iii)-shaped, not (i): **warm/persistent query engine or a cached census** to kill the + ~150 s per-call fixed cost, and/or a **bigger engine-arm budget** than 300 s; plus a **clean (uncontended) + measurement**, because contention alone may be inflating build + fixed cost 2–3×. Scope+bound is NOT merged. + + **UNCONTENDED measurement (2026-10-11, baseline paused via SIGSTOP + container kill, then resumed).** Shipped + engine, one evidence call over 1,015 pairs: `build=48 s`, `evidence=243.79 s`, `query=0`. The evidence pass is + **the same ~244 s uncontended as ~238 s contended** → the cost is **INTRINSIC, not contention**; contention only + inflated `build` (48 → 127 s) and added run-to-run noise. The model is `evidence ≈ 150 s fixed (CPG load into the + query engine + the census the evidenceOnly batch computes) + ~93 ms/pair` (244 = 150 + 1015×0.093). Consequences: + - **~200 s of fixed overhead** (`build` + evidence-load) precedes any analysis, so a 300 s budget leaves only + ~100 s for everything else. This is why `query=0` even on a clean host. + - **Change-scoping still helps, but only by cutting the per-pair tail** (~1,015×93 ms ≈ 95 s): one evidence call + over ~30 changed pairs is ~150 s + 3 s instead of 244 s, which would free ~90 s — possibly enough for analysis, + but marginal. + - **It must be ONE call, not portioned.** The ~150 s is a first-call load, so the adaptive probe attributed it to + its 4 probe pairs (per-pair≈37 s), concluded the budget fit ~0 more, and tiered only 4 — the implemented + scope+bound is doubly wrong for this cost model. The correct shape: change-first candidates, count-capped, ONE + evidence call, `deadline=None`. + - **Pre-push viability:** the ~200 s fixed floor means the engine arm cannot be a fast (≤30 s) pre-push check on a + repo this size without a warm/persistent session or a cached census. A larger budget (e.g. 600 s) makes the + benchmark arm run but is not a realistic pre-push deadline. The lever that matters for the product is killing + the ~150 s fixed cost, not scoping pairs. - [ ] **2. Seed analysis from the change.** Prioritize (or restrict first-pass to) the changed declarations plus their call/data neighborhood over repo-wide static rank, within budget. - [ ] **3. Soundness.** Keep cross-file dataflow INTO the changed function (don't scope to literal changed lines); diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 370ccac..fe269a1 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -1276,7 +1276,15 @@ EVIDENCE_PORTION = 4000 # requests per evidence invocation; roughly ten seconds # traced as before; only the long tail of unchanged regions goes untiered, and that is recorded. A smaller portion # makes the stop granular -- 4000 is one atomic batch that could never stop before the deadline. EVIDENCE_BUDGET_FRACTION = 0.5 -BOUNDED_EVIDENCE_PORTION = 256 +# The deadline is only checked BETWEEN portions and a run_batch cannot be interrupted, so a portion's duration is +# the overshoot. A cold large CPG answered a changed-region evidenceOnly pair in SECONDS (measured ~3-5 s under +# load, 50-100x the warm few ms), so even a fixed 32-pair portion overshot a ~115 s sub-budget by a whole ~150 s +# batch and left nothing for the flow queries. So the bounded pass sizes ADAPTIVELY: a small probe first, then each +# next chunk sized from the measured per-pair rate to fit the budget left -- overshoot is one probe, at any rate. +EVIDENCE_PROBE = 4 # the opening chunk, before the per-pair rate is known +# The cap on any one chunk under bounding: the adaptive rate estimate assumes a uniform per-pair cost, so a cap +# keeps a chunk from overshooting badly if the regions after the probe turn out slower than it did. +BOUNDED_EVIDENCE_PORTION = 16 def _portioned_evidence( @@ -1284,10 +1292,14 @@ def _portioned_evidence( ) -> tuple[dict[str, Any] | None, int]: """The evidence pass in portions, merged, with the number of requests ATTEMPTED. - ``None`` only when no attempted portion answered. ``deadline`` (a monotonic time) bounds the pass: the loop - checks it before each portion after the first and stops, so the requests in the portions it reached are - tiered and the tail is left untiered. The attempted count lets the caller tell a bounded skip (budget ran - out) from a query failure (the engine was asked and did not answer). Typed as the batch returns it. + ``None`` only when no attempted portion answered. The attempted count lets the caller tell a bounded skip + (budget ran out) from a query failure (the engine was asked and did not answer). + + Without a ``deadline`` the pass runs the whole population in fixed ``portion`` chunks, as before. WITH one (a + monotonic time), it is bounded ADAPTIVELY: a small probe chunk first, then each next chunk sized from the + measured per-pair rate to fit the budget that is left, capped at ``portion``. A run_batch cannot be + interrupted, so sizing to the remaining budget -- rather than a fixed count -- keeps the overshoot to one probe + whether a pair costs milliseconds or seconds. Typed as the batch returns it. """ ids = list(requests) if deadline is None and len(ids) <= portion: @@ -1298,12 +1310,25 @@ def _portioned_evidence( merged: dict[str, Any] = {} answered_any = False attempted = 0 - for start in range(0, len(ids), portion): - if deadline is not None and start > 0 and time.monotonic() >= deadline: + start = 0 + per_pair = 0.0 + while start < len(ids): + now = time.monotonic() + if deadline is not None and start > 0 and now >= deadline: break - chunk = ids[start : start + portion] + if deadline is None: + step = portion + elif per_pair <= 0.0: + step = min(EVIDENCE_PROBE, portion) # probe first, before the rate is known + else: + step = max(1, min(portion, int((deadline - now) / per_pair))) # fit the budget that is left + chunk = ids[start : start + step] + batch_started = time.monotonic() attempted += len(chunk) part = cpg.run_batch("taint", {rid: requests[rid] for rid in chunk}) + if deadline is not None and chunk: + per_pair = (time.monotonic() - batch_started) / len(chunk) + start += len(chunk) if part is None: continue answered_any = True @@ -1402,7 +1427,7 @@ def _evidence_pass( if len(languages) > 1: out.pop(key, None) if context_rows is not None: - _context_pass(cpg, work, context_rows, unit=unit) + _context_pass(cpg, work, context_rows, unit=unit, context_files=context_files) return out, max(0, attempted - len(out)), len(requests) - attempted @@ -1412,6 +1437,7 @@ def _context_pass( context_rows: dict[QuestionIdentity, list[Mapping[str, object]]], *, unit: str, + context_files: tuple[str, ...] | None = None, ) -> None: """Collect change context for the families the evidence pass does not ask. @@ -1422,7 +1448,16 @@ def _context_pass( Each family's own query supplies its own scope. A failure here leaves the family's context absent, which the caller already treats as unresolved, so it degrades exactly as before. + + ``context_files`` scopes the pass to the regions a change TOUCHED. This pass is change context, so on a push + only the changed regions need it -- and over a whole repository it is otherwise as costly as the tier pass + (measured ~200 s on one push, every bit of it unchanged-region context nothing would consult). A changed + region's own dominance/config context is what `affected_context` reads to establish what the change reaches; + the unchanged tail is left to the same `context_projection_unavailable` it would hit under a spent budget. """ + if context_files is not None: + allowed = set(context_files) + work = [item for item in work if item[1].path in allowed] grouped = _grouped(list(work)) by_rid = {rid: (region, spec) for rid, region, spec in work} for kind in ("dominance", "config"): diff --git a/tests/test_model_change_context.py b/tests/test_model_change_context.py index 3a12388..e34bac3 100644 --- a/tests/test_model_change_context.py +++ b/tests/test_model_change_context.py @@ -93,14 +93,15 @@ def test_portioned_evidence_stops_at_deadline_keeping_the_first_portion(): seen.append(len(reqs)) return {rid: [] for rid in reqs} - reqs = {str(i): {"file": "f"} for i in range(2 * scanmod.BOUNDED_EVIDENCE_PORTION + 5)} + reqs = {str(i): {"file": "f"} for i in range(50 * scanmod.BOUNDED_EVIDENCE_PORTION)} answered, attempted = scanmod._portioned_evidence(Cpg(), reqs, deadline=_t.monotonic() - 1, portion=scanmod.BOUNDED_EVIDENCE_PORTION) - assert seen == [scanmod.BOUNDED_EVIDENCE_PORTION] # only the first portion ran before the (past) deadline - assert attempted == scanmod.BOUNDED_EVIDENCE_PORTION - assert answered is not None and len(answered) == scanmod.BOUNDED_EVIDENCE_PORTION + assert seen == [scanmod.EVIDENCE_PROBE] # a past deadline stops after only the opening probe chunk + assert attempted == scanmod.EVIDENCE_PROBE + assert answered is not None and len(answered) == scanmod.EVIDENCE_PROBE seen.clear() _, attempted_full = scanmod._portioned_evidence(Cpg(), reqs, deadline=None, portion=scanmod.BOUNDED_EVIDENCE_PORTION) assert attempted_full == len(reqs) and sum(seen) == len(reqs) # no deadline -> the whole population tiers + assert all(s <= scanmod.BOUNDED_EVIDENCE_PORTION for s in seen) # in capped chunks def test_change_scoped_scan_bounds_and_orders_the_evidence_pass_change_first(tmp_path, monkeypatch):