mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
fix(scan): portion the evidence pass so a single kill is never the whole pass
The evidence pass bypasses the sink-weighted portioning by design: it is the pass that PRODUCES the weights, and per request it is cheap, 17,700 answered in 44 s. It was still one all-or-nothing batch protected only by the ceiling. On a plugin push transaction that batch is 22,125 requests, and under the CPU and memory contention of concurrent test runs it exceeded the ceiling and cost the whole case. It now runs in portions of 4,000 requests, roughly ten seconds of work behind one JVM start, with answers merged and the census kept from the first portion that carried it. Streaming keeps what a killed portion finished; portioning keeps that kill from being the entire pass. Task 13.3 records the attempt that exposed this honestly: the case deadline was exhausted with no artifact, and the failure was mine twice over -- the container mounts src live and the backend was edited mid-run, and the evidence pass was unportioned. Neither is the algorithm. The rerun uses a frozen export of the committed source with per-invocation logging. 1,423 tests pass, nineteen skipped; mypy clean. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
a8b226a5c8
commit
a4042fdb27
@@ -987,6 +987,32 @@ def _collect(
|
||||
return work
|
||||
|
||||
|
||||
EVIDENCE_PORTION = 4000 # requests per evidence invocation; roughly ten seconds of work behind one JVM start
|
||||
|
||||
|
||||
def _portioned_evidence(cpg: Any, requests: Mapping[str, Mapping[str, object]]) -> dict[str, Any] | None:
|
||||
"""The evidence pass in portions, merged. ``None`` only when no portion answered.
|
||||
|
||||
Typed as the batch returns it -- rows are the engine's mappings, and the caller reads them as such.
|
||||
"""
|
||||
ids = list(requests)
|
||||
if len(ids) <= EVIDENCE_PORTION:
|
||||
answered: dict[str, Any] | None = cpg.run_batch("taint", requests)
|
||||
return answered
|
||||
merged: dict[str, Any] = {}
|
||||
answered_any = False
|
||||
for start in range(0, len(ids), EVIDENCE_PORTION):
|
||||
part = cpg.run_batch("taint", {rid: requests[rid] for rid in ids[start : start + EVIDENCE_PORTION]})
|
||||
if part is None:
|
||||
continue
|
||||
answered_any = True
|
||||
census = part.pop("__census__", None)
|
||||
if census is not None and "__census__" not in merged:
|
||||
merged["__census__"] = census
|
||||
merged.update(part)
|
||||
return merged if answered_any else None
|
||||
|
||||
|
||||
def _evidence_pass(
|
||||
cpg: Any,
|
||||
work: Sequence[tuple[str, ScanRegion, ArbiterSpec]],
|
||||
@@ -1008,7 +1034,12 @@ def _evidence_pass(
|
||||
rid: {**dict(params), "evidenceOnly": "true", **({"contextEvidence": "true"} if context_rows is not None else {})}
|
||||
for rid, params in grouped_taint.items()
|
||||
}
|
||||
answered = cpg.run_batch("taint", requests)
|
||||
# Portioned by request count, because evidence has no per-request weight yet -- it is the pass that
|
||||
# PRODUCES the weights. It is cheap per request, 17,700 in 44 s, and it was still one all-or-nothing
|
||||
# batch: a 22,125-request evidence pass on a plugin's push transaction exceeded its ceiling under load and
|
||||
# cost the whole scan. Streaming keeps what finished when a portion is killed; portioning keeps a single
|
||||
# kill from being the whole pass.
|
||||
answered = _portioned_evidence(cpg, requests)
|
||||
if answered is None:
|
||||
return {}, len(requests)
|
||||
census = answered.pop("__census__", None)
|
||||
|
||||
Reference in New Issue
Block a user