feat(cpg): portion the work instead of capping it -- a kill keeps what finished

A hard per-invocation ceiling is all-or-nothing. A batch killed at 301 s lost every answer it had
already computed, and the retry paid a fresh JVM start per part to ask them all again: measured, 116
requests killed at 300 s, then eight restarts carrying the same everywhere-sinks in every part.

Three coordinated changes make portioning, not the cap, the thing that governs cost.

  The taint query STREAMS. The census first, then one line per request as it is answered, flushed as
  it goes, inside the unchanged fence. The other queries do not stream and need not; the driver
  reads both forms, and a legacy single document parses exactly as before.

  The driver HARVESTS a kill. The process group is terminated first -- a first cut read the pipe
  before killing and a pipe whose writer is alive never reaches end-of-file; the suite own
  hanging-engine control sat on that read for twenty-eight minutes -- then the pipe is drained and
  every complete line becomes an answer, marked partial. The retry asks only the remainder.

  Portion sizing ADAPTS. After each portion the observed seconds per sink visit, less the fixed
  start, resizes the rest toward a 120 s target; a portion that did not answer halves the budget
  rather than being fitted, because its true cost is unknown and above the ceiling. A portion cut
  down to fit puts its remainder back at the front of the queue.

A census LINE carries nothing but the census. A first cut misread a legacy document that carried the
census beside its answers as a census line and dropped every answer it sat beside; a control caught
it. Task 7.8 in contributor-scan.

Verified: 1,423 host tests pass, nineteen skipped, including new controls for the assembler in both
forms and under a kill, for a retry that asks only the remainder, and for the sizer. Not yet
verified: the joern-gated streaming check and the plugin measurement, which wait for the transfer
replay to free the machine and are stated as open in the task.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

tests/test_cpg_backend.py

@@ -1230,3 +1230,42 @@ def test_an_engine_failure_reports_the_line_that_names_the_cause(tmp_path, monke
     assert _engine_failure(None) == "timeout"
     assert "no output" in _engine_failure(subprocess.CompletedProcess(["joern"], 3, "", ""))
     assert isinstance(JoernBackend(), JoernBackend)
+
+
+def test_a_streamed_payload_assembles_and_a_killed_one_keeps_what_finished() -> None:
+    """A kill keeps every answer that had streamed; a half-written line is dropped, not guessed at."""
+    from openultrasast.cpg.backend import BEGIN, END, PARTIAL, extract_payload
+
+    whole = f'{BEGIN}\n{{"__census__": [{{"methods": "5"}}]}}\n{{"id": "0", "rows": [1]}}\n{{"id": "1", "rows": []}}\n{END}\n'
+    assembled = extract_payload(whole)
+    assert assembled == {"__census__": [{"methods": "5"}], "0": [1], "1": []}
+
+    killed = f'{BEGIN}\n[INFO ] engine chatter\n{{"__census__": []}}\n{{"id": "0", "rows": [1]}}\n{{"id": "1", "ro'
+    harvest = extract_payload(killed)
+    assert harvest is not None and harvest.get(PARTIAL) is True
+    assert harvest["0"] == [1] and "1" not in harvest
+
+    legacy = f'{BEGIN}\n{{"0": [1], "__census__": []}}\n{END}\n'
+    assert extract_payload(legacy) == {"0": [1], "__census__": []}
+    # Half of a single document is still nothing.
+    assert extract_payload(f'{BEGIN}\n{{"0": [1], "__cen') is None
+
+
+def test_a_killed_batch_retries_only_the_remainder(tmp_path: Path) -> None:
+    """The retry asks what was NOT answered, so a kill costs the unfinished requests and nothing else."""
+    from openultrasast.cpg.backend import JoernBackend
+
+    backend = JoernBackend(runner=lambda *a, **k: None)
+    calls: list[list[str]] = []
+
+    def once(cpg_path: Path, query: str, requests: dict[str, dict[str, object]]) -> dict[str, list[object]] | None:
+        calls.append(sorted(requests))
+        if len(calls) == 1:
+            return {"__census__": [], "a": [1]}  # killed mid-way: a answered, b and c did not
+        return {rid: [rid] for rid in requests}
+
+    backend._batch_once = once  # type: ignore[method-assign]
+    answered = backend.query_batch(tmp_path / "cpg.bin", "taint", {"a": {}, "b": {}, "c": {}})
+    assert answered is not None and answered["a"] == [1] and answered["b"] == ["b"] and answered["c"] == ["c"]
+    assert calls[0] == ["a", "b", "c"]
+    assert all("a" not in retry for retry in calls[1:]), calls

tests/test_cpg_queries_compile.py

@@ -575,3 +575,49 @@ def test_prototype_pollution_is_established_on_the_shape_it_declares(tmp_path: P
     flows = [row for row in (payload.get("0") or []) if row.get("sink")]
     assert flows, "a request body merged into an object was not reported"
     assert all("merge" in str(row["sink"]) for row in flows), f"the Express setting was reported: {flows}"
+
+
+@pytest.mark.skipif(_joern() is None, reason="joern is not installed on this machine")
+def test_the_taint_query_streams_one_line_per_request(tmp_path: Path) -> None:
+    """Answers arrive as they finish, so a kill keeps what finished. The fence is unchanged."""
+    from openultrasast.cpg.backend import BEGIN, END, extract_payload
+
+    frontend = __import__("shutil").which("jssrc2cpg")
+    if frontend is None:
+        pytest.skip("jssrc2cpg is not installed")
+    (tmp_path / "a.js").write_text("function handler(req, res) { return eval(req.body.x); }\n")
+    cpg = tmp_path / "cpg.bin"
+    subprocess.run([frontend, str(tmp_path), "-o", str(cpg)], capture_output=True, text=True, timeout=600, check=False)
+    if not cpg.is_file():
+        pytest.skip("could not build a sample cpg")
+    requests = tmp_path / "requests.json"
+    requests.write_text(
+        json.dumps(
+            {
+                "0": {"sources": "req.body", "sinks": "eval", "function": "handler"},
+                "1": {"sources": "req.body", "sinks": "eval", "function": "nothing"},
+            }
+        )
+    )
+    done = subprocess.run(
+        [
+            _joern() or "joern",
+            "--script",
+            str((QUERIES / "taint.sc").resolve()),
+            "--param",
+            f"cpgFile={cpg}",
+            "--param",
+            f"requestsFile={requests}",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=900,
+        check=False,
+    )
+    assert done.returncode == 0, (done.stderr or done.stdout or "")[-500:]
+    body = done.stdout[done.stdout.find(BEGIN) + len(BEGIN) : done.stdout.find(END)]
+    lines = [line for line in body.splitlines() if line.strip().startswith("{")]
+    assert len(lines) >= 3, lines  # census, then one per request
+    assert json.loads(lines[0]).keys() == {"__census__"}
+    assembled = extract_payload(done.stdout)
+    assert assembled is not None and set(assembled) >= {"0", "1", "__census__"} and "__partial__" not in assembled

tests/test_model_scope.py

@@ -320,3 +320,21 @@ def test_expired_planning_keeps_census_without_more_fact_reads(tmp_path, monkeyp
     assert time.monotonic() < deadline + 0.5
     assert len(result.scope.selected) + len(result.scope.deferred) == 200
     assert "deadline_exhausted" in result.scope.unresolved_boundaries
+
+
+def test_a_portion_sizer_shrinks_to_budget_and_learns_from_what_answered() -> None:
+    """Portions follow the engine's observed cost; a portion that did not answer halves the budget."""
+    from openultrasast.model import scan
+
+    weights = {"w": 500, "x": 400, "y": 100, "z": 100}
+    sizer = scan._PortionSizer(weights)
+    sizer.budget = 600.0
+    pending = [{"y": {}}, {"z": {}}]
+    fitted = sizer.fit({"w": {}, "x": {}}, pending=pending)
+    assert sorted(fitted) == ["w"] and pending[0] == {"x": {}}, (fitted, pending)  # x pushed to the front, not lost
+    grown = sizer.fit({"y": {}}, pending=[{"z": {}}])
+    assert sorted(grown) == ["y", "z"]
+    sizer.observe({"w": {}}, 300.0, answered=False)
+    assert sizer.budget == 300.0
+    sizer.observe({"y": {}, "z": {}}, scan.PORTION_FIXED_SECONDS + 20.0, answered=True)  # 0.1 s per visit
+    assert 450 < sizer.budget < 550, sizer.budget