fix(cpg): reclaim the scratch tree, and sweep what earlier runs abandoned

Every build allocated a temp directory for the graph, its request files and
joern's own working copy of the CPG. Nothing ever reclaimed it. A day of
scanning left 1,055 directories and 355MB in /tmp on a disk already at 95%,
and the runs that leaked hardest were the ones that FAILED -- which is exactly
backwards, since a failed scan has the least to show for the space it took.

Two mechanisms, because they cover different deaths:

  * `CpgResult.cleanup` disposes of the tree, and `scan_repository` calls it
    when the graph has answered everything it is going to. Never raises: a scan
    that found things must not fail while tidying up.
  * A sweep at build time removes trees older than six hours. A process killed
    mid-scan never runs its cleanup -- which is how most of the 1,055 died --
    so the disposal alone is not enough. Only directories under the system temp
    root carrying this module's own prefix are touched, and a scan running in
    another process right now is left alone.

Note for the record what was NOT a product leak: 6.3GB of /tmp/joern-cli-*
was three copies of the Joern CLI I made by hand during today's container
experiments, and 63MB of workspace/ in the repo came from running joern from
the repo root instead of a scratch directory. The product already runs joern
with cwd set to the scratch tree, so its workspace is contained and now
disposed with it. Both are cleaned up; neither was the code's fault, and
conflating my litter with its leak would have hidden the real one.

Disk went from 6.1GB free to 19GB.

857 tests pass.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-10 15:54:50 +02:00
co-authored by Claude Opus 5
parent 37eca758d4
commit f496f41be1
3 changed files with 110 additions and 1 deletions
+60
View File
@@ -55,6 +55,7 @@ def test_the_seam_imports_nothing_outside_the_standard_library() -> None:
"signal",
"subprocess",
"tempfile",
"time",
"dataclasses",
"pathlib",
"typing",
@@ -725,3 +726,62 @@ def test_a_timed_out_query_kills_the_jvm_not_just_the_script(tmp_path: Path) ->
with contextlib.suppress(ProcessLookupError, PermissionError):
os.kill(child, 9)
assert not alive, "the grandchild JVM must die with the script, or every timeout leaks a gigabyte"
def test_a_built_cpg_can_dispose_of_its_scratch_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Every build allocates a temp directory for the graph, its request files and joern's own working copy.
Nothing reclaimed it. A day of scanning left 1,055 directories and 355MB behind, on a disk already at
95%, and the scans that leaked hardest were the ones that failed -- which is the wrong way round.
"""
import subprocess
from openultrasast.cpg.backend import JoernBackend
(tmp_path / "a.php").write_text("<?php function a() {}")
def runner(command, **kwargs): # type: ignore[no-untyped-def]
if "-r" in command:
return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
if "--script" in command:
body = '---OUSAST-CPG-BEGIN---\n{"files": "1", "methods": "1"}\n---OUSAST-CPG-END---\n'
return subprocess.CompletedProcess(args=command, returncode=0, stdout=body, stderr="")
Path(command[command.index("-o") + 1]).write_text("cpg")
return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="")
monkeypatch.setenv("OPENULTRASAST_JOERN_PROBE", "on")
monkeypatch.setattr("shutil.which", lambda name: f"/opt/bin/{name}")
result = JoernBackend(runner=runner).build(tmp_path, language="php")
assert result is not None and callable(result.cleanup)
scratch = result.cpg_path.parent
assert scratch.is_dir()
result.cleanup()
assert not scratch.exists(), "the scratch tree is gone once the caller is done with the graph"
def test_the_sweep_removes_only_old_scratch_and_only_ours(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A process killed mid-scan never runs its cleanup, which is how most of the 1,055 died. The sweep is
the backstop -- and it must not touch a scan running in another process, nor anything that is not ours."""
import os
import time
from openultrasast.cpg.backend import SCRATCH_MAX_AGE_SECONDS, SCRATCH_PREFIX, _sweep_stale_scratch
monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path))
old = tmp_path / f"{SCRATCH_PREFIX}old"
fresh = tmp_path / f"{SCRATCH_PREFIX}fresh"
foreign = tmp_path / "someone-elses-work"
for path in (old, fresh, foreign):
path.mkdir()
(path / "cpg.bin").write_text("x")
stale = time.time() - SCRATCH_MAX_AGE_SECONDS - 60
os.utime(old, (stale, stale))
os.utime(foreign, (stale, stale))
_sweep_stale_scratch()
assert not old.exists(), "an abandoned tree older than the ceiling is reclaimed"
assert fresh.is_dir(), "a scan running right now is left alone"
assert foreign.is_dir(), "and nothing outside this module's own prefix is ever touched"