unseen-eval: sourcing memory -- compact journal records, bounded caches, streamed history (4.6 GB -> 107 MB RSS)

The sourcing run was killed four times (exit 143/137) after growing to 4.6 GB
RSS on the 7.7 GB VM. Live after the fix: flat 107 MB through 3,029
advisories. Implemented with Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

benchmarks/unseen/journal.py

@@ -11,6 +11,20 @@ import time
 from contextlib import contextmanager
 from pathlib import Path
 
+from .source import compact_advisory, compact_repository
+from .storage import DiskMap, working_store
+
+MAX_RECORD_BYTES = 16 * 1024 * 1024
+
+
+def compact(stage, value):
+    if stage == "advisories":
+        return {"rows": [compact_advisory(row) for row in value["rows"]], "done": value["done"]}
+    if stage == "resolution" and value is not None:
+        return compact_repository(value)
+    return value
+
+
 STAGES = ("advisories", "resolution", "eligibility", "extraction", "completion")
 
 
@@ -68,13 +82,13 @@ class Journal:
         self.started = self.clock()
         self.max_seconds = None if max_hours is None else max_hours * 3600
         self.rate_limit = rate_limit or (lambda: {})
-        self.rows = {stage: {} for stage in STAGES}
+        self.rows = {}
         self.elapsed_before = 0.0
         self.last = None
         self.previous_rate = {}
 
     @contextmanager
-    def open(self, parameters):
+    def open(self, parameters, *, compatible_parameters=None):
         self.path.mkdir(parents=True, exist_ok=True, mode=0o700)
         with (self.path / "LOCK").open("a") as lock:
             try:
@@ -82,6 +96,7 @@ class Journal:
             except BlockingIOError:
                 raise ValueError("journal_busy") from None
             metadata = self.path / "inputs.json"
+            migrating = False
             if metadata.exists():
                 old = json.loads(metadata.read_bytes())
                 # Only current, fixed parameter labels can enter stdout.
@@ -89,30 +104,42 @@ class Journal:
                 if old.keys() != parameters.keys():
                     changed.append("parameter_schema")
                 if changed:
-                    raise ResumeMismatch(changed)
+                    if old != compatible_parameters:
+                        raise ResumeMismatch(changed)
+                    migrating = True
             else:
                 if any(self.path.glob("*.jsonl")):
                     raise ValueError("journal_missing_inputs")
                 atomic(metadata, encoded(parameters))
-            for stage in STAGES:
-                path = self.path / (stage + ".jsonl")
-                if not path.exists():
-                    continue
-                with path.open("rb") as handle:
-                    for line in handle:
-                        # Fail closed on torn/corrupt writes; never silently lose a completed unit.
-                        if not line.endswith(b"\n"):
-                            raise ValueError("journal_incomplete_record")
-                        row = json.loads(line)
-                        if row["id"] in self.rows[stage]:
-                            raise ValueError("journal_duplicate_unit")
-                        self.rows[stage][row["id"]] = row["value"]
-                        if row["elapsed_seconds"] >= self.elapsed_before:
-                            self.elapsed_before = row["elapsed_seconds"]
-                            self.last = {"stage": stage, "id": row["id"]}
-                            self.previous_rate = row["rate_limit"]
-            self.progress("sourcing")
-            yield self
+            with working_store(self.path) as self.db:
+                self.rows = {stage: DiskMap(self.db, stage) for stage in STAGES}
+                self._replay()
+                if migrating:
+                    atomic(self.path / "inputs-before-memory-fix.json", encoded(old))
+                    atomic(metadata, encoded(parameters))
+                self.progress("sourcing")
+                yield self
+
+    def _replay(self):
+        for stage in STAGES:
+            path = self.path / (stage + ".jsonl")
+            if not path.exists():
+                continue
+            with path.open("rb") as handle:
+                while line := handle.readline(MAX_RECORD_BYTES + 1):
+                    if len(line) > MAX_RECORD_BYTES:
+                        raise ValueError("journal_record_size_cap")
+                    # Fail closed on torn/corrupt writes; never silently lose a completed unit.
+                    if not line.endswith(b"\n"):
+                        raise ValueError("journal_incomplete_record")
+                    row = json.loads(line)
+                    if row["id"] in self.rows[stage]:
+                        raise ValueError("journal_duplicate_unit")
+                    self.rows[stage][row["id"]] = compact(stage, row["value"])
+                    if row["elapsed_seconds"] >= self.elapsed_before:
+                        self.elapsed_before = row["elapsed_seconds"]
+                        self.last = {"stage": stage, "id": row["id"]}
+                        self.previous_rate = row["rate_limit"]
 
     def boundary(self):
         reason = None
@@ -145,11 +172,15 @@ class Journal:
     def put(self, stage, key, value):
         if key in self.rows[stage]:
             raise ValueError("journal_duplicate_unit")
+        value = compact(stage, value)
         row = {"id": key, "value": value, "elapsed_seconds": self.elapsed(), "rate_limit": self.rate_limit() or self.previous_rate}
         path = self.path / (stage + ".jsonl")
         with path.open("ab") as handle:
             os.chmod(path, 0o600)
-            handle.write(encoded(row) + b"\n")
+            payload = encoded(row) + b"\n"
+            if len(payload) > MAX_RECORD_BYTES:
+                raise ValueError("journal_record_size_cap")
+            handle.write(payload)
             handle.flush()
             os.fsync(handle.fileno())
         sync_directory(self.path)