From fe2a4240181c5d11a790cea3dba00d13976eae11 Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Sat, 3 Oct 2026 22:36:23 +0200
Subject: [PATCH] learn: engine_trace fetches real source at the pinned commit,
 2.5 GB JVM heap, one CPG per language

Root causes of the 50 failed pins in trace run 1: advisory pairs were fed as
excerpts (invalid Java/JS fragments), the JVM defaulted to ~768 MB heap in a
3 GB container, and mixed-language pins were refused. Implemented with Codex.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 benchmarks/learn/engine_trace.py        | 236 ++++++++++++++++++----
 benchmarks/learn/engine_trace_worker.py |  74 ++++++-
 tests/test_engine_trace.py              | 252 +++++++++++++++++++++++-
 3 files changed, 509 insertions(+), 53 deletions(-)

diff --git a/benchmarks/learn/engine_trace.py b/benchmarks/learn/engine_trace.py
index ef74b1f..baffc74 100644
--- a/benchmarks/learn/engine_trace.py
+++ b/benchmarks/learn/engine_trace.py
@@ -1,9 +1,7 @@
-"""Resumable, serial, labelled-function Joern traces. No network or live-source mounts.
+"""Resumable, serial labelled-function traces; host fetches pinned sources, Docker stays offline.
 
-Pair pins are excerpt blob identities, NOT commits: materialize their catalog side
-and registration documents. Repository pins are exported from existing local clones.
-The analyzer is git archive HEAD src plus a hashed snapshot of this uncommitted tool's
-implementation files; this allows review/worktree use without making a commit.
+Pair blob identities resolve through catalogs to real source commits. The analyzer
+is git archive HEAD src plus a hashed snapshot of this uncommitted tool.
 """
 
 from __future__ import annotations
@@ -13,6 +11,7 @@ import fcntl
 import hashlib
 import json
 import os
+import re
 import shutil
 import subprocess
 import tempfile
@@ -21,7 +20,7 @@ import tomllib
 from collections import Counter, defaultdict
 from pathlib import Path
 
-from engine_trace_worker import safe_path, unit_record, write_json
+from engine_trace_worker import EXCLUDED_DIRS, safe_path, unit_record, write_json
 
 from openultrasast.learn.examples import read_jsonl
 from openultrasast.learn.labels import load_sources, repo_name
@@ -42,6 +41,9 @@ SNAPSHOT_FILES = (
     "src/openultrasast/model/taint.py",
     "src/openultrasast/cpg/queries/taint.sc",
 )
+MATERIALIZATION_VERSION = 2
+EXPORT_CAP = 20 * 1024 * 1024
+
 STATUSES = ("path", "asked-nothing", "failed", "timeout", "unsupported")
 
 
@@ -81,7 +83,7 @@ def join_units(units, examples, include_typescript=False):
                 "function": function,
                 "language": language,
                 "source": example["source"],
-                "excerpt": example["source"] == "pairs",
+                "excerpt": example.get("excerpt", example["source"] == "pairs" or example["source"].startswith("advisory-fixes")),
                 "pin_role": example.get("pin_role", ""),
                 "supported": spec is not None,
                 "reason": "" if spec else f"no admitted taint model: {language}/{unit['family']}",
@@ -90,23 +92,48 @@ def join_units(units, examples, include_typescript=False):
     return result
 
 
-def make_plan(units, out, only_family=None, limit=0, *, include_typescript=False):
+def make_plan(
+    units,
+    out,
+    only_family=None,
+    limit=0,
+    *,
+    include_typescript=False,
+    only_source=(),
+    rerun_status=("failed",),
+    allow_excerpt_fallback=False,
+):
     grouped = defaultdict(list)
     for unit in units:
-        if not only_family or unit["family"] == only_family:
+        if (not only_family or unit["family"] == only_family) and (not only_source or unit["source"] in only_source):
             # Include admission options even when a unit's support happens to stay
             # unchanged. Old results cannot mark a different selection as complete.
-            identity = {"unit": unit, "include_typescript": include_typescript}
+            identity = {
+                "unit": unit,
+                "include_typescript": include_typescript,
+                "materialization_version": MATERIALIZATION_VERSION,
+                "allow_excerpt_fallback": allow_excerpt_fallback,
+            }
             grouped[unit["repo"], unit["pin"]].append({**unit, "input_digest": digest(identity)})
     plan = []
     for (repo, pin), rows in sorted(grouped.items()):
-        item = {"repo": repo, "pin": pin, "units": sorted(rows, key=lambda r: r["unit"])}
+        item = {
+            "repo": repo,
+            "pin": pin,
+            "materialization_version": MATERIALIZATION_VERSION,
+            "units": sorted(rows, key=lambda r: r["unit"]),
+        }
         item["selection"] = digest(item["units"])
         done = out / pin_name(item)
         if done.exists():
             old = json.loads(done.read_text())
             recorded = {r["unit"]: r.get("input_digest") for r in old.get("units", [])}
-            if old.get("done") and all(recorded.get(r["unit"]) == r["input_digest"] for r in rows):
+            if (
+                old.get("done")
+                and old.get("materialization_version") == MATERIALIZATION_VERSION
+                and not any(r.get("status") in rerun_status for r in old.get("units", []) if r["unit"] in {u["unit"] for u in rows})
+                and all(recorded.get(r["unit"]) == r["input_digest"] for r in rows)
+            ):
                 continue
         sources = {r["source"] for r in rows}
         # Inventory medians: harvest sides 31s, advisory sides 47s; v1 pins 182s, v2 pins 498s.
@@ -133,10 +160,15 @@ def make_plan(units, out, only_family=None, limit=0, *, include_typescript=False
 
 
 class Inputs:
-    """Resolve only the explicitly listed, spent corpora. Never fetch a missing clone."""
+    """Resolve spent corpora; fetch pair source commits into the host cache."""
 
-    def __init__(self, root, cache):
+    def __init__(self, root, cache, allow_excerpt_fallback=False):
         self.cache = cache
+        self.allow_excerpt_fallback = allow_excerpt_fallback
+        self.catalog_rows = defaultdict(list)
+        for catalog in (root / "benchmarks/pairs").rglob("catalog.toml"):
+            for item in tomllib.loads(catalog.read_text()).get("pair", []):
+                self.catalog_rows[item["name"]].append((catalog.parent.name, item))
         self.pairs = []
         self.repos = defaultdict(list)
         sources = load_sources()
@@ -159,6 +191,8 @@ class Inputs:
         self.hashes = {}
 
     def pair_for(self, row):
+        if row["pin_role"] not in {"fixed", "vulnerable", "vuln"}:
+            raise ValueError(f"ambiguous pair side: {row['pin_role']}")
         side = "fixed" if row["pin_role"] == "fixed" else "vuln"
         candidates = [
             c for c in self.pairs if c.relpath == row["file"] and repo_name(c.repo or f"{c.slice}:{c.name}") == repo_name(row["repo"])
@@ -188,42 +222,146 @@ class Inputs:
                 return case, side
         raise ValueError(f"cannot uniquely resolve excerpt {row['repo']}@{row['pin']}:{row['file']}")
 
+    def catalog_for(self, case):
+        matches = [
+            (source, item)
+            for source, item in self.catalog_rows[case.name]
+            if item.get("relpath") == case.relpath and repo_name(item.get("repo", "")) == repo_name(case.repo)
+        ]
+        if len(matches) != 1:
+            raise ValueError(f"ambiguous catalog source for {case.name}")
+        return matches[0]
+
+    def source_identity(self, case, side):
+        _, item = self.catalog_for(case)
+        parent, commit = item.get("parent", ""), item.get("commit", "")
+        if side not in {"vuln", "fixed"} or not all(re.fullmatch(r"[0-9a-fA-F]{40}", sha) for sha in (parent, commit)) or parent == commit:
+            raise ValueError(f"ambiguous catalog commits for {case.name}: need distinct parent and commit SHAs")
+        repo = item.get("repo", "")
+        if not repo:
+            raise ValueError(f"missing catalog repository for {case.name}")
+        return repo, parent if side == "vuln" else commit
+
+    def fetch_source(self, repo, commit, roots):
+        clone = self.cache / "trace-sources" / digest([repo, commit, roots])
+        clone.mkdir(parents=True, exist_ok=True)
+        url = repo if repo.startswith("https://") else f"https://github.com/{repo.removesuffix('.git')}.git"
+        env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
+        env.pop("GIT_NO_LAZY_FETCH", None)  # sparse checkout may fetch the selected blobs
+
+        def git(*args):
+            return subprocess.run(["git", "-C", str(clone), *args], env=env, capture_output=True, text=True, check=True, timeout=300)
+
+        git("init")
+        git("config", "remote.origin.url", url)
+        git("config", "remote.origin.promisor", "true")
+        git("config", "remote.origin.partialclonefilter", "blob:none")
+        git("fetch", "--depth", "1", "--filter=blob:none", url, commit)
+        git("sparse-checkout", "init", "--cone")
+        git("sparse-checkout", "set", "--", *roots)
+        git("checkout", "--detach", "--force", commit)
+        actual = git("rev-parse", "HEAD").stdout.strip()
+        if actual != commit:
+            raise ValueError(f"fetched commit mismatch: {actual} != {commit}")
+        return clone
+
+    def export_local(self, clone, pin, target):
+        with tempfile.TemporaryDirectory(prefix="trace-export-", dir=target.parent) as scratch:
+            export_pin(clone, pin["pin"], Path(scratch))
+            proof = export_sources(Path(scratch), target, pin["units"])
+        return {**proof, "mode": "source", "commit": pin["pin"]}
+
     def materialize(self, pin, target):
         rows = pin["units"]
         for row in rows:
             safe_path(target, row["file"])
             if row.get("mapping_error"):
                 raise ValueError(row["mapping_error"])
-        if all(r.get("excerpt") for r in rows):
-            seen = set()
-            for row in rows:
-                case, side = self.pair_for(row)
-                if (case.name, side) not in seen:
-                    files = [(case.relpath, case.fixed_file if side == "fixed" else case.vuln_file)]
-                    files.extend((rel, fixed if side == "fixed" else vuln) for rel, vuln, fixed in case.context_files)
-                    for rel, source in files:
-                        dest = safe_path(target, rel)
-                        if dest.exists() and dest.read_bytes() != source.read_bytes():
-                            raise ValueError(f"conflicting excerpt/context at one pin: {rel}")
+        if any(r.get("excerpt") or r["source"].startswith("advisory-fixes") for r in rows):
+            resolved = [self.pair_for(row) for row in rows]
+            identities = {self.source_identity(case, side) for case, side in resolved}
+            if len(identities) != 1:
+                raise ValueError("ambiguous source commits at one pair pin")
+            repo, commit = identities.pop()
+            roots = sorted({source_root(row["file"], row["language"]) for row in rows})
+            try:
+                clone = self.fetch_source(repo, commit, roots)
+                proof = export_sources(clone, target, rows, roots)
+            except (OSError, ValueError, subprocess.SubprocessError) as exc:
+                if not self.allow_excerpt_fallback:
+                    raise
+                shutil.rmtree(target, ignore_errors=True)
+                for case, side in resolved:
                     _materialize_side(target, case, side=side)
-                    seen.add((case.name, side))
-            return
+                return {"mode": "excerpt-fallback", "reason": str(exc), "repo": repo, "commit": commit}
+            verify_functions(target, rows)
+            return {**proof, "mode": "source", "repo": repo, "commit": commit, "roots": roots}
         if all(r["source"] == "dev-php" for r in rows):
             # Export the exact requested commit from the local recipe checkout.
             for recipe in (ROOT / "benchmarks/repos").glob("*.toml"):
                 data = tomllib.loads(recipe.read_text())
                 if repo_name(str(data.get("url", ""))) == repo_name(pin["repo"]):
                     checkout = self.cache / "repos" / data["name"] / pin["pin"][:12]
-                    export_pin(checkout, pin["pin"], target)
-                    return
+                    return self.export_local(checkout, pin, target)
         for clone, _ in self.repos[repo_name(pin["repo"])]:
             check = subprocess.run(["git", "-C", str(clone), "cat-file", "-e", pin["pin"] + "^{commit}"], capture_output=True)
             if check.returncode == 0:
-                export_pin(clone, pin["pin"], target)
-                return
+                return self.export_local(clone, pin, target)
         raise ValueError(f"no local clone contains {pin['repo']}@{pin['pin']}")
 
 
+def source_root(file, language):
+    parts = Path(file).parts
+    if language == "java":
+        for i in range(len(parts) - 2):
+            if parts[i : i + 3] == ("src", "main", "java"):
+                return "/".join(parts[: i + 3])
+    return parts[0] if len(parts) > 1 else "."
+
+
+def export_sources(source, target, rows, roots=(".",), cap=EXPORT_CAP):
+    """Preserve labelled paths first; cap context and exclude non-source trees."""
+    labelled = {r["file"] for r in rows}
+    languages = {r["language"] for r in rows}
+    files = set()
+    for directory, dirs, names in os.walk(source):
+        dirs[:] = sorted(d for d in dirs if d not in EXCLUDED_DIRS and d != ".git" and not (Path(directory) / d).is_symlink())
+        for name in names:
+            file = Path(directory) / name
+            rel = file.relative_to(source).as_posix()
+            if (
+                not file.is_symlink()
+                and detect_language(file) in languages
+                and any(root == "." or rel.startswith(root + "/") for root in roots)
+            ):
+                files.add(rel)
+    missing = labelled - files
+    if missing:
+        raise ValueError(f"labelled files missing or excluded from export: {sorted(missing)}")
+    proof = {"bytes": 0, "files": 0, "cap_bytes": cap, "cap_hit": False, "excludes": sorted(EXCLUDED_DIRS)}
+    for rel in sorted(files, key=lambda rel: (rel not in labelled, rel)):
+        file = safe_path(source, rel)
+        size = file.stat().st_size
+        if proof["bytes"] + size > cap:
+            proof["cap_hit"] = True
+            if rel in labelled:
+                raise ValueError("labelled source exceeds 20 MB export cap")
+            continue
+        dest = safe_path(target, rel)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(file.read_bytes())
+        proof["bytes"] += size
+        proof["files"] += 1
+    return proof
+
+
+def verify_functions(root, rows):
+    for row in rows:
+        data = safe_path(root, row["file"]).read_text(errors="replace")
+        if not re.search(r"(?<![\w$])" + re.escape(row["function"]) + r"(?![\w$])", data):
+            raise ValueError(f"labelled function not found in fetched source: {row['file']}::{row['function']}")
+
+
 def freeze_source(target):
     target.mkdir(parents=True)
     archive = target.parent / "source.tar"
@@ -280,10 +418,6 @@ def docker_command(source, checkout, out, name, image, deadline, question_deadli
 def prepare_questions(pin, root, question_deadline):
     """Prepare metadata on the host, before entering the dependency-minimal image."""
     active = [u for u in pin["units"] if u["supported"]]
-    languages = {"javascript" if u["language"] == "typescript" else u["language"] for u in active}
-    if len(languages) != 1:
-        raise ValueError(f"one pin has multiple frontend languages: {sorted(languages)}")
-    language = next(iter(languages))
     from openultrasast.learn.features import entry_names
     from openultrasast.mapping import php_hook_callbacks
     from openultrasast.plane.harvest import declares
@@ -293,11 +427,11 @@ def prepare_questions(pin, root, question_deadline):
     entries, _ = entry_names(root, {u["file"] for u in active})
     hooks = ";".join(f"{h}:{f}" for h, functions in php_hook_callbacks(root, targets).items() for f in functions)
     requests = {}
-    specs = taint_specs(language=language)
     for u in active:
+        specs = taint_specs(language="javascript" if u["language"] == "typescript" else u["language"])
         if not declares(safe_path(root, u["file"]).read_text(errors="replace").splitlines(), u["file"], u["function"]):
             raise ValueError(f"labelled function not declared: {u['file']}::{u['function']}")
-        excerpt = u.get("excerpt", False)
+        excerpt = pin.get("materialization", {}).get("mode") == "excerpt-fallback"
         params = request_params(
             specs[u["family"]],
             file=u["file"],
@@ -318,7 +452,7 @@ def run_pin(pin, source, provenance, inputs, out, args):
         checkout, output = scratch / "case", scratch / "output"
         output.mkdir()
         try:
-            inputs.materialize(pin, checkout)
+            pin = {**pin, "materialization": inputs.materialize(pin, checkout)}
             write_json(output / "pin.json", {**pin, "questions": prepare_questions(pin, checkout, args.question_deadline)})
             name = "ousast-trace-" + pin_name(pin)[:16]
             command = docker_command(source, checkout, output, name, args.image, args.deadline, args.question_deadline)
@@ -438,12 +572,20 @@ def main():
     parser.add_argument("--question-deadline", type=int, default=120)
     parser.add_argument("--limit", type=int, default=0)
     parser.add_argument("--only-family")
+    parser.add_argument("--only-source", default="", help="comma-separated source catalog IDs")
+    parser.add_argument("--rerun-status", default="failed", help="re-run completed pins with any selected unit in these statuses")
+    parser.add_argument(
+        "--allow-excerpt-fallback", action="store_true", help="allow fragments only if fetching/exporting resolved source fails"
+    )
     parser.add_argument("--include-typescript", action="store_true")
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--summary", action="store_true")
     args = parser.parse_args()
     if min(args.deadline, args.question_deadline) <= 0 or args.limit < 0:
         parser.error("deadlines must be positive; limit must be nonnegative")
+    rerun_status = set(filter(None, args.rerun_status.split(",")))
+    if rerun_status - set(STATUSES):
+        parser.error("unknown --rerun-status value")
     if args.summary:
         print(json.dumps(summary(args.out), indent=2))
         return 0
@@ -456,17 +598,27 @@ def main():
         units = join_units(read_jsonl(args.units), examples, args.include_typescript)
     except ValueError as exc:
         parser.error(str(exc))
-    inputs = Inputs(ROOT, args.cache)
+    inputs = Inputs(ROOT, args.cache, args.allow_excerpt_fallback)
     # Resolve pair source names for leak-audit coverage before calculating the resume key.
     for unit in units:
-        if unit["source"] == "pairs":
+        if unit.get("excerpt") or unit["source"].startswith("advisory-fixes"):
             try:
                 case, _ = inputs.pair_for(unit)
-                unit["source"] = case.slice
+                catalog_source, _ = inputs.catalog_for(case)
+                unit["source"] = catalog_source if catalog_source.startswith("advisory-fixes") else case.slice
                 unit["excerpt"] = True
             except ValueError as exc:
                 unit["mapping_error"] = str(exc)
-    plan = make_plan(units, args.out, args.only_family, args.limit, include_typescript=args.include_typescript)
+    plan = make_plan(
+        units,
+        args.out,
+        args.only_family,
+        args.limit,
+        include_typescript=args.include_typescript,
+        only_source=set(filter(None, args.only_source.split(","))),
+        rerun_status=rerun_status,
+        allow_excerpt_fallback=args.allow_excerpt_fallback,
+    )
     # Planning is read-only, including unsupported pins. All persistence stays below this return.
     if args.dry_run:
         print(
diff --git a/benchmarks/learn/engine_trace_worker.py b/benchmarks/learn/engine_trace_worker.py
index eb4b136..a24f676 100644
--- a/benchmarks/learn/engine_trace_worker.py
+++ b/benchmarks/learn/engine_trace_worker.py
@@ -38,9 +38,13 @@ def safe_path(root, name):
 
 FRONTENDS = {"php2cpg", "pysrc2cpg", "jssrc2cpg", "javasrc2cpg", "c2cpg", "joern-parse"}
 MIN_CPG_BYTES = 1024
+HEAP_MB = 2560
+EXCLUDED_DIRS = frozenset({"tests", "test", "docs", "node_modules", "vendor", "dist", "build", "templates", "migrations"})
 
 
 def instrument_failure(proof):
+    if any(stage.get("oom") for stage in proof.get("jvm", [])):
+        return "JVM out of memory"
     if not proof.get("files") or not proof.get("bytes"):
         return "instrument read zero files or zero bytes"
     if proof.get("cpg_bytes", 0) <= MIN_CPG_BYTES:
@@ -60,7 +64,7 @@ class MeasuredBackend(JoernBackend):
     """Use the shipped backend with a process watchdog, no concurrent query threads."""
 
     def __init__(self, deadline, question_deadline, checkpoint):
-        super().__init__(build_timeout=deadline, query_timeout=deadline, session_transport=False)
+        super().__init__(build_timeout=deadline, query_timeout=deadline, session_transport=False, heap_mb=HEAP_MB)
         self.deadline = time.monotonic() + deadline
         self.question_deadline = question_deadline
         self.checkpoint = checkpoint
@@ -127,7 +131,15 @@ class MeasuredBackend(JoernBackend):
             process.stdout.close()
         elapsed = time.monotonic() - started
         if frontend or query:
-            stage = {"command": binary, "seconds": elapsed, "exit": process.returncode}
+            stage = {
+                "command": binary,
+                "argv": command,
+                "heap_mb": self._heap_mb(),
+                "java_tool_options": self._jvm_env()["JAVA_TOOL_OPTIONS"],
+                "seconds": elapsed,
+                "exit": process.returncode,
+                "oom": b"OutOfMemoryError" in output,
+            }
             self.stages.append(stage)
             if frontend:
                 # Capture the exact frontend output before build failure cleanup or overlays.
@@ -139,6 +151,8 @@ class MeasuredBackend(JoernBackend):
                     self.fatal = "frontend wrote no CPG"
             elif elapsed < 5:
                 self.fatal = f"implausibly fast JVM step: {command[0]} in {elapsed:.3f}s (<5s)"
+        if b"OutOfMemoryError" in output:
+            self.fatal = "JVM out of memory"
         self.checkpoint()
         return subprocess.CompletedProcess(command, process.returncode if not self.fatal else 1, output.decode(errors="replace"), "")
 
@@ -158,8 +172,17 @@ def unit_record(unit, status, reason="", **extra):
     }
 
 
-def worker(pin, root, out, deadline, question_deadline):
-    proof = {"files": [], "bytes": 0, "cpg_path": None, "cpg_bytes": 0, "jvm": []}
+def worker_language(pin, root, out, deadline, question_deadline, persist=write_json):
+    proof = {
+        "files": [],
+        "bytes": 0,
+        "cpg_path": None,
+        "cpg_bytes": 0,
+        "jvm": [],
+        "heap_mb": HEAP_MB,
+        "excludes": sorted(EXCLUDED_DIRS),
+        "materialization": pin.get("materialization", {}),
+    }
     record = {
         **pin,
         "done": False,
@@ -181,7 +204,7 @@ def worker(pin, root, out, deadline, question_deadline):
             if rid in backend.answers:
                 row["questions_completed"] = [rid]
                 row["witness_rows"] = [r for r in backend.answers[rid] if isinstance(r, dict) and "kind" not in r and "sink" in r]
-        write_json(out, record)
+        persist(out, record)
 
     backend = MeasuredBackend(deadline, question_deadline, checkpoint)
     proof["jvm"] = backend.stages
@@ -205,7 +228,7 @@ def worker(pin, root, out, deadline, question_deadline):
         record["questions"] = requests
         for unit_row in record["units"]:
             unit_row["questions"] = {unit_row["unit"]: requests[unit_row["unit"]]} if unit_row["unit"] in requests else {}
-        write_json(out, record)
+        persist(out, record)
         cpg = backend.build(root, language=language)
         if cpg is None:
             raise ValueError(backend.fatal or backend.last_failure or ("pin deadline" if backend.timed_out else "CPG build failed"))
@@ -284,7 +307,46 @@ def worker(pin, root, out, deadline, question_deadline):
         record["questions_asked"] = list(dict.fromkeys(backend.asked))
         record["questions_completed"] = [rid for u in record["units"] for rid in u["questions_completed"]]
         record["done"] = True
+        persist(out, record)
+    return record
+
+
+def worker(pin, root, out, deadline, question_deadline):
+    """Run language partitions sequentially under one shared container deadline."""
+    languages = sorted({"javascript" if u["language"] == "typescript" else u["language"] for u in pin["units"] if u["supported"]})
+    if len(languages) <= 1:
+        return worker_language(pin, root, out, deadline, question_deadline)
+    end = time.monotonic() + deadline
+    record = {
+        **pin,
+        "done": False,
+        "units": [
+            unit_record(u, "timeout" if u["supported"] else "unsupported", "pin deadline" if u["supported"] else u["reason"])
+            for u in pin["units"]
+        ],
+        "instruments": {},
+    }
+
+    def persist(_, partial):
+        updates = {u["unit"]: u for u in partial["units"]}
+        record["units"] = [updates.get(u["unit"], u) for u in record["units"]]
+        record["instruments"][language] = partial["instrument"]
+        record["questions_asked"] = [rid for u in record["units"] for rid in u["questions_asked"]]
+        record["questions_completed"] = [rid for u in record["units"] for rid in u["questions_completed"]]
         write_json(out, record)
+
+    write_json(out, record)
+    for language in languages:
+        remaining = end - time.monotonic()
+        if remaining <= 0:
+            break
+        units = [
+            u for u in pin["units"] if u["supported"] and ("javascript" if u["language"] == "typescript" else u["language"]) == language
+        ]
+        questions = {u["unit"]: pin["questions"][u["unit"]] for u in units}
+        worker_language({**pin, "units": units, "questions": questions}, root, out, remaining, question_deadline, persist)
+    record["done"] = True
+    write_json(out, record)
     return record
 
 
diff --git a/tests/test_engine_trace.py b/tests/test_engine_trace.py
index 9bd9687..862580c 100644
--- a/tests/test_engine_trace.py
+++ b/tests/test_engine_trace.py
@@ -313,7 +313,7 @@ def test_dry_run_makes_no_subprocess_calls(tmp_path, monkeypatch, capsys):
     import sys
 
     u = unit()
-    examples = [{**u, "id": "a", "candidate": "app.py::run", "source": "population-v1"}]
+    examples = [{**u, "id": "a", "candidate": "app.py::run", "source": "population-v1", "excerpt": False}]
     units = tmp_path / "units.jsonl"
     units.write_text(json.dumps(u) + "\n")
     example_file = tmp_path / "examples.jsonl"
@@ -337,13 +337,14 @@ def test_family_filtered_results_merge_without_losing_prior_units(tmp_path):
     assert trace.make_plan([unit(), unit("b", family="path")], tmp_path) == []
 
 
-def test_failed_launch_persists_failed_pin_and_resume_skips_it(tmp_path, monkeypatch):
+def test_failed_launch_persists_failed_pin_and_resume_replans_it(tmp_path, monkeypatch):
     from types import SimpleNamespace
 
     class Inputs:
         def materialize(self, pin, target):
             target.mkdir()
             (target / "app.py").write_text("def run():\n    pass\n")
+            return {"mode": "source"}
 
     commands = []
 
@@ -359,7 +360,8 @@ def test_failed_launch_persists_failed_pin_and_resume_skips_it(tmp_path, monkeyp
     assert result["units"][0]["status"] == "failed"
     assert "produced no result" in result["units"][0]["reason"]
     assert [c[1] for c in commands if c[0] == "docker"] == ["run", "rm"]
-    assert trace.make_plan([unit()], tmp_path) == []
+    assert len(trace.make_plan([unit()], tmp_path)) == 1
+    assert trace.make_plan([unit()], tmp_path, rerun_status=()) == []
 
 
 def import_container_worker():
@@ -410,7 +412,7 @@ def test_trace_dry_run_unsupported_units_write_nothing(tmp_path, monkeypatch, ca
         (out / "previous.json").write_text('{"done": true, "units": []}')
     before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in out.glob("*")}
     units = [unit("ts", file="app.ts"), unit("py", pin="python")]
-    examples = [{**u, "id": u["unit"], "candidate": u["file"] + "::run", "source": "population-v1"} for u in units]
+    examples = [{**u, "id": u["unit"], "candidate": u["file"] + "::run", "source": "population-v1", "excerpt": False} for u in units]
     units_file, examples_file = tmp_path / "units.jsonl", tmp_path / "examples.jsonl"
     units_file.write_text("\n".join(map(json.dumps, units)))
     examples_file.write_text("\n".join(map(json.dumps, examples)))
@@ -428,7 +430,7 @@ def test_trace_dry_run_unsupported_units_write_nothing(tmp_path, monkeypatch, ca
 
 def test_trace_real_unsupported_run_then_typescript_opt_in_replans(tmp_path, monkeypatch, capsys):
     u = unit(file="app.ts")
-    example = {**u, "id": "a", "candidate": "app.ts::run", "source": "population-v1"}
+    example = {**u, "id": "a", "candidate": "app.ts::run", "source": "population-v1", "excerpt": False}
     units_file, examples_file, out = tmp_path / "units.jsonl", tmp_path / "examples.jsonl", tmp_path / "results"
     units_file.write_text(json.dumps(u) + "\n")
     examples_file.write_text(json.dumps(example) + "\n")
@@ -469,6 +471,7 @@ def test_trace_resume_identity_includes_selection_even_if_support_unchanged(tmp_
     first = trace.make_plan([unit()], tmp_path)[0]
     trace.save_pin(tmp_path / trace.pin_name(first), {**first, "done": True})
     assert trace.make_plan([unit()], tmp_path) == []
+    assert trace.make_plan([unit()], tmp_path, rerun_status=()) == []
     assert len(trace.make_plan([unit()], tmp_path, include_typescript=True)) == 1
 
 
@@ -567,3 +570,242 @@ def test_frontend_measurement_survives_failed_build_cleanup(tmp_path, monkeypatc
     assert result["instrument"]["cpg_path"] == str(tmp_path / "actual.graph")
     assert result["instrument"]["cpg_bytes"] == 34000
     assert not (tmp_path / "actual.graph").exists()
+
+
+def pair_inputs(tmp_path):
+    from types import SimpleNamespace
+
+    inputs = trace.Inputs(trace.ROOT, tmp_path / "cache")
+    excerpt = tmp_path / "excerpt.py"
+    excerpt.write_text("# commit: " + "f" * 40 + "\n# parent: " + "f" * 40 + "\ndef run(): pass\n")
+    case = SimpleNamespace(name="fixture", repo="owner/repo", relpath="pkg/app.py", vuln_file=excerpt, fixed_file=excerpt, context_files=())
+    inputs.catalog_rows[case.name] = [
+        ("advisory-fixes-2", {"repo": case.repo, "relpath": case.relpath, "parent": "a" * 40, "commit": "b" * 40})
+    ]
+    return inputs, case
+
+
+@pytest.mark.parametrize("side,sha", [("vuln", "a" * 40), ("fixed", "b" * 40)])
+def test_pair_materializes_catalog_side_not_header(tmp_path, monkeypatch, side, sha):
+    inputs, case = pair_inputs(tmp_path)
+    fetched = tmp_path / "fetched"
+    (fetched / "pkg").mkdir(parents=True)
+    (fetched / case.relpath).write_text("class Handler:\n    def run(self): pass\n")
+    calls = []
+
+    def fetch(repo, commit, roots):
+        calls.append((repo, commit, roots))
+        return fetched
+
+    monkeypatch.setattr(inputs, "fetch_source", fetch)
+    monkeypatch.setattr(inputs, "pair_for", lambda row: (case, side))
+    target = tmp_path / "export"
+    proof = inputs.materialize({"units": [unit(file=case.relpath)]}, target)
+    assert calls == [("owner/repo", sha, ["pkg"])]
+    assert proof["commit"] == sha and proof["mode"] == "source"
+    assert (target / case.relpath).read_text().startswith("class Handler:")
+
+
+@pytest.mark.parametrize("change", [{"parent": ""}, {"parent": "b" * 40}, {"commit": "HEAD"}])
+def test_catalog_ambiguous_commits_refused(tmp_path, change):
+    inputs, case = pair_inputs(tmp_path)
+    inputs.catalog_rows[case.name][0][1].update(change)
+    with pytest.raises(ValueError, match="ambiguous catalog commits"):
+        inputs.source_identity(case, "vuln")
+
+
+def test_ambiguous_catalog_and_side_refused(tmp_path):
+    inputs, case = pair_inputs(tmp_path)
+    inputs.catalog_rows[case.name] *= 2
+    with pytest.raises(ValueError, match="ambiguous catalog source"):
+        inputs.source_identity(case, "fixed")
+    with pytest.raises(ValueError, match="ambiguous pair side"):
+        inputs.pair_for(unit(pin_role="unknown"))
+
+
+def test_sparse_export_filters_languages_dirs_and_caps_context(tmp_path):
+    source, target = tmp_path / "source", tmp_path / "export"
+    contents = {
+        "pkg/app.py": "def run(): pass\n",
+        "pkg/context.py": "x" * 200,
+        "pkg/other.js": "function run() {}",
+        "other/app.py": "other",
+    }
+    contents.update({f"pkg/{directory}/a.py": "excluded" for directory in worker.EXCLUDED_DIRS})
+    for rel, data in contents.items():
+        path = source / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(data)
+    proof = trace.export_sources(source, target, [unit(file="pkg/app.py")], ["pkg"], cap=50)
+    assert [p.relative_to(target).as_posix() for p in target.rglob("*") if p.is_file()] == ["pkg/app.py"]
+    assert proof["cap_hit"] and proof["bytes"] == len(contents["pkg/app.py"])
+    assert proof["excludes"] == sorted(worker.EXCLUDED_DIRS)
+    assert trace.source_root("module/src/main/java/org/Handler.java", "java") == "module/src/main/java"
+    assert trace.source_root("pkg/sub/app.ts", "typescript") == "pkg"
+    assert trace.source_root("app.js", "javascript") == "."
+
+
+def test_wrong_fetched_function_fails_without_fallback(tmp_path, monkeypatch):
+    inputs, case = pair_inputs(tmp_path)
+    source = tmp_path / "source"
+    (source / "pkg").mkdir(parents=True)
+    (source / case.relpath).write_text("def runner(): pass\n")
+    inputs.allow_excerpt_fallback = True
+    monkeypatch.setattr(inputs, "pair_for", lambda row: (case, "vuln"))
+    monkeypatch.setattr(inputs, "fetch_source", lambda *args: source)
+    with pytest.raises(ValueError, match="labelled function not found in fetched source"):
+        inputs.materialize({"units": [unit(file=case.relpath)]}, tmp_path / "export")
+
+
+def test_excerpt_fallback_requires_explicit_flag(tmp_path, monkeypatch):
+    inputs, case = pair_inputs(tmp_path)
+    monkeypatch.setattr(inputs, "pair_for", lambda row: (case, "vuln"))
+
+    def unavailable(*args):
+        raise ValueError("fetch failed")
+
+    monkeypatch.setattr(inputs, "fetch_source", unavailable)
+    pin = {"units": [unit(file=case.relpath)]}
+    with pytest.raises(ValueError, match="fetch failed"):
+        inputs.materialize(pin, tmp_path / "export")
+    inputs.allow_excerpt_fallback = True
+    proof = inputs.materialize(pin, tmp_path / "fallback")
+    assert proof["mode"] == "excerpt-fallback" and proof["reason"] == "fetch failed"
+
+
+def test_fetch_renders_sparse_pinned_host_commands(tmp_path, monkeypatch):
+    inputs, _ = pair_inputs(tmp_path)
+    commands = []
+
+    def git(cmd, **kwargs):
+        commands.append(cmd[3:])
+        assert kwargs["env"]["GIT_TERMINAL_PROMPT"] == "0"
+        assert "GIT_NO_LAZY_FETCH" not in kwargs["env"]
+        return subprocess.CompletedProcess(cmd, 0, "a" * 40 + "\n", "")
+
+    monkeypatch.setattr(trace.subprocess, "run", git)
+    clone = inputs.fetch_source("owner/repo", "a" * 40, ["module/src/main/java"])
+    assert clone.is_relative_to(inputs.cache)
+    assert ["fetch", "--depth", "1", "--filter=blob:none", "https://github.com/owner/repo.git", "a" * 40] in commands
+    assert ["sparse-checkout", "set", "--", "module/src/main/java"] in commands
+    assert ["checkout", "--detach", "--force", "a" * 40] in commands
+
+
+def test_explicit_heap_in_frontend_and_query_commands(tmp_path, monkeypatch):
+    backend = worker.MeasuredBackend(900, 120, lambda: None)
+    commands = []
+
+    def run(command, **kwargs):
+        commands.append(command)
+        return subprocess.CompletedProcess(command, 0, BEGIN + '\n{"r": []}\n' + END, "")
+
+    monkeypatch.setattr(backend, "_run", run)
+    monkeypatch.setattr("openultrasast.cpg.backend.shutil.which", lambda name: "/opt/joern-cli/" + name)
+    graph = tmp_path / "cpg.bin"
+    graph.write_bytes(b"x" * 2048)
+    backend._build_with_frontend(tmp_path, graph, tmp_path, "python")
+    backend._batch_once(graph, "taint", {"r": {}})
+    assert {Path(cmd[0]).name for cmd in commands} == {"pysrc2cpg", "joern"}
+    assert all("-J-Xmx2560m" in cmd for cmd in commands)
+    assert "-Xmx2560m" in backend._jvm_env()["JAVA_TOOL_OPTIONS"]
+
+
+@pytest.mark.parametrize("binary", ["pysrc2cpg", "joern"])
+def test_oom_output_overrides_missing_graph_or_markers(tmp_path, binary):
+    launcher = tmp_path / binary
+    launcher.write_text(f"#!{sys.executable}\nprint('java.lang.OutOfMemoryError: Java heap space')\n")
+    launcher.chmod(0o755)
+    backend = worker.MeasuredBackend(30, 10, lambda: None)
+    done = backend._run([str(launcher), "--script", "query.sc"], timeout=10)
+    assert done.returncode == 1
+    assert backend.fatal == "JVM out of memory"
+    assert worker.instrument_failure({"jvm": backend.stages}) == "JVM out of memory"
+
+
+def test_two_language_pin_builds_sequential_frontends_and_routes_questions(tmp_path, monkeypatch):
+    events = []
+
+    class LanguagesBackend(FakeBackend):
+        def build(self, root, language):
+            events.append(("build", language))
+            graph = root / (language + ".bin")
+            graph.write_bytes(b"x" * 2048)
+            self.stages = [
+                {"command": {"python": "pysrc2cpg", "javascript": "jssrc2cpg"}[language], "seconds": 20, "exit": 0, "cpg_bytes": 2048}
+            ]
+
+            def batch(query, requests):
+                events.append(("query", language, sorted(requests)))
+                self.asked.extend(requests)
+                return {
+                    **dict.fromkeys(requests, []),
+                    "__census__": [{"files": "1", "file_names": ["app.py" if language == "python" else "app.js"]}],
+                }
+
+            return CpgResult(graph, lambda q, p: None, run_batch_once=batch, cleanup=lambda: events.append(("cleanup", language)))
+
+    monkeypatch.setattr(worker, "MeasuredBackend", LanguagesBackend)
+    (tmp_path / "app.py").write_text("def run(): pass\n")
+    (tmp_path / "app.js").write_text("function run() {}\n")
+    pin = {"units": [unit(), unit("js", language="javascript", file="app.js")]}
+    pin["questions"] = trace.prepare_questions(pin, tmp_path, 120)
+    assert all(params["callDepth"] == "3" for params in pin["questions"].values())
+    result = worker.worker(pin, tmp_path, tmp_path / "result.json", 900, 120)
+    assert events == [
+        ("build", "javascript"),
+        ("query", "javascript", ["js"]),
+        ("cleanup", "javascript"),
+        ("build", "python"),
+        ("query", "python", ["a"]),
+        ("cleanup", "python"),
+    ]
+    assert all(u["status"] == "asked-nothing" for u in result["units"])
+    assert set(result["instruments"]) == {"python", "javascript"}
+    assert all(u["instrument"]["heap_mb"] == 2560 for u in result["units"])
+
+
+def test_resume_materialization_version_and_status_selection(tmp_path):
+    units = [unit(source="advisory-fixes-2"), unit("other", pin="other", source="population-v1")]
+    pin = trace.make_plan(units, tmp_path, only_source={"advisory-fixes-2"})[0]
+    record = {**pin, "done": True, "units": [trace.unit_record(pin["units"][0], "asked-nothing")]}
+    path = tmp_path / trace.pin_name(pin)
+    trace.write_json(path, record)
+    assert trace.make_plan(units, tmp_path, only_source={"advisory-fixes-2"}) == []
+    assert len(trace.make_plan(units, tmp_path, only_source={"advisory-fixes-2"}, rerun_status={"failed", "asked-nothing"})) == 1
+    trace.write_json(path, {**record, "materialization_version": trace.MATERIALIZATION_VERSION - 1})
+    assert len(trace.make_plan(units, tmp_path, only_source={"advisory-fixes-2"})) == 1
+
+
+def test_sparse_fetch_from_local_git_without_network(tmp_path, monkeypatch):
+    origin = tmp_path / "origin"
+    origin.mkdir()
+    real_run = subprocess.run
+
+    def git(*args):
+        return real_run(["git", "-C", str(origin), *args], capture_output=True, text=True, check=True).stdout.strip()
+
+    git("init")
+    git("config", "user.name", "Offline Test")
+    git("config", "user.email", "offline@example.invalid")
+    for rel in ("module/src/main/java/Handler.java", "module/src/main/java/docs/Other.java", "elsewhere/Other.java"):
+        path = origin / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text("class Handler { void run() {} }\n")
+    git("add", ".")
+    git("commit", "-m", "local fixture")
+    sha = git("rev-parse", "HEAD")
+    inputs, _ = pair_inputs(tmp_path)
+
+    def local_only(cmd, **kwargs):
+        # Substitute a local repository for the one URL; no network-capable command runs.
+        cmd = [str(origin) if arg == "https://github.com/owner/repo.git" else arg for arg in cmd]
+        return real_run(cmd, **kwargs)
+
+    monkeypatch.setattr(trace.subprocess, "run", local_only)
+    clone = inputs.fetch_source("owner/repo", sha, ["module/src/main/java"])
+    assert not (clone / "elsewhere/Other.java").exists()
+    target = tmp_path / "export"
+    labelled = unit(file="module/src/main/java/Handler.java", language="java")
+    proof = trace.export_sources(clone, target, [labelled], ["module/src/main/java"])
+    trace.verify_functions(target, [labelled])
+    assert proof["files"] == 1 and proof["bytes"] > 0