fix(hunt): diff producer records where a push removed lines; worktree checkout is proven complete

Two instrument findings from the first access-control feature table (952 rows, 74 positive rows):

1. A fix that adds a guard is, replayed base-to-head, a push that removes it. The diff producer only kept
   hunks with added lines at head, so 38 of 74 positive rows had no hunk in their file at all and 69 of 74
   were "not on a changed line", while 60% of ordinary rows were. The diff annotation separated the labels
   by construction and was blind to exactly the access-control shape. The producer (version 2) now records
   per file the head lines after which the push removed code; distance and on_changed_line count them, the
   model's hunks input renders them as removed_after_line, and diff.deletion_count is an engineered feature.

2. 150 rows (16%, every positive of one repository) read empty files from the worktree although git holds
   the blobs; the build ran while another process checked out from the same clone. _full_checkout now
   proves the checkout: git status must be clean, a dirty tree is restored once, then InstrumentFailure
   checkout_incomplete. The table script cross-checks every unread row against the blob size and refuses
   to screen a partly unread tree, and records the diff-scoped recall ceiling (positive rows a label-blind
   enumeration would reach).

Co-Authored-By: Claude Fable 5.1 <[email protected]>

src/openultrasast/hunt/features.py

@@ -20,6 +20,7 @@ ENGINEERED: dict[str, tuple[str, str]] = {
     "diff.on_changed_line": ("feature.diff.on_changed_line", "hunks"),
     "diff.hunk_count": ("feature.diff.hunks", "hunks"),
     "diff.hunk_lines": ("feature.diff.hunks", "hunks"),
+    "diff.deletion_count": ("feature.diff.deletions", "hunks"),
     "source.span_lines": ("feature.source.span", "function_text"),
     "source.is_module": ("feature.source.function", "change_summary"),
     "source.caller_count": ("feature.source.callers", "callers"),
@@ -57,6 +58,7 @@ def _engineer(row: Mapping[str, Any]) -> dict[str, Any]:
             else int(bool(row["feature.diff.on_changed_line"])),
             "diff.hunk_count": _count(hunks),
             "diff.hunk_lines": None if hunks is None else sum(end - start + 1 for start, end in hunks),
+            "diff.deletion_count": _count(row.get("feature.diff.deletions")),
             "source.span_lines": None if not span else span[1] - span[0] + 1,
             "source.is_module": None if function is None else int(function == "<module>"),
             "source.caller_count": _count(callers),

src/openultrasast/hunt/producers.py

@@ -96,6 +96,19 @@ class Context:
 
     @cached_property
     def hunks(self) -> dict[str, list[list[int]]]:
+        """Line ranges per file that the push changed or added, at head."""
+        return self._patch[0]
+
+    @cached_property
+    def deletions(self) -> dict[str, list[int]]:
+        """Per file, the head lines after which the push removed lines (0 when it removed the top of the file).
+
+        A fix that adds a guard is a push that removes it when replayed base-to-head, and that push has no added
+        line at head: without this column the diff features are blind to exactly that shape."""
+        return self._patch[1]
+
+    @cached_property
+    def _patch(self) -> tuple[dict[str, list[list[int]]], dict[str, list[int]]]:
         self.verify_head()
         # Exclude private files before git opens the patch; only hunk coordinates become features.
         patch = _git(
@@ -112,6 +125,7 @@ class Context:
         )
         self.diff_bytes = len(patch.encode("utf-8"))
         result: dict[str, list[list[int]]] = {}
+        removed: dict[str, list[int]] = {}
         path = None
         for text in patch.splitlines():
             if text.startswith("+++ "):
@@ -122,7 +136,9 @@ class Context:
                 start, count = int(match[1]), int(match[2] or "1")
                 if count:
                     result.setdefault(path, []).append([start, start + count - 1])
-        return result
+                else:
+                    removed.setdefault(path, []).append(start)
+        return result, removed
 
     def ir(self, path: str) -> FileIR:
         if path not in self.irs:
@@ -143,7 +159,7 @@ class Producer:
     compute: Compute
 
 
-def _producer(name: str, compute: Compute) -> Producer:
+def _producer(name: str, compute: Compute, version: str = "1") -> Producer:
     columns = tuple(f"feature.{name}.{column}" for column in FEATURE_NAMES[name])
     provenance = tuple(f"provenance.{name}.{column}" for column in ("ok", "reason", "version", "bytes_read"))
 
@@ -167,19 +183,22 @@ def _producer(name: str, compute: Compute) -> Producer:
             {
                 f"provenance.{name}.ok": reason is None,
                 f"provenance.{name}.reason": reason,
-                f"provenance.{name}.version": "1",
+                f"provenance.{name}.version": version,
                 f"provenance.{name}.bytes_read": bytes_read,
             }
         )
         return result
 
-    return Producer(name, "1", columns + provenance, guarded)
+    return Producer(name, version, columns + provenance, guarded)
 
 
 def _diff(root: Path, change: Change, path: str, line: int, ctx: Context) -> dict[str, object]:
     hunks = ctx.hunks.get(path, [])
-    distance = min((max(start - line, 0, line - end) for start, end in hunks), default=None)
-    return {"hunks": hunks, "on_changed_line": distance == 0, "distance": distance}
+    deletions = ctx.deletions.get(path, [])
+    # A removal sits between head lines `after` and `after + 1`; both lines count as touched.
+    ranges = hunks + [[max(after, 1), max(after, 1) + 1] for after in deletions]
+    distance = min((max(start - line, 0, line - end) for start, end in ranges), default=None)
+    return {"hunks": hunks, "deletions": deletions, "on_changed_line": distance == 0, "distance": distance}
 
 
 def _source(root: Path, change: Change, path: str, line: int, ctx: Context) -> dict[str, object]:
@@ -279,7 +298,12 @@ def _taxonomy(root: Path, change: Change, path: str, line: int, ctx: Context) ->
     return {"families": [{"id": f.id, "definition": " ".join(f.description.split())} for f in load_families().families]}
 
 
-PRODUCERS = (_producer("diff", _diff), _producer("source", _source), _producer("semantic", _semantic), _producer("taxonomy", _taxonomy))
+PRODUCERS = (
+    _producer("diff", _diff, version="2"),
+    _producer("source", _source),
+    _producer("semantic", _semantic),
+    _producer("taxonomy", _taxonomy),
+)
 
 
 def _candidates(root: Path, change: Change, ctx: Context) -> list[dict[str, Any]]:

src/openultrasast/hunt/program.py

@@ -31,7 +31,11 @@ BASELINE_INSTRUCTION = (
 # Input name -> (feature column, renderer). Order is the order the model sees them in; masked columns are absent.
 INPUTS: tuple[tuple[str, str, str], ...] = (
     ("change_summary", "", "The files and functions the push touched."),
-    ("hunks", "feature.diff.hunks", "Changed line ranges per file at head, as [start, end] pairs."),
+    (
+        "hunks",
+        "feature.diff.hunks",
+        "Per file at head: changed line ranges as [start, end] pairs, and the lines after which the push removed code.",
+    ),
     ("function_text", "feature.source.span", "The enclosing function at head, numbered lines."),
     ("callers", "feature.source.callers", "Where the enclosing function is called from: path and line."),
     ("entry_points", "feature.source.entry_points", "In-repository entry points that reach the file."),
@@ -82,7 +86,13 @@ def render_rows(
         touched = sorted({(row["path"], str(row.get("feature.source.function") or "<module>")) for row in rows})
         out["change_summary"] = "\n".join(f"{path}::{function}" for path, function in touched) or "(nothing readable)"
     if "hunks" in wanted:
-        hunks = {row["path"]: row.get("feature.diff.hunks") for row in rows if row.get("feature.diff.hunks")}
+        hunks: dict[str, dict[str, list]] = {}
+        for row in rows:
+            if row.get("feature.diff.hunks") or row.get("feature.diff.deletions"):
+                hunks[row["path"]] = {
+                    "changed": row.get("feature.diff.hunks") or [],
+                    "removed_after_line": row.get("feature.diff.deletions") or [],
+                }
         out["hunks"] = json.dumps(hunks, sort_keys=True) if hunks else "(no hunks)"
     if "function_text" in wanted:
         blocks = []

src/openultrasast/hunt/runner.py

@@ -169,9 +169,15 @@ def _full_checkout(root: Path) -> None:
     try:
         sparse = _git(root, "config", "--get", "core.sparseCheckout").strip()
     except subprocess.CalledProcessError:
-        return
+        sparse = "false"
     if sparse == "true":
         _git(root, "sparse-checkout", "disable")
+    # The proof: a fresh worktree has nothing modified. Truncated files (seen once while two processes checked out
+    # from the same clone) show up here as modified; restore them once, then refuse rather than read an unread tree.
+    if _git(root, "status", "--porcelain", "--untracked-files=no").strip():
+        _git(root, "checkout", "--", ".")
+        if _git(root, "status", "--porcelain", "--untracked-files=no").strip():
+            raise InstrumentFailure("checkout_incomplete")
 
 
 def _function_at(root: Path, finding: Mapping[str, Any]) -> str | None:

src/openultrasast/hunt/table.py

@@ -13,7 +13,7 @@ SCHEMA_VERSION = 1
 KEY_COLUMNS = ("change_id", "repo", "head", "path", "line", "kind", "fold")
 LABEL_COLUMNS = ("label", "family", "fix_span", "label_ref", "verification")
 FEATURE_NAMES = {
-    "diff": ("hunks", "on_changed_line", "distance"),
+    "diff": ("hunks", "deletions", "on_changed_line", "distance"),
     "source": ("function", "span", "callers", "entry_points", "language"),
     "semantic": ("roles", "def_use", "obligations"),
     "taxonomy": ("families",),