feat(pre-push-safety-net): bound backend execution by one deadline

src/openultrasast/cpg/backend.py

@@ -25,14 +25,17 @@ import os
 import re
 import shutil
 import signal
+import stat
 import subprocess
 import tempfile
 import time
 from collections.abc import Callable, Mapping, Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from pathlib import Path
 from typing import Protocol
 
+from ..model.contracts import ExecutionBudget
+
 logger = logging.getLogger(__name__)
 
 QUERIES_DIR = Path(__file__).resolve().parent / "queries"
@@ -239,6 +242,7 @@ class CpgResult:
     # request files and joern's own workspace copy; nothing reclaimed it, and a day of scanning left 1,055
     # directories and 355MB behind on a disk that was already at 95%. The driver disposes in a `finally`.
     cleanup: Callable[[], None] | None = None
+    execution_diagnostics: Callable[[], tuple[str, ...]] | None = None
 
 
 class CpgBackend(Protocol):
@@ -272,13 +276,41 @@ class JoernBackend:
     queries_dir: Path = field(default_factory=lambda: QUERIES_DIR)
     # Why the last build refused, so the driver can name it instead of reporting a bare `cpg_build_failed`.
     last_failure: str = ""
+    execution_budget: ExecutionBudget | None = None
+    _diagnostics: list[str] = field(default_factory=list, init=False, repr=False)
+    _scratch: list[Path] = field(default_factory=list, init=False, repr=False)
+    _cancel_deadline: float | None = field(default=None, init=False, repr=False)
 
     def available(self) -> bool:
         from .capability import has_cpg
 
         return has_cpg()
 
-    def build(self, root: Path, *, language: str = "", exclude: Sequence[str] = ()) -> CpgResult | None:
+    def build(
+        self, root: Path, *, language: str = "", exclude: Sequence[str] = (), execution_budget: ExecutionBudget | None = None
+    ) -> CpgResult | None:
+        """Bind an independent graph session to the caller's absolute transaction deadline."""
+        budget = execution_budget if execution_budget is not None else self.execution_budget
+        if budget is None:
+            return self._build_impl(root, language=language, exclude=exclude)
+        session = replace(self, execution_budget=budget, last_failure="")
+        result = None
+        try:
+            session._checkpoint()
+            result = session._build_impl(root, language=language, exclude=exclude)
+            session._checkpoint()
+            return result
+        except _DeadlineExpired:
+            result = None
+            return None
+        finally:
+            if result is None:
+                for scratch in session._scratch:
+                    session._cleanup(scratch, final=True)
+            self.last_failure = session.last_failure
+            self._diagnostics = list(session._diagnostics)
+
+    def _build_impl(self, root: Path, *, language: str = "", exclude: Sequence[str] = ()) -> CpgResult | None:
         """Build a CPG for ``root``. ``None`` on any failure — the caller degrades, never guesses.
 
         ``language`` lets a failed ``joern-parse`` retry through the frontend directly, which is what Joern
@@ -287,10 +319,13 @@ class JoernBackend:
         parse = shutil.which("joern-parse")
         if parse is None and os.environ.get("OPENULTRASAST_JOERN_PROBE", "").strip().lower() not in {"1", "on", "true", "yes"}:
             return None
-        _sweep_stale_scratch()
+        if self.execution_budget is None:
+            _sweep_stale_scratch()
         scratch = Path(tempfile.mkdtemp(prefix=SCRATCH_PREFIX))
+        if self.execution_budget is not None:
+            self._scratch.append(scratch)
         cpg_path = scratch / "cpg.bin"
-        dispose = lambda: shutil.rmtree(scratch, ignore_errors=True)  # noqa: E731 -- one expression, named
+        dispose = lambda: self._cleanup(scratch, final=True)  # noqa: E731 -- one expression, named
 
         # THE PRECONDITION. Before anything is built, make the toolchain open one file from the tree and say
         # how big it is. Every instrument failure this project has had takes the same shape -- the input was
@@ -307,6 +342,7 @@ class JoernBackend:
         if unreadable:
             logger.error("cpg build refused for %s: %s", root, unreadable)
             self.last_failure = unreadable
+            dispose()
             return None
 
         # A frontend whose failures are visible and retryable is worth more than one whose are not.
@@ -329,6 +365,7 @@ class JoernBackend:
                     run_batch=lambda query, requests: self.query_batch_across(shards, query, requests),
                     unparsed=unparsed,
                     cleanup=dispose,
+                    execution_diagnostics=lambda: tuple(self._diagnostics),
                 )
         command = [parse or "joern-parse", self._heap_flag(), str(root), "--output", str(cpg_path)]
         completed = self._run(command, timeout=self.build_timeout, cwd=scratch)
@@ -350,6 +387,7 @@ class JoernBackend:
             run_batch=lambda query, requests: self.query_batch(cpg_path, query, requests),
             unparsed=unparsed,
             cleanup=dispose,
+            execution_diagnostics=lambda: tuple(self._diagnostics),
         )
 
     def _files_too_large_to_flow(self, root: Path, language: str) -> tuple[str, ...]:
@@ -358,7 +396,7 @@ class JoernBackend:
         if not extensions:
             return ()
         found: list[str] = []
-        for path in sorted(root.rglob("*")):
+        for path in self._paths(root, "*"):
             if path.suffix.lower() not in extensions or not path.is_file():
                 continue
             try:
@@ -381,7 +419,7 @@ class JoernBackend:
         if version is not None and version >= CLOSURE_DEFECT_FIXED_IN:
             return ()  # fixed upstream; nothing to exclude and every file to keep
         found: list[str] = []
-        for path in sorted(root.rglob("*.php")):
+        for path in self._paths(root, "*.php"):
             if not path.is_file():
                 continue
             try:
@@ -394,6 +432,8 @@ class JoernBackend:
                     continue
                 depth, index = 0, opening
                 while index < len(text):
+                    if index % 4096 == 0:
+                        self._checkpoint()
                     if text[index] == "{":
                         depth += 1
                     elif text[index] == "}":
@@ -421,7 +461,7 @@ class JoernBackend:
         binary = shutil.which(interpreter)
         if binary is None:
             return ""  # absent is a different failure, and the frontend reports it plainly
-        sample = next((path for path in sorted(root.rglob("*")) if path.suffix.lower() in extensions and path.is_file()), None)
+        sample = next((path for path in self._paths(root, "*") if path.suffix.lower() in extensions and path.is_file()), None)
         if sample is None:
             return ""  # nothing of this language to read; not the interpreter's fault
         probe = "$f = $argv[1]; if (!is_readable($f) || filesize($f) < 1) { exit(3); } exit(0);"
@@ -545,7 +585,7 @@ class JoernBackend:
         extensions = _LANGUAGE_EXTENSIONS.get(language.lower())
         if not extensions:
             return None
-        expected = sum(1 for path in root.rglob("*") if path.suffix.lower() in extensions and path.is_file())
+        expected = sum(1 for path in self._paths(root, "*") if path.suffix.lower() in extensions and path.is_file())
         if not expected:
             return (0, 0)  # nothing to count is not a failed count: an empty tree's graph is complete
         payload = self.query(cpg_path, "census", {})
@@ -598,6 +638,7 @@ class JoernBackend:
         island = scratch / "excluded-root"
         made = 0
         for name in files:
+            self._checkpoint()
             source = Path(name)
             if not source.is_absolute():
                 source = root / name
@@ -624,6 +665,9 @@ class JoernBackend:
         for shard in shards:
             rows = self.query(shard, query, params)
             if rows is None:
+                if self.execution_budget is not None:
+                    self._note("shard_query_incomplete")
+                    return None
                 continue
             answered += 1
             if isinstance(rows, list):
@@ -645,6 +689,9 @@ class JoernBackend:
         for shard in shards:
             one = self.query_batch(shard, query, requests)
             if one is None:
+                if self.execution_budget is not None:
+                    self._note("shard_query_incomplete")
+                    return None
                 continue
             answered += 1
             census = one.pop("__census__", None)
@@ -692,7 +739,7 @@ class JoernBackend:
             saved.replace(cpg_path)
             return True
         finally:
-            shutil.rmtree(workspace, ignore_errors=True)
+            self._cleanup(workspace)
 
     def _build_with_retries(
         self, root: Path, cpg_path: Path, scratch: Path, language: str, exclude: Sequence[str] = ()
@@ -716,6 +763,7 @@ class JoernBackend:
         """
         best: tuple[str, ...] | None = None
         for attempt in range(FRONTEND_BUILD_ATTEMPTS):
+            self._checkpoint()
             unparsed = self._build_with_frontend(root, cpg_path, scratch, language, exclude)
             if unparsed is None:
                 return best  # the frontend is missing or failed outright; keep any earlier partial build
@@ -773,6 +821,8 @@ class JoernBackend:
         point -- "no rows" and "could not ask" look identical to a caller that only sees a dict, and a
         117k-line PHP CPG that threw while loading turned a failed scan into a clean bill of health.
         """
+        if not self._time_available():
+            return None
         script = self.queries_dir / f"{query}.sc"
         if not script.is_file():
             logger.warning("no such cpg query: %s", script)
@@ -786,6 +836,8 @@ class JoernBackend:
         # {} -- and {} reads as "no rows found". The scan reported 500 regions examined and nothing found, in
         # 0.05 seconds of taint query, which is a quiet failure of exactly the kind the rung ladder cannot
         # catch because no verdict was ever produced to label.
+        if not self._time_available():
+            return None
         requests_file = cpg_path.parent / f"{query}-requests.json"
         try:
             requests_file.write_text(payload)
@@ -810,6 +862,8 @@ class JoernBackend:
             logger.warning("cpg batch %s failed: %s", query, detail)
             return None
         parsed = extract_payload(completed.stdout or "")
+        if not self._time_available():
+            return None
         if not isinstance(parsed, Mapping):
             # Unparseable output is a failure, not an empty answer. A 117k-line PHP CPG that throws while
             # loading printed a stack trace and no payload, and returning {} made the whole scan read as a
@@ -821,6 +875,8 @@ class JoernBackend:
 
     def query(self, cpg_path: Path, query: str, params: Mapping[str, object]) -> object | None:
         """Run a shipped CPGQL script against a built CPG and return its fenced JSON payload."""
+        if not self._time_available():
+            return None
         script = self.queries_dir / f"{query}.sc"
         if not script.is_file():
             logger.warning("no such cpg query: %s", script)
@@ -835,7 +891,96 @@ class JoernBackend:
             detail = (completed.stderr or "")[-400:] if completed is not None else "timeout"
             logger.warning("cpg query %s failed: %s", query, detail)
             return None
-        return extract_payload(completed.stdout or "")
+        parsed = extract_payload(completed.stdout or "")
+        return parsed if self._time_available() else None
+
+    def _note(self, reason: str) -> None:
+        if reason not in self._diagnostics:
+            self._diagnostics.append(reason)
+        self.last_failure = reason
+
+    def _time_available(self) -> bool:
+        if self._cancel_deadline is not None:
+            return False
+        if self.execution_budget is not None and time.monotonic() >= self.execution_budget.deadline_monotonic:
+            self._note("deadline_exhausted")
+            return False
+        return True
+
+    def _checkpoint(self) -> None:
+        if not self._time_available():
+            raise _DeadlineExpired
+
+    def _paths(self, root: Path, pattern: str):  # type: ignore[no-untyped-def]
+        paths = root.rglob(pattern)
+        if self.execution_budget is None:
+            yield from sorted(paths)
+        else:
+            self._checkpoint()
+            for path in paths:
+                self._checkpoint()
+                yield path
+
+    def _cleanup_limit(self) -> float:
+        assert self.execution_budget is not None
+        ceiling = self.execution_budget.deadline_monotonic + self.execution_budget.cancellation_allowance_seconds
+        return min(ceiling, self._cancel_deadline) if self._cancel_deadline is not None else ceiling
+
+    def _cleanup(self, root: Path, *, final: bool = False) -> None:
+        if self.execution_budget is None:
+            shutil.rmtree(root, ignore_errors=True)
+            return
+        deadline = self._cleanup_limit() if final or self._cancel_deadline is not None else self.execution_budget.deadline_monotonic
+        try:
+            _remove_owned_tree(root, deadline)
+        except (OSError, _DeadlineExpired):
+            self._note("scratch_cleanup_incomplete")
+
+    def _run_bounded(self, command: list[str], *, timeout: float, cwd: Path | None) -> subprocess.CompletedProcess[str] | None:
+        assert self.execution_budget is not None
+        if not self._time_available():
+            return None
+        remaining = min(timeout, self.execution_budget.deadline_monotonic - time.monotonic())
+        if remaining <= 0:
+            self._note("deadline_exhausted")
+            return None
+        process = None
+        try:
+            if self.runner is not None:
+                result = self.runner(command, capture_output=True, text=True, timeout=remaining, check=False, cwd=str(cwd) if cwd else None)
+            else:
+                process = subprocess.Popen(  # noqa: S603 -- internal argument array
+                    command,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    cwd=str(cwd) if cwd else None,
+                    env=self._jvm_env(),
+                    start_new_session=True,
+                )
+                remaining = min(remaining, max(0.0, self.execution_budget.deadline_monotonic - time.monotonic()))
+                stdout, stderr = process.communicate(timeout=remaining)
+                result = subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
+            if not self._time_available():
+                return None
+            return result
+        except subprocess.TimeoutExpired:
+            self._cancel_deadline = min(self._cleanup_limit(), time.monotonic() + self.execution_budget.cancellation_allowance_seconds)
+            self._note("deadline_exhausted" if time.monotonic() >= self.execution_budget.deadline_monotonic else "process_timeout")
+            return None
+        except (OSError, subprocess.SubprocessError):
+            self._note("process_failed")
+            return None
+        finally:
+            if process is not None:
+                # No Popen context manager: its implicit wait has no timeout. Always signal
+                # the original group ID, even if the launcher already exited, so descendants
+                # retaining pipes (or closing them) cannot survive it.
+                if not _terminate_bounded_group(process, self._cleanup_limit()):
+                    self._note("process_cleanup_incomplete")
+                for stream in (process.stdout, process.stderr):
+                    if stream is not None:
+                        stream.close()
 
     def _heap_mb(self) -> int:
         if self.heap_mb > 0:
@@ -874,6 +1019,8 @@ class JoernBackend:
         row it would OOM rather than merely degrade -- and it would look like the SCAN needing more memory,
         which is the wrong lesson entirely.
         """
+        if self.execution_budget is not None:
+            return self._run_bounded(command, timeout=timeout, cwd=cwd)
         run = self.runner if self.runner is not None else subprocess.run
         if self.runner is not None:  # tests inject a fake runner and never spawn anything
             try:
@@ -900,6 +1047,64 @@ class JoernBackend:
             return None
 
 
+class _DeadlineExpired(Exception):
+    """Internal cooperative boundary; public builds/queries retain failure semantics."""
+
+
+def _terminate_bounded_group(process: subprocess.Popen[str], deadline: float) -> bool:
+    # SIGKILL immediately leaves the shared allowance for reaping and scratch cleanup.
+    # Unlike the legacy TERM/wait loop this also kills grandchildren after leader exit.
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    except OSError:
+        return False
+    try:
+        process.wait(timeout=max(0.0, deadline - time.monotonic()))
+    except subprocess.TimeoutExpired:
+        return False
+    return True
+
+
+def _remove_owned_tree(root: Path, deadline: float) -> None:
+    """Bounded Linux dirfd cleanup, never following substituted directory links."""
+
+    def check() -> None:
+        if time.monotonic() >= deadline:
+            raise _DeadlineExpired
+
+    def contents(fd: int) -> None:
+        with os.scandir(fd) as entries:
+            for entry in entries:
+                check()
+                info = os.stat(entry.name, dir_fd=fd, follow_symlinks=False)
+                if stat.S_ISDIR(info.st_mode):
+                    child = os.open(entry.name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
+                    try:
+                        contents(child)
+                    finally:
+                        os.close(child)
+                    check()
+                    os.rmdir(entry.name, dir_fd=fd)
+                else:
+                    os.unlink(entry.name, dir_fd=fd)
+
+    if not root.exists() and not root.is_symlink():
+        return
+    check()
+    if root.is_symlink():
+        root.unlink()
+        return
+    fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        contents(fd)
+    finally:
+        os.close(fd)
+    check()
+    root.rmdir()
+
+
 SCRATCH_PREFIX = "ousast-cpg-"
 # How long an abandoned scratch tree may survive. Long enough that a scan running in another process is never
 # disturbed, short enough that a machine scanning all day does not fill its disk.

src/openultrasast/model/scan.py

@@ -18,6 +18,7 @@ the region's risk score said beforehand.
 
 from __future__ import annotations
 
+import inspect
 import logging
 import time
 from collections.abc import Mapping, Sequence
@@ -28,6 +29,7 @@ from typing import Any
 from ..cpg.backend import CpgResult
 from .candidates import enumerate_candidates
 from .config_value import request_params as config_params
+from .contracts import ExecutionBudget
 from .dominance import request_params as dominance_params
 from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows
 from .ladder import Rung
@@ -127,6 +129,49 @@ def scan_repository(
     client: Any | None = None,
     model: str = "",
     budget: ScanBudget | None = None,
+    execution_budget: ExecutionBudget | None = None,
+) -> ModelScanResult:
+    """Run with an optional transaction deadline; graph ownership ends on every exit."""
+    started = time.monotonic()
+    owned: list[Any] = []
+    try:
+        result = _scan_repository_impl(
+            root,
+            regions,
+            backend=backend,
+            client=client if execution_budget is None else None,
+            model=model,
+            budget=budget,
+            execution_budget=execution_budget,
+            owned=owned,
+        )
+    finally:
+        for cpg in owned:
+            _dispose(cpg)
+    diagnostics = list(result.degradations)
+    if execution_budget is not None:
+        if client is not None:
+            diagnostics.append({"stage": "model", "reason": "model_withheld_execution_budget"})
+        reasons = ["deadline_exhausted"] if time.monotonic() >= execution_budget.deadline_monotonic else []
+        for cpg in owned:
+            read_diagnostics = getattr(cpg, "execution_diagnostics", None)
+            if callable(read_diagnostics):
+                reasons.extend(read_diagnostics())
+        for reason in dict.fromkeys(reasons):
+            diagnostics.append({"stage": "model", "reason": reason})
+    return replace(result, degradations=tuple(diagnostics), seconds=round(time.monotonic() - started, 2))
+
+
+def _scan_repository_impl(
+    root: Path,
+    regions: Sequence[ScanRegion],
+    *,
+    backend: Any,
+    client: Any | None = None,
+    model: str = "",
+    budget: ScanBudget | None = None,
+    execution_budget: ExecutionBudget | None = None,
+    owned: list[Any],
 ) -> ModelScanResult:
     """Arbitrate a repository's regions under one CPG and one budget."""
     limits = budget if budget is not None else ScanBudget()
@@ -140,7 +185,7 @@ def scan_repository(
     for region in regions:
         counts[region.language] = counts.get(region.language, 0) + 1
     dominant = max(counts, key=lambda name: (counts[name], name)) if counts else ""
-    cpg = _build(backend, root, dominant)
+    cpg = _build(backend, root, dominant, execution_budget=execution_budget)
     build_seconds = round(time.monotonic() - build_started, 2)
     if cpg is None:
         return ModelScanResult(
@@ -151,6 +196,8 @@ def scan_repository(
             degradations=({"stage": "model", "reason": "cpg_build_failed", "detail": getattr(backend, "last_failure", "")},),
         )
 
+    owned.append(cpg)
+
     # A CPG the frontend built with files missing is not the repository the caller asked about, and no
     # verdict over it can say anything about the code that was dropped. There is no rung for that, because
     # the ladder labels verdicts and none was ever produced here -- so it is a degradation, named, with the
@@ -198,14 +245,14 @@ def scan_repository(
     tiers: list[tuple[str, str, str, int]] = []
     per_kind: dict[str, float] = {}
     evidence_by_pair: dict[tuple[str, str, str], Evidence] = {}
-    if limits.tiering and callable(getattr(cpg, "run_batch", None)):
-        candidates = (
+    if _within_deadline(execution_budget) and limits.tiering and callable(getattr(cpg, "run_batch", None)):
+        evidence_candidates = (
             ordered_regions[: (limits.evidence_regions or len(ordered_regions))]
             if limits.order_by_evidence
             else ordered_regions[: limits.max_regions]
         )
         tier_started = time.monotonic()
-        evidence_by_pair, failed = _evidence_pass(cpg, _collect(candidates), hooks)
+        evidence_by_pair, failed = _evidence_pass(cpg, _collect(evidence_candidates), hooks)
         per_kind["evidence"] = round(time.monotonic() - tier_started, 2)
         if failed:
             degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": failed})
@@ -227,6 +274,8 @@ def scan_repository(
     query_started = time.monotonic()
     pruned = 0
     for kind, requests in _grouped(work, hook_callbacks=hooks).items():
+        if not _within_deadline(execution_budget):
+            break
         if kind == "taint" and limits.prune_tier0 and tier_by_rid:
             # Tier 0 is exact: a family with no sink in reach cannot yield a flow. Measured on a 637-file
             # plugin, that is 2,161 of 2,500 requests -- 86% of a query that timed out at forty minutes.
@@ -268,8 +317,15 @@ def scan_repository(
                 rows_by_id.update(answered_batch)
         else:  # a backend without batching still works, one call at a time
             for rid, params in requests.items():
+                if not _within_deadline(execution_budget):
+                    break
                 answered = cpg.run(kind, params)
-                rows_by_id[rid] = list(answered) if isinstance(answered, list) else []
+                if isinstance(answered, list):
+                    rows_by_id[rid] = list(answered)
+                elif execution_budget is None:
+                    rows_by_id[rid] = []
+                else:
+                    degradations.append({"stage": "model", "reason": "query_failed", "kind": kind, "requests": 1})
         per_kind[kind] = round(time.monotonic() - kind_started, 2)
     query_seconds = round(time.monotonic() - query_started, 2)
 
@@ -287,6 +343,12 @@ def scan_repository(
     judged: set[tuple[str, str | None]] = set()
     unasked: set[tuple[str, str | None]] = set()
     for rid, region, spec in work:
+        if not _within_deadline(execution_budget):
+            break
+        # Unanswered requests are deferred on the bounded path, never adjudicated
+        # from an invented empty response. Exact identities are added in task 3.2.
+        if execution_budget is not None and rid not in rows_by_id:
+            continue
         exhausted = counted is not None and counted.calls >= limits.max_model_calls
         family = getattr(spec, "family", "")
         prefetched = CpgResult(cpg_path=cpg.cpg_path, run=_prefetched(rows_by_id.get(rid, [])))
@@ -331,7 +393,6 @@ def scan_repository(
     findings = _ordered(_deduplicated(collected))
     # The graph has answered everything it is going to. Its scratch tree holds the CPG, the request files and
     # joern's own working copy -- tens of megabytes per scan -- and nothing else reclaims it.
-    _dispose(cpg)
     return ModelScanResult(
         findings=findings,
         by_rung=_tally(findings),
@@ -441,11 +502,26 @@ def _dispose(cpg: object) -> None:
             logger.warning("could not remove the cpg scratch directory: %s", exc)
 
 
-def _build(backend: Any, root: Path, language: str) -> Any:
+def _within_deadline(budget: ExecutionBudget | None) -> bool:
+    return budget is None or time.monotonic() < budget.deadline_monotonic
+
+
+def _build(backend: Any, root: Path, language: str, *, execution_budget: ExecutionBudget | None = None) -> Any:
     """Build through the backend, passing the language and the vendored trees when the backend can use them."""
     from .layout import vendored_directories
 
+    if not _within_deadline(execution_budget):
+        return None
     excluded = vendored_directories(root) if root.is_dir() else ()
+    if execution_budget is not None:
+        try:
+            inspect.signature(backend.build).bind(root, language=language, exclude=excluded, execution_budget=execution_budget)
+        except (TypeError, ValueError):
+            # An old backend cannot silently turn a bounded push into an ordinary scan.
+            return None
+        if not _within_deadline(execution_budget):
+            return None
+        return backend.build(root, language=language, exclude=excluded, execution_budget=execution_budget)
     try:
         return backend.build(root, language=language, exclude=excluded)
     except TypeError:  # a backend from before the retry existed, including every test double