search-with-proof: diagnosable failures, pip install race, no duplicate tasks

Reasons carry type plus a sanitised message tail; command failures carry phase,
exit code and bounded stderr. Scratch walk tolerates directories pip removes
mid-walk; writable /workspace/home and runtime paths. Capacity retries delete a
possibly created task before resubmitting. Docker smoke checks the image first.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/test_search_executor_smoke.py

@@ -142,9 +142,9 @@ def test_docker_managed_task_uses_docker_and_no_heap_flag(tmp_path, monkeypatch)
     )
     record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
     assert record["status"] == "ok"
-    assert [a[1] for a in calls] == ["run", "rm"]
-    assert "--heap-profile" not in calls[0]
-    assert calls[1][2] == "-f"
+    assert [a[1] for a in calls] == ["image", "run", "rm"]
+    assert "--heap-profile" not in calls[1]
+    assert calls[2][2] == "-f"
 
 
 def test_apply_exception_still_deletes(tmp_path, monkeypatch):
@@ -160,3 +160,104 @@ def test_apply_exception_still_deletes(tmp_path, monkeypatch):
     assert result["status"] == "instrument_failure"
     assert events == ["apply", "delete"]
     assert "secret.example" not in json.dumps(result)
+
+
+def test_smoke_retains_sanitized_command_failure(tmp_path, monkeypatch):
+    executor, lane, events = setup(tmp_path, monkeypatch)
+    original = executor._task_run
+
+    def fail(argv, timeout, limits):
+        if "pip" in argv:
+            return {
+                "exit_code": 17,
+                "stderr": "project failed: Bearer hidden https://host/project?q=secret",
+                "stdout": "",
+                "phase": "run",
+                "reason": "CommandFailure: exit code 17",
+            }
+        return original(argv, timeout, limits)
+
+    monkeypatch.setattr(executor, "_task_run", fail)
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    row = record["commands"][-1]
+    assert row["exit_code"] == 17 and row["phase"] == "run"
+    assert "CommandFailure" in row["reason"]
+    assert all(word not in json.dumps(record) for word in ("hidden", "secret", "project", "example"))
+    assert events == ["apply", "delete"]
+
+
+def test_docker_missing_image_is_diagnosable_without_pull(tmp_path, monkeypatch):
+    import subprocess
+
+    monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
+    calls = []
+
+    def runner(argv, **kwargs):
+        calls.append(argv)
+        return subprocess.CompletedProcess(argv, 1, "", "No such image: pinned-image")
+
+    lane = batch.DockerLane(
+        SimpleNamespace(atespace="default"),
+        batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None),
+        store=Store(Executor(tmp_path, task_boundary=True)),
+        runner=runner,
+    )
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert record["status"] == "instrument_failure"
+    assert "ValueError:" in record["reason"] and "No such image" in record["reason"]
+    assert [c[1] for c in calls] == ["image"]
+
+
+@pytest.mark.parametrize("delete_fails", [False, True])
+def test_capacity_partial_create_deleted_before_retry(tmp_path, monkeypatch, delete_fails):
+    executor, lane, events = setup(tmp_path, monkeypatch)
+    live, attempts = set(), []
+
+    def ax(*args, **kwargs):
+        events.append(args[0])
+        if args[0] == "apply":
+            assert not live
+            live.add(kwargs["manifest"]["metadata"]["name"])
+            attempts.append(1)
+            if len(attempts) == 1:
+                raise batch.NoFreeWorkers("partially provisioned")
+        elif args[0] == "delete":
+            if delete_fails:
+                raise OSError("cannot delete")
+            live.discard(args[2])
+
+    lane.ax, lane.pause = ax, lambda seconds: None
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert len(attempts) == (1 if delete_fails else 2)
+    assert record["status"] == ("instrument_failure" if delete_fails else "ok")
+    if not delete_fails:
+        assert events == ["apply", "delete", "apply", "delete"] and not live
+    else:
+        # Simulated leaked task deliberately retains its reservation; restore test global.
+        batch.AX_SLOTS.release()
+
+
+def test_apply_timeout_does_not_resubmit(tmp_path, monkeypatch):
+    import subprocess
+
+    executor, lane, events = setup(tmp_path, monkeypatch)
+
+    def ax(*args, **kwargs):
+        events.append(args[0])
+        if args[0] == "apply":
+            raise subprocess.TimeoutExpired("apply", 30)
+
+    lane.ax = ax
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert events == ["apply", "delete"]
+    assert "TimeoutExpired:" in record["reason"]
+
+
+def test_dispatch_rejects_duplicate_items_before_launch(tmp_path):
+    item = batch.Item("same", {}, "result.json", (), {}, "")
+
+    def unexpected(*args):
+        pytest.fail("duplicate inputs must be rejected before any dispatch")
+
+    with pytest.raises(ValueError, match="duplicate dispatch item"):
+        batch.dispatch([item, item], batch.Workload("image", frozenset(), 1, None), tmp_path, {"ax": unexpected}, lanes=("ax",))