search-with-proof: diagnosable failures, pip install race, no duplicate tasks

Reasons carry type plus a sanitised message tail; command failures carry phase,
exit code and bounded stderr. Scratch walk tolerates directories pip removes
mid-walk; writable /workspace/home and runtime paths. Capacity retries delete a
possibly created task before resubmitting. Docker smoke checks the image first.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/test_ax_batch.py

@@ -274,13 +274,13 @@ def test_capacity_backpressure_retries_same_manifest_until_deadline(tmp_path, mo
     )
     assert lane(item(), tmp_path, 0)["status"] == "ok"
     assert len(applies) >= 3 and len(set(applies)) == 1
-    assert len(deletes) == 1
+    assert len(deletes) == len(applies)
     busy[0] = True
     lane.pause = lambda seconds: now.__setitem__(0, now[0] + seconds)
     deletes.clear()
     result = lane(item(), tmp_path, 1)
     assert result["status"] == "capacity_timeout"
-    assert not deletes
+    assert deletes
     calls = []
 
     def no_vm(*args):

tests/test_search_executor.py

@@ -39,7 +39,7 @@ def test_paths_and_oversize(tmp_path):
 def test_environment_allowlist():
     assert clean_environment(
         {"OPENROUTER_API_KEY": "secret", "GEMINI_API_KEY": "secret", "PATH": "/evil", "AWS_SECRET_ACCESS_KEY": "secret"}
-    ) == {"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}
+    ) == {"PATH": "/venv/bin:/opt/java/openjdk/bin:/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8", "JAVA_HOME": "/opt/java/openjdk"}
 
 
 class Mailbox:
@@ -240,3 +240,79 @@ def test_sandbox_receives_exact_remaining_wall_limit(tmp_path, monkeypatch):
     InProcessExecutor(tmp_path).submit("run", {"command": ["/bin/true"]}, timeout_seconds=0.2)
     assert 0 < seen["probe"] <= 0.2
     assert 0 < seen["wall"] <= 0.2
+
+
+def test_spawn_failure_has_diagnostic(tmp_path):
+    result = InProcessExecutor(tmp_path, task_boundary=True).execute(Command(1, "run", {"command": ["/missing-executor-program"]}))
+    assert result["phase"] == "spawn"
+    assert result["exit_code"] is None
+    assert "FileNotFoundError:" in result["reason"]
+    assert "No such file" in result["reason"]
+
+
+def test_timeout_preserves_child_diagnostics(tmp_path):
+    import sys
+
+    result = InProcessExecutor(tmp_path, task_boundary=True).execute(
+        Command(
+            1,
+            "run",
+            {"command": [sys.executable, "-c", "import sys,time; print('before timeout',file=sys.stderr,flush=True); time.sleep(10)"]},
+            0.3,
+        )
+    )
+    assert result["phase"] == "timeout"
+    assert result["exit_code"] is not None
+    assert result["stderr"] == "before timeout"
+    assert result["reason"].startswith("TimeoutError:")
+
+
+def test_upload_failure_reports_phase(tmp_path, monkeypatch):
+    app = InProcessExecutor(tmp_path, task_boundary=True)
+
+    def upload(command):
+        app.phase = "result upload"
+        raise OSError("upload HTTP 403 https://host/path?X-Amz-Signature=secret")
+
+    monkeypatch.setattr(app, "_execute", upload)
+    result = app.execute(Command(1, "prepare_verification", {}))
+    assert result["phase"] == "result upload"
+    assert "HTTP 403" in result["reason"]
+    assert "secret" not in result["reason"]
+
+
+def test_diagnostics_sanitize_before_tail_and_keep_twenty_lines(monkeypatch):
+    from openultrasast.search.task_storage import diagnostic, exception_reason
+
+    monkeypatch.setenv("SERVICE_TOKEN", "environment-secret")
+    detail = (
+        "https://host/repo?X-Amz-Signature=url-secret Authorization: Bearer bearer-secret X-Amz-Signature=bare-secret environment-secret"
+    )
+    result = exception_reason(ValueError(detail))
+    assert result.startswith("ValueError:") and len(result) <= 300
+    for secret in ("url-secret", "bearer-secret", "bare-secret", "environment-secret", "host/repo"):
+        assert secret not in result
+    assert len(diagnostic("\n".join(str(i) for i in range(40)), maximum=4000).splitlines()) == 20
+    assert len(exception_reason(ValueError("x" * 1000))) <= 300
+
+
+def test_guard_io_failure_after_child_exit_is_not_success(tmp_path, monkeypatch):
+    import sys
+
+    from openultrasast.search import task_storage
+
+    checks = []
+    original = task_storage.check
+
+    def check(*args, **kwargs):
+        checks.append(1)
+        if len(checks) >= 3:
+            raise PermissionError("cache cannot be read")
+        return original(*args, **kwargs)
+
+    monkeypatch.setattr(task_storage, "check", check)
+    result = InProcessExecutor(tmp_path, task_boundary=True).execute(Command(1, "run", {"command": [sys.executable, "-c", "pass"]}))
+    assert result["error"] == "PermissionError"
+    assert result["phase"] == "scratch guard"
+    assert result["exit_code"] is not None
+    assert "cache cannot be read" in result["reason"]

tests/test_search_executor_smoke.py

@@ -142,9 +142,9 @@ def test_docker_managed_task_uses_docker_and_no_heap_flag(tmp_path, monkeypatch)
     )
     record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
     assert record["status"] == "ok"
-    assert [a[1] for a in calls] == ["run", "rm"]
-    assert "--heap-profile" not in calls[0]
-    assert calls[1][2] == "-f"
+    assert [a[1] for a in calls] == ["image", "run", "rm"]
+    assert "--heap-profile" not in calls[1]
+    assert calls[2][2] == "-f"
 
 
 def test_apply_exception_still_deletes(tmp_path, monkeypatch):
@@ -160,3 +160,104 @@ def test_apply_exception_still_deletes(tmp_path, monkeypatch):
     assert result["status"] == "instrument_failure"
     assert events == ["apply", "delete"]
     assert "secret.example" not in json.dumps(result)
+
+
+def test_smoke_retains_sanitized_command_failure(tmp_path, monkeypatch):
+    executor, lane, events = setup(tmp_path, monkeypatch)
+    original = executor._task_run
+
+    def fail(argv, timeout, limits):
+        if "pip" in argv:
+            return {
+                "exit_code": 17,
+                "stderr": "project failed: Bearer hidden https://host/project?q=secret",
+                "stdout": "",
+                "phase": "run",
+                "reason": "CommandFailure: exit code 17",
+            }
+        return original(argv, timeout, limits)
+
+    monkeypatch.setattr(executor, "_task_run", fail)
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    row = record["commands"][-1]
+    assert row["exit_code"] == 17 and row["phase"] == "run"
+    assert "CommandFailure" in row["reason"]
+    assert all(word not in json.dumps(record) for word in ("hidden", "secret", "project", "example"))
+    assert events == ["apply", "delete"]
+
+
+def test_docker_missing_image_is_diagnosable_without_pull(tmp_path, monkeypatch):
+    import subprocess
+
+    monkeypatch.setenv("S3_ENDPOINT", "https://files.example")
+    calls = []
+
+    def runner(argv, **kwargs):
+        calls.append(argv)
+        return subprocess.CompletedProcess(argv, 1, "", "No such image: pinned-image")
+
+    lane = batch.DockerLane(
+        SimpleNamespace(atespace="default"),
+        batch.Workload("image@sha256:" + "a" * 64, frozenset(), 900, None),
+        store=Store(Executor(tmp_path, task_boundary=True)),
+        runner=runner,
+    )
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert record["status"] == "instrument_failure"
+    assert "ValueError:" in record["reason"] and "No such image" in record["reason"]
+    assert [c[1] for c in calls] == ["image"]
+
+
+@pytest.mark.parametrize("delete_fails", [False, True])
+def test_capacity_partial_create_deleted_before_retry(tmp_path, monkeypatch, delete_fails):
+    executor, lane, events = setup(tmp_path, monkeypatch)
+    live, attempts = set(), []
+
+    def ax(*args, **kwargs):
+        events.append(args[0])
+        if args[0] == "apply":
+            assert not live
+            live.add(kwargs["manifest"]["metadata"]["name"])
+            attempts.append(1)
+            if len(attempts) == 1:
+                raise batch.NoFreeWorkers("partially provisioned")
+        elif args[0] == "delete":
+            if delete_fails:
+                raise OSError("cannot delete")
+            live.discard(args[2])
+
+    lane.ax, lane.pause = ax, lambda seconds: None
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert len(attempts) == (1 if delete_fails else 2)
+    assert record["status"] == ("instrument_failure" if delete_fails else "ok")
+    if not delete_fails:
+        assert events == ["apply", "delete", "apply", "delete"] and not live
+    else:
+        # Simulated leaked task deliberately retains its reservation; restore test global.
+        batch.AX_SLOTS.release()
+
+
+def test_apply_timeout_does_not_resubmit(tmp_path, monkeypatch):
+    import subprocess
+
+    executor, lane, events = setup(tmp_path, monkeypatch)
+
+    def ax(*args, **kwargs):
+        events.append(args[0])
+        if args[0] == "apply":
+            raise subprocess.TimeoutExpired("apply", 30)
+
+    lane.ax = ax
+    record = smoke.run_smoke(lane, "https://github.com/example/project", "a" * 40)
+    assert events == ["apply", "delete"]
+    assert "TimeoutExpired:" in record["reason"]
+
+
+def test_dispatch_rejects_duplicate_items_before_launch(tmp_path):
+    item = batch.Item("same", {}, "result.json", (), {}, "")
+
+    def unexpected(*args):
+        pytest.fail("duplicate inputs must be rejected before any dispatch")
+
+    with pytest.raises(ValueError, match="duplicate dispatch item"):
+        batch.dispatch([item, item], batch.Workload("image", frozenset(), 1, None), tmp_path, {"ax": unexpected}, lanes=("ax",))

tests/test_search_task_storage.py

@@ -61,7 +61,7 @@ def test_executor_checks_caches_and_stops_at_scratch_limit(workspace, monkeypatc
         "p=Path(os.environ['PIP_CACHE_DIR']); p.mkdir(); (p/'large').write_bytes(b'x'*30000); " + ("time.sleep(10)" if wait else "")
     )
     result = app.submit("run", {"command": [sys.executable, "-c", code]}, timeout_seconds=3)
-    assert result["status"] == "could_not_build" and result["reason"] == "scratch limit"
+    assert result["status"] == "could_not_build" and result["reason"] == "ScratchLimit: scratch limit"
     assert result["stopped"] and app.stopped
 
 
@@ -126,7 +126,7 @@ def test_verifier_reports_scratch_limit(workspace, monkeypatch):
     (demo / "demo.json").write_text(json.dumps(spec()["demo"]))
     monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "25000")
     result = verify_side_task(Side(repo), demo, "path")
-    assert result.outcome == "could_not_build" and result.reason == "scratch limit", result
+    assert result.outcome == "could_not_build" and result.reason == "ScratchLimit: scratch limit", result
 
 
 def test_preparation_builds_before_export_and_exports_products(workspace, monkeypatch):
@@ -165,9 +165,60 @@ def test_open_logs_count_toward_scratch_guard(workspace, monkeypatch):
     monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "20000")
     app = executor.InProcessExecutor(repo, task_boundary=True)
     result = app.submit("run", {"command": [sys.executable, "-c", "print('x'*30000)"]})
-    assert result["status"] == "could_not_build" and result["reason"] == "scratch limit"
+    assert result["status"] == "could_not_build" and result["reason"] == "ScratchLimit: scratch limit"
 
 
 def test_unreadable_scratch_is_not_reported_as_empty(tmp_path):
     with pytest.raises(OSError, match="scratch root unavailable"):
         task_storage.check(tmp_path / "missing")
+
+
+def test_pip_removing_enumerated_directory_is_not_guard_failure(workspace, monkeypatch):
+    original = task_storage.os.walk
+
+    def disappearing(root, **kwargs):
+        kwargs["onerror"](FileNotFoundError(2, "removed pip build directory", str(root / "pip-build")))
+        yield from original(root, **kwargs)
+
+    monkeypatch.setattr(task_storage.os, "walk", disappearing)
+    (workspace / "input").write_bytes(b"input")
+    assert task_storage.check(workspace) >= 5
+
+
+def test_guard_still_refuses_permission_errors(workspace, monkeypatch):
+    def unreadable(root, **kwargs):
+        kwargs["onerror"](PermissionError("permission denied"))
+        return iter(())
+
+    monkeypatch.setattr(task_storage.os, "walk", unreadable)
+    with pytest.raises(PermissionError):
+        task_storage.check(workspace)
+
+
+def test_child_environment_has_disk_home_and_owned_runtime_path(workspace):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "input").write_bytes(b"proof")
+    code = (
+        "import os; from pathlib import Path; print(Path('input').stat().st_size); "
+        "print(os.environ['PATH']); p=Path.home()/'probe'; p.write_text('ok'); print(p)"
+    )
+    result = executor.InProcessExecutor(repo, task_boundary=True).submit("run", {"command": [sys.executable, "-c", code]})
+    assert result["exit_code"] == 0
+    assert result["stdout"].startswith("5\n/venv/bin:")
+    assert (workspace / "home/probe").read_text() == "ok"
+
+
+def test_scratch_failure_preserves_exit_and_stderr(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    monkeypatch.setenv("OUSAST_SCRATCH_BYTES", "20000")
+    code = (
+        "import sys,time; from pathlib import Path; print('pip build detail',file=sys.stderr,flush=True); "
+        "Path('big').write_bytes(b'x'*30000); time.sleep(10)"
+    )
+    result = executor.InProcessExecutor(repo, task_boundary=True).submit("run", {"command": [sys.executable, "-c", code]})
+    assert result["phase"] == "scratch guard"
+    assert result["exit_code"] is not None
+    assert result["stderr"] == "pip build detail"
+    assert result["reason"].startswith("ScratchLimit:")

tests/test_search_worker.py

@@ -113,7 +113,7 @@ def test_reservation_precedes_each_call_and_refuses_over_budget(tmp_path):
     job = task()
     job.limits["max_call_usd"] = 0.000001
     result = worker(job)
-    assert result["failure"] == "BudgetExhausted"
+    assert result["failure"].startswith("BudgetExhausted: ")
     assert len(client.calls) == 1