search-with-proof: ecosystem-aware build recipes and stderr in build failures

The agent passes a manifest file or project directory; pip uses -r for
requirements files and the directory for pyproject/setup, npm ci/install runs in
the directory, composer/maven/gradle likewise. Build and verify failures carry
phase, exit code and a sanitised stderr tail.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

tests/test_ax_batch.py

@@ -385,3 +385,27 @@ def test_executor_slot_deadline_has_reason(monkeypatch):
     monkeypatch.setattr(batch, "AX_SLOTS", SimpleNamespace(acquire=lambda **kw: False))
     with pytest.raises(batch.CapacityDeadline, match="no free workers available before deadline"):
         batch.SearchExecutorTask(object(), b"archive").__enter__()
+
+
+def test_search_preparation_failure_retains_command_record():
+    import time
+    from types import SimpleNamespace
+
+    from benchmarks.ax.batch import SearchExecutorTask
+    from openultrasast.search.task_storage import CommandFailure
+
+    task = object.__new__(SearchExecutorTask)
+    task.end = time.monotonic() + 60
+    task.client = SimpleNamespace(
+        submit=lambda *args, **kwargs: {
+            "status": "could_not_build",
+            "phase": "build",
+            "exit_code": 8,
+            "reason": "CommandFailure: exit code 8",
+            "stderr": "dependency missing token=hidden",
+        }
+    )
+    with pytest.raises(CommandFailure) as caught:
+        task.prepare({})
+    assert caught.value.phase == "build" and caught.value.exit_code == 8
+    assert "dependency missing" in caught.value.stderr and "hidden" not in caught.value.stderr

tests/test_search_demo.py

@@ -71,11 +71,10 @@ def test_directory_build_recipe_accepts_checkout_root(recipe):
 
 
 @pytest.mark.parametrize("recipe", ["pip", "maven"])
-def test_manifest_recipe_requires_file_path(recipe):
+def test_manifest_recipe_accepts_project_directory(recipe):
     value = document()
     value["build"] = {"recipe": recipe, "arguments": ["."]}
-    with pytest.raises(ValueError):
-        validate_demo(value)
+    assert validate_demo(value) == value
 
 
 @pytest.mark.parametrize("oracle", [None, "injection", "unknown", [], 1])
@@ -111,3 +110,16 @@ def test_family_oracle_contract(family, allowed):
         else:
             with pytest.raises(ValueError, match="not allowed"):
                 validate_oracle(family, oracle)
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest",
+    [("pip", "package.json"), ("npm", "requirements.txt"), ("composer", "pom.xml"), ("maven", "composer.json"), ("gradle", "setup.py")],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_rejects_other_ecosystem(tmp_path, recipe, manifest, directory):
+    (tmp_path / manifest).write_text("{}")
+    value = document()
+    value["build"] = {"recipe": recipe, "arguments": ["." if directory else manifest]}
+    with pytest.raises(ValueError, match="ecosystem"):
+        validate_demo(value, checkout=tmp_path)

tests/test_search_pilot_run.py

@@ -480,3 +480,23 @@ def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, ora
     (tmp_path / "demo/demo.json").write_bytes((demos["real"] / "demo.json").read_bytes())
     result = pilot.verification()
     assert result.outcome == "demonstrated", result
+
+
+@pytest.mark.parametrize("phase", ["build_export", "verify/could_not_run"])
+def test_pilot_retains_command_diagnostics(tmp_path, phase):
+    from openultrasast.search.task_storage import CommandFailure
+    from openultrasast.search.verify import SideRecord
+
+    pilot = make_pilot(tmp_path)
+    stderr = "\n".join(f"line-{i}" for i in range(30)) + "\ntoken=hidden https://example.test/?key=hidden"
+    if phase == "build_export":
+        pilot.failure(phase, CommandFailure("build", 7, stderr))
+    else:
+        side = SideRecord("could_not_run", (), 0, "start failed", phase="start", exit_code=7, stderr=stderr)
+        pilot.failure(phase, side.reason, command=side)
+    failure = pilot.record["failures"][-1]
+    assert failure["exit_code"] == 7
+    assert failure["command_phase"] == ("build" if phase == "build_export" else "start")
+    assert "line-29" in failure["stderr"] and "line-0\n" not in failure["stderr"]
+    assert "hidden" not in failure["stderr"] and "https://" not in failure["stderr"]
+    assert len(failure["stderr"]) <= 1500

tests/test_search_probe.py

@@ -2,12 +2,22 @@ import json
 import sys
 from pathlib import Path
 
+import pytest
+
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 from openultrasast.search import probe
 from openultrasast.search._sandbox import IsolationUnavailable
 
 
+@pytest.fixture(autouse=True)
+def namespace_diagnostics(monkeypatch):
+    facts = {name: {"exit_code": 0} for name in ("user", "pid", "net", "ipc", "uts", "mount")}
+    monkeypatch.setattr(probe, "namespace_facts", lambda: facts)
+    monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "userns")
+    return facts
+
+
 def test_isolation_failure_is_recorded_and_nonzero(monkeypatch, capsys):
     def fail(**kwargs):
         raise IsolationUnavailable("gVisor denied namespace")
@@ -118,3 +128,121 @@ def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch):
     assert record["status"] == "instrument_failure"
     assert "net namespace denied" in record["exception"]
     assert record["isolation_check"]["status"] == "ok"
+
+
+@pytest.mark.parametrize("execution", ["real", "stubbed"])
+def test_task_boundary_probe_executes_packaged_pairs(tmp_path, monkeypatch, namespace_diagnostics, execution):
+    if execution == "real":
+        import socket
+
+        try:
+            with socket.socket() as listener:
+                listener.bind(("127.0.0.1", 0))
+        except PermissionError:
+            pytest.skip("host policy prohibits localhost sockets required by SSRF and Chromium")
+    else:
+        from openultrasast.search.verify import RunObservation, SideRecord
+
+        def run(side, demo, family):
+            assert (side.checkout / "app.py").read_bytes()
+            observed = not (side.checkout / "fixed").exists()
+            return SideRecord(
+                "observed",
+                (RunObservation(observed, "stubbed owned effect", 0.5),),
+                0.7,
+                "stubbed execution",
+                0,
+                0.2,
+                0.5,
+                4096,
+                "task-boundary",
+                json.loads((demo / "demo.json").read_text())["oracle"] == "xss",
+            )
+
+        monkeypatch.setattr(probe, "verify_side_task", run)
+    monkeypatch.setattr(probe._sandbox, "isolation_mode", lambda: "task-boundary")
+    monkeypatch.setattr(probe, "image_facts", lambda: {})
+    monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("task boundary must skip preflight"))
+    monkeypatch.setattr(probe, "verify", lambda *a, **kw: pytest.fail("must use task executor"))
+    record = probe.probe(tmp_path)
+    assert record["status"] == "ok", record
+    assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"}
+    assert record["isolation_mode"] == "task-boundary"
+    assert record["namespaces"] == namespace_diagnostics
+    assert set(record["oracles"]) == set(probe.FAMILIES)
+    for family, demos in record["oracles"].items():
+        assert set(demos) == {"real", "observer_access", "revision_patch"}
+        for name, result in demos.items():
+            assert (result["outcome"] == "demonstrated") == (name == "real")
+            assert result["isolation_mode"] == "task-boundary"
+            assert result["elapsed_seconds"] > 0
+        real = demos["real"]
+        assert real["chromium_no_sandbox"] == (family == "xss")
+        for index, side in enumerate(real["sides"]):
+            assert len(side["runs"]) == 3
+            assert all(run["observed"] == (index == 0) for run in side["runs"])
+            assert side["isolation_mode"] == "task-boundary"
+            assert side["chromium_no_sandbox"] == (family == "xss")
+            assert side["build_seconds"] == 0
+            assert side["ready_seconds"] > 0
+            assert side["run_seconds"] > 0
+            assert side["scratch_peak_bytes"] > 0
+    assert record["wall_seconds"] > 0
+    assert record["peak_rss_kib"]["sum"] > 0
+    assert record["scratch_disk_bytes"] > 0
+    assert record["verification_scratch_peak_bytes"] > 0
+    assert record["scratch_filesystem_used_bytes"] > 0
+
+
+@pytest.mark.parametrize("family", ["sql", "path", "command"])
+def test_task_boundary_real_local_oracles(tmp_path, family):
+    sides, demos = probe.materialise(tmp_path / family, family)
+    result = probe._verify_task_pair(sides, demos["real"], family)
+    assert result.outcome == "demonstrated", result
+    assert all(len(side.runs) == 3 for side in result.sides)
+
+
+def test_task_boundary_xss_uses_browser_mode(tmp_path, monkeypatch):
+    from openultrasast.search.oracles import BrowserExecutor
+
+    calls = []
+
+    def capture(self, document, nonce):
+        assert self.task_boundary and self.no_sandbox
+        calls.append(document)
+        return '<script>alert("ousast-xss")</script>' in document
+
+    monkeypatch.setattr(BrowserExecutor, "__call__", capture)
+    sides, demos = probe.materialise(tmp_path / "xss", "xss")
+    result = probe._verify_task_pair(sides, demos["real"], "xss")
+    assert result.outcome == "demonstrated"
+    assert result.chromium_no_sandbox
+    assert all(side.chromium_no_sandbox for side in result.sides)
+    assert len(calls) == 6
+
+
+@pytest.mark.parametrize("mismatch", ["real", "observer_access", "revision_patch", "exception"])
+def test_task_boundary_mismatch_names_first_and_finishes(tmp_path, monkeypatch, mismatch):
+    from openultrasast.search.verify import VerificationRecord
+
+    # On AX the diagnostics select the boundary before the failed bwrap preflight.
+    facts = {name: {"exit_code": 1} for name in ("user", "pid", "net", "ipc", "uts", "mount")}
+    monkeypatch.setattr(probe, "namespace_facts", lambda: facts)
+    monkeypatch.setattr(probe, "image_facts", lambda: {})
+    monkeypatch.setattr(probe._sandbox, "isolation_check", lambda **kw: pytest.fail("must skip preflight"))
+
+    def result(sides, demo, family):
+        if mismatch == "exception":
+            raise OSError("executor unavailable")
+        demonstrated = demo.name == "real"
+        if demo.name == mismatch:
+            demonstrated = not demonstrated
+        return VerificationRecord("demonstrated" if demonstrated else "inconclusive", (), 1, "", "task-boundary")
+
+    monkeypatch.setattr(probe, "_verify_task_pair", result)
+    record = probe.probe(tmp_path)
+    assert record["status"] == "instrument_failure"
+    assert record["exception"].startswith(f"sql/{'real' if mismatch == 'exception' else mismatch}:")
+    assert record["namespaces"] == facts
+    assert record["isolation_check"] == {"status": "skipped", "reason": "task-boundary"}
+    assert sum(len(demos) for demos in record["oracles"].values()) == 15

tests/test_search_task_storage.py

@@ -142,7 +142,7 @@ def test_preparation_builds_before_export_and_exports_products(workspace, monkey
     blobs, commands = [], []
     monkeypatch.setattr(executor.URLTransport, "put", lambda self, url, data, timeout: blobs.append(data))
 
-    def build(argv, timeout, limits):
+    def build(argv, timeout, limits, **kwargs):
         commands.append(argv)
         products = Path(argv[argv.index("--target") + 1])
         assert products.is_relative_to(workspace)
@@ -286,14 +286,19 @@ def test_preparation_with_many_host_processes(workspace, monkeypatch):
 
 @pytest.mark.parametrize("memory_bytes", [None, 256 * 1024**2, 5 * 1024**3])
 def test_prepare_build_virtual_memory(workspace, monkeypatch, memory_bytes):
-    from openultrasast.search.demo import BUILD_RECIPES
 
     repo = workspace / "checkout"
     repo.mkdir()
     source = "import mmap; region=mmap.mmap(-1,512*1024**2); print(len(region))"
     (repo / "app.py").write_text(source)
     print("input_bytes=", len((repo / "app.py").read_bytes()))
-    monkeypatch.setitem(BUILD_RECIPES, "npm", (sys.executable, "-c", source))
+    (repo / "package.json").write_text("{}")
+    fake_bin = workspace / "bin"
+    fake_bin.mkdir()
+    tool = fake_bin / "npm"
+    tool.write_text("#!" + sys.executable + "\n" + source)
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(fake_bin) + ":/usr/bin:/bin")
     app = executor.InProcessExecutor(repo, task_boundary=True)
     app.prepared_put_url = "https://files.example/output"
     blobs = []
@@ -317,3 +322,101 @@ def test_verifier_app_virtual_memory(workspace):
     demo.write_text(json.dumps(spec()["demo"]))
     result = verify_side_task(Side(repo), demo, "path")
     assert result.outcome == "observed", result
+
+
+@pytest.mark.parametrize(
+    "recipe,manifest,extra,expected",
+    [
+        ("pip", "pyproject.toml", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "PROJECT"]),
+        ("pip", "requirements.txt", {}, ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "MANIFEST"]),
+        (
+            "pip",
+            "pyproject.toml",
+            {"requirements.txt": ""},
+            ["-I", "-m", "pip", "install", "--no-deps", "--target", "PACKAGES", "-r", "REQUIREMENTS"],
+        ),
+        ("npm", "package.json", {}, ["install", "--ignore-scripts"]),
+        ("npm", "package.json", {"package-lock.json": "{}"}, ["ci", "--ignore-scripts"]),
+        ("composer", "composer.json", {}, ["install", "--no-interaction", "--no-plugins", "--no-scripts"]),
+        ("maven", "pom.xml", {}, ["-q", "-DskipTests", "package", "-f", "MANIFEST"]),
+    ],
+)
+@pytest.mark.parametrize("directory", [False, True])
+def test_recipe_command_with_real_fake_tool(workspace, monkeypatch, recipe, manifest, extra, expected, directory):
+    repo = workspace / "checkout"
+    project = repo / "project"
+    project.mkdir(parents=True)
+    (repo / "app.py").write_text("print('ready')")
+    contents = '[build-system]\nbuild-backend="setuptools.build_meta"\n' if manifest == "pyproject.toml" and not extra else ""
+    (project / manifest).write_text(contents)
+    for name, content in extra.items():
+        (project / name).write_text(content)
+    tools = workspace / "bin"
+    tools.mkdir()
+    tool = tools / {"pip": "python3", "maven": "mvn"}.get(recipe, recipe)
+    tool.write_text(
+        "#!" + sys.executable + "\nimport json, os, sys\nfrom pathlib import Path\n"
+        "Path('invocation.json').write_text(json.dumps([sys.argv[1:], os.getcwd()]))\n"
+    )
+    tool.chmod(0o755)
+    monkeypatch.setitem(executor.SAFE_ENV, "PATH", str(tools) + ":/usr/bin:/bin")
+    monkeypatch.setattr(executor.URLTransport, "put", lambda *args: None)
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": recipe, "arguments": ["project" if directory else "project/" + manifest]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "ok", result
+    argv, cwd = json.loads((project / "invocation.json").read_text())
+    assert cwd == str(project)
+    replacements = {"PROJECT": str(project), "MANIFEST": str(project / manifest), "REQUIREMENTS": str(project / "requirements.txt")}
+    if "--target" in argv:
+        packages = argv[argv.index("--target") + 1]
+        assert Path(packages).name == "packages" and Path(packages).is_relative_to(workspace)
+        replacements["PACKAGES"] = packages
+    assert argv == [replacements.get(arg, arg) for arg in expected]
+
+
+def test_failed_build_preserves_sanitized_tail(workspace, monkeypatch):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "gradlew").write_text(
+        "for i in $(seq 1 30); do echo line-$i >&2; done\necho 'https://example.test/?token=secret token=hidden' >&2\nexit 7\n"
+    )
+    app = executor.InProcessExecutor(repo, task_boundary=True)
+    app.prepared_put_url = "https://files.example/output"
+    request = spec()
+    request["demo"]["build"] = {"recipe": "gradle", "arguments": ["."]}
+    result = app.submit("prepare_verification", request)
+    assert result["status"] == "could_not_build" and result["exit_code"] == 7
+    assert result["phase"] == "build"
+    assert "line-30" in result["stderr"] and "line-1\n" not in result["stderr"]
+    assert "hidden" not in result["stderr"] and "https://" not in result["stderr"]
+    assert len(result["stderr"]) <= 1500 and len(result["stderr"].splitlines()) <= 20
+
+
+@pytest.mark.parametrize("mode", ["cli", "http"])
+def test_verifier_start_failure_has_diagnostics(workspace, mode):
+    repo = workspace / "checkout"
+    repo.mkdir()
+    (repo / "app.py").write_text(
+        "import sys\nprint('start failed token=hidden https://example.test/?secret=x', file=sys.stderr)\nsys.exit(9)"
+    )
+    demo = workspace / "demo.json"
+    value = spec()["demo"]
+    value["start"]["mode"] = mode
+    if mode == "http":
+        value["start"]["port"] = 19001
+        value["steps"] = [{"type": "http", "method": "GET", "path": "/"}]
+    demo.write_text(json.dumps(value))
+    result = verify_side_task(Side(repo), demo, "path")
+    assert result.outcome == "could_not_run", result
+    assert result.phase == "start" and result.exit_code == 9
+    assert "start failed" in result.stderr and "hidden" not in result.stderr and "https://" not in result.stderr
+
+
+@pytest.mark.parametrize("secret", ['token="hidden"', '"password": "hidden"', "Bearer hidden", "ghp_hidden", "npm_hidden"])
+def test_command_failure_sanitizes_tokens_and_caps_tail(secret):
+    failure = task_storage.CommandFailure("build", 1, "old\n" * 50 + "x" * 2000 + "\n" + secret)
+    assert "hidden" not in failure.stderr and "old" not in failure.stderr
+    assert len(failure.stderr) <= 1500