ci: pytest runs with the hunt and semantic extras kept; sandbox tests skip by cause where no bwrap exists

CI's first full pytest run since 2836b78 showed 65 failures in three classes. uv run re-synced the environment
to the lockfile's default extras before pytest, removing dspy again (UV_NO_SYNC=1 now). The learn slice, pair
corpus and hunt producer tests need tree-sitter (semantic extra installed). Thirty-two tests need bubblewrap
isolation, which CI's container cannot provide: they carry a sandbox marker and skip where the isolation
preflight fails, with the cause's type in the reason; proven locally by hiding bwrap (7 skipped, 23 passed).
The partial-clone extraction test now probes one lazy fetch through git itself first, so git's own stderr
reaches the next CI log instead of git_exit_128.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

dagger/ci.py

@@ -45,7 +45,10 @@ async def _run_checks() -> None:
             .with_directory("/src", source)
             .with_workdir("/src")
             .with_exec(["python", "-m", "pip", "install", "uv"])
-            .with_exec(["uv", "sync", "--group", "dev", "--extra", "hunt"])
+            # uv run re-syncs the environment to the lockfile's default extras unless told not to, which silently
+            # removed the extras below before pytest ran.
+            .with_env_variable("UV_NO_SYNC", "1")
+            .with_exec(["uv", "sync", "--group", "dev", "--extra", "hunt", "--extra", "semantic"])
             .with_exec(["uv", "run", "ruff", "format", "--check", "."])
             .with_exec(["uv", "run", "ruff", "check", "."])
             .with_exec(["uv", "run", "mypy", "src/openultrasast"])

pyproject.toml

@@ -52,6 +52,7 @@ pythonpath = ["src"]
 markers = [
     "docker: requires a docker daemon",
     "semantic: requires openultrasast[semantic] tree-sitter extra",
+    "sandbox: needs bubblewrap isolation; skipped where the isolation preflight fails (CI's container has no bwrap)",
     "ax: requires the live ax cluster of ops/ax/up.sh (skipped unless `ousast plane doctor` passes)",
 ]
 

tests/conftest.py

@@ -127,3 +127,23 @@ def plane_profile(tmp_path, monkeypatch: pytest.MonkeyPatch):
     for key in ("OUSAST_KUBE_CONTEXT", "OUSAST_MEMORY", "OUSAST_REGISTRY", "OUSAST_IMAGES", "OUSAST_EXEC"):
         monkeypatch.delenv(key, raising=False)
     return root / "test.toml"
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Tests marked ``sandbox`` need bubblewrap isolation; where the preflight fails they skip, named by cause.
+
+    CI's container has no bwrap and no user namespaces, so these tests can only run on a host; a skip there is
+    an honest statement, a failure there says nothing about the code."""
+    marked = [item for item in items if item.get_closest_marker("sandbox")]
+    if not marked:
+        return
+    from openultrasast.search import _sandbox
+
+    try:
+        _sandbox.isolation_check(timeout_seconds=30)
+    except _sandbox.IsolationUnavailable as exc:
+        reason = f"sandbox isolation unavailable ({type(exc.__cause__ or exc).__name__})"
+    else:
+        return
+    for item in marked:
+        item.add_marker(pytest.mark.skip(reason=reason))

tests/test_search_browser.py

@@ -17,6 +17,7 @@ def test_browser_launch_failure_is_not_negative(monkeypatch):
 
 
 @pytest.mark.skipif(not (shutil.which("chromium") or shutil.which("chromium-browser")), reason="chromium not installed")
+@pytest.mark.sandbox
 def test_real_browser_executes_not_reflects():
     browser = BrowserExecutor()
     payload = '<script>alert("ousast-xss")</script>'
@@ -34,6 +35,7 @@ def test_real_browser_executes_not_reflects():
 
 
 @pytest.mark.parametrize("executed", [False, True])
+@pytest.mark.sandbox
 def test_marker_is_dom_attribute_not_source_text(monkeypatch, executed):
     import re
     from urllib.parse import unquote

tests/test_search_isolation.py

@@ -164,6 +164,7 @@ def test_root_fixture_permissions_and_browser_records(monkeypatch, tmp_path):
     assert all(s.chromium_no_sandbox and s.isolation_mode == "root-no-userns" for s in record.sides)
 
 
+@pytest.mark.sandbox
 def test_devnull_can_be_opened_by_nested_launcher(tmp_path):
     """The HTTP driver's Popen opens /dev/null inside bwrap, before any socket."""
     _sandbox.isolation_check()

tests/test_search_pilot_run.py

@@ -11,6 +11,7 @@ from benchmarks.search.pilot_run import main
 
 
 @pytest.mark.parametrize("side,outcome", [("vulnerable", "demonstrated"), ("fixed", "inconclusive")])
+@pytest.mark.sandbox
 def test_probe_pilot_end_to_end(tmp_path, capsys, side, outcome):
     output = tmp_path / "records.jsonl"
     assert (
@@ -231,6 +232,7 @@ def test_shared_budget_reserved_and_settled_on_error(tmp_path):
 
 
 @pytest.mark.parametrize("verify_lane", ["vm", "ax"])
+@pytest.mark.sandbox
 def test_live_wiring_with_fake_mailbox_no_network(tmp_path, monkeypatch, verify_lane):
     import shutil
     from dataclasses import asdict
@@ -567,6 +569,7 @@ def test_live_driver_stops_remaining_pairs_on_infrastructure_failure(tmp_path, m
 
 
 @pytest.mark.parametrize("oracle", ["sql", "command"])
+@pytest.mark.sandbox
 def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, oracle):
     from openultrasast.search.probe import materialise
 

tests/test_search_probe.py

@@ -131,6 +131,7 @@ def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch):
 
 
 @pytest.mark.parametrize("execution", ["real", "stubbed"])
+@pytest.mark.sandbox
 def test_task_boundary_probe_executes_packaged_pairs(tmp_path, monkeypatch, namespace_diagnostics, execution):
     if execution == "real":
         import socket
@@ -202,6 +203,7 @@ def test_task_boundary_real_local_oracles(tmp_path, family):
     assert all(len(side.runs) == 3 for side in result.sides)
 
 
+@pytest.mark.sandbox
 def test_task_boundary_xss_uses_browser_mode(tmp_path, monkeypatch):
     from openultrasast.search.oracles import BrowserExecutor
 

tests/test_search_verify.py

@@ -24,6 +24,7 @@ def edit(demo, change):
 
 
 @pytest.mark.parametrize("family", ["path", "sql", "command", "ssrf"])
+@pytest.mark.sandbox
 def test_real_differential(tmp_path, family):
     if family == "ssrf":
         try:
@@ -59,6 +60,7 @@ def test_legacy_scripts_rejected_before_process(pair, monkeypatch):
     assert verify(*sides, demo, "path").outcome == "inconclusive"
 
 
+@pytest.mark.sandbox
 def test_safe_build_failure_inconclusive(pair):
     sides, demo = pair
     (sides[1].checkout / "app.py").unlink()
@@ -67,12 +69,14 @@ def test_safe_build_failure_inconclusive(pair):
     assert result.sides[1].outcome == "could_not_build"
 
 
+@pytest.mark.sandbox
 def test_affected_build_failure(pair):
     sides, demo = pair
     (sides[0].checkout / "app.py").unlink()
     assert verify(*sides, demo, "path").outcome == "could_not_build"
 
 
+@pytest.mark.sandbox
 def test_readiness_failure(pair):
     sides, demo = pair
     (sides[1].checkout / "broken").touch()
@@ -87,6 +91,7 @@ def test_no_oracle(pair, family):
     assert verify(*sides, demo, family).outcome == "no_oracle"
 
 
+@pytest.mark.sandbox
 def test_flaky_observation_inconclusive(pair, monkeypatch):
     from openultrasast.search import oracles
 
@@ -99,6 +104,7 @@ def test_flaky_observation_inconclusive(pair, monkeypatch):
     assert [r.observed for r in result.sides[0].runs] == [True, False, False]
 
 
+@pytest.mark.sandbox
 def test_no_inherited_environment(pair, monkeypatch):
     for key in ("PYTHONPATH", "NODE_OPTIONS", "LD_PRELOAD", "GEMINI_API_KEY"):
         monkeypatch.setenv(key, "/bogus-key")
@@ -106,12 +112,14 @@ def test_no_inherited_environment(pair, monkeypatch):
     assert verify(*sides, demo, "path").outcome == "demonstrated"
 
 
+@pytest.mark.sandbox
 def test_symlink_input_refused(pair):
     sides, demo = pair
     (sides[0].checkout / "escape").symlink_to("/etc/passwd")
     assert verify(*sides, demo, "path").outcome != "demonstrated"
 
 
+@pytest.mark.sandbox
 def test_six_fresh_canaries(pair, monkeypatch):
     from openultrasast.search import oracles
 
@@ -162,6 +170,7 @@ def test_task_side_runs_one_repetition_without_model_environment(pair, monkeypat
     assert result.isolation_mode == "task-boundary"
 
 
+@pytest.mark.sandbox
 def test_capture_canary_cannot_replay(pair):
     sides, demo = pair
     edit(demo, lambda value: value["steps"][0].update(capture={"name": "stolen", "source": "output"}))
@@ -170,6 +179,7 @@ def test_capture_canary_cannot_replay(pair):
     assert "canary values cannot be captured" in result.sides[0].reason
 
 
+@pytest.mark.sandbox
 def test_ordered_cli_capture_and_stdin(pair):
     sides, demo = pair
     app = """import json, os, sys
@@ -193,6 +203,7 @@ elif sys.argv[1:] == ['use', 'public-token'] and sys.stdin.read() == 'public-tok
     assert verify(*sides, demo, "path").outcome == "demonstrated"
 
 
+@pytest.mark.sandbox
 def test_http_steps_and_capture(pair):
     try:
         with socket.socket():
@@ -229,6 +240,7 @@ HTTPServer(('127.0.0.1', 18765), Handler).serve_forever()
     assert result.outcome == "demonstrated", result
 
 
+@pytest.mark.sandbox
 def test_private_namespace_handoff_without_sockets(tmp_path):
     import subprocess
 
@@ -311,6 +323,7 @@ def test_listener_accepts_nonce_path_for_post_without_sockets(monkeypatch, capsy
     assert capsys.readouterr().out.splitlines()[1:] == ["observed"]
 
 
+@pytest.mark.sandbox
 def test_pip_products_imported_by_runtime_and_immutable(pair, monkeypatch):
     from openultrasast.sandbox import SandboxResult
 
@@ -354,6 +367,7 @@ def test_dispatcher_incomplete_repetitions_never_demonstrate(pair, monkeypatch,
     assert verify(*sides, demo, "path", task_dispatcher=dispatch).outcome == "inconclusive"
 
 
+@pytest.mark.sandbox
 def test_json_file_accepted_without_directory_wrapper(pair):
     sides, demo = pair
     assert verify(*sides, demo / "demo.json", "path").outcome == "demonstrated"
@@ -495,6 +509,7 @@ def test_http_failure_keeps_exception_tail(tmp_path):
         ("untrusted_destination", "ssrf"),
     ],
 )
+@pytest.mark.sandbox
 def test_verifier_routes_demo_oracle(pair, monkeypatch, family, oracle):
     from openultrasast.search import verify as module
     from openultrasast.search.oracles import BrowserExecutor
@@ -525,6 +540,7 @@ def test_xss_without_browser_is_unavailable_not_no_oracle(pair):
     assert result.outcome == "inconclusive" and "runtime unavailable" in result.reason
 
 
+@pytest.mark.sandbox
 def test_app_requires_environment_and_data_files_on_both_sides(pair):
     sides, demo = pair
     for side in sides:
@@ -701,6 +717,7 @@ def test_http_driver_polls_eight_second_listener_before_get_and_steps(tmp_path,
 
 @pytest.mark.parametrize("task_boundary", [False, True])
 @pytest.mark.parametrize("config", ["arguments", "environment", "files"])
+@pytest.mark.sandbox
 def test_node_directory_option_uses_owned_fixture(pair, config, task_boundary):
     sides, demo = pair
     app = """const fs = require('fs'), path = require('path');
@@ -743,6 +760,7 @@ if (!fs.existsSync(path.join(__dirname, 'fixed')) || target.startsWith(root + '/
 
 @pytest.mark.parametrize("placeholder", ["database_path", "database_url"])
 @pytest.mark.parametrize("task_boundary", [False, True])
+@pytest.mark.sandbox
 def test_sql_cli_database_fixture(tmp_path, placeholder, task_boundary):
     sides, demos = materialise(tmp_path / "sql", "sql")
     for side in sides:
@@ -765,6 +783,7 @@ def test_sql_cli_database_fixture(tmp_path, placeholder, task_boundary):
 
 
 @pytest.mark.parametrize("task_boundary", [False, True])
+@pytest.mark.sandbox
 def test_command_marker_directory_configuration(tmp_path, task_boundary):
     sides, demos = materialise(tmp_path / "command", "command")
     for side in sides:
@@ -786,6 +805,7 @@ def test_command_marker_directory_configuration(tmp_path, task_boundary):
     assert all(not run.observed for run in records[1].runs)
 
 
+@pytest.mark.sandbox
 def test_fixture_values_and_nonce_redacted_from_diagnostics(pair, monkeypatch):
     from dataclasses import asdict
 

tests/test_search_worker.py

@@ -52,6 +52,7 @@ def test_tools_bounds_and_path_escape(tmp_path):
         worker.tool("write_demo", {"path": "../escape", "content": "bad"})
 
 
+@pytest.mark.sandbox
 def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
     monkeypatch.setenv("OPENROUTER_API_KEY", "must-not-reach-repo")
     worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
@@ -68,6 +69,7 @@ def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch):
     assert "must-not-reach-repo" not in json.dumps(result)
 
 
+@pytest.mark.sandbox
 def test_coordinator_stub_smoke_real_verify(tmp_path):
     sides, probe_demos = materialise(tmp_path / "pair", "path")
     demo = tmp_path / "demo"
@@ -131,6 +133,7 @@ def test_failed_model_call_charged_and_secrets_not_reported(tmp_path):
     assert "super-private-key" not in json.dumps(result)
 
 
+@pytest.mark.sandbox
 def test_run_timeout_truncation_and_disk_bound(tmp_path):
     worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path))
     result = worker.tool("run", {"command": ["/bin/sleep", "3"], "timeout_seconds": 1})
@@ -210,6 +213,7 @@ def test_write_demo_accepts_only_declarative_json(tmp_path):
         worker.tool("write_demo", {"path": "build.sh", "content": "touch /tmp/pwn"})
 
 
+@pytest.mark.sandbox
 def test_inprocess_executor_observes_all_brain_repository_commands(tmp_path):
     class RecordingExecutor(InProcessExecutor):
         def __init__(self, repo):

tests/test_unseen_extract.py

@@ -666,6 +666,10 @@ def test_partial_clone_lazy_fetch_preserves_extraction_content(history, tmp_path
     partial = e.Git(path, runner=runner)
     missing = partial.run("rev-list", "--objects", "--missing=print", "HEAD")
     assert sum(line.startswith("?") for line in missing.splitlines()) >= 60
+    # Instrument first: one lazy fetch through git itself, with git's own stderr in the failure (the extractor
+    # records exit codes only, so a transport refusal in another environment would otherwise read as git_exit_128).
+    probe = runner(["git", "-C", str(path), "cat-file", "-p", "HEAD:file0.py"], capture_output=True, text=True)
+    assert probe.returncode == 0, probe.stderr
     result = [e.introducing(partial, candidate), e.ordinary_history(partial, "HEAD")]
     assert partial.files_read > 0 and partial.bytes_read > 0
     assert json.dumps(result, sort_keys=True).encode() == json.dumps(baseline, sort_keys=True).encode()