diff --git a/dagger/ci.py b/dagger/ci.py index 4e377c9..80197ca 100644 --- a/dagger/ci.py +++ b/dagger/ci.py @@ -45,7 +45,10 @@ async def _run_checks() -> None: .with_directory("/src", source) .with_workdir("/src") .with_exec(["python", "-m", "pip", "install", "uv"]) - .with_exec(["uv", "sync", "--group", "dev", "--extra", "hunt"]) + # uv run re-syncs the environment to the lockfile's default extras unless told not to, which silently + # removed the extras below before pytest ran. + .with_env_variable("UV_NO_SYNC", "1") + .with_exec(["uv", "sync", "--group", "dev", "--extra", "hunt", "--extra", "semantic"]) .with_exec(["uv", "run", "ruff", "format", "--check", "."]) .with_exec(["uv", "run", "ruff", "check", "."]) .with_exec(["uv", "run", "mypy", "src/openultrasast"]) diff --git a/pyproject.toml b/pyproject.toml index 877269c..c039b0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ pythonpath = ["src"] markers = [ "docker: requires a docker daemon", "semantic: requires openultrasast[semantic] tree-sitter extra", + "sandbox: needs bubblewrap isolation; skipped where the isolation preflight fails (CI's container has no bwrap)", "ax: requires the live ax cluster of ops/ax/up.sh (skipped unless `ousast plane doctor` passes)", ] diff --git a/tests/conftest.py b/tests/conftest.py index c5a43f5..08ea116 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,3 +127,23 @@ def plane_profile(tmp_path, monkeypatch: pytest.MonkeyPatch): for key in ("OUSAST_KUBE_CONTEXT", "OUSAST_MEMORY", "OUSAST_REGISTRY", "OUSAST_IMAGES", "OUSAST_EXEC"): monkeypatch.delenv(key, raising=False) return root / "test.toml" + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Tests marked ``sandbox`` need bubblewrap isolation; where the preflight fails they skip, named by cause. + + CI's container has no bwrap and no user namespaces, so these tests can only run on a host; a skip there is + an honest statement, a failure there says nothing about the code.""" + marked = [item for item in items if item.get_closest_marker("sandbox")] + if not marked: + return + from openultrasast.search import _sandbox + + try: + _sandbox.isolation_check(timeout_seconds=30) + except _sandbox.IsolationUnavailable as exc: + reason = f"sandbox isolation unavailable ({type(exc.__cause__ or exc).__name__})" + else: + return + for item in marked: + item.add_marker(pytest.mark.skip(reason=reason)) diff --git a/tests/test_search_browser.py b/tests/test_search_browser.py index f2d2074..82a0c61 100644 --- a/tests/test_search_browser.py +++ b/tests/test_search_browser.py @@ -17,6 +17,7 @@ def test_browser_launch_failure_is_not_negative(monkeypatch): @pytest.mark.skipif(not (shutil.which("chromium") or shutil.which("chromium-browser")), reason="chromium not installed") +@pytest.mark.sandbox def test_real_browser_executes_not_reflects(): browser = BrowserExecutor() payload = '' @@ -34,6 +35,7 @@ def test_real_browser_executes_not_reflects(): @pytest.mark.parametrize("executed", [False, True]) +@pytest.mark.sandbox def test_marker_is_dom_attribute_not_source_text(monkeypatch, executed): import re from urllib.parse import unquote diff --git a/tests/test_search_isolation.py b/tests/test_search_isolation.py index efaa14e..087ac63 100644 --- a/tests/test_search_isolation.py +++ b/tests/test_search_isolation.py @@ -164,6 +164,7 @@ def test_root_fixture_permissions_and_browser_records(monkeypatch, tmp_path): assert all(s.chromium_no_sandbox and s.isolation_mode == "root-no-userns" for s in record.sides) +@pytest.mark.sandbox def test_devnull_can_be_opened_by_nested_launcher(tmp_path): """The HTTP driver's Popen opens /dev/null inside bwrap, before any socket.""" _sandbox.isolation_check() diff --git a/tests/test_search_pilot_run.py b/tests/test_search_pilot_run.py index a2aabbd..619becc 100644 --- a/tests/test_search_pilot_run.py +++ b/tests/test_search_pilot_run.py @@ -11,6 +11,7 @@ from benchmarks.search.pilot_run import main @pytest.mark.parametrize("side,outcome", [("vulnerable", "demonstrated"), ("fixed", "inconclusive")]) +@pytest.mark.sandbox def test_probe_pilot_end_to_end(tmp_path, capsys, side, outcome): output = tmp_path / "records.jsonl" assert ( @@ -231,6 +232,7 @@ def test_shared_budget_reserved_and_settled_on_error(tmp_path): @pytest.mark.parametrize("verify_lane", ["vm", "ax"]) +@pytest.mark.sandbox def test_live_wiring_with_fake_mailbox_no_network(tmp_path, monkeypatch, verify_lane): import shutil from dataclasses import asdict @@ -567,6 +569,7 @@ def test_live_driver_stops_remaining_pairs_on_infrastructure_failure(tmp_path, m @pytest.mark.parametrize("oracle", ["sql", "command"]) +@pytest.mark.sandbox def test_injection_pilot_uses_demo_choice_without_manifest_subtype(tmp_path, oracle): from openultrasast.search.probe import materialise diff --git a/tests/test_search_probe.py b/tests/test_search_probe.py index 8f7126e..9f49bf1 100644 --- a/tests/test_search_probe.py +++ b/tests/test_search_probe.py @@ -131,6 +131,7 @@ def test_oracle_namespace_failure_is_instrument_failure(tmp_path, monkeypatch): @pytest.mark.parametrize("execution", ["real", "stubbed"]) +@pytest.mark.sandbox def test_task_boundary_probe_executes_packaged_pairs(tmp_path, monkeypatch, namespace_diagnostics, execution): if execution == "real": import socket @@ -202,6 +203,7 @@ def test_task_boundary_real_local_oracles(tmp_path, family): assert all(len(side.runs) == 3 for side in result.sides) +@pytest.mark.sandbox def test_task_boundary_xss_uses_browser_mode(tmp_path, monkeypatch): from openultrasast.search.oracles import BrowserExecutor diff --git a/tests/test_search_verify.py b/tests/test_search_verify.py index 98c2556..9aecebc 100644 --- a/tests/test_search_verify.py +++ b/tests/test_search_verify.py @@ -24,6 +24,7 @@ def edit(demo, change): @pytest.mark.parametrize("family", ["path", "sql", "command", "ssrf"]) +@pytest.mark.sandbox def test_real_differential(tmp_path, family): if family == "ssrf": try: @@ -59,6 +60,7 @@ def test_legacy_scripts_rejected_before_process(pair, monkeypatch): assert verify(*sides, demo, "path").outcome == "inconclusive" +@pytest.mark.sandbox def test_safe_build_failure_inconclusive(pair): sides, demo = pair (sides[1].checkout / "app.py").unlink() @@ -67,12 +69,14 @@ def test_safe_build_failure_inconclusive(pair): assert result.sides[1].outcome == "could_not_build" +@pytest.mark.sandbox def test_affected_build_failure(pair): sides, demo = pair (sides[0].checkout / "app.py").unlink() assert verify(*sides, demo, "path").outcome == "could_not_build" +@pytest.mark.sandbox def test_readiness_failure(pair): sides, demo = pair (sides[1].checkout / "broken").touch() @@ -87,6 +91,7 @@ def test_no_oracle(pair, family): assert verify(*sides, demo, family).outcome == "no_oracle" +@pytest.mark.sandbox def test_flaky_observation_inconclusive(pair, monkeypatch): from openultrasast.search import oracles @@ -99,6 +104,7 @@ def test_flaky_observation_inconclusive(pair, monkeypatch): assert [r.observed for r in result.sides[0].runs] == [True, False, False] +@pytest.mark.sandbox def test_no_inherited_environment(pair, monkeypatch): for key in ("PYTHONPATH", "NODE_OPTIONS", "LD_PRELOAD", "GEMINI_API_KEY"): monkeypatch.setenv(key, "/bogus-key") @@ -106,12 +112,14 @@ def test_no_inherited_environment(pair, monkeypatch): assert verify(*sides, demo, "path").outcome == "demonstrated" +@pytest.mark.sandbox def test_symlink_input_refused(pair): sides, demo = pair (sides[0].checkout / "escape").symlink_to("/etc/passwd") assert verify(*sides, demo, "path").outcome != "demonstrated" +@pytest.mark.sandbox def test_six_fresh_canaries(pair, monkeypatch): from openultrasast.search import oracles @@ -162,6 +170,7 @@ def test_task_side_runs_one_repetition_without_model_environment(pair, monkeypat assert result.isolation_mode == "task-boundary" +@pytest.mark.sandbox def test_capture_canary_cannot_replay(pair): sides, demo = pair edit(demo, lambda value: value["steps"][0].update(capture={"name": "stolen", "source": "output"})) @@ -170,6 +179,7 @@ def test_capture_canary_cannot_replay(pair): assert "canary values cannot be captured" in result.sides[0].reason +@pytest.mark.sandbox def test_ordered_cli_capture_and_stdin(pair): sides, demo = pair app = """import json, os, sys @@ -193,6 +203,7 @@ elif sys.argv[1:] == ['use', 'public-token'] and sys.stdin.read() == 'public-tok assert verify(*sides, demo, "path").outcome == "demonstrated" +@pytest.mark.sandbox def test_http_steps_and_capture(pair): try: with socket.socket(): @@ -229,6 +240,7 @@ HTTPServer(('127.0.0.1', 18765), Handler).serve_forever() assert result.outcome == "demonstrated", result +@pytest.mark.sandbox def test_private_namespace_handoff_without_sockets(tmp_path): import subprocess @@ -311,6 +323,7 @@ def test_listener_accepts_nonce_path_for_post_without_sockets(monkeypatch, capsy assert capsys.readouterr().out.splitlines()[1:] == ["observed"] +@pytest.mark.sandbox def test_pip_products_imported_by_runtime_and_immutable(pair, monkeypatch): from openultrasast.sandbox import SandboxResult @@ -354,6 +367,7 @@ def test_dispatcher_incomplete_repetitions_never_demonstrate(pair, monkeypatch, assert verify(*sides, demo, "path", task_dispatcher=dispatch).outcome == "inconclusive" +@pytest.mark.sandbox def test_json_file_accepted_without_directory_wrapper(pair): sides, demo = pair assert verify(*sides, demo / "demo.json", "path").outcome == "demonstrated" @@ -495,6 +509,7 @@ def test_http_failure_keeps_exception_tail(tmp_path): ("untrusted_destination", "ssrf"), ], ) +@pytest.mark.sandbox def test_verifier_routes_demo_oracle(pair, monkeypatch, family, oracle): from openultrasast.search import verify as module from openultrasast.search.oracles import BrowserExecutor @@ -525,6 +540,7 @@ def test_xss_without_browser_is_unavailable_not_no_oracle(pair): assert result.outcome == "inconclusive" and "runtime unavailable" in result.reason +@pytest.mark.sandbox def test_app_requires_environment_and_data_files_on_both_sides(pair): sides, demo = pair for side in sides: @@ -701,6 +717,7 @@ def test_http_driver_polls_eight_second_listener_before_get_and_steps(tmp_path, @pytest.mark.parametrize("task_boundary", [False, True]) @pytest.mark.parametrize("config", ["arguments", "environment", "files"]) +@pytest.mark.sandbox def test_node_directory_option_uses_owned_fixture(pair, config, task_boundary): sides, demo = pair app = """const fs = require('fs'), path = require('path'); @@ -743,6 +760,7 @@ if (!fs.existsSync(path.join(__dirname, 'fixed')) || target.startsWith(root + '/ @pytest.mark.parametrize("placeholder", ["database_path", "database_url"]) @pytest.mark.parametrize("task_boundary", [False, True]) +@pytest.mark.sandbox def test_sql_cli_database_fixture(tmp_path, placeholder, task_boundary): sides, demos = materialise(tmp_path / "sql", "sql") for side in sides: @@ -765,6 +783,7 @@ def test_sql_cli_database_fixture(tmp_path, placeholder, task_boundary): @pytest.mark.parametrize("task_boundary", [False, True]) +@pytest.mark.sandbox def test_command_marker_directory_configuration(tmp_path, task_boundary): sides, demos = materialise(tmp_path / "command", "command") for side in sides: @@ -786,6 +805,7 @@ def test_command_marker_directory_configuration(tmp_path, task_boundary): assert all(not run.observed for run in records[1].runs) +@pytest.mark.sandbox def test_fixture_values_and_nonce_redacted_from_diagnostics(pair, monkeypatch): from dataclasses import asdict diff --git a/tests/test_search_worker.py b/tests/test_search_worker.py index 50534e2..019d325 100644 --- a/tests/test_search_worker.py +++ b/tests/test_search_worker.py @@ -52,6 +52,7 @@ def test_tools_bounds_and_path_escape(tmp_path): worker.tool("write_demo", {"path": "../escape", "content": "bad"}) +@pytest.mark.sandbox def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch): monkeypatch.setenv("OPENROUTER_API_KEY", "must-not-reach-repo") worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path)) @@ -68,6 +69,7 @@ def test_run_scrubs_model_key_in_real_sandbox(tmp_path, monkeypatch): assert "must-not-reach-repo" not in json.dumps(result) +@pytest.mark.sandbox def test_coordinator_stub_smoke_real_verify(tmp_path): sides, probe_demos = materialise(tmp_path / "pair", "path") demo = tmp_path / "demo" @@ -131,6 +133,7 @@ def test_failed_model_call_charged_and_secrets_not_reported(tmp_path): assert "super-private-key" not in json.dumps(result) +@pytest.mark.sandbox def test_run_timeout_truncation_and_disk_bound(tmp_path): worker = Worker(StubModel([]), repo=tmp_path, demo=tmp_path / "demo", executor=InProcessExecutor(tmp_path)) result = worker.tool("run", {"command": ["/bin/sleep", "3"], "timeout_seconds": 1}) @@ -210,6 +213,7 @@ def test_write_demo_accepts_only_declarative_json(tmp_path): worker.tool("write_demo", {"path": "build.sh", "content": "touch /tmp/pwn"}) +@pytest.mark.sandbox def test_inprocess_executor_observes_all_brain_repository_commands(tmp_path): class RecordingExecutor(InProcessExecutor): def __init__(self, repo): diff --git a/tests/test_unseen_extract.py b/tests/test_unseen_extract.py index bf7555a..4537576 100644 --- a/tests/test_unseen_extract.py +++ b/tests/test_unseen_extract.py @@ -666,6 +666,10 @@ def test_partial_clone_lazy_fetch_preserves_extraction_content(history, tmp_path partial = e.Git(path, runner=runner) missing = partial.run("rev-list", "--objects", "--missing=print", "HEAD") assert sum(line.startswith("?") for line in missing.splitlines()) >= 60 + # Instrument first: one lazy fetch through git itself, with git's own stderr in the failure (the extractor + # records exit codes only, so a transport refusal in another environment would otherwise read as git_exit_128). + probe = runner(["git", "-C", str(path), "cat-file", "-p", "HEAD:file0.py"], capture_output=True, text=True) + assert probe.returncode == 0, probe.stderr result = [e.introducing(partial, candidate), e.ordinary_history(partial, "HEAD")] assert partial.files_read > 0 and partial.bytes_read > 0 assert json.dumps(result, sort_keys=True).encode() == json.dumps(baseline, sort_keys=True).encode()