Open_Ultra_SAST/tests/test_sandbox_runner.py

from __future__ import annotations

import os
import stat
import subprocess
from pathlib import Path
from typing import Any

import pytest

from openultrasast.sandbox import (
    DockerCliRunner,
    FakeSandboxRunner,
    SandboxJob,
    SandboxResult,
    SandboxRunner,
    build_docker_argv,
    resolve_sandbox_runner,
)
from openultrasast.sandbox import runner as runner_mod


def _job(tmp_path: Path, **overrides: object) -> SandboxJob:
    repo = tmp_path / "repo"
    repo.mkdir(exist_ok=True)
    (repo / "app.py").write_text("print(1)\n")
    values: dict[str, object] = {
        "image": "python:3.12-alpine",
        "command": ("python", "/scratch/case.py"),
        "repo_root": repo,
        "scratch_files": {"case.py": "print('ok')\n"},
        "timeout_seconds": 30,
        "memory_mb": 256,
        "pids_limit": 128,
    }
    values.update(overrides)
    return SandboxJob(**values)  # type: ignore[arg-type]


def _flag_value(argv: list[str], flag: str) -> str:
    try:
        return argv[argv.index(flag) + 1]
    except (ValueError, IndexError) as exc:
        raise AssertionError(f"{flag} missing from {argv}") from exc


def _mount_options(spec: str) -> dict[str, str]:
    options: dict[str, str] = {}
    for part in spec.split(","):
        if "=" in part:
            key, value = part.split("=", 1)
            options[key] = value
        else:
            options[part] = "true"
    return options


def _workspace_mount(argv: list[str]) -> dict[str, str]:
    specs: list[str] = []
    for index, item in enumerate(argv):
        if item in {"--mount", "-v", "--volume"} and index + 1 < len(argv):
            specs.append(argv[index + 1])
    for spec in specs:
        options = _mount_options(spec)
        target = options.get("dst") or options.get("destination") or options.get("target")
        if target == "/workspace" or spec.endswith(":/workspace") or ":/workspace:" in spec:
            return options if target == "/workspace" else {"raw": spec}
        if "/workspace" in spec:
            return options if options else {"raw": spec}
    raise AssertionError(f"no /workspace mount in {argv}")


def _seed_source(argv: list[str]) -> Path:
    mounts = [argv[index + 1] for index, item in enumerate(argv) if item == "--mount" and index + 1 < len(argv)]
    seed_mounts = [spec for spec in mounts if "/scratch-in" in spec]
    assert seed_mounts, argv
    options = _mount_options(seed_mounts[0])
    source = Path(options.get("src") or options.get("source") or "")
    assert source.is_dir(), argv
    return source


def _copy_script(argv: list[str]) -> str:
    try:
        return argv[argv.index("-c") + 1]
    except (ValueError, IndexError) as exc:
        raise AssertionError(f"no /bin/sh -c wrapper in {argv}") from exc


def _posix_mode(path: Path) -> int:
    return path.stat().st_mode & 0o777


def test_build_docker_argv_starts_with_docker_run_rm_init(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    assert argv[0] == "docker"
    assert argv[1] == "run"
    assert "--rm" in argv
    assert "--init" in argv


def test_build_docker_argv_includes_network_none_and_read_only(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    joined = " ".join(argv)
    assert "--read-only" in argv
    assert "--network none" in joined or "--network=none" in argv
    assert "--network host" not in joined
    assert "--network=host" not in argv


def test_build_docker_argv_excludes_docker_sock(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    joined = " ".join(argv)
    assert "docker.sock" not in joined
    assert "/var/run/docker.sock" not in joined
    assert "-v" not in argv
    assert "--volume" not in argv


def test_build_docker_argv_forbids_privileged_and_host_network(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    assert "--privileged" not in argv
    assert "--pid" not in argv or _flag_value(argv, "--pid") != "host"
    assert "--ipc" not in argv or _flag_value(argv, "--ipc") != "host"
    assert "--userns" not in argv or "host" not in _flag_value(argv, "--userns")


def test_build_docker_argv_scratch_is_writable_tmpfs(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    tmpfs = _flag_value(argv, "--tmpfs")
    assert tmpfs.startswith("/scratch")
    assert "rw" in tmpfs
    assert "exec" in tmpfs


def test_build_docker_argv_applies_memory_and_pids_limits(tmp_path: Path) -> None:
    job = _job(tmp_path, memory_mb=512, pids_limit=64)
    argv = build_docker_argv(job)
    memory = _flag_value(argv, "--memory")
    assert "512" in memory
    assert memory.endswith("m") or memory.endswith("M") or memory.endswith("b")
    assert _flag_value(argv, "--pids-limit") == "64"


def test_build_docker_argv_drops_capabilities_and_new_privileges(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    assert _flag_value(argv, "--cap-drop") == "ALL"
    security = _flag_value(argv, "--security-opt")
    assert "no-new-privileges" in security


def test_build_docker_argv_uses_non_root_user(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    user = _flag_value(argv, "--user")
    uid = user.split(":", 1)[0]
    assert uid not in {"0", "root"}
    assert int(uid) != 0


def test_build_docker_argv_bind_mounts_repo_read_only_at_workspace(tmp_path: Path) -> None:
    job = _job(tmp_path)
    argv = build_docker_argv(job)
    mount = _workspace_mount(argv)
    source = mount.get("src") or mount.get("source")
    assert source == str(job.repo_root.resolve())
    assert mount.get("dst") == "/workspace" or mount.get("destination") == "/workspace" or mount.get("target") == "/workspace"
    assert mount.get("ro") == "true" or mount.get("readonly") == "true"
    assert "rw" not in mount


def test_build_docker_argv_source_tree_is_not_writable(tmp_path: Path) -> None:
    argv = build_docker_argv(_job(tmp_path))
    mount = _workspace_mount(argv)
    assert mount.get("ro") == "true" or mount.get("readonly") == "true"
    assert "--read-only" in argv
    joined = " ".join(argv)
    assert f"{_job(tmp_path).repo_root.resolve()}:/workspace:rw" not in joined
    assert f"{_job(tmp_path).repo_root.resolve()}:/workspace " not in joined + " "


def test_build_docker_argv_ends_with_image_and_command(tmp_path: Path) -> None:
    job = _job(tmp_path)
    argv = build_docker_argv(job)
    assert argv[-len(job.command) :] == list(job.command)
    assert argv[-len(job.command) - 1] == job.image


def test_fake_sandbox_runner_records_jobs_and_returns_programmed_result(tmp_path: Path) -> None:
    programmed = SandboxResult(exit_code=7, stdout="out", stderr="err", timed_out=False)
    runner: SandboxRunner = FakeSandboxRunner(programmed)
    job = _job(tmp_path)
    result = runner.run(job)
    assert result == programmed
    assert isinstance(runner, FakeSandboxRunner)
    assert runner.jobs == [job]


def test_resolve_runner_fake_via_env(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv(runner_mod.RUNNER_ENV, "fake")
    assert isinstance(resolve_sandbox_runner(), FakeSandboxRunner)


def test_resolve_runner_defaults_to_docker_cli(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.delenv(runner_mod.RUNNER_ENV, raising=False)
    assert isinstance(resolve_sandbox_runner(), DockerCliRunner)


def test_fake_sandbox_runner_can_be_reprogrammed(tmp_path: Path) -> None:
    runner = FakeSandboxRunner()
    first = _job(tmp_path, command=("echo", "one"))
    second = _job(tmp_path, command=("echo", "two"))
    timeout_result = SandboxResult(exit_code=-1, stdout="", stderr="deadline", timed_out=True)
    runner.result = timeout_result
    assert runner.run(first).timed_out is True
    runner.result = SandboxResult(exit_code=0, stdout="ok", stderr="", timed_out=False)
    assert runner.run(second).stdout == "ok"
    assert runner.jobs == [first, second]


def test_fake_sandbox_runner_does_not_need_docker_daemon(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    def boom(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]:
        raise AssertionError("unit tests must not invoke the docker daemon")

    monkeypatch.setattr(subprocess, "run", boom)
    monkeypatch.setattr(runner_mod.subprocess, "run", boom)
    programmed = SandboxResult(exit_code=0, stdout="fake", stderr="", timed_out=False)
    runner = FakeSandboxRunner(programmed)
    assert runner.run(_job(tmp_path)) == programmed


def test_docker_cli_runner_uses_constructed_argv_and_timeout(tmp_path: Path) -> None:
    seen: dict[str, Any] = {}
    job = _job(tmp_path, scratch_files={}, timeout_seconds=19)

    def fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
        seen["command"] = list(command)
        seen["timeout"] = kwargs.get("timeout")
        seen["check"] = kwargs.get("check")
        seen["capture_output"] = kwargs.get("capture_output")
        seen["text"] = kwargs.get("text")
        return subprocess.CompletedProcess(command, 3, stdout="hello", stderr="warn")

    result = DockerCliRunner(runner=fake_run).run(job)
    assert seen["command"] == build_docker_argv(job)
    assert seen["timeout"] == 19
    assert seen["check"] is False
    assert seen["capture_output"] is True
    assert seen["text"] is True
    assert result == SandboxResult(exit_code=3, stdout="hello", stderr="warn", timed_out=False)
    assert "--network" in seen["command"]
    assert "--read-only" in seen["command"]
    assert "docker.sock" not in " ".join(seen["command"])


def test_docker_cli_runner_maps_timeout_expired_to_timed_out(tmp_path: Path) -> None:
    job = _job(tmp_path, scratch_files={}, timeout_seconds=5)

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        raise subprocess.TimeoutExpired(cmd=command, timeout=5, output="partial", stderr="slow")

    result = DockerCliRunner(runner=fake_run).run(job)
    assert result.timed_out is True
    assert result.stdout == "partial"
    assert result.stderr == "slow"


def test_docker_cli_runner_does_not_invoke_daemon_when_runner_injected(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    def boom(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]:
        raise AssertionError("unit tests must not invoke the docker daemon")

    monkeypatch.setattr(subprocess, "run", boom)
    monkeypatch.setattr(runner_mod.subprocess, "run", boom)

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

    result = DockerCliRunner(runner=fake_run).run(_job(tmp_path, scratch_files={}))
    assert result.timed_out is False
    assert result.exit_code == 0


def test_docker_cli_runner_does_not_write_the_source_tree(tmp_path: Path) -> None:
    job = _job(tmp_path)
    original = (job.repo_root / "app.py").read_text()
    before = {path.relative_to(job.repo_root): path.read_text() for path in job.repo_root.rglob("*") if path.is_file()}

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        joined = " ".join(command)
        assert "docker.sock" not in joined
        assert "--read-only" in command
        return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

    DockerCliRunner(runner=fake_run).run(job)
    assert (job.repo_root / "app.py").read_text() == original
    after = {path.relative_to(job.repo_root): path.read_text() for path in job.repo_root.rglob("*") if path.is_file()}
    assert after == before
    assert "case.py" not in {path.name for path in job.repo_root.iterdir()}


def test_docker_cli_runner_seeds_scratch_files_outside_the_repo(tmp_path: Path) -> None:
    job = _job(tmp_path, scratch_files={"nested/case.py": "print('seeded')\n"})
    seen: dict[str, Any] = {}

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        seen["command"] = list(command)
        mounts = [command[index + 1] for index, item in enumerate(command) if item == "--mount" and index + 1 < len(command)]
        seed_mounts = [spec for spec in mounts if "/scratch-in" in spec]
        assert seed_mounts, command
        options = _mount_options(seed_mounts[0])
        source = Path(options.get("src") or options.get("source") or "")
        assert source.is_dir()
        assert job.repo_root.resolve() not in source.resolve().parents
        assert source.resolve() != job.repo_root.resolve()
        assert (source / "nested" / "case.py").read_text() == "print('seeded')\n"
        assert options.get("ro") == "true" or options.get("readonly") == "true"
        assert "--tmpfs" in command
        assert _flag_value(command, "--tmpfs").startswith("/scratch")
        return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

    result = DockerCliRunner(runner=fake_run).run(job)
    assert result.exit_code == 0
    argv = seen["command"]
    assert "docker.sock" not in " ".join(argv)
    assert "--read-only" in argv
    assert job.image in argv


def test_docker_cli_runner_scratch_seed_is_world_readable_for_container_user(tmp_path: Path) -> None:
    job = _job(tmp_path, scratch_files={"nested/dir/case.py": "print('seeded')\n"})
    seen: dict[str, Path] = {}

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        source = _seed_source(command)
        seen["source"] = source
        assert _posix_mode(source) == 0o755
        for dirpath, _dirnames, filenames in os.walk(source):
            directory = Path(dirpath)
            assert stat.S_ISDIR(directory.stat().st_mode)
            assert _posix_mode(directory) == 0o755, directory
            for name in filenames:
                file_path = directory / name
                assert _posix_mode(file_path) == 0o644, file_path
        return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

    DockerCliRunner(runner=fake_run).run(job)
    assert "source" in seen


def test_docker_cli_runner_scratch_copy_uses_cp_r_not_cp_a(tmp_path: Path) -> None:
    job = _job(tmp_path, scratch_files={"nested/case.py": "print('seeded')\n"})
    seen: dict[str, str] = {}

    def fake_run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]:
        script = _copy_script(command)
        seen["script"] = script
        assert command[command.index(job.image) + 1] == "/bin/sh"
        assert "cp -a" not in script
        assert "cp -a" not in " ".join(command)
        assert script.startswith("cp -r /scratch-in/. /scratch/")
        assert 'exec "$@"' in script
        return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

    DockerCliRunner(runner=fake_run).run(job)
    assert seen["script"] == 'cp -r /scratch-in/. /scratch/ && exec "$@"'