Open_Ultra_SAST/tests/test_regress_verdicts.py

"""Map sandbox outcomes to the four verdicts and the worth-fixing gate (task 5.3)."""

from __future__ import annotations

from pathlib import Path

import pytest

from openultrasast.regress import RegressionRunner
from openultrasast.regress.verdict import (
    ALREADY_COVERED,
    INCONCLUSIVE,
    NOT_TRIGGERABLE,
    TRIGGERABLE,
    is_worth_fixing,
    verdict_from_result,
)
from openultrasast.sandbox import FakeSandboxRunner, SandboxJob, SandboxResult

_REACHABLE = "reachable"
_INFERRED = "inferred-file-surface"
_UNKNOWN = "unknown"


def _result(*, exit_code: int = 0, stdout: str = "", stderr: str = "", timed_out: bool = False) -> SandboxResult:
    return SandboxResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=timed_out)


def _job(tmp_path: Path, snippet: str) -> SandboxJob:
    repo = tmp_path / "repo"
    repo.mkdir()
    return SandboxJob(
        image="python:3.12-alpine",
        command=("python", "/scratch/case.py"),
        repo_root=repo,
        scratch_files={"case.py": snippet},
        timeout_seconds=30,
        memory_mb=256,
        pids_limit=128,
    )


# observation kwargs, sandbox result, reachability, expected verdict, expected reason, worth-fixing
_CASES = (
    pytest.param(
        {"recipe_missing": True},
        None,
        _REACHABLE,
        INCONCLUSIVE,
        "missing_recipe",
        False,
        id="recipe-missing-inconclusive",
    ),
    pytest.param(
        {"sandbox_missing": True},
        None,
        _REACHABLE,
        INCONCLUSIVE,
        "sandbox_unavailable",
        False,
        id="sandbox-missing-inconclusive",
    ),
    pytest.param(
        {},
        _result(exit_code=-1, timed_out=True),
        _REACHABLE,
        INCONCLUSIVE,
        "timeout",
        False,
        id="timeout-inconclusive",
    ),
    pytest.param(
        {"safety_rejected": True},
        None,
        _INFERRED,
        INCONCLUSIVE,
        "safety_rejected",
        False,
        id="safety-reject-inconclusive",
    ),
    pytest.param(
        {"covering_test": True},
        _result(exit_code=0, stdout="ok\n"),
        _REACHABLE,
        ALREADY_COVERED,
        "covered_test_passed",
        False,
        id="covering-test-passed-already-covered",
    ),
    pytest.param(
        {},
        _result(exit_code=1, stderr="AssertionError: boom\n"),
        _REACHABLE,
        TRIGGERABLE,
        "assertion",
        True,
        id="assertion-triggerable-reachable-worth-fixing",
    ),
    pytest.param(
        {},
        _result(exit_code=1, stderr="ERROR: AddressSanitizer: heap-buffer-overflow\n"),
        _INFERRED,
        TRIGGERABLE,
        "sanitizer_abort",
        True,
        id="sanitizer-triggerable-inferred-worth-fixing",
    ),
    pytest.param(
        {},
        _result(exit_code=139, stderr="Segmentation fault\n"),
        _UNKNOWN,
        TRIGGERABLE,
        "crash",
        False,
        id="crash-triggerable-unknown-not-worth-fixing",
    ),
    pytest.param(
        {},
        _result(exit_code=0, stdout="ok\n"),
        _REACHABLE,
        NOT_TRIGGERABLE,
        "exit_zero",
        False,
        id="exit-zero-not-triggerable",
    ),
    pytest.param(
        {"covering_test": True},
        _result(exit_code=0),
        _UNKNOWN,
        ALREADY_COVERED,
        "covered_test_passed",
        False,
        id="already-covered-unknown-does-not-fail-build",
    ),
    pytest.param(
        {},
        _result(exit_code=0),
        _INFERRED,
        NOT_TRIGGERABLE,
        "exit_zero",
        False,
        id="not-triggerable-inferred-does-not-fail-build",
    ),
    pytest.param(
        {},
        _result(exit_code=-1, timed_out=True),
        _INFERRED,
        INCONCLUSIVE,
        "timeout",
        False,
        id="inconclusive-inferred-does-not-fail-build",
    ),
)


@pytest.mark.parametrize(
    ("flags", "result", "reachability", "expected_verdict", "expected_reason", "expected_worth_fixing"),
    _CASES,
)
def test_sandbox_outcomes_map_to_verdicts_and_worth_fixing(
    flags: dict[str, bool],
    result: SandboxResult | None,
    reachability: str,
    expected_verdict: str,
    expected_reason: str,
    expected_worth_fixing: bool,
) -> None:
    mapped = verdict_from_result(result, **flags)

    assert mapped.verdict == expected_verdict
    assert mapped.reason == expected_reason
    assert mapped.reason
    assert is_worth_fixing(mapped.verdict, reachability) is expected_worth_fixing


def test_table_covers_four_verdicts_and_worth_fixing_boolean() -> None:
    verdicts = {case.values[3] for case in _CASES}
    worth_fixing_values = {case.values[5] for case in _CASES}

    assert verdicts == {TRIGGERABLE, NOT_TRIGGERABLE, ALREADY_COVERED, INCONCLUSIVE}
    assert worth_fixing_values == {True, False}


def test_runner_maps_timeout_crash_and_exit_zero(tmp_path: Path) -> None:
    snippet = "assert 1 + 1 == 2\n"
    job = _job(tmp_path, snippet)
    timeout = FakeSandboxRunner(_result(exit_code=-1, timed_out=True))
    crash = FakeSandboxRunner(_result(exit_code=1, stderr="AssertionError\n"))
    ok = FakeSandboxRunner(_result(exit_code=0))

    timed_out = RegressionRunner(timeout).run_snippet(snippet, job)
    triggered = RegressionRunner(crash).run_snippet(snippet, job)
    not_triggered = RegressionRunner(ok).run_snippet(snippet, job)

    assert timed_out.verdict == INCONCLUSIVE
    assert timed_out.reason == "timeout"
    assert triggered.verdict == TRIGGERABLE
    assert not_triggered.verdict == NOT_TRIGGERABLE
    assert is_worth_fixing(timed_out.verdict, _REACHABLE) is False
    assert is_worth_fixing(triggered.verdict, _REACHABLE) is True
    assert is_worth_fixing(not_triggered.verdict, _REACHABLE) is False