feat(three-stage-scan): map sandbox outcomes to worth-fixing verdicts

.kiro/specs/three-stage-scan/tasks.md

@@ -102,7 +102,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
 
 ## Phase 3 — Core: Regression verdicts
 
-- [ ] 5. Produce worth-fixing verdicts from sandboxed candidates
+- [x] 5. Produce worth-fixing verdicts from sandboxed candidates
 - [x] 5.1 Select a bounded candidate set from hotspots and severe reachable inventory
   - Selection honors max-candidates and always includes reachable severity-5 inventory even when the ledger demoted the hotspot.
   - Observable completion: a unit test with six hotspots and max-candidates three returns three, plus the sev-5 reachable extra when present.

src/openultrasast/regress/__init__.py

@@ -1,17 +1,33 @@
 """Isolated regression snippets and verdicts."""
 
-from .candidate import INCONCLUSIVE, MISSING_RECIPE, SAFETY_REJECTED, RegressionRunner, RegressionVerdict, select_candidates
+from .candidate import RegressionRunner, select_candidates
 from .recipes import recipe_for
 from .safety import UnsafeSnippetError, check_snippet_safety
+from .verdict import (
+    ALREADY_COVERED,
+    INCONCLUSIVE,
+    MISSING_RECIPE,
+    NOT_TRIGGERABLE,
+    SAFETY_REJECTED,
+    TRIGGERABLE,
+    RegressionVerdict,
+    is_worth_fixing,
+    verdict_from_result,
+)
 
 __all__ = [
+    "ALREADY_COVERED",
     "INCONCLUSIVE",
     "MISSING_RECIPE",
+    "NOT_TRIGGERABLE",
     "SAFETY_REJECTED",
+    "TRIGGERABLE",
     "RegressionRunner",
     "RegressionVerdict",
     "UnsafeSnippetError",
     "check_snippet_safety",
+    "is_worth_fixing",
     "recipe_for",
     "select_candidates",
+    "verdict_from_result",
 ]

src/openultrasast/regress/candidate.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
-from dataclasses import dataclass
 from pathlib import Path
 
 from ..complexity.ledger import hotspot_key, select_forced_candidates
@@ -13,12 +12,26 @@ from ..findings import StaticFinding
 from ..sandbox import SandboxJob, SandboxResult, SandboxRunner
 from .recipes import recipe_for
 from .safety import UnsafeSnippetError, check_snippet_safety
+from .verdict import (
+    INCONCLUSIVE,
+    MISSING_RECIPE,
+    NOT_TRIGGERABLE,
+    SAFETY_REJECTED,
+    TRIGGERABLE,
+    RegressionVerdict,
+    verdict_from_result,
+)
 
-INCONCLUSIVE = "inconclusive"
+__all__ = [
-SAFETY_REJECTED = "safety_rejected"
+    "INCONCLUSIVE",
-MISSING_RECIPE = "missing_recipe"
+    "MISSING_RECIPE",
-TRIGGERABLE = "triggerable"
+    "NOT_TRIGGERABLE",
-NOT_TRIGGERABLE = "not_triggerable"
+    "SAFETY_REJECTED",
+    "TRIGGERABLE",
+    "RegressionRunner",
+    "RegressionVerdict",
+    "select_candidates",
+]
 
 
 def select_candidates(
@@ -73,12 +86,6 @@ def _hotspot_from_forced_finding(finding: StaticFinding) -> Hotspot:
     )
 
 
-@dataclass(frozen=True)
-class RegressionVerdict:
-    verdict: str
-    reason: str = ""
-
-
 class RegressionRunner:
     """Run a snippet in the sandbox only after the structural safety check passes."""
 
@@ -96,22 +103,18 @@ class RegressionRunner:
     ) -> RegressionVerdict:
         job = recipe_for(language, snippet, image, sandbox_limits, repo_root=repo_root)
         if job is None:
-            return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
+            return verdict_from_result(recipe_missing=True)
         return self.run_snippet(snippet, job)
 
     def run_snippet(self, snippet: str, job: SandboxJob | None) -> RegressionVerdict:
         if job is None:
-            return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
+            return verdict_from_result(recipe_missing=True)
         try:
             check_snippet_safety(snippet)
         except UnsafeSnippetError:
-            return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED)
+            return verdict_from_result(safety_rejected=True)
         return _verdict_from_result(self._sandbox.run(job))
 
 
 def _verdict_from_result(result: SandboxResult) -> RegressionVerdict:
-    if result.timed_out:
+    return verdict_from_result(result)
-        return RegressionVerdict(verdict=INCONCLUSIVE, reason="timeout")
-    if result.exit_code != 0:
-        return RegressionVerdict(verdict=TRIGGERABLE, reason="nonzero_exit")
-    return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason="exit_zero")

src/openultrasast/regress/verdict.py

@@ -0,0 +1,94 @@
+"""Map sandbox outcomes to verdicts and the worth-fixing gate."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ..sandbox import SandboxResult
+
+TRIGGERABLE = "triggerable"
+NOT_TRIGGERABLE = "not_triggerable"
+ALREADY_COVERED = "already_covered"
+INCONCLUSIVE = "inconclusive"
+
+MISSING_RECIPE = "missing_recipe"
+SAFETY_REJECTED = "safety_rejected"
+SANDBOX_UNAVAILABLE = "sandbox_unavailable"
+TIMEOUT = "timeout"
+COVERED_TEST_PASSED = "covered_test_passed"
+ASSERTION = "assertion"
+SANITIZER_ABORT = "sanitizer_abort"
+CRASH = "crash"
+NONZERO_EXIT = "nonzero_exit"
+EXIT_ZERO = "exit_zero"
+
+WORTH_FIXING_REACHABILITY = frozenset({"reachable", "inferred-file-surface"})
+
+_SANITIZER_MARKERS = (
+    "AddressSanitizer",
+    "UndefinedBehaviorSanitizer",
+    "ThreadSanitizer",
+    "MemorySanitizer",
+    "LeakSanitizer",
+)
+_ASSERTION_MARKERS = (
+    "AssertionError",
+    "Assertion failed",
+    "assertion failed",
+    "assert failed",
+)
+_CRASH_MARKERS = (
+    "Fatal Python error",
+    "Segmentation fault",
+    "SIGSEGV",
+    "SIGABRT",
+    "Aborted",
+)
+
+
+@dataclass(frozen=True)
+class RegressionVerdict:
+    verdict: str
+    reason: str = ""
+
+
+def is_worth_fixing(verdict: str, reachability: str) -> bool:
+    """True iff the candidate is triggerable on a reachable or inferred-file-surface finding."""
+    return verdict == TRIGGERABLE and reachability in WORTH_FIXING_REACHABILITY
+
+
+def verdict_from_result(
+    result: SandboxResult | None = None,
+    *,
+    recipe_missing: bool = False,
+    sandbox_missing: bool = False,
+    safety_rejected: bool = False,
+    covering_test: bool = False,
+) -> RegressionVerdict:
+    """Map a sandbox observation onto one of the four regression verdicts."""
+    if recipe_missing:
+        return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
+    if sandbox_missing:
+        return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE)
+    if safety_rejected:
+        return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED)
+    if result is None:
+        return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE)
+    if result.timed_out:
+        return RegressionVerdict(verdict=INCONCLUSIVE, reason=TIMEOUT)
+    if covering_test and result.exit_code == 0:
+        return RegressionVerdict(verdict=ALREADY_COVERED, reason=COVERED_TEST_PASSED)
+    if result.exit_code != 0:
+        return RegressionVerdict(verdict=TRIGGERABLE, reason=_nonzero_reason(result))
+    return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=EXIT_ZERO)
+
+
+def _nonzero_reason(result: SandboxResult) -> str:
+    text = f"{result.stdout}\n{result.stderr}".lower()
+    if any(marker.lower() in text for marker in _SANITIZER_MARKERS):
+        return SANITIZER_ABORT
+    if any(marker.lower() in text for marker in _ASSERTION_MARKERS):
+        return ASSERTION
+    if any(marker.lower() in text for marker in _CRASH_MARKERS):
+        return CRASH
+    return NONZERO_EXIT

tests/test_regress_verdicts.py

@@ -0,0 +1,201 @@
+"""Map sandbox outcomes to the four verdicts and the worth-fixing gate (task 5.3)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from openultrasast.regress import RegressionRunner
+from openultrasast.regress.verdict import (
+    ALREADY_COVERED,
+    INCONCLUSIVE,
+    NOT_TRIGGERABLE,
+    TRIGGERABLE,
+    is_worth_fixing,
+    verdict_from_result,
+)
+from openultrasast.sandbox import FakeSandboxRunner, SandboxJob, SandboxResult
+
+_REACHABLE = "reachable"
+_INFERRED = "inferred-file-surface"
+_UNKNOWN = "unknown"
+
+
+def _result(*, exit_code: int = 0, stdout: str = "", stderr: str = "", timed_out: bool = False) -> SandboxResult:
+    return SandboxResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=timed_out)
+
+
+def _job(tmp_path: Path, snippet: str) -> SandboxJob:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    return SandboxJob(
+        image="python:3.12-alpine",
+        command=("python", "/scratch/case.py"),
+        repo_root=repo,
+        scratch_files={"case.py": snippet},
+        timeout_seconds=30,
+        memory_mb=256,
+        pids_limit=128,
+    )
+
+
+# observation kwargs, sandbox result, reachability, expected verdict, expected reason, worth-fixing
+_CASES = (
+    pytest.param(
+        {"recipe_missing": True},
+        None,
+        _REACHABLE,
+        INCONCLUSIVE,
+        "missing_recipe",
+        False,
+        id="recipe-missing-inconclusive",
+    ),
+    pytest.param(
+        {"sandbox_missing": True},
+        None,
+        _REACHABLE,
+        INCONCLUSIVE,
+        "sandbox_unavailable",
+        False,
+        id="sandbox-missing-inconclusive",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=-1, timed_out=True),
+        _REACHABLE,
+        INCONCLUSIVE,
+        "timeout",
+        False,
+        id="timeout-inconclusive",
+    ),
+    pytest.param(
+        {"safety_rejected": True},
+        None,
+        _INFERRED,
+        INCONCLUSIVE,
+        "safety_rejected",
+        False,
+        id="safety-reject-inconclusive",
+    ),
+    pytest.param(
+        {"covering_test": True},
+        _result(exit_code=0, stdout="ok\n"),
+        _REACHABLE,
+        ALREADY_COVERED,
+        "covered_test_passed",
+        False,
+        id="covering-test-passed-already-covered",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=1, stderr="AssertionError: boom\n"),
+        _REACHABLE,
+        TRIGGERABLE,
+        "assertion",
+        True,
+        id="assertion-triggerable-reachable-worth-fixing",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=1, stderr="ERROR: AddressSanitizer: heap-buffer-overflow\n"),
+        _INFERRED,
+        TRIGGERABLE,
+        "sanitizer_abort",
+        True,
+        id="sanitizer-triggerable-inferred-worth-fixing",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=139, stderr="Segmentation fault\n"),
+        _UNKNOWN,
+        TRIGGERABLE,
+        "crash",
+        False,
+        id="crash-triggerable-unknown-not-worth-fixing",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=0, stdout="ok\n"),
+        _REACHABLE,
+        NOT_TRIGGERABLE,
+        "exit_zero",
+        False,
+        id="exit-zero-not-triggerable",
+    ),
+    pytest.param(
+        {"covering_test": True},
+        _result(exit_code=0),
+        _UNKNOWN,
+        ALREADY_COVERED,
+        "covered_test_passed",
+        False,
+        id="already-covered-unknown-does-not-fail-build",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=0),
+        _INFERRED,
+        NOT_TRIGGERABLE,
+        "exit_zero",
+        False,
+        id="not-triggerable-inferred-does-not-fail-build",
+    ),
+    pytest.param(
+        {},
+        _result(exit_code=-1, timed_out=True),
+        _INFERRED,
+        INCONCLUSIVE,
+        "timeout",
+        False,
+        id="inconclusive-inferred-does-not-fail-build",
+    ),
+)
+
+
+@pytest.mark.parametrize(
+    ("flags", "result", "reachability", "expected_verdict", "expected_reason", "expected_worth_fixing"),
+    _CASES,
+)
+def test_sandbox_outcomes_map_to_verdicts_and_worth_fixing(
+    flags: dict[str, bool],
+    result: SandboxResult | None,
+    reachability: str,
+    expected_verdict: str,
+    expected_reason: str,
+    expected_worth_fixing: bool,
+) -> None:
+    mapped = verdict_from_result(result, **flags)
+
+    assert mapped.verdict == expected_verdict
+    assert mapped.reason == expected_reason
+    assert mapped.reason
+    assert is_worth_fixing(mapped.verdict, reachability) is expected_worth_fixing
+
+
+def test_table_covers_four_verdicts_and_worth_fixing_boolean() -> None:
+    verdicts = {case.values[3] for case in _CASES}
+    worth_fixing_values = {case.values[5] for case in _CASES}
+
+    assert verdicts == {TRIGGERABLE, NOT_TRIGGERABLE, ALREADY_COVERED, INCONCLUSIVE}
+    assert worth_fixing_values == {True, False}
+
+
+def test_runner_maps_timeout_crash_and_exit_zero(tmp_path: Path) -> None:
+    snippet = "assert 1 + 1 == 2\n"
+    job = _job(tmp_path, snippet)
+    timeout = FakeSandboxRunner(_result(exit_code=-1, timed_out=True))
+    crash = FakeSandboxRunner(_result(exit_code=1, stderr="AssertionError\n"))
+    ok = FakeSandboxRunner(_result(exit_code=0))
+
+    timed_out = RegressionRunner(timeout).run_snippet(snippet, job)
+    triggered = RegressionRunner(crash).run_snippet(snippet, job)
+    not_triggered = RegressionRunner(ok).run_snippet(snippet, job)
+
+    assert timed_out.verdict == INCONCLUSIVE
+    assert timed_out.reason == "timeout"
+    assert triggered.verdict == TRIGGERABLE
+    assert not_triggered.verdict == NOT_TRIGGERABLE
+    assert is_worth_fixing(timed_out.verdict, _REACHABLE) is False
+    assert is_worth_fixing(triggered.verdict, _REACHABLE) is True
+    assert is_worth_fixing(not_triggered.verdict, _REACHABLE) is False