marius/Open_Ultra_SAST
1
0
Fork 0
mirror of https://github.com/norandom/OpenUltraSAST.git synced 2026-10-16 03:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(three-stage-scan): map sandbox outcomes to worth-fixing verdicts

Browse Source
This commit is contained in:
Marius Ciepluch
2026-09-03 22:59:01 +02:00
parent 62c7990e49
commit 331b62f1c1
5 changed files with 336 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.kiro/specs/three-stage-scan/tasks.md
+1 -1
View File
@@ -102,7 +102,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
## Phase 3 — Core: Regression verdicts
- [ ] 5. Produce worth-fixing verdicts from sandboxed candidates
- [x] 5. Produce worth-fixing verdicts from sandboxed candidates
- [x] 5.1 Select a bounded candidate set from hotspots and severe reachable inventory
- Selection honors max-candidates and always includes reachable severity-5 inventory even when the ledger demoted the hotspot.
- Observable completion: a unit test with six hotspots and max-candidates three returns three, plus the sev-5 reachable extra when present.
src/openultrasast/regress/__init__.py
+17 -1
View File
@@ -1,17 +1,33 @@
"""Isolated regression snippets and verdicts."""
from .candidate import INCONCLUSIVE, MISSING_RECIPE, SAFETY_REJECTED, RegressionRunner, RegressionVerdict, select_candidates
from .candidate import RegressionRunner, select_candidates
from .recipes import recipe_for
from .safety import UnsafeSnippetError, check_snippet_safety
from .verdict import (
ALREADY_COVERED,
INCONCLUSIVE,
MISSING_RECIPE,
NOT_TRIGGERABLE,
SAFETY_REJECTED,
TRIGGERABLE,
RegressionVerdict,
is_worth_fixing,
verdict_from_result,
)
__all__ = [
"ALREADY_COVERED",
"INCONCLUSIVE",
"MISSING_RECIPE",
"NOT_TRIGGERABLE",
"SAFETY_REJECTED",
"TRIGGERABLE",
"RegressionRunner",
"RegressionVerdict",
"UnsafeSnippetError",
"check_snippet_safety",
"is_worth_fixing",
"recipe_for",
"select_candidates",
"verdict_from_result",
]
src/openultrasast/regress/candidate.py
+23 -20
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from ..complexity.ledger import hotspot_key, select_forced_candidates
@@ -13,12 +12,26 @@ from ..findings import StaticFinding
from ..sandbox import SandboxJob, SandboxResult, SandboxRunner
from .recipes import recipe_for
from .safety import UnsafeSnippetError, check_snippet_safety
from .verdict import (
INCONCLUSIVE,
MISSING_RECIPE,
NOT_TRIGGERABLE,
SAFETY_REJECTED,
TRIGGERABLE,
RegressionVerdict,
verdict_from_result,
)
INCONCLUSIVE = "inconclusive"
SAFETY_REJECTED = "safety_rejected"
MISSING_RECIPE = "missing_recipe"
TRIGGERABLE = "triggerable"
NOT_TRIGGERABLE = "not_triggerable"
__all__ = [
"INCONCLUSIVE",
"MISSING_RECIPE",
"NOT_TRIGGERABLE",
"SAFETY_REJECTED",
"TRIGGERABLE",
"RegressionRunner",
"RegressionVerdict",
"select_candidates",
]
def select_candidates(
@@ -73,12 +86,6 @@ def _hotspot_from_forced_finding(finding: StaticFinding) -> Hotspot:
)
@dataclass(frozen=True)
class RegressionVerdict:
verdict: str
reason: str = ""
class RegressionRunner:
"""Run a snippet in the sandbox only after the structural safety check passes."""
@@ -96,22 +103,18 @@ class RegressionRunner:
) -> RegressionVerdict:
job = recipe_for(language, snippet, image, sandbox_limits, repo_root=repo_root)
if job is None:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
return verdict_from_result(recipe_missing=True)
return self.run_snippet(snippet, job)
def run_snippet(self, snippet: str, job: SandboxJob | None) -> RegressionVerdict:
if job is None:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
return verdict_from_result(recipe_missing=True)
try:
check_snippet_safety(snippet)
except UnsafeSnippetError:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED)
return verdict_from_result(safety_rejected=True)
return _verdict_from_result(self._sandbox.run(job))
def _verdict_from_result(result: SandboxResult) -> RegressionVerdict:
if result.timed_out:
return RegressionVerdict(verdict=INCONCLUSIVE, reason="timeout")
if result.exit_code != 0:
return RegressionVerdict(verdict=TRIGGERABLE, reason="nonzero_exit")
return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason="exit_zero")
return verdict_from_result(result)
src/openultrasast/regress/verdict.py
+94
View File
@@ -0,0 +1,94 @@
"""Map sandbox outcomes to verdicts and the worth-fixing gate."""
from __future__ import annotations
from dataclasses import dataclass
from ..sandbox import SandboxResult
TRIGGERABLE = "triggerable"
NOT_TRIGGERABLE = "not_triggerable"
ALREADY_COVERED = "already_covered"
INCONCLUSIVE = "inconclusive"
MISSING_RECIPE = "missing_recipe"
SAFETY_REJECTED = "safety_rejected"
SANDBOX_UNAVAILABLE = "sandbox_unavailable"
TIMEOUT = "timeout"
COVERED_TEST_PASSED = "covered_test_passed"
ASSERTION = "assertion"
SANITIZER_ABORT = "sanitizer_abort"
CRASH = "crash"
NONZERO_EXIT = "nonzero_exit"
EXIT_ZERO = "exit_zero"
WORTH_FIXING_REACHABILITY = frozenset({"reachable", "inferred-file-surface"})
_SANITIZER_MARKERS = (
"AddressSanitizer",
"UndefinedBehaviorSanitizer",
"ThreadSanitizer",
"MemorySanitizer",
"LeakSanitizer",
)
_ASSERTION_MARKERS = (
"AssertionError",
"Assertion failed",
"assertion failed",
"assert failed",
)
_CRASH_MARKERS = (
"Fatal Python error",
"Segmentation fault",
"SIGSEGV",
"SIGABRT",
"Aborted",
)
@dataclass(frozen=True)
class RegressionVerdict:
verdict: str
reason: str = ""
def is_worth_fixing(verdict: str, reachability: str) -> bool:
"""True iff the candidate is triggerable on a reachable or inferred-file-surface finding."""
return verdict == TRIGGERABLE and reachability in WORTH_FIXING_REACHABILITY
def verdict_from_result(
result: SandboxResult | None = None,
*,
recipe_missing: bool = False,
sandbox_missing: bool = False,
safety_rejected: bool = False,
covering_test: bool = False,
) -> RegressionVerdict:
"""Map a sandbox observation onto one of the four regression verdicts."""
if recipe_missing:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE)
if sandbox_missing:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE)
if safety_rejected:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED)
if result is None:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE)
if result.timed_out:
return RegressionVerdict(verdict=INCONCLUSIVE, reason=TIMEOUT)
if covering_test and result.exit_code == 0:
return RegressionVerdict(verdict=ALREADY_COVERED, reason=COVERED_TEST_PASSED)
if result.exit_code != 0:
return RegressionVerdict(verdict=TRIGGERABLE, reason=_nonzero_reason(result))
return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=EXIT_ZERO)
def _nonzero_reason(result: SandboxResult) -> str:
text = f"{result.stdout}\n{result.stderr}".lower()
if any(marker.lower() in text for marker in _SANITIZER_MARKERS):
return SANITIZER_ABORT
if any(marker.lower() in text for marker in _ASSERTION_MARKERS):
return ASSERTION
if any(marker.lower() in text for marker in _CRASH_MARKERS):
return CRASH
return NONZERO_EXIT
tests/test_regress_verdicts.py
+201
View File
@@ -0,0 +1,201 @@
"""Map sandbox outcomes to the four verdicts and the worth-fixing gate (task 5.3)."""
from __future__ import annotations
from pathlib import Path
import pytest
from openultrasast.regress import RegressionRunner
from openultrasast.regress.verdict import (
ALREADY_COVERED,
INCONCLUSIVE,
NOT_TRIGGERABLE,
TRIGGERABLE,
is_worth_fixing,
verdict_from_result,
)
from openultrasast.sandbox import FakeSandboxRunner, SandboxJob, SandboxResult
_REACHABLE = "reachable"
_INFERRED = "inferred-file-surface"
_UNKNOWN = "unknown"
def _result(*, exit_code: int = 0, stdout: str = "", stderr: str = "", timed_out: bool = False) -> SandboxResult:
return SandboxResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=timed_out)
def _job(tmp_path: Path, snippet: str) -> SandboxJob:
repo = tmp_path / "repo"
repo.mkdir()
return SandboxJob(
image="python:3.12-alpine",
command=("python", "/scratch/case.py"),
repo_root=repo,
scratch_files={"case.py": snippet},
timeout_seconds=30,
memory_mb=256,
pids_limit=128,
)
# observation kwargs, sandbox result, reachability, expected verdict, expected reason, worth-fixing
_CASES = (
pytest.param(
{"recipe_missing": True},
None,
_REACHABLE,
INCONCLUSIVE,
"missing_recipe",
False,
id="recipe-missing-inconclusive",
),
pytest.param(
{"sandbox_missing": True},
None,
_REACHABLE,
INCONCLUSIVE,
"sandbox_unavailable",
False,
id="sandbox-missing-inconclusive",
),
pytest.param(
{},
_result(exit_code=-1, timed_out=True),
_REACHABLE,
INCONCLUSIVE,
"timeout",
False,
id="timeout-inconclusive",
),
pytest.param(
{"safety_rejected": True},
None,
_INFERRED,
INCONCLUSIVE,
"safety_rejected",
False,
id="safety-reject-inconclusive",
),
pytest.param(
{"covering_test": True},
_result(exit_code=0, stdout="ok\n"),
_REACHABLE,
ALREADY_COVERED,
"covered_test_passed",
False,
id="covering-test-passed-already-covered",
),
pytest.param(
{},
_result(exit_code=1, stderr="AssertionError: boom\n"),
_REACHABLE,
TRIGGERABLE,
"assertion",
True,
id="assertion-triggerable-reachable-worth-fixing",
),
pytest.param(
{},
_result(exit_code=1, stderr="ERROR: AddressSanitizer: heap-buffer-overflow\n"),
_INFERRED,
TRIGGERABLE,
"sanitizer_abort",
True,
id="sanitizer-triggerable-inferred-worth-fixing",
),
pytest.param(
{},
_result(exit_code=139, stderr="Segmentation fault\n"),
_UNKNOWN,
TRIGGERABLE,
"crash",
False,
id="crash-triggerable-unknown-not-worth-fixing",
),
pytest.param(
{},
_result(exit_code=0, stdout="ok\n"),
_REACHABLE,
NOT_TRIGGERABLE,
"exit_zero",
False,
id="exit-zero-not-triggerable",
),
pytest.param(
{"covering_test": True},
_result(exit_code=0),
_UNKNOWN,
ALREADY_COVERED,
"covered_test_passed",
False,
id="already-covered-unknown-does-not-fail-build",
),
pytest.param(
{},
_result(exit_code=0),
_INFERRED,
NOT_TRIGGERABLE,
"exit_zero",
False,
id="not-triggerable-inferred-does-not-fail-build",
),
pytest.param(
{},
_result(exit_code=-1, timed_out=True),
_INFERRED,
INCONCLUSIVE,
"timeout",
False,
id="inconclusive-inferred-does-not-fail-build",
),
)
@pytest.mark.parametrize(
("flags", "result", "reachability", "expected_verdict", "expected_reason", "expected_worth_fixing"),
_CASES,
)
def test_sandbox_outcomes_map_to_verdicts_and_worth_fixing(
flags: dict[str, bool],
result: SandboxResult | None,
reachability: str,
expected_verdict: str,
expected_reason: str,
expected_worth_fixing: bool,
) -> None:
mapped = verdict_from_result(result, **flags)
assert mapped.verdict == expected_verdict
assert mapped.reason == expected_reason
assert mapped.reason
assert is_worth_fixing(mapped.verdict, reachability) is expected_worth_fixing
def test_table_covers_four_verdicts_and_worth_fixing_boolean() -> None:
verdicts = {case.values[3] for case in _CASES}
worth_fixing_values = {case.values[5] for case in _CASES}
assert verdicts == {TRIGGERABLE, NOT_TRIGGERABLE, ALREADY_COVERED, INCONCLUSIVE}
assert worth_fixing_values == {True, False}
def test_runner_maps_timeout_crash_and_exit_zero(tmp_path: Path) -> None:
snippet = "assert 1 + 1 == 2\n"
job = _job(tmp_path, snippet)
timeout = FakeSandboxRunner(_result(exit_code=-1, timed_out=True))
crash = FakeSandboxRunner(_result(exit_code=1, stderr="AssertionError\n"))
ok = FakeSandboxRunner(_result(exit_code=0))
timed_out = RegressionRunner(timeout).run_snippet(snippet, job)
triggered = RegressionRunner(crash).run_snippet(snippet, job)
not_triggered = RegressionRunner(ok).run_snippet(snippet, job)
assert timed_out.verdict == INCONCLUSIVE
assert timed_out.reason == "timeout"
assert triggered.verdict == TRIGGERABLE
assert not_triggered.verdict == NOT_TRIGGERABLE
assert is_worth_fixing(timed_out.verdict, _REACHABLE) is False
assert is_worth_fixing(triggered.verdict, _REACHABLE) is True
assert is_worth_fixing(not_triggered.verdict, _REACHABLE) is False