From 331b62f1c1972d4f3472e955f81332b6b6a0f659 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:59:01 +0200 Subject: [PATCH] feat(three-stage-scan): map sandbox outcomes to worth-fixing verdicts --- .kiro/specs/three-stage-scan/tasks.md | 2 +- src/openultrasast/regress/__init__.py | 18 ++- src/openultrasast/regress/candidate.py | 43 +++--- src/openultrasast/regress/verdict.py | 94 ++++++++++++ tests/test_regress_verdicts.py | 201 +++++++++++++++++++++++++ 5 files changed, 336 insertions(+), 22 deletions(-) create mode 100644 src/openultrasast/regress/verdict.py create mode 100644 tests/test_regress_verdicts.py diff --git a/.kiro/specs/three-stage-scan/tasks.md b/.kiro/specs/three-stage-scan/tasks.md index 2cbd736..f1100ab 100644 --- a/.kiro/specs/three-stage-scan/tasks.md +++ b/.kiro/specs/three-stage-scan/tasks.md @@ -102,7 +102,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex ## Phase 3 — Core: Regression verdicts -- [ ] 5. Produce worth-fixing verdicts from sandboxed candidates +- [x] 5. Produce worth-fixing verdicts from sandboxed candidates - [x] 5.1 Select a bounded candidate set from hotspots and severe reachable inventory - Selection honors max-candidates and always includes reachable severity-5 inventory even when the ledger demoted the hotspot. - Observable completion: a unit test with six hotspots and max-candidates three returns three, plus the sev-5 reachable extra when present. diff --git a/src/openultrasast/regress/__init__.py b/src/openultrasast/regress/__init__.py index 7584d71..4097be5 100644 --- a/src/openultrasast/regress/__init__.py +++ b/src/openultrasast/regress/__init__.py @@ -1,17 +1,33 @@ """Isolated regression snippets and verdicts.""" -from .candidate import INCONCLUSIVE, MISSING_RECIPE, SAFETY_REJECTED, RegressionRunner, RegressionVerdict, select_candidates +from .candidate import RegressionRunner, select_candidates from .recipes import recipe_for from .safety import UnsafeSnippetError, check_snippet_safety +from .verdict import ( + ALREADY_COVERED, + INCONCLUSIVE, + MISSING_RECIPE, + NOT_TRIGGERABLE, + SAFETY_REJECTED, + TRIGGERABLE, + RegressionVerdict, + is_worth_fixing, + verdict_from_result, +) __all__ = [ + "ALREADY_COVERED", "INCONCLUSIVE", "MISSING_RECIPE", + "NOT_TRIGGERABLE", "SAFETY_REJECTED", + "TRIGGERABLE", "RegressionRunner", "RegressionVerdict", "UnsafeSnippetError", "check_snippet_safety", + "is_worth_fixing", "recipe_for", "select_candidates", + "verdict_from_result", ] diff --git a/src/openultrasast/regress/candidate.py b/src/openultrasast/regress/candidate.py index 43f4c47..d4fb045 100644 --- a/src/openultrasast/regress/candidate.py +++ b/src/openultrasast/regress/candidate.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from dataclasses import dataclass from pathlib import Path from ..complexity.ledger import hotspot_key, select_forced_candidates @@ -13,12 +12,26 @@ from ..findings import StaticFinding from ..sandbox import SandboxJob, SandboxResult, SandboxRunner from .recipes import recipe_for from .safety import UnsafeSnippetError, check_snippet_safety +from .verdict import ( + INCONCLUSIVE, + MISSING_RECIPE, + NOT_TRIGGERABLE, + SAFETY_REJECTED, + TRIGGERABLE, + RegressionVerdict, + verdict_from_result, +) -INCONCLUSIVE = "inconclusive" -SAFETY_REJECTED = "safety_rejected" -MISSING_RECIPE = "missing_recipe" -TRIGGERABLE = "triggerable" -NOT_TRIGGERABLE = "not_triggerable" +__all__ = [ + "INCONCLUSIVE", + "MISSING_RECIPE", + "NOT_TRIGGERABLE", + "SAFETY_REJECTED", + "TRIGGERABLE", + "RegressionRunner", + "RegressionVerdict", + "select_candidates", +] def select_candidates( @@ -73,12 +86,6 @@ def _hotspot_from_forced_finding(finding: StaticFinding) -> Hotspot: ) -@dataclass(frozen=True) -class RegressionVerdict: - verdict: str - reason: str = "" - - class RegressionRunner: """Run a snippet in the sandbox only after the structural safety check passes.""" @@ -96,22 +103,18 @@ class RegressionRunner: ) -> RegressionVerdict: job = recipe_for(language, snippet, image, sandbox_limits, repo_root=repo_root) if job is None: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE) + return verdict_from_result(recipe_missing=True) return self.run_snippet(snippet, job) def run_snippet(self, snippet: str, job: SandboxJob | None) -> RegressionVerdict: if job is None: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE) + return verdict_from_result(recipe_missing=True) try: check_snippet_safety(snippet) except UnsafeSnippetError: - return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED) + return verdict_from_result(safety_rejected=True) return _verdict_from_result(self._sandbox.run(job)) def _verdict_from_result(result: SandboxResult) -> RegressionVerdict: - if result.timed_out: - return RegressionVerdict(verdict=INCONCLUSIVE, reason="timeout") - if result.exit_code != 0: - return RegressionVerdict(verdict=TRIGGERABLE, reason="nonzero_exit") - return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason="exit_zero") + return verdict_from_result(result) diff --git a/src/openultrasast/regress/verdict.py b/src/openultrasast/regress/verdict.py new file mode 100644 index 0000000..4ec569d --- /dev/null +++ b/src/openultrasast/regress/verdict.py @@ -0,0 +1,94 @@ +"""Map sandbox outcomes to verdicts and the worth-fixing gate.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..sandbox import SandboxResult + +TRIGGERABLE = "triggerable" +NOT_TRIGGERABLE = "not_triggerable" +ALREADY_COVERED = "already_covered" +INCONCLUSIVE = "inconclusive" + +MISSING_RECIPE = "missing_recipe" +SAFETY_REJECTED = "safety_rejected" +SANDBOX_UNAVAILABLE = "sandbox_unavailable" +TIMEOUT = "timeout" +COVERED_TEST_PASSED = "covered_test_passed" +ASSERTION = "assertion" +SANITIZER_ABORT = "sanitizer_abort" +CRASH = "crash" +NONZERO_EXIT = "nonzero_exit" +EXIT_ZERO = "exit_zero" + +WORTH_FIXING_REACHABILITY = frozenset({"reachable", "inferred-file-surface"}) + +_SANITIZER_MARKERS = ( + "AddressSanitizer", + "UndefinedBehaviorSanitizer", + "ThreadSanitizer", + "MemorySanitizer", + "LeakSanitizer", +) +_ASSERTION_MARKERS = ( + "AssertionError", + "Assertion failed", + "assertion failed", + "assert failed", +) +_CRASH_MARKERS = ( + "Fatal Python error", + "Segmentation fault", + "SIGSEGV", + "SIGABRT", + "Aborted", +) + + +@dataclass(frozen=True) +class RegressionVerdict: + verdict: str + reason: str = "" + + +def is_worth_fixing(verdict: str, reachability: str) -> bool: + """True iff the candidate is triggerable on a reachable or inferred-file-surface finding.""" + return verdict == TRIGGERABLE and reachability in WORTH_FIXING_REACHABILITY + + +def verdict_from_result( + result: SandboxResult | None = None, + *, + recipe_missing: bool = False, + sandbox_missing: bool = False, + safety_rejected: bool = False, + covering_test: bool = False, +) -> RegressionVerdict: + """Map a sandbox observation onto one of the four regression verdicts.""" + if recipe_missing: + return RegressionVerdict(verdict=INCONCLUSIVE, reason=MISSING_RECIPE) + if sandbox_missing: + return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE) + if safety_rejected: + return RegressionVerdict(verdict=INCONCLUSIVE, reason=SAFETY_REJECTED) + if result is None: + return RegressionVerdict(verdict=INCONCLUSIVE, reason=SANDBOX_UNAVAILABLE) + if result.timed_out: + return RegressionVerdict(verdict=INCONCLUSIVE, reason=TIMEOUT) + if covering_test and result.exit_code == 0: + return RegressionVerdict(verdict=ALREADY_COVERED, reason=COVERED_TEST_PASSED) + if result.exit_code != 0: + return RegressionVerdict(verdict=TRIGGERABLE, reason=_nonzero_reason(result)) + return RegressionVerdict(verdict=NOT_TRIGGERABLE, reason=EXIT_ZERO) + + +def _nonzero_reason(result: SandboxResult) -> str: + text = f"{result.stdout}\n{result.stderr}".lower() + if any(marker.lower() in text for marker in _SANITIZER_MARKERS): + return SANITIZER_ABORT + if any(marker.lower() in text for marker in _ASSERTION_MARKERS): + return ASSERTION + if any(marker.lower() in text for marker in _CRASH_MARKERS): + return CRASH + return NONZERO_EXIT diff --git a/tests/test_regress_verdicts.py b/tests/test_regress_verdicts.py new file mode 100644 index 0000000..f2e143f --- /dev/null +++ b/tests/test_regress_verdicts.py @@ -0,0 +1,201 @@ +"""Map sandbox outcomes to the four verdicts and the worth-fixing gate (task 5.3).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from openultrasast.regress import RegressionRunner +from openultrasast.regress.verdict import ( + ALREADY_COVERED, + INCONCLUSIVE, + NOT_TRIGGERABLE, + TRIGGERABLE, + is_worth_fixing, + verdict_from_result, +) +from openultrasast.sandbox import FakeSandboxRunner, SandboxJob, SandboxResult + +_REACHABLE = "reachable" +_INFERRED = "inferred-file-surface" +_UNKNOWN = "unknown" + + +def _result(*, exit_code: int = 0, stdout: str = "", stderr: str = "", timed_out: bool = False) -> SandboxResult: + return SandboxResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=timed_out) + + +def _job(tmp_path: Path, snippet: str) -> SandboxJob: + repo = tmp_path / "repo" + repo.mkdir() + return SandboxJob( + image="python:3.12-alpine", + command=("python", "/scratch/case.py"), + repo_root=repo, + scratch_files={"case.py": snippet}, + timeout_seconds=30, + memory_mb=256, + pids_limit=128, + ) + + +# observation kwargs, sandbox result, reachability, expected verdict, expected reason, worth-fixing +_CASES = ( + pytest.param( + {"recipe_missing": True}, + None, + _REACHABLE, + INCONCLUSIVE, + "missing_recipe", + False, + id="recipe-missing-inconclusive", + ), + pytest.param( + {"sandbox_missing": True}, + None, + _REACHABLE, + INCONCLUSIVE, + "sandbox_unavailable", + False, + id="sandbox-missing-inconclusive", + ), + pytest.param( + {}, + _result(exit_code=-1, timed_out=True), + _REACHABLE, + INCONCLUSIVE, + "timeout", + False, + id="timeout-inconclusive", + ), + pytest.param( + {"safety_rejected": True}, + None, + _INFERRED, + INCONCLUSIVE, + "safety_rejected", + False, + id="safety-reject-inconclusive", + ), + pytest.param( + {"covering_test": True}, + _result(exit_code=0, stdout="ok\n"), + _REACHABLE, + ALREADY_COVERED, + "covered_test_passed", + False, + id="covering-test-passed-already-covered", + ), + pytest.param( + {}, + _result(exit_code=1, stderr="AssertionError: boom\n"), + _REACHABLE, + TRIGGERABLE, + "assertion", + True, + id="assertion-triggerable-reachable-worth-fixing", + ), + pytest.param( + {}, + _result(exit_code=1, stderr="ERROR: AddressSanitizer: heap-buffer-overflow\n"), + _INFERRED, + TRIGGERABLE, + "sanitizer_abort", + True, + id="sanitizer-triggerable-inferred-worth-fixing", + ), + pytest.param( + {}, + _result(exit_code=139, stderr="Segmentation fault\n"), + _UNKNOWN, + TRIGGERABLE, + "crash", + False, + id="crash-triggerable-unknown-not-worth-fixing", + ), + pytest.param( + {}, + _result(exit_code=0, stdout="ok\n"), + _REACHABLE, + NOT_TRIGGERABLE, + "exit_zero", + False, + id="exit-zero-not-triggerable", + ), + pytest.param( + {"covering_test": True}, + _result(exit_code=0), + _UNKNOWN, + ALREADY_COVERED, + "covered_test_passed", + False, + id="already-covered-unknown-does-not-fail-build", + ), + pytest.param( + {}, + _result(exit_code=0), + _INFERRED, + NOT_TRIGGERABLE, + "exit_zero", + False, + id="not-triggerable-inferred-does-not-fail-build", + ), + pytest.param( + {}, + _result(exit_code=-1, timed_out=True), + _INFERRED, + INCONCLUSIVE, + "timeout", + False, + id="inconclusive-inferred-does-not-fail-build", + ), +) + + +@pytest.mark.parametrize( + ("flags", "result", "reachability", "expected_verdict", "expected_reason", "expected_worth_fixing"), + _CASES, +) +def test_sandbox_outcomes_map_to_verdicts_and_worth_fixing( + flags: dict[str, bool], + result: SandboxResult | None, + reachability: str, + expected_verdict: str, + expected_reason: str, + expected_worth_fixing: bool, +) -> None: + mapped = verdict_from_result(result, **flags) + + assert mapped.verdict == expected_verdict + assert mapped.reason == expected_reason + assert mapped.reason + assert is_worth_fixing(mapped.verdict, reachability) is expected_worth_fixing + + +def test_table_covers_four_verdicts_and_worth_fixing_boolean() -> None: + verdicts = {case.values[3] for case in _CASES} + worth_fixing_values = {case.values[5] for case in _CASES} + + assert verdicts == {TRIGGERABLE, NOT_TRIGGERABLE, ALREADY_COVERED, INCONCLUSIVE} + assert worth_fixing_values == {True, False} + + +def test_runner_maps_timeout_crash_and_exit_zero(tmp_path: Path) -> None: + snippet = "assert 1 + 1 == 2\n" + job = _job(tmp_path, snippet) + timeout = FakeSandboxRunner(_result(exit_code=-1, timed_out=True)) + crash = FakeSandboxRunner(_result(exit_code=1, stderr="AssertionError\n")) + ok = FakeSandboxRunner(_result(exit_code=0)) + + timed_out = RegressionRunner(timeout).run_snippet(snippet, job) + triggered = RegressionRunner(crash).run_snippet(snippet, job) + not_triggered = RegressionRunner(ok).run_snippet(snippet, job) + + assert timed_out.verdict == INCONCLUSIVE + assert timed_out.reason == "timeout" + assert triggered.verdict == TRIGGERABLE + assert not_triggered.verdict == NOT_TRIGGERABLE + assert is_worth_fixing(timed_out.verdict, _REACHABLE) is False + assert is_worth_fixing(triggered.verdict, _REACHABLE) is True + assert is_worth_fixing(not_triggered.verdict, _REACHABLE) is False