feat(three-stage-scan): prove split-sink triggerable in sandbox

This commit is contained in:
Marius Ciepluch
2026-09-04 11:54:01 +02:00
parent 9af356797f
commit ac3da41c95
3 changed files with 45 additions and 2 deletions
+4 -2
View File
@@ -165,7 +165,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
## Phase 5 — Validation: Grounded corpora and CI
- [ ] 7. Replace cheat-sheet success for stages 2 and 3
- [x] 7. Replace cheat-sheet success for stages 2 and 3
- [x] 7.1 Add split-sink fixtures for Python, JavaScript, Java, and C/C++
- Each language family has at least one case where source and sink are on different lines or assigned before use.
- Observable completion: stage-1 pattern scan misses at least one planted sink in each family, documented in the manifest expected set.
@@ -185,7 +185,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
- _Requirements: 8.3, 8.4, 8.6_
- _Depends: 7.1, 2.2_
- [ ] 7.4 Prove stage 3 can trigger a planted miss when the sandbox is available
- [x] 7.4 Prove stage 3 can trigger a planted miss when the sandbox is available
- Docker-marked tests skip when the probe fails; when they run, at least one stage-1 miss is `triggerable`.
- Default pytest does not require a daemon. The pull-request job stays stage 1; a nightly or labeled job may request deep under time and candidate caps.
- Observable completion: fake-runner tests always run; docker tests skip-or-pass; the default CI workflow still invokes quick-only detection plus the new split-sink map gate.
@@ -199,3 +199,5 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
- Map `top_k` default is 20 (not named in design.md); `max_hunter_hotspots=8` and `max_candidates=5` follow design.
- `collect_signals` adjacent-test detection is false unless the caller also passes `repo_files`; task 2.2 must pass enumerated repo paths.
- ToolHunter rejects JSON dumps unless a repo tool was invoked; findings are always `suspicion` with prefix `tool-hunter:`.
- Split-sink map gate treats "high band" as the highest band the map emitted on that corpus (small fixtures often top out at medium).
- `@pytest.mark.docker` tests skip when `SandboxProbe` is unavailable; default CI does not require a daemon.
+1
View File
@@ -27,6 +27,7 @@ ousast = "openultrasast.cli:main"
[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = ["src"]
markers = ["docker: requires a docker daemon"]
[tool.ruff]
line-length = 140
+40
View File
@@ -0,0 +1,40 @@
"""Stage-3 regression on split-sink: fake runner always; live Docker skips without a daemon."""
from pathlib import Path
import pytest
from openultrasast.config import SandboxConfig
from openultrasast.regress.recipes import recipe_for
from openultrasast.regress.safety import check_snippet_safety
from openultrasast.regress.verdict import TRIGGERABLE, verdict_from_result
from openultrasast.sandbox import DockerCliRunner, FakeSandboxRunner, SandboxProbe, SandboxResult
_SPLIT_SINK_PYTHON = Path("benchmarks/fixtures/split-sink-python")
_LIMITS = SandboxConfig(memory_mb=256, timeout_seconds=30, pids_limit=128)
_PYTHON_IMAGE = "python:3.12-alpine"
_SNIPPET = 'term = "x\'; drop"\nquery = "select * from items where title like \'%" + term + "%\'"\nassert "\'" not in query\n'
def _split_sink_job():
check_snippet_safety(_SNIPPET)
job = recipe_for("python", _SNIPPET, _PYTHON_IMAGE, _LIMITS, repo_root=_SPLIT_SINK_PYTHON.resolve())
assert job is not None
return job
def test_fake_runner_marks_split_sink_snippet_triggerable() -> None:
fake = FakeSandboxRunner(SandboxResult(exit_code=1, stdout="", stderr="AssertionError", timed_out=False))
result = fake.run(_split_sink_job())
verdict = verdict_from_result(result)
assert verdict.verdict == TRIGGERABLE
assert fake.jobs
@pytest.mark.docker
def test_docker_marks_split_sink_python_triggerable() -> None:
if not SandboxProbe().available():
pytest.skip("docker daemon unavailable")
result = DockerCliRunner().run(_split_sink_job())
verdict = verdict_from_result(result)
assert verdict.verdict == TRIGGERABLE, (result.exit_code, result.stdout, result.stderr)