mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(three-stage-scan): prove split-sink triggerable in sandbox
This commit is contained in:
@@ -165,7 +165,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
|
||||
|
||||
## Phase 5 — Validation: Grounded corpora and CI
|
||||
|
||||
- [ ] 7. Replace cheat-sheet success for stages 2 and 3
|
||||
- [x] 7. Replace cheat-sheet success for stages 2 and 3
|
||||
- [x] 7.1 Add split-sink fixtures for Python, JavaScript, Java, and C/C++
|
||||
- Each language family has at least one case where source and sink are on different lines or assigned before use.
|
||||
- Observable completion: stage-1 pattern scan misses at least one planted sink in each family, documented in the manifest expected set.
|
||||
@@ -185,7 +185,7 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
|
||||
- _Requirements: 8.3, 8.4, 8.6_
|
||||
- _Depends: 7.1, 2.2_
|
||||
|
||||
- [ ] 7.4 Prove stage 3 can trigger a planted miss when the sandbox is available
|
||||
- [x] 7.4 Prove stage 3 can trigger a planted miss when the sandbox is available
|
||||
- Docker-marked tests skip when the probe fails; when they run, at least one stage-1 miss is `triggerable`.
|
||||
- Default pytest does not require a daemon. The pull-request job stays stage 1; a nightly or labeled job may request deep under time and candidate caps.
|
||||
- Observable completion: fake-runner tests always run; docker tests skip-or-pass; the default CI workflow still invokes quick-only detection plus the new split-sink map gate.
|
||||
@@ -199,3 +199,5 @@ This plan ships the three-stage scan without rebuilding CWE policy or the regex
|
||||
- Map `top_k` default is 20 (not named in design.md); `max_hunter_hotspots=8` and `max_candidates=5` follow design.
|
||||
- `collect_signals` adjacent-test detection is false unless the caller also passes `repo_files`; task 2.2 must pass enumerated repo paths.
|
||||
- ToolHunter rejects JSON dumps unless a repo tool was invoked; findings are always `suspicion` with prefix `tool-hunter:`.
|
||||
- Split-sink map gate treats "high band" as the highest band the map emitted on that corpus (small fixtures often top out at medium).
|
||||
- `@pytest.mark.docker` tests skip when `SandboxProbe` is unavailable; default CI does not require a daemon.
|
||||
|
||||
@@ -27,6 +27,7 @@ ousast = "openultrasast.cli:main"
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
markers = ["docker: requires a docker daemon"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 140
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Stage-3 regression on split-sink: fake runner always; live Docker skips without a daemon."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.config import SandboxConfig
|
||||
from openultrasast.regress.recipes import recipe_for
|
||||
from openultrasast.regress.safety import check_snippet_safety
|
||||
from openultrasast.regress.verdict import TRIGGERABLE, verdict_from_result
|
||||
from openultrasast.sandbox import DockerCliRunner, FakeSandboxRunner, SandboxProbe, SandboxResult
|
||||
|
||||
_SPLIT_SINK_PYTHON = Path("benchmarks/fixtures/split-sink-python")
|
||||
_LIMITS = SandboxConfig(memory_mb=256, timeout_seconds=30, pids_limit=128)
|
||||
_PYTHON_IMAGE = "python:3.12-alpine"
|
||||
_SNIPPET = 'term = "x\'; drop"\nquery = "select * from items where title like \'%" + term + "%\'"\nassert "\'" not in query\n'
|
||||
|
||||
|
||||
def _split_sink_job():
|
||||
check_snippet_safety(_SNIPPET)
|
||||
job = recipe_for("python", _SNIPPET, _PYTHON_IMAGE, _LIMITS, repo_root=_SPLIT_SINK_PYTHON.resolve())
|
||||
assert job is not None
|
||||
return job
|
||||
|
||||
|
||||
def test_fake_runner_marks_split_sink_snippet_triggerable() -> None:
|
||||
fake = FakeSandboxRunner(SandboxResult(exit_code=1, stdout="", stderr="AssertionError", timed_out=False))
|
||||
result = fake.run(_split_sink_job())
|
||||
verdict = verdict_from_result(result)
|
||||
assert verdict.verdict == TRIGGERABLE
|
||||
assert fake.jobs
|
||||
|
||||
|
||||
@pytest.mark.docker
|
||||
def test_docker_marks_split_sink_python_triggerable() -> None:
|
||||
if not SandboxProbe().available():
|
||||
pytest.skip("docker daemon unavailable")
|
||||
result = DockerCliRunner().run(_split_sink_job())
|
||||
verdict = verdict_from_result(result)
|
||||
assert verdict.verdict == TRIGGERABLE, (result.exit_code, result.stdout, result.stderr)
|
||||
Reference in New Issue
Block a user