mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(three-stage-scan): add fail-on worth-fixing
This commit is contained in:
+18
-1
@@ -54,7 +54,24 @@ def test_surface_exposes_no_shell_or_exec_tool() -> None:
|
||||
for name in server.tool_names:
|
||||
assert not any(token in name.lower() for token in forbidden), name
|
||||
for tool in McpServer()._tools.values(): # no tool accepts a free-form command argument
|
||||
assert "command" not in tool.input_schema.get("properties", {})
|
||||
properties = tool.input_schema.get("properties", {})
|
||||
assert "command" not in properties
|
||||
assert "shell" not in properties
|
||||
assert "docker" not in properties
|
||||
|
||||
|
||||
def test_scan_rejects_deep_and_keeps_cli_only_modes() -> None:
|
||||
server = McpServer()
|
||||
listed = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
|
||||
assert listed is not None
|
||||
scan_tool = next(tool for tool in listed["result"]["tools"] if tool["name"] == "openultrasast.scan")
|
||||
mode_enum = scan_tool["inputSchema"]["properties"]["mode"]["enum"]
|
||||
assert mode_enum == ["quick", "standard"]
|
||||
assert "deep" not in mode_enum
|
||||
|
||||
result = _call(server, "openultrasast.scan", {"path": ".", "mode": "deep"})
|
||||
assert result["isError"]
|
||||
assert "deep" in result["error"]
|
||||
|
||||
|
||||
def test_notifications_get_no_response() -> None:
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.findings import StaticFinding
|
||||
@@ -63,11 +66,45 @@ def test_markdown_report_includes_verification_status(tmp_path: Path) -> None:
|
||||
def test_scan_exit_code_policy() -> None:
|
||||
finding = _finding()
|
||||
verification = verify_finding(finding)
|
||||
worth_fixing = SimpleNamespace(worth_fixing=True)
|
||||
|
||||
assert scan_exit_code([finding], [verification], "never") == 0
|
||||
assert scan_exit_code([finding], [verification], "findings") == 1
|
||||
assert scan_exit_code([], [], "findings") == 0
|
||||
assert scan_exit_code([finding], [verification], "verified") == 1
|
||||
assert scan_exit_code([], [], "verified") == 0
|
||||
# findings/verified ignore worth-fixing verdicts
|
||||
assert scan_exit_code([], [], "findings", worth_fixing_verdicts=[worth_fixing]) == 0
|
||||
assert scan_exit_code([], [], "verified", worth_fixing_verdicts=[worth_fixing]) == 0
|
||||
assert scan_exit_code([finding], [verification], "findings", worth_fixing_verdicts=[worth_fixing]) == 1
|
||||
assert scan_exit_code([finding], [verification], "verified", worth_fixing_verdicts=[worth_fixing]) == 1
|
||||
|
||||
|
||||
def test_scan_exit_code_worth_fixing() -> None:
|
||||
finding = _finding()
|
||||
verification = verify_finding(finding)
|
||||
worth_fixing = SimpleNamespace(worth_fixing=True)
|
||||
not_worth_fixing = SimpleNamespace(worth_fixing=False)
|
||||
|
||||
assert scan_exit_code([finding], [verification], "worth-fixing") == 0
|
||||
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[]) == 0
|
||||
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[not_worth_fixing]) == 0
|
||||
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[worth_fixing]) == 1
|
||||
|
||||
|
||||
def test_scan_exit_code_unknown_policy() -> None:
|
||||
with pytest.raises(ValueError, match="unknown fail policy"):
|
||||
scan_exit_code([], [], "maybe")
|
||||
|
||||
|
||||
def test_scan_help_lists_worth_fixing(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
main(["scan", "--help"])
|
||||
assert exc.value.code == 0
|
||||
help_text = capsys.readouterr().out
|
||||
assert "worth-fixing" in help_text
|
||||
assert "findings" in help_text
|
||||
assert "verified" in help_text
|
||||
|
||||
|
||||
def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
@@ -87,6 +124,15 @@ def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, m
|
||||
assert manifest["findings"][0]["artifact_refs"]["verification_json"] == "verification.json"
|
||||
|
||||
|
||||
def test_cli_scan_fail_on_worth_fixing_exits_zero_without_verdicts(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
|
||||
assert main(["scan", str(repo), "--mode", "quick", "--fail-on", "worth-fixing"]) == 0
|
||||
|
||||
|
||||
def _finding() -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="python-unsafe-eval:app.py:3",
|
||||
|
||||
Reference in New Issue
Block a user