feat(three-stage-scan): add fail-on worth-fixing

This commit is contained in:
Marius Ciepluch
2026-09-03 19:44:04 +02:00
parent 926c8c273e
commit c5a80c2817
5 changed files with 82 additions and 5 deletions
+18 -1
View File
@@ -54,7 +54,24 @@ def test_surface_exposes_no_shell_or_exec_tool() -> None:
for name in server.tool_names:
assert not any(token in name.lower() for token in forbidden), name
for tool in McpServer()._tools.values(): # no tool accepts a free-form command argument
assert "command" not in tool.input_schema.get("properties", {})
properties = tool.input_schema.get("properties", {})
assert "command" not in properties
assert "shell" not in properties
assert "docker" not in properties
def test_scan_rejects_deep_and_keeps_cli_only_modes() -> None:
server = McpServer()
listed = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
assert listed is not None
scan_tool = next(tool for tool in listed["result"]["tools"] if tool["name"] == "openultrasast.scan")
mode_enum = scan_tool["inputSchema"]["properties"]["mode"]["enum"]
assert mode_enum == ["quick", "standard"]
assert "deep" not in mode_enum
result = _call(server, "openultrasast.scan", {"path": ".", "mode": "deep"})
assert result["isError"]
assert "deep" in result["error"]
def test_notifications_get_no_response() -> None:
+46
View File
@@ -1,5 +1,8 @@
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from openultrasast.cli import main
from openultrasast.findings import StaticFinding
@@ -63,11 +66,45 @@ def test_markdown_report_includes_verification_status(tmp_path: Path) -> None:
def test_scan_exit_code_policy() -> None:
finding = _finding()
verification = verify_finding(finding)
worth_fixing = SimpleNamespace(worth_fixing=True)
assert scan_exit_code([finding], [verification], "never") == 0
assert scan_exit_code([finding], [verification], "findings") == 1
assert scan_exit_code([], [], "findings") == 0
assert scan_exit_code([finding], [verification], "verified") == 1
assert scan_exit_code([], [], "verified") == 0
# findings/verified ignore worth-fixing verdicts
assert scan_exit_code([], [], "findings", worth_fixing_verdicts=[worth_fixing]) == 0
assert scan_exit_code([], [], "verified", worth_fixing_verdicts=[worth_fixing]) == 0
assert scan_exit_code([finding], [verification], "findings", worth_fixing_verdicts=[worth_fixing]) == 1
assert scan_exit_code([finding], [verification], "verified", worth_fixing_verdicts=[worth_fixing]) == 1
def test_scan_exit_code_worth_fixing() -> None:
finding = _finding()
verification = verify_finding(finding)
worth_fixing = SimpleNamespace(worth_fixing=True)
not_worth_fixing = SimpleNamespace(worth_fixing=False)
assert scan_exit_code([finding], [verification], "worth-fixing") == 0
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[]) == 0
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[not_worth_fixing]) == 0
assert scan_exit_code([finding], [verification], "worth-fixing", worth_fixing_verdicts=[worth_fixing]) == 1
def test_scan_exit_code_unknown_policy() -> None:
with pytest.raises(ValueError, match="unknown fail policy"):
scan_exit_code([], [], "maybe")
def test_scan_help_lists_worth_fixing(capsys: pytest.CaptureFixture[str]) -> None:
with pytest.raises(SystemExit) as exc:
main(["scan", "--help"])
assert exc.value.code == 0
help_text = capsys.readouterr().out
assert "worth-fixing" in help_text
assert "findings" in help_text
assert "verified" in help_text
def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
@@ -87,6 +124,15 @@ def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, m
assert manifest["findings"][0]["artifact_refs"]["verification_json"] == "verification.json"
def test_cli_scan_fail_on_worth_fixing_exits_zero_without_verdicts(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
repo = tmp_path / "repo"
repo.mkdir()
(repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n")
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
assert main(["scan", str(repo), "--mode", "quick", "--fail-on", "worth-fixing"]) == 0
def _finding() -> StaticFinding:
return StaticFinding(
finding_id="python-unsafe-eval:app.py:3",