feat(three-stage-scan): report map and honor fail-on worth-fixing

tests/test_complexity_ledger.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from types import SimpleNamespace
 
 from openultrasast.complexity.ledger import (
     NOT_TRIGGERABLE_DELTA,
@@ -9,6 +10,7 @@ from openultrasast.complexity.ledger import (
     hotspot_key,
     load_ledger,
     must_keep_as_candidate,
+    persist_verdicts,
     record_verdict,
     select_forced_candidates,
     write_ledger,
@@ -248,6 +250,25 @@ def test_overlay_reorders_and_rebands_without_dropping_hits() -> None:
     assert overlaid[0].inventory_finding_ids == ("low:1",)
 
 
+def test_persist_verdicts_records_triggerable_and_skips_inconclusive(tmp_path: Path) -> None:
+    path = tmp_path / ".openultrasast" / "calibration" / "complexity_ledger.json"
+    persist_verdicts(
+        path,
+        [
+            SimpleNamespace(path="app.py", function_name="admin", verdict="triggerable"),
+            SimpleNamespace(path="safe.py", function_name=None, verdict="not_triggerable"),
+            SimpleNamespace(path="other.py", function_name="skip", verdict="inconclusive"),
+        ],
+    )
+    loaded = load_ledger(path)
+
+    assert loaded[hotspot_key("app.py", "admin")].last_verdict == "triggerable"
+    assert loaded[hotspot_key("app.py", "admin")].score_delta == TRIGGERABLE_DELTA
+    assert loaded[hotspot_key("safe.py", None)].last_verdict == "not_triggerable"
+    assert loaded[hotspot_key("safe.py", None)].score_delta == NOT_TRIGGERABLE_DELTA
+    assert hotspot_key("other.py", "skip") not in loaded
+
+
 def test_build_complexity_map_without_ledger_path_skips_overlay(tmp_path: Path) -> None:
     target = _target("src/app.py", loc=20)
     finding = _hit("src/app.py", "rule:src/app.py:1")

tests/test_reports.py

@@ -6,7 +6,10 @@ from types import SimpleNamespace
 import pytest
 
 from openultrasast.cli import main
+from openultrasast.complexity.hints import TestHint
+from openultrasast.complexity.map import ComplexityMap, Hotspot
 from openultrasast.findings import StaticFinding
+from openultrasast.regress import CandidateVerdict
 from openultrasast.reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report
 from openultrasast.run import ScanRun
 from openultrasast.stages import Stage, plan_for_mode, record_completed, stages_payload
@@ -96,6 +99,69 @@ def test_markdown_report_includes_verification_status(tmp_path: Path) -> None:
     assert "worth_fixing" not in text.lower()
 
 
+def test_markdown_report_includes_complexity_map_and_worth_fixing_sections(tmp_path: Path) -> None:
+    finding = _finding()
+    output = tmp_path / "report.md"
+    hint = TestHint(
+        path="app.py",
+        function_name="admin",
+        gap="no_adjacent_test",
+        test_kind="http-contract",
+        reason="no adjacent test file for app.py::admin; recommend http-contract.",
+    )
+    hotspot = Hotspot(
+        path="app.py",
+        function_name="admin",
+        score=6.5,
+        band="high",
+        signals={"loc": 12, "has_adjacent_test": False},
+        rationale="high band: nested public route",
+        test_hint=hint,
+        inventory_finding_ids=(finding.finding_id,),
+    )
+    verdict = CandidateVerdict(
+        path="app.py",
+        function_name="admin",
+        language="python",
+        verdict="triggerable",
+        reason="crash",
+        inventory_finding_ids=(finding.finding_id,),
+        worth_fixing=True,
+    )
+    skipped = CandidateVerdict(
+        path="safe.py",
+        function_name=None,
+        language="python",
+        verdict="not_triggerable",
+        reason="exit_zero",
+        inventory_finding_ids=(),
+        worth_fixing=False,
+    )
+
+    write_markdown_report(
+        [finding],
+        output,
+        [verify_finding(finding)],
+        complexity_map=ComplexityMap(hotspots=(hotspot,), heuristic_only=True),
+        verdicts=(verdict, skipped),
+    )
+
+    text = output.read_text()
+    inventory_at = text.index("## Inventory")
+    map_at = text.index("## Complexity map")
+    worth_at = text.index("## Worth fixing")
+    assert inventory_at < map_at < worth_at
+    map_section = text[map_at:worth_at]
+    worth_section = text[worth_at:]
+    assert "app.py::admin" in map_section
+    assert "Gap: `no_adjacent_test`" in map_section
+    assert "Recommended test: `http-contract`" in map_section
+    assert "triggerable" in worth_section
+    assert "crash" in worth_section
+    assert "safe.py" not in worth_section
+    assert "## Inventory" in text
+
+
 def test_markdown_report_labels_hits_as_inventory_not_worth_fixing(tmp_path: Path) -> None:
     finding = _finding()
     output = tmp_path / "report.md"
@@ -277,6 +343,11 @@ def test_standard_scan_writes_complexity_map_without_docker(tmp_path: Path, monk
     assert manifest["complexity"]["hotspot_count"] == len(complexity_map["hotspots"])
     assert manifest["complexity"]["heuristic_only"] is True
     assert manifest["artifacts"]["complexity_map"] == "complexity_map.json"
+    report = (run_dir / "report.md").read_text()
+    assert "## Inventory" in report
+    assert "## Complexity map" in report
+    assert "Gap:" in report
+    assert "## Worth fixing" not in report
 
 
 def test_standard_scan_records_hunter_model_unavailable_when_model_unset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -337,6 +408,38 @@ def test_manifest_includes_complexity_summary(tmp_path: Path) -> None:
     assert payload["artifacts"]["complexity_map"] == "complexity_map.json"
 
 
+def test_manifest_includes_worth_fixing_count(tmp_path: Path) -> None:
+    run = ScanRun(scan_id="scan-1", root=tmp_path / "run", target=tmp_path / "repo")
+    run.root.mkdir()
+    finding = _finding()
+    artifacts = {
+        "findings": run.root / "findings.json",
+        "verification": run.root / "verification.json",
+        "markdown": run.root / "report.md",
+        "sarif": run.root / "report.sarif",
+        "verdicts": run.root / "verdicts.json",
+    }
+    output = run.root / "manifest.json"
+
+    write_manifest(
+        run=run,
+        findings=[finding],
+        verifications=[verify_finding(finding)],
+        artifact_paths=artifacts,
+        path=output,
+        stages={"requested": ["static", "map", "regress"], "completed": ["static", "map", "regress"], "skipped": []},
+        worth_fixing={
+            "count": 1,
+            "verdicts": [{"path": "app.py", "function_name": "admin", "verdict": "triggerable", "reason": "crash"}],
+        },
+    )
+
+    payload = json.loads(output.read_text())
+    assert payload["stages"]["completed"] == ["static", "map", "regress"]
+    assert payload["worth_fixing"]["count"] == 1
+    assert payload["worth_fixing"]["verdicts"][0]["verdict"] == "triggerable"
+
+
 def _finding() -> StaticFinding:
     return StaticFinding(
         finding_id="python-unsafe-eval:app.py:3",

tests/test_scan_deep.py

@@ -9,6 +9,7 @@ from pathlib import Path
 import pytest
 
 from openultrasast.cli import main
+from openultrasast.sandbox import FakeSandboxRunner, SandboxResult
 from openultrasast.sandbox import runner as sandbox_runner
 
 _VULN = "@app.route('/admin')\ndef admin():\n    return eval(request.data)\n"
@@ -48,6 +49,20 @@ def _latest_run(repo: Path) -> Path:
     return sorted((repo / ".runs").iterdir())[-1]
 
 
+def _crash_runner() -> FakeSandboxRunner:
+    return FakeSandboxRunner(SandboxResult(exit_code=139, stdout="", stderr="Segmentation fault\n", timed_out=False))
+
+
+def _enable_crash_sandbox(monkeypatch: pytest.MonkeyPatch) -> FakeSandboxRunner:
+    from openultrasast import cli
+
+    fake = _crash_runner()
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+    monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1")
+    monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake)
+    return fake
+
+
 def test_deep_scan_skips_regress_when_sandbox_unavailable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     repo = _repo(tmp_path)
     monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
@@ -99,6 +114,20 @@ def test_deep_scan_with_fake_runner_writes_verdicts(tmp_path: Path, monkeypatch:
         assert "path" in item
     assert manifest["artifacts"]["verdicts"] == "verdicts.json"
     assert manifest["artifacts"]["complexity_map"] == "complexity_map.json"
+    assert manifest["worth_fixing"]["count"] == sum(1 for item in payload["verdicts"] if item.get("worth_fixing"))
+    report = (run_dir / "report.md").read_text()
+    inventory_at = report.index("## Inventory")
+    map_at = report.index("## Complexity map")
+    worth_at = report.index("## Worth fixing")
+    assert inventory_at < map_at < worth_at
+    map_section = report[map_at:worth_at]
+    assert "app.py" in map_section
+    assert "Gap:" in map_section
+    ledger_path = repo / ".openultrasast" / "calibration" / "complexity_ledger.json"
+    assert ledger_path.is_file()
+    ledger = json.loads(ledger_path.read_text())
+    assert ledger
+    assert any(entry.get("last_verdict") in {"triggerable", "not_triggerable"} for entry in ledger.values())
 
 
 def test_deep_scan_unsafe_hunter_snippet_is_safety_rejected_without_docker_job(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -132,3 +161,40 @@ def test_deep_scan_unsafe_hunter_snippet_is_safety_rejected_without_docker_job(t
     for item in payload["verdicts"]:
         if item["reason"] == "safety_rejected":
             assert item["verdict"] == "inconclusive"
+
+
+def test_deep_scan_fail_on_worth_fixing_exits_one_when_triggerable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path)
+    _enable_crash_sandbox(monkeypatch)
+    docker_argv = _guard_docker(monkeypatch)
+
+    assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "worth-fixing"]) == 1
+    assert docker_argv == []
+
+    run_dir = _latest_run(repo)
+    payload = json.loads((run_dir / "verdicts.json").read_text())
+    manifest = json.loads((run_dir / "manifest.json").read_text())
+    report = (run_dir / "report.md").read_text()
+    worth_fixing = [item for item in payload["verdicts"] if item.get("worth_fixing")]
+
+    assert worth_fixing
+    assert len(worth_fixing) >= 1
+    assert any(item["verdict"] == "triggerable" for item in worth_fixing)
+    assert manifest["worth_fixing"]["count"] == len(worth_fixing)
+    assert "## Complexity map" in report
+    assert "## Worth fixing" in report
+    assert "triggerable" in report
+    ledger = json.loads((repo / ".openultrasast" / "calibration" / "complexity_ledger.json").read_text())
+    assert any(entry.get("last_verdict") == "triggerable" for entry in ledger.values())
+
+
+def test_deep_scan_fail_on_never_exits_zero_with_worth_fixing_verdict(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = _repo(tmp_path)
+    _enable_crash_sandbox(monkeypatch)
+    docker_argv = _guard_docker(monkeypatch)
+
+    assert main(["scan", str(repo), "--mode", "deep", "--fail-on", "never"]) == 0
+    assert docker_argv == []
+
+    payload = json.loads((_latest_run(repo) / "verdicts.json").read_text())
+    assert any(item.get("worth_fixing") for item in payload["verdicts"])