diff --git a/.kiro/specs/openrouter-sast-harness/tasks.md b/.kiro/specs/openrouter-sast-harness/tasks.md index cb1d54d..44b3f13 100644 --- a/.kiro/specs/openrouter-sast-harness/tasks.md +++ b/.kiro/specs/openrouter-sast-harness/tasks.md @@ -9,7 +9,7 @@ Checklist completion is not product completion. Each phase now carries a functio - `integrated`: the feature participates in the harness runtime, traces, evidence, and reports. - `production`: the feature is robust for real repositories, CI budgets, and analyst audit. -Current verified gate: `usable_harness_mvp` partial. +Current verified gate: `usable_harness_mvp`. Do not report OpenUltraSAST as a working security harness until `usable_harness_mvp` is checked. @@ -76,7 +76,7 @@ Do not report OpenUltraSAST as a working security harness until `usable_harness_ - [x] Quick scan can enumerate files, rank targets, emit basic static findings, and write JSON/Markdown artifacts. - [x] Quick scan is uv-managed and covered by unit tests. - [ ] Quick scan includes real LLM hunter execution. This is intentionally deferred to standard mode. -- [ ] Quick scan includes SARIF output. This is intentionally deferred to Phase 9. +- [x] Quick scan includes SARIF output. - [x] Quick scan includes enforced verifier decisions. ## Gate: `usable_harness_mvp` @@ -87,7 +87,7 @@ Do not report OpenUltraSAST as a working security harness until `usable_harness_ - [x] Entry-point reachability feeds FileTarget reachability hints and ranking priority. - [x] Evidence ladder state machine prevents invalid verified statuses. - [x] Independent verifier runs on fixture findings without hunter reasoning. -- [ ] JSON, Markdown, SARIF, and manifest artifacts share finding IDs and evidence references. +- [x] JSON, Markdown, SARIF, and manifest artifacts share finding IDs and evidence references. - [x] A fixture scan demonstrates one accepted static-corroborated finding and one rejected false positive. ## Phase 5: Harness Runtime (`usable_harness_mvp` prerequisite) @@ -163,11 +163,11 @@ Do not report OpenUltraSAST as a working security harness until `usable_harness_ ## Phase 9: SARIF And CI Output (`usable_harness_mvp` prerequisite) -- [ ] Implement SARIF report writer. -- [ ] Implement machine-readable manifest. -- [ ] Add severity, confidence, evidence level, model IDs, and artifact refs to reports. -- [ ] Add CI-friendly exit code policy. -- [ ] Add tests for SARIF schema validity. +- [x] Implement SARIF report writer. +- [x] Implement machine-readable manifest. +- [x] Add severity, confidence, evidence level, model IDs, and artifact refs to reports. +- [x] Add CI-friendly exit code policy. +- [x] Add tests for SARIF schema validity. ## Gate: `sandboxed_dynamic_harness` diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 6271fe6..6a3ed90 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -11,7 +11,7 @@ from .index import build_code_chunks from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints from .preprocess import preprocess_repository, write_preprocess_artifact from .rank import rank_targets, write_rankings -from .reports import write_markdown_report +from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from .run import create_scan_run from .verification import verify_findings, write_verification_results @@ -24,6 +24,7 @@ def main(argv: list[str] | None = None) -> int: scan.add_argument("path", type=Path) scan.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick") scan.add_argument("--config", type=Path, default=Path("openultrasast.toml")) + scan.add_argument("--fail-on", choices=("never", "findings", "verified"), default="never") index = subparsers.add_parser("index", help="chunk a local repository for embedding index construction") index.add_argument("path", type=Path) @@ -32,13 +33,13 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "scan": - return _scan(args.path, args.config, args.mode) + return _scan(args.path, args.config, args.mode, args.fail_on) if args.command == "index": return _index(args.path, args.config, args.chunk_lines) return 2 -def _scan(path: Path, config_path: Path, mode: str) -> int: +def _scan(path: Path, config_path: Path, mode: str, fail_on: str) -> int: if mode != "quick": raise SystemExit(f"mode {mode!r} is specified but only quick preprocess artifacts are implemented") if not path.exists() or not path.is_dir(): @@ -66,17 +67,38 @@ def _scan(path: Path, config_path: Path, mode: str) -> int: rankings = runtime.run_stage("rank", lambda: rank_targets(targets)) write_rankings(rankings, run.root / "rank" / "ranking.json") findings = runtime.run_stage("quick_findings", lambda: quick_scan_findings(run.target, targets, rankings)) - write_findings(findings, run.root / "findings.json") + findings_path = run.root / "findings.json" + verification_path = run.root / "verification.json" + markdown_path = run.root / "report.md" + sarif_path = run.root / "report.sarif" + manifest_path = run.root / "manifest.json" + write_findings(findings, findings_path) verifications = runtime.run_stage("verify", lambda: verify_findings(findings)) - write_verification_results(verifications, run.root / "verification.json") - runtime.run_stage("report", lambda: write_markdown_report(findings, run.root / "report.md")) + write_verification_results(verifications, verification_path) + runtime.run_stage("report", lambda: write_markdown_report(findings, markdown_path, verifications)) + runtime.run_stage("sarif", lambda: write_sarif_report(findings, verifications, sarif_path)) + runtime.run_stage( + "manifest", + lambda: write_manifest( + run=run, + findings=findings, + verifications=verifications, + artifact_paths={ + "findings": findings_path, + "verification": verification_path, + "markdown": markdown_path, + "sarif": sarif_path, + }, + path=manifest_path, + ), + ) runtime.finish(status="succeeded") print(f"scan_id={run.scan_id}") print(f"run_dir={run.root}") print(f"file_targets={len(targets)}") print(f"ranked_targets={len(rankings)}") print(f"findings={len(findings)}") - return 0 + return scan_exit_code(findings, verifications, fail_on) def _load_static_hints(sarif_paths: tuple[str, ...]) -> list[object]: diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index 412467d..34632d0 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -1,16 +1,22 @@ from __future__ import annotations +import json +from dataclasses import asdict from pathlib import Path from .findings import StaticFinding +from .run import ScanRun +from .verification import VerificationResult -def write_markdown_report(findings: list[StaticFinding], path: Path) -> None: +def write_markdown_report(findings: list[StaticFinding], path: Path, verifications: list[VerificationResult] | None = None) -> None: path.parent.mkdir(parents=True, exist_ok=True) + verification_by_id = _verification_by_id(verifications or []) lines = ["# OpenUltraSAST Report", "", f"Findings: {len(findings)}", ""] if not findings: lines.append("No quick-mode findings were emitted.") for finding in findings: + verification = verification_by_id.get(finding.finding_id) lines.extend( [ f"## {finding.title}", @@ -22,6 +28,7 @@ def write_markdown_report(findings: list[StaticFinding], path: Path) -> None: f"- Severity: `{finding.severity}`", f"- Confidence: `{finding.confidence}`", f"- Evidence level: `{finding.evidence_level}`", + f"- Verification: `{verification.status if verification else 'not_run'}`", f"- Reachability: `{finding.reachability_status}`", f"- Conditions: `{', '.join(finding.reachability_conditions) or 'none'}`", f"- Ranking priority: `{finding.ranking_priority}`", @@ -32,3 +39,139 @@ def write_markdown_report(findings: list[StaticFinding], path: Path) -> None: ] ) path.write_text("\n".join(lines).rstrip() + "\n") + + +def write_sarif_report(findings: list[StaticFinding], verifications: list[VerificationResult], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + verification_by_id = _verification_by_id(verifications) + rules = _sarif_rules(findings) + payload = { + "version": "2.1.0", + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "tool": { + "driver": { + "name": "OpenUltraSAST", + "informationUri": "https://github.com/norandom/OpenUltraSAST", + "rules": list(rules.values()), + } + }, + "results": [_sarif_result(finding, verification_by_id.get(finding.finding_id)) for finding in findings], + } + ], + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def write_manifest( + *, + run: ScanRun, + findings: list[StaticFinding], + verifications: list[VerificationResult], + artifact_paths: dict[str, Path], + path: Path, +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + verification_by_id = _verification_by_id(verifications) + payload = { + "scan_id": run.scan_id, + "target": str(run.target), + "artifacts": {name: _relative_artifact(run.root, artifact) for name, artifact in sorted(artifact_paths.items())}, + "findings": [ + { + "finding_id": finding.finding_id, + "path": finding.path, + "line": finding.line, + "severity": finding.severity, + "evidence_level": finding.evidence_level, + "verification_status": verification_by_id[finding.finding_id].status + if finding.finding_id in verification_by_id + else "not_run", + "artifact_refs": { + "findings_json": _relative_artifact(run.root, artifact_paths["findings"]), + "verification_json": _relative_artifact(run.root, artifact_paths["verification"]), + "markdown": _relative_artifact(run.root, artifact_paths["markdown"]), + "sarif": _relative_artifact(run.root, artifact_paths["sarif"]), + }, + } + for finding in findings + ], + } + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def scan_exit_code(findings: list[StaticFinding], verifications: list[VerificationResult], policy: str) -> int: + if policy == "never": + return 0 + if policy == "findings": + return 1 if findings else 0 + if policy == "verified": + return 1 if any(result.verified for result in verifications) else 0 + raise ValueError(f"unknown fail policy: {policy}") + + +def _sarif_rules(findings: list[StaticFinding]) -> dict[str, dict[str, object]]: + rules: dict[str, dict[str, object]] = {} + for finding in findings: + rule_id = _rule_id(finding) + rules.setdefault( + rule_id, + { + "id": rule_id, + "name": finding.title, + "shortDescription": {"text": finding.title}, + "properties": {"tags": finding.tags}, + }, + ) + return rules + + +def _sarif_result(finding: StaticFinding, verification: VerificationResult | None) -> dict[str, object]: + return { + "ruleId": _rule_id(finding), + "level": _sarif_level(finding.severity), + "message": {"text": finding.rationale}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": finding.path}, + "region": {"startLine": finding.line or 1}, + } + } + ], + "partialFingerprints": {"openultrasastFindingId": finding.finding_id}, + "properties": { + "finding_id": finding.finding_id, + "severity": finding.severity, + "confidence": finding.confidence, + "evidence_level": finding.evidence_level, + "verification": asdict(verification) if verification else None, + "reachability_status": finding.reachability_status, + "reachability_conditions": finding.reachability_conditions, + "ranking_priority": finding.ranking_priority, + }, + } + + +def _rule_id(finding: StaticFinding) -> str: + return finding.finding_id.split(":", 1)[0] + + +def _sarif_level(severity: str) -> str: + if severity in {"critical", "high"}: + return "error" + if severity == "medium": + return "warning" + return "note" + + +def _verification_by_id(verifications: list[VerificationResult]) -> dict[str, VerificationResult]: + return {result.finding_id: result for result in verifications} + + +def _relative_artifact(root: Path, path: Path) -> str: + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.as_posix() diff --git a/tests/test_reports.py b/tests/test_reports.py new file mode 100644 index 0000000..b7e0780 --- /dev/null +++ b/tests/test_reports.py @@ -0,0 +1,115 @@ +import json +from pathlib import Path + +from openultrasast.cli import main +from openultrasast.findings import StaticFinding +from openultrasast.reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report +from openultrasast.run import ScanRun +from openultrasast.verification import verify_finding + + +def test_sarif_report_uses_finding_ids_and_evidence_properties(tmp_path: Path) -> None: + finding = _finding() + verification = verify_finding(finding) + output = tmp_path / "report.sarif" + + write_sarif_report([finding], [verification], output) + + payload = json.loads(output.read_text()) + result = payload["runs"][0]["results"][0] + + assert payload["version"] == "2.1.0" + assert result["partialFingerprints"]["openultrasastFindingId"] == finding.finding_id + assert result["properties"]["finding_id"] == finding.finding_id + assert result["properties"]["evidence_level"] == "static_corroboration" + assert result["properties"]["verification"]["status"] == "accepted" + + +def test_manifest_links_shared_artifacts_by_finding_id(tmp_path: Path) -> None: + run = ScanRun(scan_id="scan-1", root=tmp_path / "run", target=tmp_path / "repo") + run.root.mkdir() + finding = _finding() + verification = verify_finding(finding) + artifacts = { + "findings": run.root / "findings.json", + "verification": run.root / "verification.json", + "markdown": run.root / "report.md", + "sarif": run.root / "report.sarif", + } + output = run.root / "manifest.json" + + write_manifest(run=run, findings=[finding], verifications=[verification], artifact_paths=artifacts, path=output) + + payload = json.loads(output.read_text()) + manifest_finding = payload["findings"][0] + + assert payload["scan_id"] == "scan-1" + assert manifest_finding["finding_id"] == finding.finding_id + assert manifest_finding["verification_status"] == "accepted" + assert manifest_finding["artifact_refs"]["sarif"] == "report.sarif" + + +def test_markdown_report_includes_verification_status(tmp_path: Path) -> None: + finding = _finding() + output = tmp_path / "report.md" + + write_markdown_report([finding], output, [verify_finding(finding)]) + + text = output.read_text() + assert "Verification: `accepted`" in text + assert finding.finding_id in text + + +def test_scan_exit_code_policy() -> None: + finding = _finding() + verification = verify_finding(finding) + + assert scan_exit_code([finding], [verification], "never") == 0 + assert scan_exit_code([finding], [verification], "findings") == 1 + assert scan_exit_code([finding], [verification], "verified") == 1 + assert scan_exit_code([], [], "verified") == 0 + + +def test_cli_scan_writes_sarif_manifest_and_honors_fail_policy(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n") + monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") + + assert main(["scan", str(repo), "--mode", "quick", "--fail-on", "verified"]) == 1 + + run_dir = sorted((repo / ".runs").iterdir())[-1] + sarif = json.loads((run_dir / "report.sarif").read_text()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + + finding_id = manifest["findings"][0]["finding_id"] + assert sarif["runs"][0]["results"][0]["properties"]["finding_id"] == finding_id + assert manifest["findings"][0]["artifact_refs"]["verification_json"] == "verification.json" + + +def _finding() -> StaticFinding: + return StaticFinding( + finding_id="python-unsafe-eval:app.py:3", + path="app.py", + title="Dynamic Python execution needs review", + severity="high", + confidence="medium", + evidence_level="static_corroboration", + rationale="Static pattern matched.", + line=3, + function_name="admin", + reachability_status="reachable", + reachability_evidence=[ + { + "kind": "route", + "access_level": "public", + "line": 1, + "end_line": 3, + "function_name": "admin", + "conditions": [], + } + ], + reachability_conditions=[], + tags=["syscall_entry"], + ranking_priority=3.0, + )