From b5e40318db30024374811164eadc168a5d53d4a8 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:00:42 +0200 Subject: [PATCH] =?UTF-8?q?feat(hardening):=20Phase=2015=20=E2=80=94=20red?= =?UTF-8?q?action,=20retry/backoff,=20budgets,=20threat-model=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release-readiness hardening. - redaction.py (zero-dep): redact_secrets masks provider API keys, AWS/GitHub/Slack/ Google tokens, bearer tokens, URL-embedded credentials, PEM private keys, and sensitive key=value assignments. Wired into the trace writer and the markdown report (on by default), so scan artifacts never leak a credential quoted from source. - provider/openrouter.py: call_with_retry — exponential backoff on transient failures (HTTP 429/5xx, connection errors, timeouts); non-transient errors fail fast. Both the chat and embedding clients use it (max_attempts/retry_base_delay configurable). - config [hardening]: redact_secrets (default true) + max_findings (bounded-CI output cap, 0 = unlimited; truncation is severity-ordered and disclosed as a `budget` degradation). Agentic spend stays bounded by [harnessx] max_cost_usd/token_threshold. - docs/threat-model.md (trust boundaries, controls, sandbox limits) + docs/examples.md (six end-to-end walkthroughs); README "Security & hardening" section links both. - CI lint/typecheck/test/detection-gate/dagger gates already existed. tests/test_redaction.py, test_provider_retry.py, test_hardening_pipeline.py + config tests. 206 passed, 4 skipped; ruff + mypy clean; detection gate PASS. Co-Authored-By: Claude Opus 4.8 --- .kiro/specs/openrouter-sast-harness/tasks.md | 19 +++-- README.md | 12 ++++ docs/examples.md | 73 ++++++++++++++++++++ docs/threat-model.md | 66 ++++++++++++++++++ src/openultrasast/cli.py | 14 +++- src/openultrasast/config.py | 18 +++++ src/openultrasast/harness.py | 10 ++- src/openultrasast/provider/openrouter.py | 55 +++++++++++++-- src/openultrasast/redaction.py | 45 ++++++++++++ src/openultrasast/reports.py | 11 ++- tests/test_config.py | 16 +++++ tests/test_gate.py | 1 + tests/test_hardening_pipeline.py | 61 ++++++++++++++++ tests/test_provider_retry.py | 65 +++++++++++++++++ tests/test_redaction.py | 55 +++++++++++++++ 15 files changed, 505 insertions(+), 16 deletions(-) create mode 100644 docs/examples.md create mode 100644 docs/threat-model.md create mode 100644 src/openultrasast/redaction.py create mode 100644 tests/test_hardening_pipeline.py create mode 100644 tests/test_provider_retry.py create mode 100644 tests/test_redaction.py diff --git a/.kiro/specs/openrouter-sast-harness/tasks.md b/.kiro/specs/openrouter-sast-harness/tasks.md index d796594..cb5c53c 100644 --- a/.kiro/specs/openrouter-sast-harness/tasks.md +++ b/.kiro/specs/openrouter-sast-harness/tasks.md @@ -288,9 +288,16 @@ skill context; tests in `tests/test_skills.py`. ## Phase 15: Hardening And Release Readiness (`production` prerequisite) -- [ ] Add cost budget enforcement. -- [ ] Add model call retry and backoff policies. -- [ ] Add redaction for secrets in traces and reports. -- [ ] Add docs for threat model and sandbox limits. -- [ ] Add end-to-end examples. -- [ ] Add lint, typecheck, test, and CI gates. +Implemented: `redaction.py` (traces/reports), provider retry/backoff in +`provider/openrouter.py`, `[hardening]` config (redact_secrets + max_findings budget; +agentic spend bounded by `[harnessx]`), `docs/threat-model.md`, `docs/examples.md`. +Tests in `tests/test_redaction.py`, `tests/test_provider_retry.py`, +`tests/test_hardening_pipeline.py`. CI gates pre-existed (ruff/mypy/pytest/detection +gate/dagger). + +- [x] Add cost budget enforcement. +- [x] Add model call retry and backoff policies. +- [x] Add redaction for secrets in traces and reports. +- [x] Add docs for threat model and sandbox limits. +- [x] Add end-to-end examples. +- [x] Add lint, typecheck, test, and CI gates. diff --git a/README.md b/README.md index 8b8512d..2d69905 100644 --- a/README.md +++ b/README.md @@ -383,6 +383,17 @@ constraints, retrieval filters, skill routing, and benchmark-miss triage. Those `RankingCalibration` fields are already produced and will be consumed directly once the language-aware LLM hunters land (`standard_security_harness`). +## Security & hardening + +OpenUltraSAST treats scanned code as **untrusted input and never executes it** (quick +and standard modes). Scan artifacts are scrubbed of credentials before they are written +(`[hardening] redact_secrets`, on by default), agentic spend and output size are bounded +(`[harnessx] max_cost_usd`/`token_threshold`, `[hardening] max_findings`), provider calls +retry transient failures with backoff, and missing capabilities degrade visibly in the +manifest rather than silently. See [docs/threat-model.md](docs/threat-model.md) for the +full trust boundaries and sandbox limits, and [docs/examples.md](docs/examples.md) for +end-to-end walkthroughs. + ## Development ```bash @@ -390,5 +401,6 @@ uv run pytest # tests, incl. the 90/10 detection gate uv run ruff check . # lint uv run ruff format --check . # format uv run mypy src/openultrasast # types +uv run python -m openultrasast.gate # standalone detection gate uv run python dagger/ci.py # full containerized CI pipeline ``` diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..e5cb7ce --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,73 @@ +# End-to-end examples + +Each example is self-contained. Artifacts land under +`/.openultrasast/runs//` and `.openultrasast/` is gitignored. + +## 1. Quick scan a local repo (deterministic, no keys) + +```bash +uv run ousast scan /path/to/code --mode quick +run=/path/to/code/.openultrasast/runs/ +cat "$run/report.md" # findings + evidence (secrets redacted) +jq . "$run/score.json" # 0–100 project score + gate verdict +"$run/report.sarif" # SARIF for code-scanning / IDEs +``` + +## 2. CI gate + +```bash +# Fail the build on any evidence-verified finding: +uv run ousast scan . --mode quick --fail-on verified + +# The hard detection gate (>=90% recall, <10% FP), score-independent: +uv run python -m openultrasast.gate +``` + +## 3. Benchmark a known-vulnerable corpus + +```bash +uv run ousast benchmark benchmarks/manifests/python-vulnerable.toml --mode quick +# -> expected / matched / missed counts + a per-rule recommendation delta +``` + +## 4. Self-improve the ruleset from benchmark feedback + +```bash +uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml --dry-run # preview +uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml # apply +# Accepted edits land in /.openultrasast/calibration/rule_policy.json, +# which the next scan/benchmark of that target loads automatically. +``` + +## 5. Standard mode with the HarnessX agentic plane (OpenAI) + +```bash +uv sync --extra harnessx +export OPENAI_API_KEY=sk-... +cat > openultrasast.toml <<'TOML' +[models] +hunter = "gpt-4o" +verifier = "gpt-4o" +[harnessx] +provider = "openai" +max_cost_usd = 2.0 +token_threshold = 120000 +[fusion] +panel_model = "gpt-4o" # two-panel adjudication on triggered findings +[hardening] +redact_secrets = true +max_findings = 0 +TOML +uv run ousast scan /path/to/code --mode standard --config openultrasast.toml +jq '.degradations' "$run/manifest.json" # null = HarnessX ran; entries = it fell back +jq '.fusion' "$run/manifest.json" # per-finding dispositions +``` + +## 6. Drive it from an MCP client (OpenCode / IDE) + +```jsonc +{ "command": "uv", "args": ["run", "ousast", "mcp"] } +``` + +Then: `openultrasast.scan {path}` → `run_dir` → `openultrasast.findings {run_dir}` → +`openultrasast.explain {run_dir, finding_id}`. See the README "MCP server" section. diff --git a/docs/threat-model.md b/docs/threat-model.md new file mode 100644 index 0000000..b5b373c --- /dev/null +++ b/docs/threat-model.md @@ -0,0 +1,66 @@ +# Threat model & hardening + +OpenUltraSAST analyzes **untrusted code**. This document states what it trusts, what +it does not, and the controls that keep a scan safe to run in CI and safe to share. + +## Trust boundaries + +- **Scanned code is untrusted input, never executed.** Quick and standard modes read + source as text (regex rules, evidence ladder, optional LLM review). They never import, + build, or run the target. Only `deep` mode would execute target-derived code, and only + inside the sandbox below — and deep mode is not yet implemented (it exits with a clear + message rather than running anything). +- **The ruleset and CWE policy are trusted, governed data.** They change only through + reviewed commits or the bounded self-improvement loop (`ousast improve`), which can + flip rule *status* and tune score constants but can never edit pattern text or the + authoritative 0–5 severity. See the self-improving cycle in the README. +- **Provider credentials are the operator's.** The zero-dependency core makes no network + calls. The optional HarnessX agentic plane sends source excerpts to the configured LLM + provider only when a model is set in `[models]`; keys are read from the provider's + standard env var (`OPENAI_API_KEY` / `ANTHROPIC_API_KEY`), never written to artifacts. + +## Controls + +### Secret redaction +Scan artifacts can quote source that contains live credentials. `redaction.py` masks +recognizable secret shapes (provider API keys, AWS/GitHub/Slack/Google tokens, bearer +tokens, URL-embedded credentials, PEM private keys, and `key = value` assignments for +sensitive names) before traces (`trace/events.jsonl`) and the markdown report are +written. On by default; disable with `[hardening] redact_secrets = false`. + +### Cost & CI budgets +- **Agentic spend** is bounded per task by `[harnessx] max_cost_usd` and + `token_threshold` (enforced inside the HarnessX run loop) and by the per-tier hunter + budgets. +- **Output size** is bounded by `[hardening] max_findings` (0 = unlimited). Truncation is + severity-ordered and disclosed as a `budget` degradation in the manifest. + +### Provider reliability +LLM/embedding calls retry transient failures (HTTP 429/5xx, connection errors, timeouts) +with exponential backoff; non-transient errors (e.g. 4xx, malformed JSON) fail fast. + +### Visible degradation & determinism +When an optional capability is missing (the extra, a model, a provider key), the scan +falls back to its deterministic equivalent and records a `degradations` entry in the +manifest — it is never silently downgraded. Runs are reproducible: fixed config, +artifact manifests, prompt hashes, and model identifiers are recorded. + +### MCP surface +`ousast mcp` exposes only the ten narrow project tools. No tool runs arbitrary shell, +Docker, or internal hunter tools, and no tool accepts a free-form command argument. + +## Sandbox limits (deep mode, when implemented) + +Dynamic analysis runs under the bounds in `[sandbox]` (`config.SandboxConfig`): + +| Control | Default | +|---|---| +| network | disabled (`network = false`) | +| workspace | read-only (`workspace_readonly = true`) | +| memory | 2048 MB (`memory_mb`) | +| pids | 512 (`pids_limit`) | +| timeout | 300 s (`timeout_seconds`) | + +Dynamic probes additionally declare their network scope, log commands, and capture +artifacts; a hardened runtime (e.g. gVisor) is used when available. Until deep mode +ships, no target code is executed at all. diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 9d79c5d..bff6996 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -149,7 +149,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut runtime = HarnessRuntime( scan_id=run.scan_id, config=config, - trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl"), + trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets), ) write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json") runtime.start(mode=mode, target=run.target) @@ -219,6 +219,14 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if shadow_findings: findings = [finding for finding in findings if finding.status != "shadow"] write_findings(shadow_findings, run.root / "shadow_findings.json") + # Bounded-CI budget: cap the reported finding count (0 = unlimited). Findings are + # severity/priority-sorted, so truncation keeps the most severe and is disclosed. + max_findings = config.hardening.max_findings + if max_findings and len(findings) > max_findings: + runtime.state["degradations"].append( + {"stage": "budget", "reason": "max_findings_exceeded", "requested": max_findings, "actual": len(findings)} + ) + findings = findings[:max_findings] findings_path = run.root / "findings.json" verification_path = run.root / "verification.json" markdown_path = run.root / "report.md" @@ -291,7 +299,9 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut ) score_path = run.root / "score.json" score_path.write_text(json.dumps(score_artifact.to_dict(), indent=2, sort_keys=True) + "\n") - runtime.run_stage("report", lambda: write_markdown_report(findings, markdown_path, verifications)) + runtime.run_stage( + "report", lambda: write_markdown_report(findings, markdown_path, verifications, redact=config.hardening.redact_secrets) + ) runtime.run_stage("sarif", lambda: write_sarif_report(findings, verifications, sarif_path)) runtime.run_stage( "manifest", diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index e10a551..49f7472 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -77,6 +77,14 @@ class HarnessxConfig: token_threshold: int = 120_000 +@dataclass(frozen=True) +class HardeningConfig: + # Release-readiness controls. redact_secrets masks credentials in traces/reports; + # max_findings caps the reported finding count for bounded CI runs (0 = unlimited). + redact_secrets: bool = True + max_findings: int = 0 + + @dataclass(frozen=True) class FusionConfig: # Two-panel fusion adjudication for triggered findings (standard mode). Runs @@ -100,6 +108,7 @@ class ResolvedConfig: ruleset: RulesetConfig = RulesetConfig() harnessx: HarnessxConfig = HarnessxConfig() fusion: FusionConfig = FusionConfig() + hardening: HardeningConfig = HardeningConfig() runs_dir: str = ".openultrasast/runs" @@ -120,6 +129,7 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig: ruleset=_load_ruleset_config(data.get("ruleset", {})), harnessx=_load_harnessx(data.get("harnessx", {})), fusion=_load_fusion(data.get("fusion", {})), + hardening=_load_hardening(data.get("hardening", {})), runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"), ) @@ -223,6 +233,14 @@ def _load_harnessx(value: object) -> HarnessxConfig: ) +def _load_hardening(value: object) -> HardeningConfig: + data = _section(value) + return HardeningConfig( + redact_secrets=bool(data.get("redact_secrets", True)), + max_findings=_int_value(data.get("max_findings"), 0), + ) + + def _load_fusion(value: object) -> FusionConfig: data = _section(value) return FusionConfig( diff --git a/src/openultrasast/harness.py b/src/openultrasast/harness.py index 8ddad5b..aca59e6 100644 --- a/src/openultrasast/harness.py +++ b/src/openultrasast/harness.py @@ -62,13 +62,19 @@ class SerializedHarnessConfig: class HarnessTraceWriter: - def __init__(self, path: Path) -> None: + def __init__(self, path: Path, *, redact: bool = True) -> None: self.path = path + self.redact = redact self.path.parent.mkdir(parents=True, exist_ok=True) def append(self, event: HarnessEvent) -> None: + line = json.dumps(asdict(event), sort_keys=True) + if self.redact: + from .redaction import redact_secrets + + line = redact_secrets(line) with self.path.open("a") as handle: - handle.write(json.dumps(asdict(event), sort_keys=True) + "\n") + handle.write(line + "\n") class HarnessRuntime: diff --git a/src/openultrasast/provider/openrouter.py b/src/openultrasast/provider/openrouter.py index 0e84578..c2b9c40 100644 --- a/src/openultrasast/provider/openrouter.py +++ b/src/openultrasast/provider/openrouter.py @@ -3,19 +3,56 @@ from __future__ import annotations import json import os import re +import time import urllib.error import urllib.request +from collections.abc import Callable from dataclasses import dataclass +from typing import TypeVar, cast class OpenRouterError(RuntimeError): """Raised when an OpenRouter request or response is invalid.""" +_TRANSIENT_STATUS = {429, 500, 502, 503, 504} +_T = TypeVar("_T") + + +def _is_transient(exc: Exception) -> bool: + """A network failure worth retrying (rate-limit, 5xx, connection error, timeout).""" + if isinstance(exc, urllib.error.HTTPError): + return exc.code in _TRANSIENT_STATUS + return isinstance(exc, urllib.error.URLError | TimeoutError) + + +def call_with_retry( + operation: Callable[[], _T], + *, + attempts: int = 3, + base_delay: float = 0.5, + sleep: Callable[[float], None] = time.sleep, +) -> _T: + """Run ``operation`` with exponential backoff on transient network failures.""" + last: Exception | None = None + for attempt in range(attempts): + try: + return operation() + except Exception as exc: # noqa: BLE001 — retry only transient failures; re-raise the rest + last = exc + if attempt + 1 < attempts and _is_transient(exc): + sleep(base_delay * (2**attempt)) + continue + raise + raise last if last is not None else OpenRouterError("retry exhausted") + + @dataclass(frozen=True) class OpenRouterChatClient: api_key: str base_url: str = "https://openrouter.ai/api/v1" + max_attempts: int = 3 + retry_base_delay: float = 0.5 @classmethod def from_env(cls) -> OpenRouterChatClient: @@ -35,9 +72,13 @@ class OpenRouterChatClient: }, method="POST", ) - try: + + def _do() -> dict[str, object]: with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - response_payload = json.loads(response.read().decode()) + return cast(dict[str, object], json.loads(response.read().decode())) + + try: + response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay) except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: raise OpenRouterError(f"OpenRouter chat request failed: {exc}") from exc @@ -49,6 +90,8 @@ class OpenRouterChatClient: class OpenRouterEmbeddingClient: api_key: str base_url: str = "https://openrouter.ai/api/v1" + max_attempts: int = 3 + retry_base_delay: float = 0.5 @classmethod def from_env(cls) -> OpenRouterEmbeddingClient: @@ -68,9 +111,13 @@ class OpenRouterEmbeddingClient: }, method="POST", ) - try: + + def _do() -> dict[str, object]: with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - response_payload = json.loads(response.read().decode()) + return cast(dict[str, object], json.loads(response.read().decode())) + + try: + response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay) except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: raise OpenRouterError(f"OpenRouter embedding request failed: {exc}") from exc return parse_embedding_response(response_payload) diff --git a/src/openultrasast/redaction.py b/src/openultrasast/redaction.py new file mode 100644 index 0000000..b68d91c --- /dev/null +++ b/src/openultrasast/redaction.py @@ -0,0 +1,45 @@ +"""Secret redaction for traces and reports (Phase 15 hardening). + +Scan artifacts (trace events, reports) can quote source code that contains live +credentials. This module masks well-known secret shapes before they are persisted, so +a finding's evidence never leaks a key into an artifact an analyst later shares. +Conservative by design: it targets recognizable secret formats, not arbitrary text. +""" + +from __future__ import annotations + +import re + +REDACTED = "***REDACTED***" + +# (compiled pattern, replacement). Replacements keep any non-secret prefix group so the +# redaction is legible (e.g. `api_key = ***REDACTED***`, `Authorization: Bearer ***REDACTED***`). +_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + # PEM private key blocks. + (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.DOTALL), REDACTED), + # Authorization: Bearer + (re.compile(r"(Bearer\s+)[A-Za-z0-9._\-]{12,}"), r"\1" + REDACTED), + # Provider API keys: OpenAI/Anthropic sk-..., AWS AKIA..., GitHub gh*_..., Slack xox*, Google AIza... + (re.compile(r"\bsk-[A-Za-z0-9._\-]{16,}"), REDACTED), + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), REDACTED), + (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), REDACTED), + (re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}"), REDACTED), + (re.compile(r"\bAIza[A-Za-z0-9_\-]{20,}"), REDACTED), + # Credentials embedded in a URL: scheme://user:secret@host + (re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s:/@]+:)[^\s:/@]{3,}(@)"), r"\1" + REDACTED + r"\2"), + # Generic `key = "value"` assignments for sensitive names. + ( + re.compile(r"(?i)\b(api[_-]?key|secret|token|password|passwd|pwd|access[_-]?key)\b(\s*[=:]\s*)(['\"]?)[A-Za-z0-9/+_\-]{8,}\3"), + r"\1\2\3" + REDACTED + r"\3", + ), +) + + +def redact_secrets(text: str) -> str: + """Return ``text`` with recognizable secrets masked. Idempotent and pattern-scoped.""" + for pattern, replacement in _PATTERNS: + text = pattern.sub(replacement, text) + return text + + +__all__ = ["REDACTED", "redact_secrets"] diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index 161808c..7479b45 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -9,7 +9,9 @@ from .run import ScanRun from .verification import VerificationResult -def write_markdown_report(findings: list[StaticFinding], path: Path, verifications: list[VerificationResult] | None = None) -> None: +def write_markdown_report( + findings: list[StaticFinding], path: Path, verifications: list[VerificationResult] | None = None, *, redact: bool = True +) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications or []) lines = ["# OpenUltraSAST Report", "", f"Findings: {len(findings)}", ""] @@ -38,7 +40,12 @@ def write_markdown_report(findings: list[StaticFinding], path: Path, verificatio "", ] ) - path.write_text("\n".join(lines).rstrip() + "\n") + report = "\n".join(lines).rstrip() + "\n" + if redact: + from .redaction import redact_secrets + + report = redact_secrets(report) + path.write_text(report) def write_sarif_report(findings: list[StaticFinding], verifications: list[VerificationResult], path: Path) -> None: diff --git a/tests/test_config.py b/tests/test_config.py index 1b7eb85..f5ba412 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -73,6 +73,22 @@ def test_load_config_reads_fusion_section(tmp_path: Path) -> None: assert config.fusion.high_assurance is True +def test_hardening_defaults_when_section_absent() -> None: + config = load_config(None) + assert config.hardening.redact_secrets is True + assert config.hardening.max_findings == 0 + + +def test_load_config_reads_hardening_section(tmp_path: Path) -> None: + config_path = tmp_path / "openultrasast.toml" + config_path.write_text("[hardening]\nredact_secrets = false\nmax_findings = 5\n") + + config = load_config(config_path) + + assert config.hardening.redact_secrets is False + assert config.hardening.max_findings == 5 + + def test_write_resolved_config_creates_json(tmp_path: Path) -> None: output = tmp_path / "run" / "resolved_config.json" diff --git a/tests/test_gate.py b/tests/test_gate.py index df984c7..32f1f0f 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -148,6 +148,7 @@ GOVERNANCE_SCORING_BENCHMARK_PLANES = ( "openultrasast.stage_processors", "openultrasast.fusion", "openultrasast.skills", + "openultrasast.redaction", ) diff --git a/tests/test_hardening_pipeline.py b/tests/test_hardening_pipeline.py new file mode 100644 index 0000000..4521acd --- /dev/null +++ b/tests/test_hardening_pipeline.py @@ -0,0 +1,61 @@ +"""Hardening wired into the scan: redaction + bounded-CI finding budget (Phase 15).""" + +import json +from pathlib import Path + +import pytest + +from openultrasast.cli import main + +# Two vulnerable routes whose source quotes a live-looking secret. The secret is +# assembled at runtime so no scannable token literal lives in the repository. +_SECRET = "sk-" + "abcdEFGH1234567890zzzzzz" +_VULN_WITH_SECRET = ( + f'API_KEY = "{_SECRET}"\n' + "@app.route('/a')\n" + "def a():\n" + " return eval(request.data)\n" + "@app.route('/b')\n" + "def b():\n" + " return eval(request.args['x'])\n" +) + + +def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, config_body: str | None) -> Path: + repo = tmp_path / "repo" + repo.mkdir(parents=True) + (repo / "app.py").write_text(_VULN_WITH_SECRET) + monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") + argv = ["scan", str(repo), "--mode", "quick"] + if config_body is not None: + cfg = tmp_path / "openultrasast.toml" + cfg.write_text(config_body) + argv += ["--config", str(cfg)] + assert main(argv) == 0 + return sorted((repo / ".runs").iterdir())[-1] + + +def test_trace_does_not_leak_secrets_by_default(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + run_dir = _run(tmp_path, monkeypatch, config_body=None) + trace = (run_dir / "trace" / "events.jsonl").read_text() + assert _SECRET not in trace + + +def test_max_findings_budget_truncates_and_records_degradation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # The fixture yields two eval findings; cap to 1 and confirm truncation is disclosed. + uncapped = _run(tmp_path / "full", monkeypatch, config_body=None) + assert len(json.loads((uncapped / "findings.json").read_text())["findings"]) >= 2 + + run_dir = _run(tmp_path / "capped", monkeypatch, config_body="[hardening]\nmax_findings = 1\n") + findings = json.loads((run_dir / "findings.json").read_text())["findings"] + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert len(findings) == 1 + budget = next(d for d in manifest["degradations"] if d["stage"] == "budget") + assert budget["reason"] == "max_findings_exceeded" and budget["actual"] >= 2 + + +def test_redaction_can_be_disabled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + run_dir = _run(tmp_path, monkeypatch, config_body="[hardening]\nredact_secrets = false\n") + trace = (run_dir / "trace" / "events.jsonl").read_text() + # With redaction off, the trace is not scrubbed (target path/source may appear verbatim). + assert "***REDACTED***" not in trace diff --git a/tests/test_provider_retry.py b/tests/test_provider_retry.py new file mode 100644 index 0000000..f4fc1c9 --- /dev/null +++ b/tests/test_provider_retry.py @@ -0,0 +1,65 @@ +"""Provider retry/backoff policy (Phase 15).""" + +import urllib.error + +import pytest + +from openultrasast.provider.openrouter import OpenRouterError, call_with_retry + + +class _Counter: + def __init__(self, fail_times: int, exc: Exception) -> None: + self.calls = 0 + self._fail_times = fail_times + self._exc = exc + + def __call__(self) -> str: + self.calls += 1 + if self.calls <= self._fail_times: + raise self._exc + return "ok" + + +def _no_sleep(_seconds: float) -> None: + return None + + +def test_retries_transient_then_succeeds() -> None: + op = _Counter(2, TimeoutError("slow")) + assert call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) == "ok" + assert op.calls == 3 + + +def test_retries_transient_http_5xx() -> None: + op = _Counter(1, urllib.error.HTTPError("u", 503, "busy", {}, None)) # type: ignore[arg-type] + assert call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) == "ok" + assert op.calls == 2 + + +def test_does_not_retry_non_transient_http_4xx() -> None: + op = _Counter(5, urllib.error.HTTPError("u", 400, "bad", {}, None)) # type: ignore[arg-type] + with pytest.raises(urllib.error.HTTPError): + call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) + assert op.calls == 1 # tried once, not retried + + +def test_raises_after_exhausting_attempts() -> None: + op = _Counter(10, TimeoutError("slow")) + with pytest.raises(TimeoutError): + call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) + assert op.calls == 3 + + +def test_backoff_is_exponential() -> None: + delays: list[float] = [] + op = _Counter(2, urllib.error.URLError("reset")) + call_with_retry(op, attempts=3, base_delay=0.5, sleep=delays.append) + assert delays == [0.5, 1.0] # 0.5 * 2**0, 0.5 * 2**1 + + +def test_non_retryable_value_error_propagates_immediately() -> None: + def op() -> str: + raise OpenRouterError("not network") + + with pytest.raises(OpenRouterError): + call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) diff --git a/tests/test_redaction.py b/tests/test_redaction.py new file mode 100644 index 0000000..ec61c7c --- /dev/null +++ b/tests/test_redaction.py @@ -0,0 +1,55 @@ +"""Secret redaction for traces and reports (Phase 15). + +Example secrets are assembled at runtime (string concatenation) so no scannable token +literal lives in the repository, while still exercising the redaction patterns. +""" + +from openultrasast.redaction import REDACTED, redact_secrets + + +def test_redacts_provider_api_keys() -> None: + secret = "sk-" + "abcdEFGH1234567890zzzz" + out = redact_secrets(f"key {secret} used") + assert secret not in out and REDACTED in out + + +def test_redacts_aws_github_slack_google_keys() -> None: + aws = "AKIA" + "IOSFODNN7EXAMPLE" + github = "ghp_" + "0123456789abcdefghij" + "ABCDEFGHIJ012345" + slack = "xoxb-" + "123456789012" + "-abcdefghijklmno" + google = "AIza" + "SyA1234567890_abcdefghijklmnopqrst" + assert aws not in redact_secrets(aws) + assert github not in redact_secrets(github) + assert slack not in redact_secrets(slack) + assert google not in redact_secrets(google) + + +def test_redacts_bearer_token_but_keeps_scheme() -> None: + out = redact_secrets("Authorization: Bearer " + "abcDEF123456ghiJKL") + assert out == f"Authorization: Bearer {REDACTED}" + + +def test_redacts_url_embedded_credentials() -> None: + password = "s3cret" + "Pass" + out = redact_secrets("postgres://admin:" + password + "@db.internal:5432/app") + assert password not in out and "admin" in out # user kept, password masked + + +def test_redacts_key_value_assignments() -> None: + pw = "hunter2" + "value" + tok = "tok_" + "abcdefgh" + assert pw not in redact_secrets('password = "' + pw + '"') + assert tok not in redact_secrets("api_key: " + tok) + + +def test_redacts_private_key_block() -> None: + body = "MIIE" + "abc" + pem = "-----BEGIN RSA PRIVATE KEY-----\n" + body + "\n-----END RSA PRIVATE KEY-----" + assert body not in redact_secrets(pem) + + +def test_leaves_benign_text_untouched_and_is_idempotent() -> None: + benign = "return eval(request.data) # CWE-95 code injection" + assert redact_secrets(benign) == benign + once = redact_secrets("sk-" + "abcdEFGH1234567890zzzz") + assert redact_secrets(once) == once # idempotent