mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 12:09:27 +00:00
Compare commits
@@ -24,17 +24,50 @@ jobs:
|
||||
- run: uv run ruff check .
|
||||
- run: uv run mypy src/openultrasast
|
||||
- run: uv run pytest
|
||||
- name: Detection gate (>=90% recall, <10% FP — hard, score-independent)
|
||||
run: uv run python -m openultrasast.gate
|
||||
- run: uv run python -m compileall src tests
|
||||
|
||||
dagger:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# Must match dagger-io (0.21.7) and dagger/_engine/_version.py CLI_VERSION
|
||||
# so the engine image tag is deterministic. The dagger-for-github action
|
||||
# runs `docker image inspect registry.dagger.io/engine:v${DAGGER_VERSION}`
|
||||
# and skips its own (flaky) pull when the image is already in the local
|
||||
# docker cache, so the pinned tag below MUST equal the action's version.
|
||||
DAGGER_VERSION: 0.21.7
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Pre-pull dagger engine image (retry transient registry timeouts)
|
||||
timeout-minutes: 8
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image="registry.dagger.io/engine:v${DAGGER_VERSION}"
|
||||
attempts=5
|
||||
for i in $(seq 1 "${attempts}"); do
|
||||
echo "::group::Pull attempt ${i}/${attempts}: ${image}"
|
||||
if timeout 90 docker pull "${image}"; then
|
||||
echo "Pulled ${image} on attempt ${i}"
|
||||
docker image inspect "${image}" >/dev/null
|
||||
echo "::endgroup::"
|
||||
exit 0
|
||||
fi
|
||||
echo "Pull attempt ${i} failed."
|
||||
echo "::endgroup::"
|
||||
if [ "${i}" -lt "${attempts}" ]; then
|
||||
echo "Retrying in 20s..."
|
||||
sleep 20
|
||||
fi
|
||||
done
|
||||
echo "All ${attempts} pull attempts failed for ${image}" >&2
|
||||
exit 1
|
||||
- uses: dagger/dagger-for-github@v7
|
||||
with:
|
||||
version: latest
|
||||
version: 0.21.7
|
||||
verb: run
|
||||
args: uv run python dagger/ci.py
|
||||
|
||||
@@ -27,5 +27,7 @@ jobs:
|
||||
- run: uv build
|
||||
- uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: dist/*
|
||||
files: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
generate_release_notes: true
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# DocuHarnessX ↔ OpenUltraSAST — Blueprint Alignment
|
||||
|
||||
How OpenUltraSAST's HarnessX integration (Phase 3+) aligns with **DocuHarnessX**
|
||||
(`/home/mc/Source/DocuHarnessX`), the author's other HarnessX project. Both are HarnessX
|
||||
bundles steered from claude-code/opencode via Kiro spec-driven development. Goal: one shared
|
||||
architecture the steering layer drives identically.
|
||||
|
||||
## Orientation
|
||||
|
||||
DocuHarnessX is **agent-first** — an 8-stage pipeline (`ingest → analyze → classify → plan →
|
||||
write → review → assemble → deploy`) that *is* the agent loop. OpenUltraSAST is a
|
||||
**deterministic-pipeline + agentic-plane hybrid** — a sync `HarnessRuntime` driving 13 stages,
|
||||
with HarnessX confined to the one stage that is genuinely an agent loop (the hunter pool). They
|
||||
converge on the same load-bearing patterns and diverge only where SAST's determinism and
|
||||
supply-chain posture require it.
|
||||
|
||||
**What "self-improving multi-agent system" means in DocuHarnessX (important):** it does **not**
|
||||
use `meta_harness` / `MetaAgent.evolve` at all (the `Train` loop is explicit future / out of
|
||||
scope). Its self-improvement is realized two ways: (1) an in-product **fail-closed LLM-judge
|
||||
quality firewall** (COBESY: fixed criteria, threshold-re-derived verdicts, unavailable → fail),
|
||||
and (2) the **claude-code/opencode Kiro steering loop** (fresh implementer → adversarial
|
||||
reviewer → debugger subagents per task). OpenUltraSAST inherits both substrates and *additionally*
|
||||
builds the `MetaAgent.evolve` loop DocuHarnessX leaves unbuilt.
|
||||
|
||||
## Shared blueprint (the spine)
|
||||
|
||||
| Concern | DocuHarnessX | OpenUltraSAST | Verdict |
|
||||
|---|---|---|---|
|
||||
| Composition seam | `make_docgen` in `docuharnessx/bundle.py` — the only HarnessX import site; `control \| stages_builder()`; returns model-less `HarnessConfig` | `build_sast()` in `harness/build_sast.py` (mirrors `examples/coding/build_coding`), lazy behind `_has_harnessx()` | **shared** |
|
||||
| Model binding | CLI `prepare_run()`: `model_config.agentic(make_docgen(...))`; model on `harness.model_config.main`, never in config | `HxScanOrchestrator`: `asyncio.run(model_config.agentic(cfg).run(ScanTask(...)))` | **shared** |
|
||||
| Processor base | `NoOpStage(MultiHookProcessor)`, content-neutral on `step_end`, real module-level classes for `_target_` resolution | `MultiHookProcessor(skip_model=True)` + `SlotContractMixin`, real module-level classes | **shared** |
|
||||
| Inter-stage seam | `RunContext` over append-only slot constants; frozen versioned handoff dataclasses | typed `sast.*` slots + `SlotContractMixin` (`reads_slots`/`writes_slots`) | **shared** |
|
||||
| Domain / "ontology" seam | `.docuharnessx/ontology.yaml` → `Vocabulary` (Role×Subject×Intent), loaded at run start | central **CWE policy + ruleset** (`policy/CWE_Score.tsv` + `ruleset/*.toml` + `rule_policy.json`), loaded at run start | **analogous** |
|
||||
| Acceptance firewall | COBESY all-of gate; judge `passed` untrusted/re-derived; unavailable → fail | `VerifyProcessor` (llm-judge + structural fallback, rejects unknown reachability, fail-closed) | **analogous** |
|
||||
| Steering | Kiro SDD; `CLAUDE.md`; `.kiro/{steering,specs}`; `kiro-impl` fresh-subagent + adversarial review; `kiro-verify-completion` | same Kiro convention; `.kiro/specs/harnessx-self-improving-rulesets/*`; OpenCode steers, kiro-* maintainer-only | **shared** |
|
||||
| Self-improvement (`evolve`) | **none** — future seam | `improve/evolve.py → MetaAgent.evolve`, bounded `rule`/`policy` levers, `rule_policy.json` ledger | **diverges (OUS builds it)** |
|
||||
| Dependency posture | `harnessx` direct git dep + pyyaml | core zero-dep; `harnessx` optional SHA-pinned extra, lazy | **diverges (by design)** |
|
||||
|
||||
## Three principled divergences (and reconciliation)
|
||||
|
||||
1. **Agent-first vs hybrid.** OUS keeps the sync `HarnessRuntime` through Phases 0–4 and confines
|
||||
HarnessX to the hunter pool (the runtimes are "incompatible at every load-bearing point —
|
||||
bridge, do not map"). Reconcile by treating `build_sast()` as building **only the hunter
|
||||
sub-harness**, applying DocuHarnessX's `make_docgen` pattern *at the hunter-pool boundary*.
|
||||
Optional Phase 5 converges OUS toward DocuHarnessX's agent-first end-state (retire
|
||||
`HarnessRuntime`) — the divergence is temporal, and DocuHarnessX is the north star.
|
||||
2. **Editable ontology vs authoritative severity.** DocuHarnessX's whole vocabulary is editable
|
||||
presets. OUS splits its domain seam into a **mutable detection vocabulary** (`ruleset/*.toml`,
|
||||
loop- and human-editable) and an **immutable severity/scope authority** (`CWE_Score.tsv`,
|
||||
verycode-upstream, the loop must never edit). Where DocuHarnessX relies on convention ("never a
|
||||
hardcoded literal"), OUS relies on a **validator** (`EvolveValidator._policy` forbids editing the
|
||||
0–5 severity) because a loop, not just a human, edits it.
|
||||
3. **Firewall-as-reward vs reward/feasibility split.** DocuHarnessX's firewall *is* the signal
|
||||
(binary pass/fail). OUS must feed `MetaAgent.evolve`, so it keeps a **scalar reward**
|
||||
(`project_score`) **separate from** a **hard feasibility constraint** (≥90% recall / <10% FP),
|
||||
un-folded so the loop can't game the number. Reuse DocuHarnessX's fail-closed firewall verbatim
|
||||
at the *per-finding* `VerifyProcessor` granularity; layer the reward/feasibility split on top.
|
||||
|
||||
## Phase 3 blueprint for OpenUltraSAST (mirror DocuHarnessX)
|
||||
|
||||
- **`build_sast()` factory** = the single HarnessX composition import site (DocuHarnessX's "blast
|
||||
radius is this file" rule on `bundle.py`); `control | hunter_stages_builder()`; raised loop
|
||||
thresholds (large-repo scanning revisits shapes, like large-repo docs); model bound later in
|
||||
`HxScanOrchestrator`. OUS delta: the `_has_harnessx()` guard + raise-then-fallback to
|
||||
`run_hunter_pool()`.
|
||||
- **Ontology seam** = CWE policy + ruleset as OUS's "ontology": `load_ruleset(path, ledger)` +
|
||||
`load_policy(tsv)` loaded into `sast.*` slots at run start; `resolve_severity()` is the single
|
||||
severity authority (mirrors DocuHarnessX's "every transform reads the loaded store, never a
|
||||
literal"); absent ledger → byte-identical baseline; unmapped enabled-rule CWE → fail-loud
|
||||
(stricter than DocuHarnessX, because severity is authoritative).
|
||||
- **Processor set**: register HarnessX classes directly **only inside the hunter sub-harness**
|
||||
(LoopDetection, CostGuard/TokenBudget, ToolFailureGuard, ToolFilter, SelfVerify, EpisodeMetrics/
|
||||
OTel/Checkpoint); everything deterministic is a `skip_model=True` `MultiHookProcessor` over slots
|
||||
(SarifIngest/Preprocess/EntryPoint/Rank/Calibrate/QuickFindings/PolicyScoring/Verify/Score) with
|
||||
stable `make_<stage>_processor()` factories so a spec replaces exactly one stub in place.
|
||||
- **`MetaAgent.evolve` loop** (Phase 4, on the same substrate): two bounded levers (`rule`,
|
||||
`policy`); reward = `project_score`; feasibility = recall/FP gate (un-folded); signal path
|
||||
benchmark per-rule precision/recall + `rule_policy_delta` → `rule_signals.json` →
|
||||
`MetaAgent.evolve`; gate = validity (canonicalize → contract → `_rules`/`_policy` → replay) →
|
||||
novelty/evidence → re-benchmark → attribution → accept or **byte-for-byte revert** (the
|
||||
structural analog of DocuHarnessX's `DEFAULT_UNAVAILABLE_VERDICT="fail"`); ledger =
|
||||
`rule_policy.json` (OUS's resume-safe analog of DocuHarnessX's markdown round journal).
|
||||
- **Steering surface**: reuse DocuHarnessX's `kiro-*` skills verbatim (maintainer-only); add
|
||||
SAST-domain `.kiro/steering/` rules pinning the detection gate as a hard invariant, the zero-dep
|
||||
guard, and the `pip --dry-run` audit gate; traceable per-module docstrings citing spec Req/Decision
|
||||
numbers.
|
||||
|
||||
## Net
|
||||
|
||||
The two projects already share the spine — one composition factory, model bound in the CLI via
|
||||
`ModelConfig.agentic`, content-neutral processors over typed slots, a project-loaded domain seam,
|
||||
a fail-closed acceptance firewall, and the Kiro fresh-subagent steering loop. OpenUltraSAST diverges
|
||||
in exactly three principled places (hybrid runtime confining HarnessX to the hunter pool; it *builds*
|
||||
the `MetaAgent.evolve` loop with a reward/feasibility split + `rule_policy.json` ledger; zero-dep core
|
||||
with HarnessX opt-in). Adopting DocuHarnessX's load-bearing patterns at the points above gives the
|
||||
two projects one architecture that opencode/claude-code steers identically.
|
||||
@@ -0,0 +1,116 @@
|
||||
# HarnessX Dependency Dry-Run Audit (Task 5.1)
|
||||
|
||||
Recorded realized transitive dependency graph and the accept-vs-vendor decision for
|
||||
adopting HarnessX (`git+https://github.com/Darwin-Agent/HarnessX.git`) in Phase 3.
|
||||
|
||||
## Method
|
||||
|
||||
```bash
|
||||
uv venv /tmp/hx-audit --python 3.11
|
||||
uv pip install --dry-run --python /tmp/hx-audit/bin/python <HarnessX checkout>
|
||||
```
|
||||
|
||||
- **118 transitive packages** resolved from HarnessX's **30 direct** `[project.dependencies]`
|
||||
(all mandatory upstream; the only extra is `dev`).
|
||||
- Pinned versions observed: `harnessx 0.1.0`, and engine image `registry.dagger.io`/SDK
|
||||
unrelated; LLM SDKs `anthropic 0.111.0`, `openai 2.43.0`, `litellm 1.89.3`.
|
||||
|
||||
### Realized transitive graph (118)
|
||||
|
||||
```
|
||||
aiofiles aiohappyeyeballs aiohttp aiosignal annotated-doc annotated-types anthropic
|
||||
antlr4-python3-runtime anyio attrs bracex certifi cffi charset-normalizer click cryptography
|
||||
distro docker dockerfile-parse docstring-parser e2b et-xmlfile fastapi fastuuid filelock
|
||||
frozenlist fsspec googleapis-common-protos greenlet grpcio h11 h2 harnessx hf-xet hpack
|
||||
html2text httpcore httptools httpx httpx-sse huggingface-hub hydra-core hyperframe idna
|
||||
importlib-metadata jinja2 jiter jsonschema jsonschema-specifications litellm loguru lxml
|
||||
markdown-it-py markupsafe mcp mdurl multidict omegaconf openai openpyxl opentelemetry-api
|
||||
opentelemetry-exporter-otlp opentelemetry-exporter-otlp-proto-common
|
||||
opentelemetry-exporter-otlp-proto-grpc opentelemetry-exporter-otlp-proto-http opentelemetry-proto
|
||||
opentelemetry-sdk opentelemetry-semantic-conventions packaging pdfminer-six pdfplumber pillow
|
||||
playwright prompt-toolkit propcache protobuf pycparser pydantic pydantic-core pydantic-settings
|
||||
pyee pygments pyjwt pypdfium2 python-dateutil python-docx python-dotenv python-multipart
|
||||
python-pptx pyyaml referencing regex reportlab requests rich rpds-py shellingham six sniffio
|
||||
sse-starlette starlette structlog tiktoken tokenizers tqdm typer typing-extensions
|
||||
typing-inspection urllib3 uvicorn uvloop watchfiles wcmatch wcwidth websockets xlsxwriter yarl zipp
|
||||
```
|
||||
|
||||
## Key finding: heavy backends are lazy; ~half the tree is dead weight
|
||||
|
||||
Every heavy backend is **lazily imported** — `docker`/`e2b` are `TYPE_CHECKING`-guarded and
|
||||
imported inside provider factories; `playwright` is imported inside `_get_local_page()`;
|
||||
office/PDF libs inside `tools/builtin/read.py` parse functions; `sandbox/__init__.py` registers
|
||||
docker/e2b via `try/except`. The `gateway/` is a **separate distribution**. So the 118-package
|
||||
tree is an **install-time** cost; at **runtime only what a given agent path touches is loaded**.
|
||||
|
||||
This is the answer to "I don't need most of these": neither OpenUltraSAST (SAST) nor the
|
||||
author's DocuHarnessX (docs) loads the bulk of it — HarnessX simply declares every capability as
|
||||
a *mandatory* dependency.
|
||||
|
||||
| Category | Packages | SAST verdict |
|
||||
|---|---|---|
|
||||
| LLM SDKs | anthropic, openai, litellm, tiktoken, tokenizers, huggingface-hub, hf-xet, jiter, regex, tqdm, fsspec | **needed** |
|
||||
| HTTP / async / pydantic core | httpx(+core/h11/h2/hpack/hyperframe), anyio, aiohttp(+glue), pydantic(+core/settings), certifi, urllib3, requests, sniffio | **needed** |
|
||||
| Config / CLI / logging | hydra-core, omegaconf, antlr4-runtime, typer, click, rich, prompt-toolkit, structlog, loguru, jsonschema(+glue) | **needed** |
|
||||
| Browser automation | **playwright, pyee, greenlet, html2text** | **dead weight** (+ out-of-band Chromium binary download via `playwright install`) |
|
||||
| Sandbox backends | docker, dockerfile-parse, **e2b** | docker = useful (isolate hunter); **e2b = dead weight** |
|
||||
| Web gateway / Lab UI | **fastapi, uvicorn, starlette, sse-starlette, httptools, websockets, uvloop, watchfiles, python-multipart, aiofiles** | **dead weight** (server surface; never started by a headless embed) |
|
||||
| gRPC + OpenTelemetry | grpcio, protobuf, googleapis-common-protos, opentelemetry-* (8) | **optional** (tracing only) |
|
||||
| Office / PDF | **python-pptx, python-docx, openpyxl, reportlab, pdfplumber, pdfminer-six, pypdfium2, pillow, xlsxwriter, lxml, et-xmlfile** | **dead weight** (SAST reads source, not documents) |
|
||||
| MCP | mcp, httpx-sse | optional (only if the agent calls MCP tools) |
|
||||
|
||||
Roughly **55–60 packages are opt-in capability weight** pulled in mandatorily, including a
|
||||
browser binary and a Docker SDK — an awkward footprint for a supply-chain auditing tool.
|
||||
|
||||
## Decision
|
||||
|
||||
**ACCEPT HarnessX as the optional, SHA-pinned extra `openultrasast[harnessx]`, lazy-imported
|
||||
behind `_has_harnessx()`. Keep a pre-specified vendored lean subset as a documented, triggered
|
||||
fallback. Pursue upstream dependency-extras as the preferred long-term slimming path.**
|
||||
|
||||
Rationale:
|
||||
1. **Footprint is fully opt-in.** Core `dependencies = []` stays zero-dep; the governance /
|
||||
scoring / benchmark / detection planes never pull HarnessX. Only `openultrasast[harnessx]`
|
||||
installs the tree, and lazy imports keep heavy backends cold unless the path runs.
|
||||
2. **Author precedent.** DocuHarnessX already ships `harnessx` as a direct git dependency — the
|
||||
`ModelConfig.agentic` integration is proven and maintained. OpenUltraSAST reuses the
|
||||
mechanism but tightens the posture (opt-in, not direct).
|
||||
3. **Lower maintenance than vendoring.** Accept + SHA-pin gives reproducibility without a fork;
|
||||
vendoring is paid only if the *installed* graph is itself judged unacceptable.
|
||||
|
||||
### Preferred long-term: upstream dependency-extras (helps both projects)
|
||||
|
||||
The clean fix for "I don't need most of these" is to modularize HarnessX's own dependencies into
|
||||
extras — e.g. `harnessx[browser]`, `harnessx[sandbox-e2b]`, `harnessx[gateway]`, `harnessx[office]`,
|
||||
`harnessx[otel]` — leaving a lean agent-runtime core (`anthropic`/`openai`/`litellm`/`pydantic`/
|
||||
`httpx`/`hydra`/`structlog`/`mcp`). Since the author owns the `Darwin-Agent/HarnessX` line, this is
|
||||
actionable and benefits DocuHarnessX equally. OpenUltraSAST would then depend on
|
||||
`harnessx` (lean core) under its `[harnessx]` extra and add only the subsets it needs.
|
||||
|
||||
### Vendored lean subset (fallback, fully specified)
|
||||
|
||||
If triggered, vendor `harnessx/` only:
|
||||
|
||||
- **KEEP**: `core/`, `processors/{control,evaluation,observability,tools,context,memory,multi_model}`,
|
||||
`meta_harness/`, `rl/`, `providers/`, `tools/builtin/` (minus `browser.py`/`web_*`),
|
||||
`sandbox/{base,local,docker}.py`, plus `config`/`tracing`/`plugins`/`bundles`/`workspace` as needed.
|
||||
- **REMOVE**: `frontend/`, `gateway/`, `harnessx/api/` (drops the web-server stack), `examples/`,
|
||||
`docs/`, `benchmarks/`, `container/`, `scripts/`, `recipe/` (incl. the `recipe/verl_harnessX/verl`
|
||||
submodule — clone non-recursive regardless), `extensions/skills/{docx,pdf,pptx,xlsx}` (drops
|
||||
office/PDF), `harnessx/sandbox/e2b.py` (drops e2b), `harnessx/tools/builtin/browser.py`
|
||||
(drops playwright + browser binary). Net ≈ 40–60% fewer packages, no browser binary.
|
||||
|
||||
## Residual risks (security-tool-specific) and mitigations
|
||||
|
||||
| Risk | Mitigation |
|
||||
|---|---|
|
||||
| `playwright install` fetches a Chromium binary out-of-band (unaudited, breaks hermetic builds) | Browser path is lazy and never invoked; document that `[harnessx]` does not trigger a browser download; lean subset deletes `browser.py` |
|
||||
| `docker` SDK can reach the Docker daemon socket (privilege boundary) | Docker sandbox **default-off**, `LocalSandbox` default; run untrusted target code only inside the sandbox; SHA-pin docker/cryptography/cffi |
|
||||
| FastAPI/uvicorn listening-socket surface in a security tool | `api/` server only starts on the explicit `serve` path; add a guard test asserting no server starts on import; lean subset removes `api/` |
|
||||
| 118-package CVE surface in a supply-chain tool | SHA-pin the entire extra; keep core **zero-dep**; add a **zero-dep CI guard test** (importing `openultrasast` without the extra pulls nothing outside stdlib) |
|
||||
| `recipe/verl` git submodule recursion | Init non-recursively; never in the accept-as-extra install path |
|
||||
|
||||
> Decision date 2026-06-22. The `pip install --dry-run` audit is itself the Phase-3 gate: it is
|
||||
> re-run before the packaging task (5.2), and if the realized graph pulls a browser binary /
|
||||
> Docker SDK / FastAPI stack as *mandatory* (non-lazy) deps judged unacceptable, the spec switches
|
||||
> to the vendored lean subset under the same extra name.
|
||||
@@ -0,0 +1,611 @@
|
||||
# Design Document
|
||||
|
||||
## Overview
|
||||
|
||||
**Purpose.** This feature splits OpenUltraSAST's single overloaded regex layer — which today is detector, severity authority, and volume knob at once — into three governed owners: **rules detect**, a **central CWE policy decides severity and scope**, and a **single project score** becomes the optimization target. On top of that governance plane it adopts the HarnessX runtime exactly where it pays off — the LLM hunter loop, the LLM-judge verifier, and a meta-harness self-improvement loop that adapts rules and policy gates under hard bounds.
|
||||
|
||||
**Users.** OpenUltraSAST maintainers gain a staged, reversible HarnessX adoption and an automatic, gated rule/policy tuning loop. Security policy owners gain a versioned, central CWE-to-severity policy. Security reviewers and CI consumers gain a bounded 0-100 AppSec project score with a two-condition gate.
|
||||
|
||||
**Impact.** Today `findings.PatternRule` carries a string `severity`, `quick_scan_findings` reads a module-level `PATTERN_RULES`, `calibration.py` only demotes ranking (floored at `MIN_CALIBRATED_PRIORITY = 0.1`, keyed on `tags[0]`), `benchmark.py:368` writes an inert `next_improvement_candidate="rules_or_language_hunter"`, and `harness.HarnessRuntime.run_stage` is a sync dict-state driver. This design turns rules into TOML data, removes `PatternRule.severity`, sources severity from a vendored verycode `CWE_Score.tsv`, computes a project score, makes the benchmark actionable, and confines HarnessX behind an optional, SHA-pinned, lazy-imported extra — while `dependencies = []` and `requires-python = ">=3.11"` stay intact and the **≥90% recall / <10% FP** detection gate holds at every phase.
|
||||
|
||||
### Goals
|
||||
|
||||
- Rules become versioned data (`ruleset/<lang>/*.toml`) with a loop-controlled `status` and no rule-local severity.
|
||||
- One CWE = one governed severity, sourced from `policy/CWE_Score.tsv` via `resolve_severity()`.
|
||||
- A single 0-100 `project_score` from `policy.severity × reachability`, with a two-condition CI gate.
|
||||
- HarnessX adopted at the hunter loop, the verifier, and the meta-harness, behind `openultrasast[harnessx]`.
|
||||
- A bounded self-improvement loop (`rule`, `policy` levers) that auto-reverts byte-for-byte under the gate.
|
||||
- The governance + scoring + benchmark planes stay **zero-dependency** and **byte-identical by default** through Phases 0-2.
|
||||
|
||||
### Non-Goals
|
||||
|
||||
- The loop authoring or rewriting detection pattern text (human PR only).
|
||||
- The loop editing the authoritative 0-5 CWE severity (verycode-upstream authority).
|
||||
- A full single-step HarnessX rewrite of all deterministic stages and weight-level RL (deferred to optional Phase 5+).
|
||||
- Changing the reachability vocabulary, which stays exactly `reachable | inferred-file-surface | unknown`.
|
||||
- Dynamic/DAST scoring — report-only for a static tool.
|
||||
|
||||
## Boundary Commitments
|
||||
|
||||
### This Spec Owns
|
||||
|
||||
- The **RulesetStore** (`ruleset/store.py` + `ruleset/<lang>/*.toml`): `load_ruleset()`, the `PatternRule` schema (detection-only), and the `enabled|shadow|disabled` status vocabulary.
|
||||
- The **PolicyStore** (`policy/verycode.py` + `policy/CWE_Score.tsv`): `load_policy()`, `CwePolicy`, `resolve_severity()`, and the CWE-120→CWE-121 remap.
|
||||
- The **ScoreModel** (`scoring/project_score.py`): `SEV_WEIGHT`, `REACH_MULT`, `finding_penalty`, `project_score`, the score artifact, and the two-condition CI gate.
|
||||
- The **loop ledger** (`.openultrasast/calibration/rule_policy.json`): the loop-owned overlay of `status`/evidence-floor/threshold/precision.
|
||||
- The **HarnessX seam**: `harness/build_sast.py`, `HxScanOrchestrator`, the `MultiHookProcessor` SAST processors, the `SlotContractMixin`, and the `improve/` meta-harness driver.
|
||||
- The **benchmark→signal** conversion: `rule_id` attribution, per-rule precision/recall, `rule_policy_delta`, and `rule_signals.json`.
|
||||
|
||||
### Out of Boundary
|
||||
|
||||
- The detection-pattern **content** of rules (authored by humans via PR; the loop may only toggle status / raise floors).
|
||||
- The authoritative **0-5 severity** numbers in `CWE_Score.tsv` (verycode-upstream; PR gaps there).
|
||||
- Upstream **HarnessX internals** beyond the minimal meta-harness patch (`_VALID_LEVERS`, `_render_task_brief`, `EvolveValidator`, changeset buckets, `contract_autocheck`).
|
||||
- The DAST/dynamic execution engine; dynamic-only CWEs are report-only here.
|
||||
- Retiring `HarnessRuntime` and adopting `rl/` — explicitly Phase 5+, optional.
|
||||
|
||||
### Allowed Dependencies
|
||||
|
||||
- **Stdlib only** for the governance/scoring/benchmark planes: `tomllib`, `csv`, `json`, `math`, `dataclasses`.
|
||||
- **HarnessX** only behind the `[harnessx]` extra, pinned to a commit SHA, imported lazily behind `_has_harnessx()`.
|
||||
- The existing `HarnessRuntime` as the top-level sync driver through Phases 0-4 (not removed by this spec until Phase 5).
|
||||
- The existing reachability vocabulary emitted by `findings.py` (consumed, never redefined).
|
||||
|
||||
### Revalidation Triggers
|
||||
|
||||
- **Contract shape changes** to `PatternRule` (removing `severity`), `CwePolicy`, `VerificationResult`, `BenchmarkMiss`/`BenchmarkFalsePositive` (adding `rule_id`), or the `project_score` artifact JSON.
|
||||
- **Data ownership change**: severity moving from rule → policy; calibration re-keying from `tags[0]` → `rule_id`.
|
||||
- **Dependency direction change**: any new import of HarnessX outside a lazy `_has_harnessx()` guard; any addition to core `dependencies`.
|
||||
- **Runtime prerequisite change**: the startup CWE-resolution invariant (fail-loud if an enabled rule's CWE is unmapped); the `pip install --dry-run` audit gate before Phase 3.
|
||||
- **Lever surface change**: any addition to `journal._VALID_LEVERS` or the `EvolveValidator` bounds.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Existing Architecture Analysis
|
||||
|
||||
| Existing element | File | Constraint to respect / debt to fix |
|
||||
|---|---|---|
|
||||
| `HarnessRuntime.run_stage(name, callback)` | `harness.py:74` | Sync, dict `state` (`{"degradations": []}`), single `HarnessEvent` (`scan_id`), `ProcessorSpec.reads/writes` allow-list. Incompatible with HarnessX async-generator + message-window contract — **bridge, do not map**. Stays as the top-level driver through Phase 4. |
|
||||
| `PatternRule.severity: str` + module-level `PATTERN_RULES` | `findings.py:31,34` | The fused defect: regex carries its own severity string ("high"/"medium"/"low") and is the only volume control. **Remove `severity`; externalize patterns to TOML.** |
|
||||
| `quick_scan_findings(root, targets, rankings)` | `findings.py` / `cli.py:126` | Reads `PATTERN_RULES` implicitly. Becomes `quick_scan_findings(..., ruleset)` wired like `static_hints`. |
|
||||
| `calibrate_rankings` demotion | `calibration.py:146` | `max(MIN_CALIBRATED_PRIORITY, calibrated - 0.5)`, keyed on `finding.tags[0]` (`calibration.py:262`). Demotes ranking, never the rule. **Re-key to `rule_id`; write status flips to ledger.** |
|
||||
| `verify_findings` rejects on `reachability_status == "unknown"` | `verification.py:91` | Structural verifier. Keep as **zero-dep fallback**; add LLM-judge path. |
|
||||
| `benchmark.py:368` `next_improvement_candidate="rules_or_language_hunter"` | `benchmark.py` | Inert stub written to `calibration_records.json`. **Make actionable** via `rule_id` attribution + `rule_policy_delta`. |
|
||||
| `cli.py` `run_stage(...)` call chain | `cli.py:104-145` | The 13-stage sync pipeline. Stages migrate to HarnessX hooks incrementally; the hunter pool first. |
|
||||
|
||||
### Architecture Pattern & Boundary Map
|
||||
|
||||
**Selected pattern.** Three planes with one-directional authority: **governance data drives execution; execution emits signals; the improvement loop edits governance under a gate.** Nothing in execution mutates governance directly. The governance + scoring planes are **pure stdlib, zero HarnessX** (non-negotiable, so CI gating runs in the default install). HarnessX lives only in the execution and improvement planes, behind the extra.
|
||||
|
||||
**Boundaries.** RulesetStore detects, PolicyStore decides severity/scope, ScoreModel optimizes — three owners, three change channels (human PR for patterns/severity; loop-via-`EvolveValidator` for status/floors/score-constants).
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph GOV["GOVERNANCE PLANE — stdlib, versioned, audited, ZERO-DEP"]
|
||||
direction LR
|
||||
POL["PolicyStore<br/>policy/verycode.py<br/>CWE_Score.tsv -> CwePolicy<br/>severity 0-5, static/dynamic"]
|
||||
RUL["RulesetStore<br/>ruleset/store.py<br/>*.toml: pattern, cwe, status"]
|
||||
LED["rule_policy.json<br/>loop-owned: status/floor/precision"]
|
||||
SCO["ScoreModel<br/>scoring/project_score.py<br/>policy-weighted -> 0-100 + gate"]
|
||||
FPL["FP ledger<br/>false_positive_learnings.json"]
|
||||
end
|
||||
subgraph EXE["EXECUTION PLANE — HarnessX runtime, OPTIONAL EXTRA"]
|
||||
direction LR
|
||||
DET["Deterministic stages<br/>skip_model MultiHookProcessors"]
|
||||
HUNT["hunter_pool<br/>HxScanOrchestrator (real LLM loop)"]
|
||||
VER["VerifyProcessor<br/>llm_judge helpers (+ structural fallback)"]
|
||||
PSP["PolicyScoringProcessor + ScoreProcessor"]
|
||||
end
|
||||
subgraph IMP["IMPROVEMENT PLANE — meta_harness, OPTIONAL EXTRA"]
|
||||
EVO["improve/evolve.py -> MetaAgent.evolve()<br/>levers: rule | policy"]
|
||||
BEN["BenchmarkHarness<br/>recall/FP + per-rule attribution"]
|
||||
end
|
||||
POL --> DET & PSP & SCO
|
||||
RUL --> DET & HUNT
|
||||
LED --> RUL
|
||||
DET --> HUNT --> VER --> PSP --> SCO
|
||||
VER --> FPL
|
||||
EXE -->|trajectories + verdicts| BEN
|
||||
BEN -->|rule_signals.json + deltas| EVO
|
||||
EVO -->|gated edits| RUL & POL & LED
|
||||
SCO -->|gate verdict| CI[("CI gate")]
|
||||
BEN -->|>=90% recall / <10% FP| CI
|
||||
```
|
||||
|
||||
**New components rationale.** PolicyStore/ScoreModel/RulesetStore exist to break the fused regex defect; `HxScanOrchestrator`/`VerifyProcessor` exist to host the agent loop and judge; `improve/evolve.py`+`EvolveValidator` exist to make adaptation safe (bounded, gated, reversible). **Steering compliance:** zero-dep default install preserved; detection gate kept as a hard constraint separate from the score reward.
|
||||
|
||||
### Technology Stack
|
||||
|
||||
| Layer | Choice / Version | Role in Feature | Notes |
|
||||
|-------|------------------|-----------------|-------|
|
||||
| CLI | `cli.py` (`_run_scan`) | Wires ruleset/policy; Phase 3 routes hunter pool to `HxScanOrchestrator` | Unchanged entrypoints when extra absent |
|
||||
| Governance | Python 3.11 stdlib (`tomllib`, `csv`, `dataclasses`) | RulesetStore + PolicyStore load | Zero-dep, fail-loud on unmapped CWE |
|
||||
| Scoring | Python stdlib `math` | `project_score` exponential decay | Always on, zero-dep |
|
||||
| Execution runtime | HarnessX `@ git+...@<SHA>` (optional extra) | Hunter loop, judge verifier, processors | Lazy import behind `_has_harnessx()` |
|
||||
| Self-improvement | HarnessX `meta_harness` (extra/vendored) | `MetaAgent.evolve` + `EvolveValidator` + journal | 5/6-hook minimal patch |
|
||||
| Data / Storage | TOML rulesets, TSV policy, JSON ledger/artifacts | Versioned governance + loop overlay | All `_target_`/JSON-serializable for resume |
|
||||
| Events | HarnessX lifecycle hooks (`on_task_start`…`on_task_end`) | Stage→hook mapping | `HarnessRuntime` sync driver retained Phases 0-4 |
|
||||
| Packaging | `pyproject.toml` `[project.optional-dependencies]` | `harnessx` extra, pinned SHA | Core `dependencies = []` unchanged |
|
||||
|
||||
## File Structure Plan
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
src/openultrasast/
|
||||
├── ruleset/
|
||||
│ ├── store.py # load_ruleset()/write_ruleset(); PatternRule (NO severity); status/min_evidence_level/precision_estimate/version
|
||||
│ ├── python/*.toml # rules-as-data, one [[rule]] per detector
|
||||
│ ├── c/*.toml # incl. CWE-120 -> CWE-121 remapped buffer rules
|
||||
│ └── <lang>/*.toml # same pattern per language
|
||||
├── policy/
|
||||
│ ├── verycode.py # load_policy() -> dict[str, CwePolicy]; resolve_severity(); CwePolicy dataclass
|
||||
│ └── CWE_Score.tsv # vendored verycode data (167 CWEs; header "Flaw Severity " has trailing space)
|
||||
├── scoring/
|
||||
│ └── project_score.py # SEV_WEIGHT, REACH_MULT, finding_penalty(), project_score(), gate()
|
||||
├── harness/ # Phase 3 (extra)
|
||||
│ ├── build_sast.py # build_sast() factory mirrors examples/coding/build_coding
|
||||
│ ├── task.py # ScanTask(BaseTask)
|
||||
│ ├── run.py # HxScanOrchestrator: asyncio.run around model_config.agentic(cfg).run(...)
|
||||
│ ├── slot_contract.py # SlotContractMixin: reads_slots/writes_slots on MultiHookProcessor
|
||||
│ └── processors.py # SarifIngest/Preprocess/EntryPoint/Rank/Calibrate/QuickFindings/PolicyScoring/Verify/Score processors
|
||||
└── improve/ # Phase 4 (extra)
|
||||
├── evolve.py # orchestrator: benchmark -> delta -> rule_signals.json -> MetaAgent.evolve() -> re-benchmark -> accept/revert
|
||||
├── rule_validator.py # EvolveValidator._rules / ._policy bound checks
|
||||
└── rule_autocheck.py # ContractAutoCheckProcessor _check_rule/_check_policy branches
|
||||
|
||||
.openultrasast/calibration/
|
||||
└── rule_policy.json # loop-owned ledger: status/threshold/evidence-floor/precision per rule_id
|
||||
```
|
||||
|
||||
### Modified Files
|
||||
|
||||
- `findings.py` — extract `PATTERN_RULES` to TOML; **delete `PatternRule.severity`**; `quick_scan_findings(root, targets, rankings, ruleset)` takes ruleset arg.
|
||||
- `calibration.py` — re-key from `tags[0]` (line 262) to `rule_id`; write `rule_policy.json` status flips, not only ranking demotion (line 146).
|
||||
- `benchmark.py` — add `rule_id` to `BenchmarkMiss`/`BenchmarkFalsePositive`; per-rule precision/recall in `BenchmarkMetrics`; emit `rule_policy_delta`; replace the inert stub at line 368; add `BenchEvaluator` subclass.
|
||||
- `verification.py` — add LLM-judge path (`VerifyProcessor` over `llm_judge` helpers); keep structural verifier (`reachability_status == "unknown"` reject, line 91) as zero-dep fallback.
|
||||
- `config.py` — ruleset/policy/score paths + gate thresholds (`K`, `MIN_SCORE`, `min_emit_priority`, `min_emit_precision`).
|
||||
- `cli.py` — `_run_scan` wires ruleset/policy; Phase 3 routes the `hunter_pool` `run_stage` call (line 122) to `HxScanOrchestrator`.
|
||||
- `harness.py` — retired in Phase 5 only.
|
||||
- `pyproject.toml` — add `[project.optional-dependencies] harnessx = ["harnessx @ git+https://github.com/Darwin-Agent/HarnessX.git@<SHA>"]`; core `dependencies = []` unchanged.
|
||||
- `harnessx/meta_harness/{journal.py, agent.py, validate_workflow.py, processors/contract_autocheck.py}` — vendored/patched seam (Phase 4).
|
||||
|
||||
## System Flows
|
||||
|
||||
### Scan pipeline on HarnessX (Phase 3 seam)
|
||||
|
||||
The top-level driver stays sync (`HarnessRuntime`); only the `hunter_pool` stage opens an async surface via `asyncio.run`. Deterministic stages run as `skip_model=True` processors carrying work in typed slots; the `SlotContractMixin` validates `reads_slots`/`writes_slots` on every hook (preserving the `ProcessorSpec.reads/writes` discipline). If a HarnessX capability is unavailable at runtime, the stage falls back to its existing equivalent and records a degradation in the manifest.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["on_task_start: SarifIngestProcessor<br/>-> slot sast.static_hints"] --> B["Preprocess -> sast.targets"]
|
||||
B --> C["EntryPoint -> sast.entry_points"]
|
||||
C --> D["on_step_start: Rank -> sast.rankings"]
|
||||
D --> E["Calibrate (MemoryRetrieval pattern over FP ledger)<br/>demotes sast.rankings"]
|
||||
E --> F{"HarnessX extra<br/>present & capable?"}
|
||||
F -->|yes| G["hunter_pool: HxScanOrchestrator<br/>asyncio.run(model_config.agentic(cfg).run(ScanTask))<br/>control/tools/obs processors -> StatefulTrajectory"]
|
||||
F -->|no / capability missing| G2["fallback run_hunter_pool()<br/>record degradation in manifest"]
|
||||
G --> H["QuickFindings: regex over load_ruleset()<br/>-> sast.findings (NO severity yet)"]
|
||||
G2 --> H
|
||||
H --> I["PolicyScoringProcessor: resolve_severity(policy, rule.cwe)<br/>+ emission decision; drop shadow findings"]
|
||||
I --> J{"reachability_status<br/>== unknown?"}
|
||||
J -->|yes| K["VerifyProcessor: REJECTED (structural fallback)<br/>or llm_judge verdict"]
|
||||
J -->|no| K2["VerifyProcessor: llm_judge build/parse/call<br/>(structural fallback if extra absent)"]
|
||||
K --> L["record_calibration: lesson/missing -> FalsePositiveLearning<br/>+ per-rule outcome -> rule_policy.json"]
|
||||
K2 --> L
|
||||
L --> M["on_task_end: ScoreProcessor -> project_score + gate"]
|
||||
M --> N["report / sarif / manifest (incl. score block)"]
|
||||
```
|
||||
|
||||
### Self-improvement loop
|
||||
|
||||
The benchmark is the **regression suite governing the data stores**. The objective is `maximize project_score subject to recall ≥ 90% AND FP < 10%` as a hard feasibility constraint — kept separate so the loop cannot "improve" the number by shadowing rules that hurt real security. Every change is `shadow`-staged first, novelty/evidence/replay-gated, and reverted byte-for-byte on any failure.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant B as BenchmarkHarness
|
||||
participant O as improve/evolve.py
|
||||
participant M as MetaAgent.evolve()
|
||||
participant V as EvolveValidator (_rules/_policy)
|
||||
participant J as journal (compute_attribution/fill_gating)
|
||||
participant L as rule_policy.json
|
||||
B->>O: misses(recall) + FPs/verifier-rejects(precision) + per-rule metrics
|
||||
O->>O: write rule_signals.json [{rule_id, signal, cwe, evidence, suggested_action}]
|
||||
O->>M: evolve(brief + rule_signals.json)
|
||||
M->>M: write ruleset/*.toml + policy/overrides.toml + candidates.md + journal Round N
|
||||
M->>V: validity(canon -> contract -> _rules/_policy -> replay) + policy(novelty + evidence)
|
||||
alt validation fails
|
||||
V-->>M: StrictValidationError(kind="rule_change") -> round rejected
|
||||
else passes
|
||||
V->>B: re-run benchmark on candidate
|
||||
B->>J: passed_now vs passed_before
|
||||
J->>J: compute_attribution(predicted): flipped/still_F/regressed/still_T/absent
|
||||
alt recall<90% OR FP>=10% OR score regressed OR unpredicted regression
|
||||
J->>L: fill_gating(reverted) -> restore config byte-for-byte
|
||||
else
|
||||
J->>L: fill_gating(accepted) -> persist status/floor/precision
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
## Requirements Traceability
|
||||
|
||||
| Requirement | Summary | Components | Interfaces | Flows |
|
||||
|-------------|---------|------------|------------|-------|
|
||||
| 1.1, 1.4 | Hunter pool on real `Harness.run()`; async confined to one stage | `HxScanOrchestrator` | `run.py:HxScanOrchestrator.run()` | Scan pipeline |
|
||||
| 1.2 | Verify via llm-judge + structural fallback | `VerifyProcessor` | `processors.py:VerifyProcessor` | Scan pipeline |
|
||||
| 1.3 | Extra absent → unchanged quick/standard | `cli._run_scan`, `_has_harnessx()` | capability guard | Scan pipeline (`F` branch) |
|
||||
| 1.5 | Slot allow-list validated per hook | `SlotContractMixin` | `slot_contract.py` | Scan pipeline |
|
||||
| 1.6 | Deterministic stages `skip_model` | SAST processors | `MultiHookProcessor(skip_model=True)` | Scan pipeline |
|
||||
| 1.7 | Event/`scan_id` rename to HarnessX hooks/`run_id` | `build_sast`, journal | hook map table | Scan pipeline |
|
||||
| 1.8 | Capability-missing fallback + manifest degradation | `HxScanOrchestrator` | manifest `degradations` | Scan pipeline (`G2`) |
|
||||
| 2.1, 2.4 | Rules-as-data; 3 statuses; no rule severity | RulesetStore | `load_ruleset()`, `PatternRule` | — |
|
||||
| 2.2, 2.3 | Severity from policy; legacy strings discarded | PolicyStore, `PolicyScoringProcessor` | `resolve_severity()` | Scan pipeline (`I`) |
|
||||
| 2.5 | Shadow excluded from report+score | `PolicyScoringProcessor`, ScoreModel | emission filter | Scan pipeline (`I`,`M`) |
|
||||
| 2.6 | CwePolicy 0-5 + static/dynamic; dynamic report-only | PolicyStore | `CwePolicy` | — |
|
||||
| 2.7 | Fail-loud on unmapped enabled-rule CWE | PolicyStore startup invariant | `load_policy()` check | — |
|
||||
| 2.8 | CWE-120 → CWE-121 remap | RulesetStore | TOML data | — |
|
||||
| 2.9, 5.9, 7.5 | Governance/score/benchmark zero-dep | all GOV components | stdlib-only | — |
|
||||
| 3.1 | Two levers `rule`/`policy` | `improve/evolve.py` | `_VALID_LEVERS` | Self-improvement |
|
||||
| 3.2, 3.3 | No pattern text / no 0-5 severity edits | `EvolveValidator` | `_rules`/`_policy` | Self-improvement |
|
||||
| 3.4 | No enabled→disabled jump (shadow mandatory) | `EvolveValidator._rules` | bound check | Self-improvement |
|
||||
| 3.5 | Parse/status/bounds/dup/CWE-resolve checks | `EvolveValidator._rules` | validity phase | Self-improvement |
|
||||
| 3.6 | Novelty gate on reverted re-proposals | `journal.check_novelty` | novelty gate | Self-improvement |
|
||||
| 3.7 | Replay smoke gate (ruleset boots) | `run_synthetic_task_smoke_gate` | replay phase | Self-improvement |
|
||||
| 3.8, 3.9 | `StrictValidationError`; journal each edit | `EvolveValidator`, journal | `fill_gating` | Self-improvement |
|
||||
| 4.1, 4.2 | `rule_id` attribution; per-rule metrics; delta artifact | BenchmarkHarness | `rule_policy_delta` | Self-improvement |
|
||||
| 4.3 | Delta → `rule_signals.json` (miss/fp) | `improve/evolve.py` | signal writer | Self-improvement |
|
||||
| 4.4, 4.5 | Hard accept gate; byte-for-byte revert | `improve/evolve.py`, journal | `fill_gating(reverted)` | Self-improvement |
|
||||
| 4.6 | Auto-shadow precision-draggers | `improve/evolve.py` | status flip | Self-improvement |
|
||||
| 4.7 | Coverage-gap nominates rule (human PR) | `improve/evolve.py` | `rule_signals.json` | Self-improvement |
|
||||
| 4.8, 6.5 | Score=reward, gate=feasibility (separate) | `improve/evolve.py` | objective split | Self-improvement |
|
||||
| 5.1, 5.2 | 0-100 score; reachability multipliers | ScoreModel | `project_score()` | Scan pipeline (`M`) |
|
||||
| 5.3 | Dynamic-only/unmapped → 0 penalty, report-only | ScoreModel | `finding_penalty()` | Scan pipeline (`M`) |
|
||||
| 5.4 | Shadow excluded from score input | ScoreModel | pre-call filter | Scan pipeline (`M`) |
|
||||
| 5.5 | Score artifact + manifest block | ScoreModel | score JSON | Scan pipeline (`N`) |
|
||||
| 5.6 | FP lowers effective reachability, not delete | `calibration.py` | reachability knob | Self-improvement |
|
||||
| 5.7 | Hard gate: sev5 + reachable → fail | CI Score Gate | `gate()` | — |
|
||||
| 5.8 | Score gate advisory-first, then blocking | CI Score Gate | `gate()` | — |
|
||||
| 6.1, 6.4, 6.6 | Per-merge recall/FP gate + HX tolerance | CI Pipeline | benchmark gate | — |
|
||||
| 6.2 | Byte-identical baseline Phases 0-2 | BenchmarkHarness | golden diff | Migration |
|
||||
| 6.3 | Gate must pass on re-benchmarked candidate | `improve/evolve.py` | accept gate | Self-improvement |
|
||||
| 7.1, 7.6 | Empty core deps; pinned extra; non-recursive submodule | Packaging Config | `pyproject.toml` | Migration |
|
||||
| 7.2 | Lazy import behind capability check | `_has_harnessx()` | guard | — |
|
||||
| 7.3, 7.4, 7.7 | Dry-run audit; vendored fallback trigger; pre-specified | Packaging Config | audit gate | Migration (Phase 3 gate) |
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
| Component | Domain/Layer | Intent | Req Coverage | Key Dependencies (P0/P1) | Contracts |
|
||||
|-----------|--------------|--------|--------------|--------------------------|-----------|
|
||||
| RulesetStore | Governance | Rules-as-data + status | 2.1,2.4,2.8 | tomllib (P0), rule_policy.json (P1) | State |
|
||||
| PolicyStore | Governance | CWE→severity/scope authority | 2.2,2.6,2.7 | CWE_Score.tsv (P0) | State |
|
||||
| ScoreModel | Scoring | 0-100 score + gate | 5.1-5.9 | PolicyStore (P0) | Batch, State |
|
||||
| HxScanOrchestrator | Execution | Hunter loop on HarnessX | 1.1,1.4,1.8 | HarnessX (P0), RulesetStore (P1) | Service, Event |
|
||||
| PolicyScoringProcessor | Execution | Resolve severity + emission | 2.2,2.3,2.5 | PolicyStore (P0) | Event |
|
||||
| VerifyProcessor | Execution | LLM-judge + structural fallback | 1.2 | llm_judge (P1) | Event |
|
||||
| EvolveValidator (`_rules`/`_policy`) | Improvement | Bound-check loop edits | 3.2-3.8 | journal (P0), PolicyStore (P1) | Service |
|
||||
| improve/evolve.py | Improvement | Benchmark→signal→evolve→gate | 4.1-4.8,6.3 | BenchmarkHarness (P0), MetaAgent (P0) | Batch |
|
||||
|
||||
### Governance Layer
|
||||
|
||||
#### RulesetStore
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Load detection rules from versioned TOML; carry loop-controlled status; carry no severity |
|
||||
| Requirements | 2.1, 2.4, 2.8, 2.9 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- `load_ruleset(path) -> tuple[PatternRule, ...]` replaces module-level `PATTERN_RULES`; loaded with `tomllib` (zero-dep).
|
||||
- `PatternRule` carries `rule_id`, `languages`, `cwe`, `tags`, `pattern`, `status`, `min_evidence_level`, `precision_estimate`, `version` — **no `severity` field**.
|
||||
- `status ∈ {enabled, shadow, disabled}` is the volume control; the loop overlays `status`/`min_evidence_level`/`precision_estimate` from `rule_policy.json`.
|
||||
- C/C++ buffer rules (`c-unsafe-copy`, `c-unsafe-memory-copy`, `c-unsafe-scanf`) declare `cwe = "CWE-121"` (remapped from unmapped CWE-120).
|
||||
|
||||
**Dependencies** — Inbound: `quick_scan_findings` (P0). Outbound: `rule_policy.json` overlay (P1). External: `tomllib` (P0).
|
||||
|
||||
**Contracts**: State [x]
|
||||
|
||||
##### Service Interface
|
||||
```python
|
||||
def load_ruleset(path: Path, ledger: Path | None = None) -> tuple[PatternRule, ...]: ...
|
||||
def write_ruleset(path: Path, rules: Iterable[PatternRule]) -> None: ...
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PatternRule:
|
||||
rule_id: str
|
||||
languages: tuple[str, ...]
|
||||
cwe: str # the ONLY policy join key
|
||||
tags: tuple[str, ...]
|
||||
pattern: str # human-PR-only; loop must never edit
|
||||
status: str = "enabled" # enabled | shadow | disabled (loop-controlled)
|
||||
min_evidence_level: str = "static_corroboration"
|
||||
precision_estimate: float = 0.0
|
||||
version: str = "1"
|
||||
```
|
||||
- **Preconditions:** every loaded rule has a non-empty `pattern` and a `CWE-NNN` `cwe`.
|
||||
- **Postconditions:** ledger overlay applied; `status`/floors reflect `rule_policy.json`.
|
||||
- **Invariants:** no `severity` attribute exists; `status` is one of the three literals.
|
||||
|
||||
#### PolicyStore
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Authoritative CWE→severity/scope; resolve finding severity |
|
||||
| Requirements | 2.2, 2.3, 2.6, 2.7, 2.8 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- `load_policy() -> dict[str, CwePolicy]` keyed `"CWE-NNN"` from `CWE_Score.tsv` (167 CWEs; severity column header is literally `"Flaw Severity "` with a trailing space — parse defensively).
|
||||
- `resolve_severity(policy, rule.cwe, target, ranking)` replaces every literal `rule.severity` read. **Policy always wins**; any legacy rule severity string is discarded.
|
||||
- **Startup invariant (fail-loud):** every `enabled` rule's `cwe` must resolve in PolicyStore, else the scan aborts. `dynamic`-only CWEs are report-only.
|
||||
|
||||
**Dependencies** — Inbound: `PolicyScoringProcessor`, ScoreModel (P0). External: `csv`, `dataclasses` (P0).
|
||||
|
||||
**Contracts**: State [x]
|
||||
|
||||
##### Service Interface
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class CwePolicy:
|
||||
flaw_category: str
|
||||
severity: int # 0..5 — AUTHORITATIVE; overrides any rule string; loop must never edit
|
||||
static: bool # SAST scope gate
|
||||
dynamic: bool # DAST scope — report out-of-scope, never scored by static tool
|
||||
|
||||
def load_policy(tsv: Path = CWE_SCORE_TSV) -> dict[str, CwePolicy]: ...
|
||||
def resolve_severity(policy: Mapping[str, CwePolicy], cwe: str, target, ranking) -> int: ...
|
||||
def assert_rules_resolve(rules, policy) -> None: # raises at startup if an enabled rule's CWE is unmapped
|
||||
...
|
||||
```
|
||||
- **Invariants:** one CWE ⇒ one severity; two rules for the same CWE always receive identical severity.
|
||||
|
||||
### Scoring Layer
|
||||
|
||||
#### ScoreModel
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Compute a 0-100 project score and the two-condition gate verdict |
|
||||
| Requirements | 5.1-5.9 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- Pure stdlib (`math`), always on, zero-dep — runs in the default install with no HarnessX.
|
||||
- Excludes `shadow`-status findings before computing; assigns 0 penalty to dynamic-only/unmapped CWEs (reports them instead).
|
||||
- The `REACH_MULT` keys are exactly the three values `findings.py` emits; the reachability multiplier is the **FP-calibration knob** (a confirmed FP lowers effective reachability, never deletes a rule).
|
||||
- Emits the scan-level artifact and the manifest score block; computes the hard gate (sev5 + reachable) and the score gate (advisory-first).
|
||||
|
||||
**Contracts**: Batch [x] / State [x]
|
||||
|
||||
##### Service Interface
|
||||
```python
|
||||
SEV_WEIGHT = {5: 50, 4: 25, 3: 10, 2: 2, 1: 1, 0: 0}
|
||||
REACH_MULT = {"reachable": 1.0, "inferred-file-surface": 0.6, "unknown": 0.4}
|
||||
|
||||
def finding_penalty(f, rule_cwe: str, policy) -> float:
|
||||
pol = policy.get(rule_cwe)
|
||||
if pol is None or not pol.static: # dynamic-only / unmapped -> report, don't score
|
||||
return 0.0
|
||||
return SEV_WEIGHT[pol.severity] * REACH_MULT.get(f.reachability_status, 0.4)
|
||||
|
||||
def project_score(findings, rule_cwe_by_id, policy, K: float = 60.0) -> int:
|
||||
# shadow-status findings excluded before this call
|
||||
P = sum(finding_penalty(f, rule_cwe_by_id[f.finding_id], policy) for f in findings)
|
||||
return round(100 * math.exp(-P / K)) # P=0 -> 100; one critical-reachable -> ~43
|
||||
|
||||
def gate(findings, score, *, min_score: int, block_severity_reachable: int = 5,
|
||||
blocking: bool) -> dict: ...
|
||||
```
|
||||
- **Postconditions:** artifact JSON (see Data Models) written next to per-finding output and merged into the manifest.
|
||||
|
||||
### Execution Layer
|
||||
|
||||
#### HxScanOrchestrator
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Run the hunter pool on a real HarnessX agent loop, confining async to one stage |
|
||||
| Requirements | 1.1, 1.4, 1.7, 1.8 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- Invoked from the existing `hunter_pool` `run_stage` callback (`cli.py:122`); wraps `asyncio.run(model_config.agentic(harness_config).run(ScanTask(...)))`.
|
||||
- `build_sast()` mirrors `examples/coding/build_coding`; `ModelConfig(main=hunter, judge=…, ranker=…, patcher=…).agentic(cfg)` keeps model info out of `HarnessConfig`.
|
||||
- Each hunter sub-harness registers `LoopDetectionProcessor`, `CostGuardProcessor`/`TokenBudgetProcessor`, `ToolFailureGuard`, `ToolFilterProcessor`, `SelfVerifyProcessor`, `EpisodeMetricsProcessor`/`OTelProcessor`/`CheckpointProcessor`.
|
||||
- Produces a `StatefulTrajectory` per target; rewards backfilled from verifier verdicts. On capability-missing, falls back to `run_hunter_pool()` and appends to manifest `degradations`.
|
||||
|
||||
**Dependencies** — External: HarnessX core + processors (P0, lazy). Inbound: `cli._run_scan` (P0).
|
||||
|
||||
**Contracts**: Service [x] / Event [x]
|
||||
|
||||
##### Event Contract
|
||||
- Hook mapping: `stage_start`→`on_step_start`, `stage_end`→`on_step_end`, `scan_id`→`session_id`/`run_id`; `before/after_model|tool` direct. `HARNESSX_CONTRACT_MODE` (message-window) + `OUS_SLOT_CONTRACT` (key-level via `SlotContractMixin`) split the old single `contract_mode`.
|
||||
|
||||
#### PolicyScoringProcessor
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Resolve policy severity per finding and make the emission decision |
|
||||
| Requirements | 2.2, 2.3, 2.5 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- Runs `on_after_tool`/`on_step_end`; reads `sast.rankings` + PolicyStore; writes resolved severity + emit-decision.
|
||||
- Drops `shadow` findings from the reported set and the score input while recording their outcome for precision tracking.
|
||||
- Applies `min_emit_priority`/`min_emit_precision` thresholds — the new volume controls that replace "every regex match becomes a finding".
|
||||
|
||||
**Contracts**: Event [x]
|
||||
|
||||
#### VerifyProcessor
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | LLM-judge verification with a zero-dep structural fallback |
|
||||
| Requirements | 1.2 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- Wraps `build_judge_prompt`/`parse_judge_response`/`_call_judge` (`llm_judge.py`) as a `judge`-role sub-harness; maps the strict-JSON verdict (`plausible|unsupported|hedging|…` + `lesson`/`missing`) to `VerificationResult`.
|
||||
- **Fallback:** the existing structural verifier (`verification.py:91`, rejects on `reachability_status == "unknown"`) is used verbatim when the extra is absent or the judge is unavailable.
|
||||
|
||||
**Contracts**: Event [x]
|
||||
|
||||
### Improvement Layer
|
||||
|
||||
#### EvolveValidator (`_rules` / `_policy`)
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Enforce the bounded-lever safety contract on every loop edit |
|
||||
| Requirements | 3.2-3.8 |
|
||||
|
||||
**Responsibilities & Constraints** (the gate — runs in the *validity* phase beside `_contract`/`_replay`)
|
||||
- `_rules`: edit parses; `status ∈ {enabled,shadow,disabled}`; `tighten`/`loosen` adjusts only threshold/evidence-floor **within configured bounds**; `add` doesn't duplicate a `rule_id`; **every `cwe` resolves in PolicyStore**; **pattern text unchanged** (reject otherwise); **no `enabled→disabled` jump** (require `shadow`).
|
||||
- `_policy`: touches only evidence floor, scope, `K`/`MIN_SCORE` within bounds; **never the 0-5 severity**.
|
||||
- On any failure: `raise StrictValidationError(kind="rule_change")` → the whole round is rejected.
|
||||
|
||||
**Contracts**: Service [x]
|
||||
|
||||
##### Service Interface
|
||||
```python
|
||||
_VALID_LEVERS = frozenset({
|
||||
"configuration", "control", "action", "instruction",
|
||||
"rule", # status/evidence-floor/threshold in ruleset/ — NEVER pattern text
|
||||
"policy", # evidence floor + scope + K/MIN_SCORE — NEVER the 0-5 CWE severity
|
||||
})
|
||||
|
||||
class EvolveValidator:
|
||||
def _rules(self, edit) -> None: ... # raises StrictValidationError(kind="rule_change")
|
||||
def _policy(self, edit) -> None: ...
|
||||
```
|
||||
|
||||
#### improve/evolve.py (orchestrator)
|
||||
|
||||
| Field | Detail |
|
||||
|-------|--------|
|
||||
| Intent | Drive benchmark→signal→evolve→re-benchmark→accept/revert under the gate |
|
||||
| Requirements | 4.1-4.8, 6.3 |
|
||||
|
||||
**Responsibilities & Constraints**
|
||||
- Converts `rule_policy_delta` + FP learnings → `rule_signals.json` (recall side `signal="miss"`; precision side `signal="verifier_reject"|"fp"`); calls `MetaAgent.evolve()`.
|
||||
- **Acceptance gate (hard):** round accepted **only if** aggregate recall ≥ 90% **AND** FP < 10% **AND** `project_score` did not regress on the held set **AND** no unpredicted regression; otherwise `fill_gating(outcome="reverted")` restores config byte-for-byte.
|
||||
- Precision-dragging rules are **auto-shadowed**, never deleted; coverage gaps nominate a rule via `rule_signals.json` (pattern authoring stays human-PR).
|
||||
- Objective: `maximize project_score s.t. recall/FP gate` — score is the reward, the gate is the feasibility region; never folded together.
|
||||
|
||||
**Contracts**: Batch [x]
|
||||
|
||||
## Data Models
|
||||
|
||||
### Ruleset policy schema (detection only — TOML)
|
||||
|
||||
```toml
|
||||
[[rule]]
|
||||
rule_id = "python-sql-injection"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-89" # the ONLY policy join key
|
||||
tags = ["injection"]
|
||||
pattern = '(\.execute)\s*\(\s*[^,)]*%' # human-PR-only
|
||||
status = "enabled" # enabled | shadow | disabled (loop-controlled)
|
||||
min_evidence_level = "static_corroboration" # loop may raise within bounds
|
||||
precision_estimate = 0.62 # backfilled from benchmarks
|
||||
version = "3"
|
||||
```
|
||||
|
||||
Status semantics: `enabled` fires/scores/reports; `shadow` fires and records outcomes but is excluded from report and score (mandatory staging state for any loop-proposed change); `disabled` does not fire.
|
||||
|
||||
Loop ledger (`rule_policy.json`) — loop-owned overlay, re-keyed on `rule_id`:
|
||||
```json
|
||||
{ "python-sql-injection": { "status": "shadow", "min_evidence_level": "dynamic_corroboration",
|
||||
"precision_estimate": 0.41, "outcome": "accepted", "round": 7 } }
|
||||
```
|
||||
|
||||
### Project-score schema (artifact + manifest block)
|
||||
|
||||
```json
|
||||
{ "project_score": 73, "max_severity": 5, "penalty_total": 30.0,
|
||||
"by_category": {"Command or Argument Injection": 1, "Cryptographic Issues": 2},
|
||||
"out_of_scope_dynamic_only": 0,
|
||||
"unmapped_cwe": [],
|
||||
"gate": {"min_score": 80, "block_severity_reachable": 5, "blocking": false, "passed": false} }
|
||||
```
|
||||
|
||||
### Consistency & Integrity
|
||||
|
||||
- **Authority direction:** governance data is the source of truth; execution reads it, emits findings/signals; only the loop (via `EvolveValidator`) writes `status`/floors/score-constants back, never the pattern or the 0-5 severity.
|
||||
- **Temporal/versioning:** `PatternRule.version` and `rule_policy.json.round` give per-rule audit; the journal records each lever edit; `fill_gating(reverted)` restores byte-for-byte.
|
||||
- **Serialization (resume-safe):** all scan slots (`sast.targets`, `sast.findings`, `sast.rankings`, verifications) use plain dicts/dataclasses with `_target_` registration so they survive `wake()`.
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Error Strategy
|
||||
|
||||
| Error type | Condition | Response |
|
||||
|---|---|---|
|
||||
| **Startup config error** | An `enabled` rule's `cwe` is unmapped in PolicyStore | `assert_rules_resolve` raises; **fail loud**, abort scan before any work (Req 2.7) |
|
||||
| **Malformed ruleset/policy TOML** | `tomllib` parse failure | Fail loud at load; CI catches before merge |
|
||||
| **Loop edit violates bounds** | Pattern edit / severity edit / enabled→disabled / duplicate `rule_id` / unmapped CWE | `EvolveValidator` raises `StrictValidationError(kind="rule_change")`; entire round rejected (Req 3.8) |
|
||||
| **Reverted re-proposal w/o rationale** | Novelty gate hit | `journal.check_novelty` blocks; loop continues with other candidates (Req 3.6) |
|
||||
| **Replay smoke gate failure** | Edited ruleset/policy won't boot | Round rejected before benchmark re-run (Req 3.7) |
|
||||
| **Acceptance-gate failure** | recall<90% OR FP≥10% OR score regressed OR unpredicted regression | `fill_gating(reverted)` restores config byte-for-byte (Req 4.5) |
|
||||
| **HarnessX capability missing at runtime** | Extra absent or sub-harness unavailable | Stage falls back to existing equivalent; record degradation in manifest (Req 1.8) |
|
||||
| **Slot contract violation** | Stage reads/writes an undeclared slot | `SlotContractMixin` raises under `OUS_SLOT_CONTRACT=strict`; warns under `warn` (Req 1.5) |
|
||||
| **Slot serialization loss on resume** | Non-`_target_` slot restores as `None` | Per-slot-type round-trip test guards; `_target_` registration required |
|
||||
|
||||
### Monitoring
|
||||
|
||||
- `EpisodeMetricsProcessor.episode_summary` → manifest metrics block; `OTelProcessor` OTLP export when configured.
|
||||
- `HarnessJournal` writes per-run `{run_id}.jsonl` / `_trace.jsonl` / `_state.json`, rotated on `SegmentBoundaryEvent`.
|
||||
- The score artifact `unmapped_cwe` list surfaces policy-completeness drift; the journal changeset table surfaces every rule/policy lever edit per round.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
- `load_policy()` parses the `"Flaw Severity "` trailing-space header and yields 167 `CwePolicy` entries with correct `static`/`dynamic` flags.
|
||||
- `project_score()` boundary cases: P=0 → 100; one severity-5 reachable finding → ~43; one low finding → ~97; shadow findings excluded.
|
||||
- `resolve_severity()` returns policy severity (not the legacy rule string) and is identical for two rules sharing a CWE.
|
||||
- `EvolveValidator._rules` rejects: pattern edit, severity edit, `enabled→disabled` jump, duplicate `rule_id`, unmapped CWE.
|
||||
- `finding_penalty()` returns 0.0 for dynamic-only and unmapped CWEs.
|
||||
|
||||
### Integration Tests
|
||||
- **Byte-identical baseline (Req 6.2):** with the default ruleset and an empty `rule_policy.json`, `benchmark.py` output matches the pre-feature golden corpus exactly.
|
||||
- **Startup invariant (Req 2.7):** an enabled rule with an unmapped CWE aborts the scan with a loud error.
|
||||
- **Self-improvement round (Req 4.4/4.5):** a deliberately noisy rule is auto-shadowed when it drags precision; a round that regresses recall is reverted byte-for-byte.
|
||||
- **Capability fallback (Req 1.8):** extra absent → `quick`/`standard` run through the sync driver unchanged; degradation recorded when a HX stage is requested but unavailable.
|
||||
- **Slot round-trip (Req on resume):** each scan slot type serializes and restores without loss after `wake()`.
|
||||
|
||||
### E2E / Gate Tests
|
||||
- **CI detection gate (Req 6.1):** the pipeline fails the merge if recall<90% or FP≥10% on the corpus.
|
||||
- **Two-condition score gate (Req 5.7/5.8):** hard gate fails on a sev5+reachable finding; score gate is advisory until `K`/`MIN_SCORE` are calibrated, then blocking.
|
||||
- **Zero-dep guard (Req 7.5):** with HarnessX absent, governance/scoring/benchmark planes operate and CI gating runs in the default install.
|
||||
|
||||
### Performance / Tolerance
|
||||
- **HX tolerance (Req 6.4):** an enabled HarnessX-backed stage keeps recall/FP within the configured tolerance of the zero-dep baseline.
|
||||
- Hunter-loop cost/token budgets enforced by `CostGuardProcessor`/`TokenBudgetProcessor`.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- **Supply-chain posture (Req 7):** core `dependencies = []` is preserved; HarnessX ships only as a SHA-pinned optional extra, lazy-imported behind `_has_harnessx()`. A `pip install --dry-run` audit runs before Phase 3; if the realized transitive graph pulls browser binaries / Docker SDK / a FastAPI web stack as *mandatory* deps and maintainers judge it unacceptable for a supply-chain auditing tool, the pre-specified **vendored lean subset** (Option C) is switched in under the same extra name — a packaging change, not a redesign. Submodule init is **non-recursive** so the large `verl` RL fork is never pulled.
|
||||
- **Loop containment:** the meta-agent can never invent or rewrite detectors (no pattern edits) or rewrite the severity authority (no 0-5 edits); `shadow`-first staging, novelty/evidence/replay gates, and byte-for-byte revert bound its blast radius — directly preventing the "too influential" defect from re-entering via the loop.
|
||||
- **Gate integrity:** the recall/FP gate is a hard constraint that is never folded into the score reward, so the loop cannot "improve" the number by shadowing rules that hurt real security.
|
||||
|
||||
## Migration Strategy
|
||||
|
||||
Every phase is independently shippable, test-gated, and reversible. Phases 0-2 deliver governance value at **zero dependency cost** and stay byte-identical by default. Phases 3-4 are the HarnessX adoption behind the extra. Phase 5 is the optional native end-state.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
P0["Phase 0<br/>Policy + Score<br/>zero-dep, gate advisory"] --> P1["Phase 1<br/>Rules-as-data<br/>zero-dep, byte-identical"]
|
||||
P1 --> P2["Phase 2<br/>Benchmark->signal + ledger<br/>zero-dep"]
|
||||
P2 --> GATE{"pip --dry-run audit<br/>(D2 decision)"}
|
||||
GATE -->|graph acceptable| P3B["Phase 3<br/>[harnessx] extra (Option B)<br/>hunter loop + verify"]
|
||||
GATE -->|graph unacceptable| P3C["Phase 3<br/>vendored lean subset (Option C)<br/>same extra name"]
|
||||
P3B --> P4["Phase 4<br/>Self-improvement loop live"]
|
||||
P3C --> P4
|
||||
P4 --> P5["Phase 5 (optional)<br/>full single-step harness + RL<br/>retire HarnessRuntime"]
|
||||
```
|
||||
|
||||
| Phase | Scope | Dep cost | Merge gate | Reversibility |
|
||||
|---|---|---|---|---|
|
||||
| **0 — Policy & score** | Vendor `CWE_Score.tsv`; `policy/verycode.py` + `scoring/project_score.py`; CWE-120→121 remap; startup CWE-resolution invariant; emit score artifact + advisory gate. `HarnessRuntime`/`PATTERN_RULES` unchanged. | **zero** | benchmark byte-identical; score is additive output | gate disable flag |
|
||||
| **1 — Rules as data** | Extract `PATTERN_RULES`→`ruleset/<lang>/*.toml`; delete `PatternRule.severity`; `load_ruleset()`; `resolve_severity`; add `status` + emit thresholds. | **zero** | benchmark **byte-identical** to Phase 0 | keep `PATTERN_RULES` fallback |
|
||||
| **2 — Benchmark→signal + ledger** | `rule_id` on misses/FPs; per-rule precision/recall; emit `rule_policy_delta`; `rule_policy.json` ledger; calibration writes status flips re-keyed on `rule_id`. | **zero** | empty ledger == current behavior | empty ledger = no-op |
|
||||
| **3 — HX execution adoption** | Run `pip --dry-run` audit (D2 gate). Add `[harnessx]` extra (pinned SHA, lazy, zero-dep CI guard); `build_sast()` + `HxScanOrchestrator`; hunter pool on real `Harness.run()`; `VerifyProcessor`; `PolicyScoring`/`ScoreProcessor`. `HarnessRuntime` stays. | **opt-in heavy** | extra-absent: unchanged; extra-present: recall/FP within tolerance | feature flag off |
|
||||
| **4 — Self-improvement live** | Vendor/patch the meta-harness seam; orchestrator benchmark→delta→signal→`evolve()`→re-benchmark→accept/revert. | **opt-in** | ≥90% recall / <10% FP per-round; auto-revert | don't run `evolve` → ledger-only changes |
|
||||
| **5 — Full harness + RL (optional)** | Migrate deterministic stages to single-step harness + `SlotContractMixin`; retire `HarnessRuntime`; promote score gate to blocking; optional `rl/`. | **opt-in** | golden-diff vs Phase 3 empty; gate held | per-stage revert |
|
||||
|
||||
**Rollback triggers.** Any phase that drops recall<90% or raises FP≥10% on the corpus blocks the merge; any self-improvement round failing the acceptance gate reverts byte-for-byte; an unacceptable `pip --dry-run` graph switches Phase 3 to the vendored subset.
|
||||
@@ -0,0 +1,115 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
OpenUltraSAST today fuses three responsibilities into one regex layer: detection patterns also carry their own severity strings and act as the sole volume control, so a noisy rule can dominate scan output, set its own criticality, and fire forever with no central governance. This feature splits those responsibilities into three governed owners — rules detect, a central CWE policy decides severity and scope, and a single project score becomes the optimization target — and adopts the HarnessX runtime exactly where it pays off: the LLM hunter loop, the LLM-judge verifier, and a meta-harness self-improvement loop that adapts rules and policy gates under hard bounds.
|
||||
|
||||
The work spans five user asks. First, OpenUltraSAST shall adopt HarnessX directly at the hunter loop and migrate the existing scan flow stage-by-stage onto HarnessX processors. Second, detection rules shall become versioned data governed centrally, with severity sourced from the verycode CWE policy rather than from the rule itself. Third, the HarnessX meta-harness shall adapt rules and policy gates within bounded levers. Fourth, benchmark results shall drive that adaptation as actionable signals with a hard acceptance gate. Fifth, a Veracode-style project score shall be computed from policy-weighted, reachability-adjusted findings and gated in CI.
|
||||
|
||||
Two cross-cutting constraints govern every part of this feature. The existing detection gate of at least 90% recall and under 10% false-positive rate on the benchmark corpus must never regress, at any phase, for any change. And the HarnessX runtime must be adopted in a way that preserves the project's zero-dependency posture for the governance and scoring planes, so an explicit dependency-weight decision controls whether HarnessX ships as an optional extra or as a vendored lean subset.
|
||||
|
||||
## Boundary Context
|
||||
|
||||
- **In scope**: detection rules as versioned data; a central CWE-to-severity policy with static/dynamic scope; a single project score (0-100) with a two-condition CI gate; HarnessX adoption at the hunter loop, the verifier, and the self-improvement meta-harness; benchmark-to-signal conversion with per-rule attribution; bounded loop authority over rule status, evidence floors, thresholds, and score constants; a dependency-weight audit and decision; preservation of the recall/FP detection gate.
|
||||
- **Out of scope**: authoring new detection pattern text by the loop (human pull-request only); editing the authoritative 0-5 CWE severity by the loop (verycode-upstream authority); a full single-step HarnessX rewrite of all deterministic stages and weight-level reinforcement learning (deferred, optional, later phase); changing the reachability vocabulary, which remains exactly `reachable | inferred-file-surface | unknown`; dynamic/DAST scoring, which is report-only for a static tool.
|
||||
- **Adjacent expectations**: the governance and scoring planes shall run with no HarnessX present and `dependencies = []`; CI gating (score plus benchmark) shall run in the zero-dependency default install; HarnessX, when present, shall be confined behind a capability check and lazy imports so heavy code paths stay cold in the default install.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: HarnessX Direct Adoption and Scan-Flow Migration
|
||||
|
||||
**Objective:** As an OpenUltraSAST maintainer, I want the scan flow to adopt the HarnessX runtime directly at the LLM hunter loop and migrate the existing stages onto HarnessX processors in a staged way, so that the project gains the HarnessX agent loop, journal, and trajectory machinery without a high-risk big-bang rewrite.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. Where the HarnessX extra is installed, the Scan Orchestrator shall run the hunter pool stage on a real HarnessX `Harness.run()` loop with the HarnessX control, tool, and observability processors registered in the hunter sub-harness.
|
||||
2. Where the HarnessX extra is installed, the Verify Processor shall produce verification verdicts using the HarnessX llm-judge helpers and shall retain the structural verifier as a zero-dependency fallback.
|
||||
3. While the HarnessX extra is absent, the Scan Runtime shall execute `quick` and `standard` scans through the existing sync driver with behavior unchanged from the pre-adoption baseline.
|
||||
4. When the hunter pool stage runs under HarnessX, the Scan Orchestrator shall confine all asynchronous execution to that single stage so that the deterministic stages remain synchronous.
|
||||
5. The Scan Runtime shall preserve each stage's declared read and write slot allow-list and shall validate slot access on every lifecycle hook, so that the existing provenance and audit discipline is retained under the HarnessX runtime.
|
||||
6. Where a deterministic stage runs under HarnessX, the Scan Runtime shall execute it with model invocation disabled so that no deterministic stage incurs a model call.
|
||||
7. When migrating event names, the Scan Runtime shall map the existing `stage_start`/`stage_end` to the HarnessX step-level hooks and `scan_id` to the HarnessX run identifier without losing trace continuity.
|
||||
8. If a HarnessX-backed stage is enabled and the corresponding HarnessX capability is unavailable at runtime, then the Scan Runtime shall fall back to the existing equivalent stage and shall record the degradation in the scan manifest.
|
||||
|
||||
### Requirement 2: Central Security-Affine Ruleset Governance
|
||||
|
||||
**Objective:** As a security policy owner, I want detection rules to be versioned data whose severity and scope come from a central CWE policy rather than from the rule itself, so that one CWE maps to one governed severity and no rule can be too influential by setting its own criticality or volume.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The Ruleset Store shall load detection rules from versioned data files in which each rule declares a pattern, a CWE, tags, and a loop-controlled `status`, and shall not carry any rule-local severity field.
|
||||
2. When resolving the severity of a finding, the Policy Scoring Processor shall read severity exclusively from the Policy Store keyed on the rule's CWE and shall discard any severity string carried in legacy rule data.
|
||||
3. While two enabled rules map to the same CWE, the Policy Scoring Processor shall assign both findings the identical policy-governed severity so that two rules for one CWE can no longer disagree.
|
||||
4. The Ruleset Store shall support exactly three rule statuses — `enabled`, `shadow`, and `disabled` — where `enabled` fires and scores and reports, `shadow` fires and records outcomes but is excluded from the report and the score, and `disabled` does not fire.
|
||||
5. When a finding's rule status is `shadow`, the Policy Scoring Processor shall exclude that finding from the reported findings and from the project-score input while still recording its outcome for precision tracking.
|
||||
6. The Policy Store shall expose, per CWE, an authoritative 0-5 severity plus a static scope flag and a dynamic scope flag, and shall mark dynamic-only CWEs as report-only and never scored by the static tool.
|
||||
7. If an enabled rule's CWE does not resolve in the Policy Store at startup, then the Scan Runtime shall fail loud and shall not proceed with the scan.
|
||||
8. The Ruleset Store shall remap the C/C++ buffer-handling rules that reference the unmapped CWE-120 to CWE-121 (severity 5) so that those rules contribute a nonzero penalty to the project score.
|
||||
9. The Ruleset Store and the Policy Store shall be loaded with the standard library only and shall require no third-party dependency.
|
||||
|
||||
### Requirement 3: Bounded Rule and Policy Adaptation in the Self-Improvement Loop
|
||||
|
||||
**Objective:** As an OpenUltraSAST maintainer, I want the HarnessX meta-harness to adapt rules and policy gates only through two bounded levers, so that the loop can suppress, stage, or raise the bar on noisy rules without ever inventing detectors or rewriting the severity authority.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The Self-Improvement Loop shall expose exactly two new edit levers: a `rule` lever covering rule status, evidence floor, and threshold, and a `policy` lever covering evidence floor, scope, and the score constants `K` and `MIN_SCORE`.
|
||||
2. The Rule Edit Validator shall reject any loop edit that modifies detection pattern text, so that pattern authoring remains human-pull-request only.
|
||||
3. The Policy Edit Validator shall reject any loop edit that modifies the authoritative 0-5 CWE severity, so that the severity authority remains upstream-governed.
|
||||
4. If the loop proposes moving a rule directly from `enabled` to `disabled`, then the Rule Edit Validator shall reject the edit and require `shadow` as a mandatory intermediate staging state.
|
||||
5. When validating a proposed edit, the Rule Edit Validator shall assert that the edit parses, that `status` is one of `enabled`/`shadow`/`disabled`, that any threshold or evidence-floor tightening or loosening stays within configured bounds, that an added rule does not duplicate an existing rule identifier, and that every referenced CWE resolves in the Policy Store.
|
||||
6. If the loop re-proposes an edit that was previously reverted without supplying a retry rationale, then the Self-Improvement Loop shall block the re-proposal via the novelty gate.
|
||||
7. Before accepting any round, the Self-Improvement Loop shall run a replay smoke gate that proves the edited ruleset and policy still load and boot.
|
||||
8. When any loop edit is rejected during validation, the Self-Improvement Loop shall raise a rule-change validation error and reject the entire round.
|
||||
9. The Self-Improvement Loop shall record each rule and policy edit, its lever, and its outcome in the journal so that every change is attributable and reversible.
|
||||
|
||||
### Requirement 4: Benchmark-Driven Iteration with a Hard Acceptance Gate
|
||||
|
||||
**Objective:** As an OpenUltraSAST maintainer, I want benchmark results to become actionable per-rule signals that drive the self-improvement loop under a hard acceptance gate, so that detection quality improves automatically while regressions are reverted byte-for-byte.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The Benchmark Harness shall attribute every recorded miss and every recorded false positive to a `rule_id` and shall aggregate per-rule precision and recall in its metrics.
|
||||
2. The Benchmark Harness shall emit a per-rule recommendation artifact alongside its records, recommending enable, disable, shadow, threshold, or evidence-floor changes per rule.
|
||||
3. When the orchestrator converts benchmark output to loop input, it shall write recall-side items as miss signals and precision-side items as verifier-reject or false-positive signals into the loop's signal file.
|
||||
4. After the meta-agent edits rules or policy, the Self-Improvement Loop shall re-run the benchmark and shall accept the round only if aggregate recall is at least 90% AND aggregate false-positive rate is under 10% AND the project score did not regress on the held set AND no unpredicted regression occurred.
|
||||
5. If a re-run round fails any acceptance condition, then the Self-Improvement Loop shall revert the round and restore the rule and policy configuration byte-for-byte.
|
||||
6. If a rule drags aggregate precision below the gate, then the Self-Improvement Loop shall auto-shadow that rule rather than delete it, so that recall is preserved while the false-positive cost is removed.
|
||||
7. When a coverage gap surfaces as a persistent miss, the Self-Improvement Loop shall nominate a new or loosened rule through the signal file while leaving pattern authoring to a human pull request.
|
||||
8. The Self-Improvement Loop shall treat maximizing the project score as the optimization objective and the recall/FP gate as a separate hard feasibility constraint, so that the loop cannot improve the score by shadowing rules in a way that harms real security.
|
||||
|
||||
### Requirement 5: Project Scores via verycode Policies
|
||||
|
||||
**Objective:** As a security reviewer, I want a single 0-100 project score computed from policy-weighted, reachability-adjusted findings and gated in CI, so that a scan produces a bounded AppSec quality signal alongside its findings.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The Score Model shall compute a single 0-100 project score from policy-governed severity weights multiplied by a reachability multiplier over the reported findings, using the standard library only.
|
||||
2. The Score Model shall use reachability multipliers keyed exactly on the three emitted reachability values `reachable`, `inferred-file-surface`, and `unknown`.
|
||||
3. When a finding maps to a CWE that is dynamic-only or unmapped in the Policy Store, the Score Model shall assign that finding a zero score penalty and shall report it as out-of-scope or unmapped rather than scoring it.
|
||||
4. The Score Model shall exclude findings whose rule status is `shadow` from the score input before computing the project score.
|
||||
5. The Score Model shall emit a scan-level score artifact containing the project score, the maximum severity, the total penalty, a by-category breakdown, the out-of-scope dynamic-only count, the unmapped CWE list, and the gate verdict, and shall include the score block in the scan manifest.
|
||||
6. When confirming a false positive, the Calibration subsystem shall lower the finding's effective reachability multiplier rather than delete the rule, so that recall stays high while the score stops over-penalizing.
|
||||
7. If any reported finding maps to policy severity 5 AND has `reachability_status` equal to `reachable`, then the CI Score Gate shall fail the build.
|
||||
8. Where the score gate is configured as blocking, the CI Score Gate shall fail the build when the project score is below `MIN_SCORE`; until the score constants are calibrated against the corpus, the CI Score Gate shall compute and report the score advisory-first without blocking.
|
||||
9. The Score Model and the CI Score Gate shall run in the zero-dependency default install with no HarnessX present.
|
||||
|
||||
### Requirement 6: Detection Gate Non-Regression
|
||||
|
||||
**Objective:** As an OpenUltraSAST maintainer, I want the existing detection gate of at least 90% recall and under 10% false positives on the benchmark corpus to hold across every phase and every change, so that no governance, scoring, runtime, or self-improvement change can silently degrade detection quality.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The CI Pipeline shall evaluate aggregate recall and aggregate false-positive rate on the benchmark corpus on every merge and shall fail the merge if recall is below 90% or the false-positive rate is at or above 10%.
|
||||
2. While the default ruleset and an empty loop ledger are in effect, the Benchmark Harness shall produce results byte-identical to the pre-feature baseline for the governance and rules-as-data phases.
|
||||
3. When the self-improvement loop proposes any rule or policy change, the Self-Improvement Loop shall require the recall/FP gate to pass on the re-benchmarked candidate before the change can be accepted.
|
||||
4. If a HarnessX-backed stage is enabled, then the CI Pipeline shall confirm recall and false-positive rate stay within the configured tolerance of the equivalent zero-dependency baseline.
|
||||
5. The CI Pipeline shall treat the recall/FP gate as a hard constraint that is never folded into the project-score reward, so that improving the score can never substitute for meeting the detection gate.
|
||||
6. If any change would reduce recall below 90% or raise the false-positive rate to 10% or above, then the CI Pipeline shall block the change regardless of any project-score improvement.
|
||||
|
||||
### Requirement 7: HarnessX Dependency-Weight Decision
|
||||
|
||||
**Objective:** As an OpenUltraSAST maintainer, I want an explicit dependency-weight decision that keeps the core install zero-dependency and chooses between an optional HarnessX extra and a vendored lean subset based on an audited dependency graph, so that a supply-chain auditing tool never silently inherits an unacceptable transitive dependency footprint.
|
||||
|
||||
#### Acceptance Criteria
|
||||
1. The Packaging Configuration shall keep the core `dependencies` list empty and shall expose HarnessX only as an optional extra pinned to a specific commit revision.
|
||||
2. The Runtime shall guard every HarnessX import behind a capability check and shall import HarnessX lazily so that heavy code paths stay cold in the default install.
|
||||
3. Before adopting the HarnessX execution stages, the maintainer shall run a dependency dry-run audit and shall record the realized transitive dependency graph.
|
||||
4. If the audited transitive graph pulls a browser-binary, container-SDK, or web-server stack as a mandatory dependency and the maintainers judge it unacceptable for a supply-chain auditing tool, then the Packaging Configuration shall switch to a vendored lean subset of HarnessX under the same extra name with the heavy subtrees removed.
|
||||
5. While HarnessX is absent, the CI Pipeline shall run a zero-dependency guard test confirming the governance, scoring, and benchmark planes operate with no HarnessX present.
|
||||
6. Where the project is developed via a submodule, the Runtime setup shall initialize HarnessX non-recursively so that the large reinforcement-learning fork subtree is not pulled.
|
||||
7. The Packaging Configuration shall keep the vendored lean subset pre-specified so that switching from the optional extra to the vendored subset is a packaging change and not a redesign.
|
||||
@@ -0,0 +1,191 @@
|
||||
# Research & Design Decisions
|
||||
|
||||
---
|
||||
**Purpose**: Capture the discovery findings and architectural investigation behind the `harnessx-self-improving-rulesets` spec. This is the background-notes companion to `design.md` — it records *what was read*, *what was found in the source*, and *why* the three resolved decisions came out the way they did. `design.md` carries the committed architecture; this file carries the evidence and the rejected alternatives.
|
||||
---
|
||||
|
||||
## Summary
|
||||
- **Feature**: `harnessx-self-improving-rulesets`
|
||||
- **Discovery Scope**: Complex Integration (adopt an external agent-runtime library + re-architect the in-house rule/policy/scoring planes)
|
||||
- **Key Findings**:
|
||||
- HarnessX and OpenUltraSAST (OUS) already both have a thing called a "harness" with "processors" and "events", but they are **incompatible at every load-bearing point**: HarnessX is async-generator / message-window-contract / transcript-state; OUS is sync / declared-`reads`/`writes`-contract / dict-state. There is no drop-in.
|
||||
- HarnessX's real, reusable value for OUS is concentrated in three places: the **LLM hunter agent loop** (control/tools/observability processors), the **llm-judge verifier** (`build_judge_prompt`/`parse_judge_response`/`_call_judge`), and the **meta-harness self-improvement engine** (journal / novelty / evidence / replay / attribution). Everything else (lab web UI, browser tools, sandboxes, office skills) is unused weight.
|
||||
- OUS today fuses **detector + severity authority + volume control** into one regex layer (`PATTERN_RULES` in `findings.py`); its self-improvement loop only demotes *ranking* (floored at `0.1`, keyed on `tags[0]`), never the rules; and its benchmark already names the ruleset as the thing to fix (`next_improvement_candidate="rules_or_language_hunter"`) but writes it to an inert JSON. The signals exist; the loop back onto the rules does not.
|
||||
- The `verycode` policy data (`CWE_Score.tsv`, 167 CWEs, severity 0-5 + static/dynamic scope) is **complete and authoritative**, but the `verycode-score` scoring *algorithm was never implemented* (`// scorer code here` placeholder). The score formula in the design is a labeled, defensible reconstruction.
|
||||
- OUS today has **zero runtime dependencies** (`dependencies = []`); HarnessX's `dependencies` list is ~30 heavyweight packages (Playwright, Docker SDK, FastAPI/uvicorn, multi-LLM SDKs, office-doc suite) with **no lean extra** to pip-install a subset — a direct hard dependency would invert the project's supply-chain posture for a security tool.
|
||||
|
||||
## Research Log
|
||||
|
||||
### HarnessX core composition & runtime model
|
||||
- **Context**: We need to know whether OUS's `harness.HarnessRuntime` can be replaced by, or interoperate with, HarnessX's `Harness`, and how an external project builds and runs a HarnessX harness.
|
||||
- **Sources Consulted**: `/tmp/HarnessX/harnessx/core/{harness,builder,processor,events,runloop,state,contract_check,config_schema,config_store,model_config,trajectory}.py` (read in full); `/home/mc/Source/OpenUltraSAST/src/openultrasast/harness.py`.
|
||||
- **Findings**:
|
||||
- Public composition is three objects + one method: `HarnessBuilder.build() -> HarnessConfig`; `ModelConfig.agentic(HarnessConfig) -> Harness`; `await Harness.run(task) -> HarnessResult`. **`HarnessConfig` carries NO model info** — model is always supplied separately via `ModelConfig` (key→provider map, `main` required, non-`main` keys auto-get sub-harnesses).
|
||||
- `HarnessBuilder` is an immutable fluent factory: `add(proc)`, `slot(**kw)`, `add_tool`, `plugin(source)`, `__or__`/`merge` (collects conflicts → `HarnessConflictError`), `build()` (topo-sorts processors per hook by `_order` then Kahn within a bucket by `_after`). Declarative alt: `build_from_config(dict)` from a flat `processors:` list of `_target_` dicts.
|
||||
- The run loop (`run_loop`) is a fixed async agent loop: `task_start` → `while True:` (step: `step_start` → `before_model` → call provider → `after_model` → per-tool `before_tool`/exec/`after_tool` → `step_end`) → `task_end`. `get_procs(key)` runs `"*"` (MultiHook) processors on every hook, then hook-specific lists.
|
||||
- State is **two message tracks** (`raw_messages` append-only, `messages` effective context) + typed `slots: dict[str, StateSlot]` (the nearest analogue to OUS's flat `state` dict, but typed and `_target_`-serializable). `StatefulTrajectory` is a first-class product (`s_t = (z_t, Δz_t, a_t, o_t, e_t, r_t)`), built inline during `run_loop`, with `backfill_rewards` and `to_training_records`/`to_rl_records`.
|
||||
- Hook-name alignment vs OUS: 5/8 match exactly (`task_start`, `before_model`, `after_model`, `before_tool`, `after_tool`, `task_end`); the gap is `stage_start`/`stage_end` ↔ `step_start`/`step_end`. Semantically an OUS "stage" is a coarse pipeline phase wrapping a callback (`run_stage(name, callback)`), whereas a HarnessX "step" is one model-call iteration — not the same concept.
|
||||
- **Implications**: A whole-pipeline swap is high risk. The cheapest correct adoption confines HarnessX's async loop to the one OUS stage that *is* an agent loop (the hunter pool) and keeps the sync driver for deterministic stages. This directly motivates Decision D1 (phased seam, not big-bang).
|
||||
|
||||
### The two incompatible processor contracts
|
||||
- **Context**: OUS's defining feature is a declared `reads`/`writes` per-key state contract. We must know if HarnessX preserves it.
|
||||
- **Sources Consulted**: `/tmp/HarnessX/harnessx/core/{processor,contract_check}.py`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/harness.py:49-118`.
|
||||
- **Findings**:
|
||||
- HarnessX `Processor` is `async def process(event) -> AsyncIterator[Event]` — an async generator that yields transformed events. The recommended base is `MultiHookProcessor` with `on_*` hook methods, an `@on(EventClass)` decorator, and class-attr ordering hints (`_order`, `_after`, `_singleton_group`, `_hook`). There is **no `spec`, no `handles`, no `reads`/`writes`.**
|
||||
- HarnessX's contract governs **message-window mutation**, not state keys: enforced in `ProcessorChain.process` via `_validate_messages_contract` (system stays at index 0 on `step_start`; `before_model` net length ∈ {0,+1}; appended message must be role `user`; no out-of-window edits) and per-chain `check_post_hook_invariants`. Mode is the env var `HARNESSX_CONTRACT_MODE ∈ {warn(default), strict}` — not a constructor arg.
|
||||
- OUS `Processor` is **synchronous**, `def handle(event, state: dict) -> dict | None`, with `spec: ProcessorSpec(name, version, handles, reads, writes)`; `_run_processor` enforces *state-key* discipline (reads must pre-exist; writes must be declared; return must be a dict). `ContractMode` is a `"strict"|"warn"` constructor arg.
|
||||
- **Implications**: The two contracts do not map onto each other. OUS's per-key allow-list (its provenance/audit discipline) has **no home in HarnessX**. To keep that discipline we must reimplement it as a `SlotContractMixin` on our HarnessX processors (`reads_slots`/`writes_slots` class attrs validated in a base `on_*` wrapper), gated by our own `OUS_SLOT_CONTRACT` env. This is Decision D7.
|
||||
|
||||
### HarnessX built-in processors — reuse map
|
||||
- **Context**: Which HarnessX processors can OUS register directly, which are reusable as patterns/helpers, and which are irrelevant?
|
||||
- **Sources Consulted**: `/tmp/HarnessX/harnessx/processors/` (control, tools, multi_model, observability, evaluation, memory, context); `/home/mc/Source/OpenUltraSAST/src/openultrasast/cli.py` (`_run_scan`).
|
||||
- **Findings**:
|
||||
- **Register-direct only inside the hunter loop** (the one real agent loop): `LoopDetectionProcessor`, `CostGuardProcessor`/`TokenBudgetProcessor`, `ToolFailureGuard`, `SelfVerifyProcessor`, `ToolFilterProcessor`/`ToolWhitelistProcessor`, `ModelRouterProcessor`, `EpisodeMetricsProcessor`/`OTelProcessor`/`CheckpointProcessor`.
|
||||
- **Reuse as helper functions** for `verify`: the `llm_judge` code (`build_judge_prompt`, `parse_judge_response` with the verdict whitelist, the two-attempt repair loop `_call_judge`) plus the strict-JSON verdict schema (`verdict/confidence/cause/missing_capability`/`lesson`/`missing`). The judge is explicitly told "ground truth is intentionally withheld" — matching OUS's no-oracle scanning.
|
||||
- **Reuse as pattern** for `calibrate`/`record_calibration`: `MemoryRetrievalProcessor` + `MemoryPolicy` (retrieve/add, should_retrieve/should_store) is the clean interface to refactor the FP ledger toward; `MemoryExtractionProcessor` mirrors distilling verifier outcomes into durable learnings. Storage stays a custom JSON ledger.
|
||||
- **Keep custom** for every SAST-domain stage: `static_mapping`, `preprocess`, `entry_point_mapping`, `quick_findings`, `report`, `sarif`, `manifest`, and the benchmark recall/FP scoring (HarnessX `EvalResult.score` is a single float — too coarse for the recall/FP gate).
|
||||
- **Implications**: Adoption is bimodal — register HarnessX classes directly only in the hunter sub-harness; everywhere else port logic as helpers or patterns. This is the backbone of the §3 flow-migration table in `design.md`.
|
||||
|
||||
### HarnessX self-improvement (meta_harness) and the rule-adaptation seam
|
||||
- **Context**: Can an external signal (a verifier rejection or a benchmark miss) drive a *detection-rule* change through HarnessX's self-improvement loop, and how does it compare to OUS's current calibration?
|
||||
- **Sources Consulted**: `/tmp/HarnessX/harnessx/meta_harness/{agent,journal,replay,validate_workflow}.py`, `processors/contract_autocheck.py`, `workers/trajectory_digester.py`, `rl/{builder,config,task}.py`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/{calibration,benchmark,findings,rank,cli}.py`.
|
||||
- **Findings**:
|
||||
- HarnessX self-improves a *harness* round by round. The unit of change is a `HarnessConfig` (the meta-agent literally rewrites the inner agent's config/processors/templates as files); the unit of memory is a markdown **journal** with per-`## Round N` YAML frontmatter; the unit of safety is a multi-phase **validation gate**.
|
||||
- The journal classifies every change by a **lever** from `_VALID_LEVERS = {configuration, control, action, instruction}` and tracks `predicted_affected` task_ids plus a `gating_attribution` (`flipped/still_F/regressed/still_T/absent`). `build_context` renders a per-lever **time-decayed Beta posterior** scoreboard (`_DECAY=0.9`) and an explicit "Reverted hypotheses — do not re-propose" list.
|
||||
- `EvolveValidator.run` enforces validity (canonicalize → contract → diff → replay smoke gate), policy (novelty + evidence, only when the diff is non-empty), advisory. `check_novelty` hard-blocks re-proposing a reverted hypothesis or reverted `(levers, predicted_affected)` signature unless a `retry_rationale` is present. `_evidence` requires a `candidates.md` `## Candidate C-N` cross-referenced by the journal. `run_synthetic_task_smoke_gate` proves the evolved config boots end-to-end. `ContractAutoCheckProcessor` gives in-session `[AUTO-CHECK]` feedback on every authored `processors/*.py`/`tools/*.py` edit.
|
||||
- **The concrete seam to drive a rule change**: add a lever value to `journal._VALID_LEVERS`; carry the external signal in via a `rule_signals.json` read-context artifact + `_render_task_brief` deliverable; add a validity-phase check (`EvolveValidator._rules`) that raises `StrictValidationError(kind="rule_change")`; add changeset buckets (`compute_changeset` + `journal._CHANGESET_KEYS_ORDER`); extend `contract_autocheck` to classify rule files. The novelty/evidence/replay/attribution machinery then applies for free.
|
||||
- **OUS by comparison**: detection rules in `findings.py` are static and never modified. Calibration (`calibration.py`) only does `calibrated = max(0.1, calibrated - 0.5)` on a *file's* ranking priority (keyed on `tags[0]`, not `rule_id`) plus a prompt-constraint string — it never disables/tightens/adds a rule, has no novelty gate, no attribution. Benchmark misses produce a `BenchmarkCalibrationRecord` whose `next_improvement_candidate` is hard-coded to `"rules_or_language_hunter"` and written to an **inert** `calibration_records.json` that nothing consumes. The recall side is a fully open loop.
|
||||
- **Implications**: OUS has exactly the *signals* HarnessX needs (verifier rejections = precision side; benchmark misses = recall side) but no loop closing back onto the rules. The meta_harness is the engine that closes it. This motivates Decision D3 (which levers, and their hard bounds) and the §5/§6 design.
|
||||
|
||||
### The verycode policy & scoring model
|
||||
- **Context**: The design needs a central CWE→severity authority and a project score. What does `verycode` actually provide?
|
||||
- **Sources Consulted**: `/tmp/verycode-policies/{README.md,CWE_Score.tsv}`; `gh repo view norandom/verycode-score` (incl. `action.yaml`, `main.go`); `gh repo view norandom/verycode-lib`; `/home/mc/Source/OpenUltraSAST/src/openultrasast/findings.py`.
|
||||
- **Findings**:
|
||||
- `CWE_Score.tsv` is the real artifact: 167 data rows, one per CWE, columns `Flaw Category | CWE ID | CWE Name | Flaw Severity (0-5) | Static (X/empty) | Dynamic (X/empty)`. Severity distribution: 12×5, 15×4, 97×3, 31×2, 0×1, 12×0. Scope: 154 Static, 45 Dynamic, 13 Dynamic-only, none neither. (Header column 4 is literally `"Flaw Severity "` with a trailing space.)
|
||||
- **The scoring algorithm does not exist.** `norandom/verycode-score` is the unmodified `the-gophers/go-action` "Tweeter" template with `// scorer code here` and a `--policy policies/standard.json` flag; `norandom/verycode-lib` is a one-line stub README. The data shape + the "GitHub Advanced Security findings, weighted by CWE severity" descriptions imply a Veracode-style 0-100 pipeline; the design reconstructs the formula and labels it as such.
|
||||
- **Real coverage gap**: CWE-120 (Classic Buffer Overflow) is **absent from the TSV**, but 3 high-value C/C++ rules use it (`c-unsafe-copy`, `c-unsafe-memory-copy`, `c-unsafe-scanf`). Unmapped → those rules would contribute 0 to the score. CWE-121 (Stack Buffer Overflow) is present at severity 5 and is the precise classic-overflow CWE.
|
||||
- **Severity mismatch**: OUS rule strings disagree with policy (rules label CWE-78/95/134 "high"; policy rates them 5/critical; rules label CWE-94/502 "high"; policy rates them 3/medium). The policy is the authoritative source.
|
||||
- The OUS reachability vocabulary is exactly `reachable | inferred-file-surface | unknown` (verified in `findings.py`), which maps cleanly onto a `REACH_MULT = {reachable:1.0, inferred-file-surface:0.6, unknown:0.4}` exploitability multiplier — and that multiplier is the natural FP-calibration knob (lower effective reachability instead of deleting a rule).
|
||||
- **Implications**: Severity must flow from PolicyStore (`CWE_Score.tsv`), not from the rule. Day-one fixes: remap the 3 CWE-120 rules to CWE-121 (Decision D5); discard rule string-severities (Decision D4). The score is a stdlib-only reconstruction (Decision D8 ships it advisory-first until `K`/`MIN_SCORE` are calibrated).
|
||||
|
||||
### Installability & dependency weight
|
||||
- **Context**: How does an external project depend on HarnessX, and what does it cost a zero-dependency security tool?
|
||||
- **Sources Consulted**: `/tmp/HarnessX/pyproject.toml`, `harnessx/__init__.py`, `core/builder.py`, `plugins/{base,loader,discovery,convert}.py`, `examples/{coding,research,assistant}/harness.py`, `extensions/plugins/workflow/plugin.py`, `.gitmodules`; `/home/mc/Source/OpenUltraSAST/pyproject.toml`.
|
||||
- **Findings**:
|
||||
- HarnessX is **library-first** and **effectively not on PyPI** (`name="harnessx"`, `version="0.1.0"`, Beta; consumed from Git). `requires-python = ">=3.11"` — exact match with OUS, no floor conflict. Depend via a SHA-pinned Git URL (the version string won't move).
|
||||
- Custom stages map cleanly onto HarnessX's plugin model: each SAST stage is a `MultiHookProcessor`; group stages + tools + skills into a `HarnessPlugin` mounted via `builder.plugin(...)`; load in-process (instance/class) rather than via directory/`plugin.json`.
|
||||
- **Dependency-weight is the headline risk.** HarnessX's `dependencies` (always-installed, not extras) is ~30 heavyweight packages: `anthropic`/`openai`/`litellm`/`tiktoken`, `fastapi`/`uvicorn`/`websockets`, `playwright` (ships browser binaries), `docker`/`e2b`, `hydra-core`/`omegaconf`/`structlog`/`loguru`, `opentelemetry-*`, `mcp`, `python-pptx`/`python-docx`/`openpyxl`/`reportlab`/`pdfplumber`. **There is no lean extra** — `pip install harnessx` pulls everything. OUS currently has `dependencies = []`.
|
||||
- **Mitigating fact**: HarnessX uses **pervasive lazy imports** (heavy modules imported inside functions). So the heavy deps are *installed* but only *imported on demand* — which makes both an optional-extra opt-in and a vendored subset genuinely viable.
|
||||
- **Submodule caveat**: `.gitmodules` declares a nested `recipe/verl_harnessX/verl` → a large verl RL fork OUS does not need; never `--recursive` init.
|
||||
- **Implications**: A hard dependency is off the table (it inverts the zero-dep stance and pins to an unmovable Beta). Options narrow to (B) optional extra and (C) vendored lean subset. The lazy-import architecture is what makes B keep heavy code cold by default and makes C a surgical packaging switch. This is Decision D2.
|
||||
|
||||
### Current OUS rule-system problem analysis
|
||||
- **Context**: Establish, in code terms, why the user's complaint ("rules are too influential and not centrally managed") is true.
|
||||
- **Sources Consulted**: `/home/mc/Source/OpenUltraSAST/src/openultrasast/{findings,rank,calibration,benchmark,verification,harness,config,cli}.py`.
|
||||
- **Findings** (six code-level defects):
|
||||
1. **Pattern matches dominate output.** In quick mode, every non-comment regex match becomes a finding (`quick_scan_findings`); the only gate is language. Ranking only sorts; verification rejects only on `reachability_status == "unknown"`. A noisy regex sets output volume and FP rate directly — the rule *is* the result.
|
||||
2. **Severity is rule-local.** `PatternRule.severity` is a string literal copied straight onto the finding (`_finding_from_match`). Two rules for the same CWE can disagree; nothing reconciles them.
|
||||
3. **No central store.** The ruleset is a Python tuple compiled into the package; `config.py` has no rule/policy section. You cannot enable/disable/tune a rule without editing source and cutting a release.
|
||||
4. **Calibration demotes ranking, not rules.** `calibrate_ranking` only subtracts `learning.demotion` (default 0.5) from a *file's* `priority`, floored at `MIN_CALIBRATED_PRIORITY = 0.1`, keyed on `tags[0]`. A chronically wrong rule fires at full severity forever; learning targets the wrong object.
|
||||
5. **No CWE-policy linkage.** `cwe` is free-text used only for rationale text and loose benchmark matching. No table maps CWE → governed severity/enablement/evidence floor.
|
||||
6. **No project-score weighting.** Severity and emission are global constants; the ranking signal (`composite_priority`) only re-sorts. A low-severity crypto finding in a fuzzable network parser is never promoted; a high-severity finding in dead code is never demoted.
|
||||
- **Implications**: The regex layer is simultaneously **detector, severity authority, and volume knob**, while ranking/verification/calibration/benchmark can only act *around* the rules, never *on* them. The fix is to split those three roles into three owners (rules detect / policy decides severity & scope / score is the optimization target) with a one-directional authority model (governance drives execution; execution emits signals; the loop edits governance under a gate). This is the organizing principle of `design.md`.
|
||||
|
||||
## Architecture Pattern Evaluation
|
||||
|
||||
These are the three "architect angles" that were considered for the overall shape. Each is internally coherent; the synthesis in `design.md` borrows from all three and resolves their three disagreements (D1/D2/D3).
|
||||
|
||||
| Option | Description | Strengths | Risks / Limitations | Notes |
|
||||
|--------|-------------|-----------|---------------------|-------|
|
||||
| **Incremental seam** | Keep the OUS sync driver; run HarnessX only inside the `hunter_pool` callback via `asyncio.run`. Rules→data + policy + score added around the existing pipeline. Dependency = optional extra (B) with vendored (C) as escape hatch. Loop edits a `"rule"` lever (status/threshold). | Smallest blast radius; confines the async surface to one stage; keeps the recall/FP gate green per-merge; preserves zero-dep default. | Two harness contracts coexist for a while; `HarnessRuntime` lingers; deterministic stages don't gain HarnessX trajectories yet. | Chosen as the base trajectory (Phases 0-4). |
|
||||
| **Native rewrite** | Delete `HarnessRuntime` entirely; the *whole* scan becomes a single-step HarnessX harness with `skip_model=True` + slot dataflow; vendored lean subset (C) as the committed default. Loop edits a `"rule"` lever; patterns stay human-PR'd. | Cleanest end-state; one runtime, one contract; full trajectory/journal coverage of every stage; smallest auditable dep graph. | Touches every stage at once — too much blast radius to keep the gate green per-merge; vendoring is permanent fork-maintenance burden. | Demoted to optional Phase 5+; its end-state (single-step harness, retire `HarnessRuntime`) is the long-term target. |
|
||||
| **Policy-first** | The whole scan runs as processors on a fixed loop; central CWE policy is the centerpiece; loop edits `"rule"` + `"policy"` levers (policy = evidence floors/scope, *not* the 0-5 CWE severity). Dependency = optional extra (B) default, vendored (C) fallback. | Strongest governance story; two levers give the loop the right surface to "stop rules being too influential" without inventing detectors or rewriting severity. | Same whole-pipeline-at-once risk as native if taken literally; needs careful bounds so the loop can't corrupt the ruleset. | Contributed the two-lever model (D3) and the bounded-authority safety rules. |
|
||||
|
||||
## Design Decisions
|
||||
|
||||
The three architect angles agreed on the destination (rules→governed data; severity from a central CWE policy; a gated project score; HarnessX hosting the hunter loop and a rule-adapting self-improvement loop). They disagreed on three load-bearing choices. Each is resolved below with the recall/FP gate as the deciding tie-breaker. (D4-D9 are smaller decisions that follow.)
|
||||
|
||||
### Decision D1: Scope of the HarnessX migration
|
||||
- **Context**: How much of the scan moves onto HarnessX, and how fast?
|
||||
- **Alternatives Considered**:
|
||||
1. Incremental seam — sync driver stays; HarnessX only in the hunter callback.
|
||||
2. Native rewrite — delete `HarnessRuntime`; whole scan becomes a single-step harness immediately.
|
||||
3. Policy-first whole-loop — whole scan as processors on a fixed loop.
|
||||
- **Selected Approach**: **Phased.** Adopt HarnessX directly *at the hunter loop* first (Phase 3); migrate deterministic stages onto a single-step harness only later and optionally (Phase 5+), retiring `HarnessRuntime` only after the seam is proven and the gate has held.
|
||||
- **Rationale**: The sync→async + contract mismatch is the single highest-risk item in the whole effort. The native rewrite touches every stage at once — too much blast radius to keep ≥90% recall / <10% FP green per-merge. The incremental seam ships the *actual* HarnessX value (agent loop, journal, `StatefulTrajectory`) at exactly the one stage that is already an agent loop.
|
||||
- **Trade-offs**: Two harness contracts coexist through Phases 3-4 and `HarnessRuntime` lingers; in exchange every phase is independently shippable, test-gated, and reversible.
|
||||
- **Follow-up**: Golden-output diff gate at each phase; confine `asyncio.run` to the hunter callback; `SlotContractMixin` to preserve key-level discipline.
|
||||
|
||||
### Decision D2: Dependency strategy
|
||||
- **Context**: A zero-dependency security tool considering a ~30-package agent runtime with no lean extra.
|
||||
- **Alternatives Considered**:
|
||||
1. (A) Hard dependency in main deps.
|
||||
2. (B) Optional extra `openultrasast[harnessx]`, SHA-pinned, lazy-imported.
|
||||
3. (C) Vendored lean subset under `src/openultrasast/_vendor/harnessx/`.
|
||||
- **Selected Approach**: **(B) is the shipping default; (C) is a pre-committed, triggered fallback** gated by an explicit `pip install --dry-run` audit before Phase 3. Core `dependencies = []` stays; all HarnessX imports are lazy behind a `_has_harnessx()` check.
|
||||
- **Rationale**: HarnessX's pervasive lazy imports mean the heavy deps are *installed* but only *imported on demand*, so a clean opt-in extra already keeps heavy code paths cold in the default install — with the least maintenance burden, while tracking upstream. (A) permanently inverts the project's stance and pins to an unmovable Beta `0.1.0`. (C) adds permanent fork-maintenance and should only be paid for when the *installed* graph (not the *imported* graph) is itself unacceptable.
|
||||
- **Trade-offs**: With B, enabling the extra still inherits the full HarnessX dep set (no lean upstream extra exists); every HarnessX import must be guarded.
|
||||
- **Follow-up**: Run the dry-run audit and record the realized transitive graph. **Decision rule**: if it pulls Playwright browser binaries / Docker SDK / a FastAPI web stack as *mandatory* transitive deps and maintainers judge that unacceptable, switch to (C) — vendor `harnessx/{core,plugins,processors,bundles,meta_harness,workspace,tools/base}` + the `llm_judge`/`journal`/`validate_workflow`/`agent`/`contract_autocheck` modules, delete `lab/`/browser/sandbox/office subtrees, re-pin a minimal set. The lazy-import design makes the deletion surgical. Never use the recursive submodule path (`recipe/verl_harnessX/verl`).
|
||||
|
||||
### Decision D3: What the self-improvement loop is allowed to edit
|
||||
- **Context**: The user's core complaint is that rules are "too influential." The loop must be able to *govern* rules without becoming a new way for them to run wild.
|
||||
- **Alternatives Considered**:
|
||||
1. Single `"rule"` lever (status/threshold).
|
||||
2. `"rule"` lever; patterns stay human-PR'd (native angle).
|
||||
3. `"rule"` + `"policy"` levers, where policy = evidence floors/scope, *not* CWE severity (policy-first angle).
|
||||
- **Selected Approach**: **Two bounded levers.** `"rule"` = `status` (enabled/shadow/disabled) + evidence-floor + threshold, **never pattern text**. `"policy"` = evidence floor + scope + score constants `K`/`MIN_SCORE`, **never the authoritative 0-5 CWE severity**. Patterns and CWE severities are human-PR-only.
|
||||
- **Rationale**: This is the precise answer to "stop rules being too influential" — the loop may *suppress / stage / raise-the-bar*, never *invent detectors* or *rewrite the severity authority*. Bounds + mandatory `shadow` staging + auto-revert keep loop authority safe; the meta_harness novelty/evidence/replay/attribution gates apply for free.
|
||||
- **Trade-offs**: The loop cannot fix a bad pattern itself (it can only `shadow` it and nominate a human PR), and cannot re-weight a CWE — both are deliberate guardrails.
|
||||
- **Follow-up**: Implement `EvolveValidator._rules`/`._policy` raising `StrictValidationError(kind="rule_change")`; enforce the four "may NOT" bounds (no pattern edits, no 0-5 severity edits, no `enabled→disabled` without `shadow`, no reverted re-proposal without `retry_rationale`).
|
||||
|
||||
### Decision D4: Severity authority
|
||||
- **Context**: Two rules for the same CWE can disagree on severity today.
|
||||
- **Selected Approach**: PolicyStore (`CWE_Score.tsv`) is authoritative; `resolve_severity(policy, rule.cwe, target, ranking)` replaces the literal `rule.severity` read; `PatternRule.severity` is removed. Policy always wins; rule string-severities are discarded.
|
||||
- **Rationale**: One CWE = one governed severity, reconciled centrally; ends rule disagreement.
|
||||
- **Trade-offs**: Loses the per-rule severity hint (acceptable — it was an authoring accident, not a governed decision).
|
||||
|
||||
### Decision D5: The CWE-120 coverage gap
|
||||
- **Context**: 3 C/C++ buffer rules use CWE-120, which is absent from `CWE_Score.tsv`.
|
||||
- **Selected Approach**: Remap `c-unsafe-copy`/`c-unsafe-memory-copy`/`c-unsafe-scanf` to **CWE-121** (Stack Buffer Overflow, severity 5) in the ruleset TOML.
|
||||
- **Rationale**: CWE-121 is the precise classic-overflow CWE and is already at severity 5; this avoids forking the upstream policy file. Otherwise those high-value rules contribute 0 to the score.
|
||||
- **Follow-up**: Add a hard CI invariant — every `enabled` rule's `cwe` must resolve in PolicyStore (fail loud at startup); log `unmapped_cwe` in the score artifact; PR genuine gaps upstream to verycode-policies.
|
||||
|
||||
### Decision D6: Response to a confirmed false positive
|
||||
- **Context**: How does the system react to an FP without hurting recall?
|
||||
- **Selected Approach**: Lower the finding's effective **reachability multiplier** and/or `shadow` the rule — **never delete it**.
|
||||
- **Rationale**: The score stops over-penalizing while the rule keeps firing (tracked, just not scored/reported), preserving ≥90% recall and pushing FP <10%. This is exactly the recall/FP balance the project gates on.
|
||||
|
||||
### Decision D7: Where OUS's `reads`/`writes` contract lives
|
||||
- **Context**: HarnessX has no per-key state contract; OUS's defining discipline must survive.
|
||||
- **Selected Approach**: Reimplement as a `SlotContractMixin` on our `MultiHookProcessor`s (`reads_slots`/`writes_slots` class attrs validated in a base `on_*` wrapper), gated by `OUS_SLOT_CONTRACT=strict|warn`.
|
||||
- **Rationale**: The two contracts are incompatible; the mixin preserves provenance/audit discipline inside HarnessX's runtime without trying to bend the message-window contract.
|
||||
|
||||
### Decision D8: Project-score gate rollout
|
||||
- **Context**: The verycode score formula is a reconstruction (no upstream implementation).
|
||||
- **Selected Approach**: Ship the score gate **advisory-first** (computed and reported, not blocking); promote to blocking only after `K`/`MIN_SCORE` are calibrated against the benchmark corpus. Score computation is stdlib-only and always on (zero-dep), so CI gating runs in the default install with no HarnessX present.
|
||||
- **Rationale**: The formula needs corpus calibration before it can safely block a merge; keeping it zero-dep keeps the CI gate independent of the optional harness.
|
||||
|
||||
### Decision D9: RL adoption
|
||||
- **Context**: HarnessX has an `rl/` subsystem (weight-level improvement) alongside the config-level meta_harness.
|
||||
- **Selected Approach**: Optional, Phase 5+.
|
||||
- **Rationale**: Config-level adaptation via the meta_harness is sufficient to close the rule/policy loop; weight-level RL over hunter trajectories is purely additive and not on the critical path.
|
||||
|
||||
## Risks & Mitigations
|
||||
- **Sync↔async + contract incompatibility (High)** — confine the async surface to the hunter stage via `asyncio.run` in its callback (Phase 3); `SlotContractMixin` for key-level discipline; deterministic stages run `skip_model`; golden-output diff gate at each phase.
|
||||
- **Dependency posture inversion (High)** — governance/scoring planes stay zero-dep; HarnessX strictly behind the extra + lazy imports + pinned SHA; CI zero-dep guard test; `pip --dry-run` audit before Phase 3; vendored-subset fallback pre-specified.
|
||||
- **Benchmark-gate regression from rule auto-tuning (High)** — the recall/FP gate is the per-round acceptance test (aggregate + per-rule, hard constraint); `shadow` before `disabled`; novelty + attribution block thrash and reverted re-proposals; auto-revert byte-for-byte.
|
||||
- **Score gaming the gate (High)** — keep the gate as a *hard constraint*, never part of the reward; track unpredicted regressions; add a score-regression check to acceptance.
|
||||
- **Meta-agent corrupts the ruleset, re-introducing "too influential" (Medium)** — bounded levers (D3): no pattern rewrites, no severity edits, mandatory `shadow` staging, no `enabled→disabled` jump; recall gate auto-reverts; every change journaled and novelty-checked.
|
||||
- **Policy completeness / CWE drift (Medium)** — day-one CWE-120→CWE-121 remap; hard CI invariant that every enabled rule's CWE resolves; `unmapped_cwe` logged; treat verycode-policies as authoritative-upstream and PR gaps there.
|
||||
- **Frozen Beta upstream (Medium)** — pin SHA; document in `VENDOR.md`; keep the meta_harness patch minimal/isolated; rule lever degrades gracefully (absent patch = no rule lever, loop still does ranking demotion); vendored subset removes upstream-instability exposure for depended modules.
|
||||
- **Slot serialization loss on resume (Medium)** — all scan slots use plain dicts/dataclasses with `_target_` registration; per-slot-type round-trip serialization test (non-serializable content restores as `None` on `wake()`).
|
||||
- **Two-store coordination drift (Low-Med)** — a single `resolve_severity` owns the rule-status + policy-gate combination; the changeset records both lever edits per round; the benchmark regression suite catches interaction regressions.
|
||||
|
||||
## References
|
||||
- HarnessX core — `/tmp/HarnessX/harnessx/core/{harness,builder,processor,events,runloop,state,contract_check,model_config,trajectory}.py` — composition/run-loop/contract/state model.
|
||||
- HarnessX processors — `/tmp/HarnessX/harnessx/processors/{evaluation/llm_judge.py, evaluation/strategies/evaluators/prm.py, observability/{otel_proc,episode_metrics,checkpoint}.py, memory/memory_retrieval.py, control/loop_detection.py}` — reuse map (verify/calibrate/observability).
|
||||
- HarnessX meta_harness — `/tmp/HarnessX/harnessx/meta_harness/{agent,journal,replay,validate_workflow}.py`, `processors/contract_autocheck.py`, `workers/trajectory_digester.py`; `rl/{builder,config,task}.py` — self-improvement engine + rule-adaptation seam.
|
||||
- HarnessX packaging — `/tmp/HarnessX/pyproject.toml`, `harnessx/__init__.py`, `plugins/{base,loader,discovery,convert}.py`, `examples/{coding,research,assistant}/harness.py`, `extensions/plugins/workflow/plugin.py`, `.gitmodules` — installability + dependency weight.
|
||||
- verycode policy — `/tmp/verycode-policies/{CWE_Score.tsv,README.md}`; `gh repo view norandom/verycode-score` (`action.yaml`, `main.go` placeholder); `gh repo view norandom/verycode-lib` (stub) — policy data (complete) vs scoring algorithm (never implemented).
|
||||
- OUS current system — `/home/mc/Source/OpenUltraSAST/src/openultrasast/{findings,rank,calibration,benchmark,verification,harness,config,cli}.py` — `PATTERN_RULES`, ranking-only calibration (`MIN_CALIBRATED_PRIORITY=0.1`), inert `next_improvement_candidate`, sync `HarnessRuntime`, no rule/policy config section.
|
||||
- Companion design — `.kiro/specs/harnessx-self-improving-rulesets/design.md` (committed architecture); detection gate per `MEMORY.md`: ≥90% recall / <10% FP on the benchmark corpus.
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"feature_name": "harnessx-self-improving-rulesets",
|
||||
"created_at": "2026-06-22T10:07:18Z",
|
||||
"updated_at": "2026-06-22T16:45:00Z",
|
||||
"language": "en",
|
||||
"phase": "implementation",
|
||||
"approvals": {
|
||||
"requirements": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"design": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
},
|
||||
"tasks": {
|
||||
"generated": true,
|
||||
"approved": true
|
||||
}
|
||||
},
|
||||
"ready_for_implementation": true
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
# Implementation Plan
|
||||
|
||||
This plan splits the fused regex layer into three governed owners (rules detect, CWE policy decides severity/scope, project score is the optimization target) and adopts HarnessX behind an optional extra. Tasks are grouped into four phases. Each phase is independently shippable and keeps the ≥90% recall / <10% FP detection gate green. Phases 1-2 (Foundation) and the early Core work stay zero-dependency and byte-identical by default; the HarnessX execution and self-improvement work (Integration) lives behind a lazy, SHA-pinned extra; Validation locks the cross-cutting gates and the dependency-weight decision.
|
||||
|
||||
## Phase 1 — Foundation: Governance & Scoring Planes (zero-dep)
|
||||
|
||||
- [x] 1. Stand up the central CWE policy authority
|
||||
- [x] 1.1 Vendor the verycode CWE severity data and build the Policy Store loader
|
||||
- Vendor the verycode CWE-to-severity dataset into the governance plane and parse it with the standard library only, tolerating the trailing-space severity column header.
|
||||
- Expose, per CWE, an authoritative 0-5 severity, a static scope flag, and a dynamic scope flag; mark dynamic-only CWEs as report-only.
|
||||
- Observable completion: loading the policy yields the full set of CWE policy entries with correct static/dynamic flags and zero third-party imports, verified by a unit test.
|
||||
- _Requirements: 2.6, 2.9_
|
||||
- _Boundary: PolicyStore_
|
||||
|
||||
- [x] 1.2 Expose policy-governed severity resolution
|
||||
- Resolve a finding's severity exclusively from the Policy Store keyed on the rule's CWE, discarding any severity string carried in legacy rule data.
|
||||
- Guarantee that two findings whose rules map to the same CWE receive identical severity.
|
||||
- Observable completion: a resolution call returns the policy severity (never a legacy string) and is identical for two rules sharing a CWE, proven by a unit test.
|
||||
- _Requirements: 2.2, 2.3_
|
||||
- _Boundary: PolicyStore_
|
||||
|
||||
- [x] 1.3 Enforce the fail-loud startup CWE-resolution invariant
|
||||
- Assert at startup that every enabled rule's CWE resolves in the Policy Store; abort the scan loudly before any work when an enabled rule's CWE is unmapped.
|
||||
- Observable completion: an enabled rule with an unmapped CWE aborts the scan with a loud error and no findings are produced, proven by an integration test.
|
||||
- _Requirements: 2.7_
|
||||
- _Boundary: PolicyStore_
|
||||
- _Depends: 1.1_
|
||||
|
||||
- [x] 2. Build the project Score Model and CI gates (zero-dep)
|
||||
- [x] 2.1 Compute the 0-100 policy-weighted, reachability-adjusted project score
|
||||
- Compute a single 0-100 project score from policy-governed severity weights multiplied by a reachability multiplier over reported findings, using the standard library only.
|
||||
- Key the reachability multipliers exactly on the three emitted reachability values (`reachable`, `inferred-file-surface`, `unknown`) without redefining the vocabulary.
|
||||
- Assign a zero score penalty to findings whose CWE is dynamic-only or unmapped, reporting them as out-of-scope or unmapped instead of scoring them.
|
||||
- Observable completion: a no-penalty corpus scores 100, a single severity-5 reachable finding scores roughly 43, and dynamic-only/unmapped findings contribute zero penalty, proven by boundary unit tests.
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.9_
|
||||
- _Boundary: ScoreModel_
|
||||
- _Depends: 1.1_
|
||||
|
||||
- [x] 2.2 Emit the scan-level score artifact and manifest score block
|
||||
- Emit a score artifact containing the project score, the maximum severity, the total penalty, a by-category breakdown, the out-of-scope dynamic-only count, the unmapped-CWE list, and the gate verdict; include the score block in the scan manifest.
|
||||
- Observable completion: a scan writes a score artifact with every required field and merges the score block into the manifest, verified against the documented artifact schema.
|
||||
- _Requirements: 5.5_
|
||||
- _Boundary: ScoreModel_
|
||||
- _Depends: 2.1_
|
||||
|
||||
- [x] 2.3 Implement the two-condition CI score gate (advisory-first)
|
||||
- Fail the build when any reported finding maps to policy severity 5 and has reachability `reachable`.
|
||||
- When configured as blocking, fail the build when the project score is below the minimum; until the score constants are calibrated against the corpus, compute and report the score advisory-first without blocking.
|
||||
- Observable completion: the hard gate fails on a severity-5 reachable finding while the score gate reports advisory-only by default, proven by gate tests.
|
||||
- _Requirements: 5.7, 5.8, 5.9_
|
||||
- _Boundary: CI Score Gate_
|
||||
- _Depends: 2.1_
|
||||
|
||||
- [x] 2.4 Centralize ruleset/policy/score configuration
|
||||
- Add configuration entries for the ruleset, policy, and score data locations plus the gate thresholds and emission floors used by later phases.
|
||||
- Observable completion: the scan reads the score constants and emission thresholds from configuration rather than hardcoded literals, confirmed by a config-driven test.
|
||||
- _Requirements: 5.8_
|
||||
- _Boundary: Packaging Configuration, ScoreModel_
|
||||
|
||||
## Phase 2 — Core: Rules-as-Data and Benchmark Signals (zero-dep, byte-identical)
|
||||
|
||||
- [x] 3. Convert detection rules into versioned, governed data
|
||||
- [x] 3.1 Externalize detection patterns into a versioned Ruleset Store
|
||||
- Load detection rules from versioned data files in which each rule declares a pattern, a CWE, tags, and a loop-controlled status, carrying no rule-local severity field.
|
||||
- Remove the rule-local severity field from the in-memory rule model so severity can only come from policy.
|
||||
- Observable completion: the existing pattern rules load from versioned data files with no severity attribute present, proven by a unit test asserting the absence of the field.
|
||||
- _Requirements: 2.1, 2.9_
|
||||
- _Boundary: RulesetStore_
|
||||
|
||||
- [x] 3.2 Implement the three-status volume control
|
||||
- Support exactly three rule statuses — `enabled`, `shadow`, `disabled` — where enabled fires/scores/reports, shadow fires and records outcomes but is excluded from report and score, and disabled does not fire.
|
||||
- Exclude shadow-status findings from the reported set and from the score input while still recording their outcome for precision tracking.
|
||||
- Observable completion: an enabled rule appears in report and score, a shadow rule is excluded from both yet records an outcome, and a disabled rule never fires, proven by a unit test across all three statuses.
|
||||
- _Requirements: 2.4, 2.5, 5.4_
|
||||
- _Boundary: RulesetStore, ScoreModel_
|
||||
- _Depends: 3.1, 2.1_
|
||||
|
||||
- [x] 3.3 Remap the unmapped C/C++ buffer rules to a scored CWE
|
||||
- Remap the C/C++ buffer-handling rules that reference the unmapped CWE-120 to CWE-121 so they resolve in policy and contribute a nonzero penalty.
|
||||
- Observable completion: the remapped buffer rules resolve to a severity-5 policy entry and produce a nonzero score penalty, verified by a unit test.
|
||||
- _Requirements: 2.8_
|
||||
- _Boundary: RulesetStore_
|
||||
- _Depends: 3.1, 1.1_
|
||||
|
||||
- [x] 3.4 Wire findings and severity resolution through the governance plane
|
||||
- Pass the loaded ruleset into the finding-detection stage instead of reading an implicit module-level rule list, and resolve every finding's severity through the Policy Store.
|
||||
- Apply emission thresholds so not every regex match becomes a reported finding, replacing the old "rule is its own volume control" behavior.
|
||||
- Observable completion: a scan produces findings with policy-resolved severity and shadow findings filtered out, while the default ruleset keeps benchmark output byte-identical to the pre-feature baseline.
|
||||
- _Requirements: 2.2, 2.3, 2.5, 6.2_
|
||||
- _Boundary: PolicyScoringProcessor, RulesetStore, PolicyStore_
|
||||
- _Depends: 3.1, 3.2, 1.2_
|
||||
|
||||
- [x] 4. Make benchmark output an actionable per-rule signal
|
||||
- [x] 4.1 Attribute misses and false positives to rules
|
||||
- Attribute every recorded miss and every recorded false positive to a rule identifier and aggregate per-rule precision and recall in the benchmark metrics.
|
||||
- Observable completion: a benchmark run reports per-rule precision/recall and every miss and false positive carries a rule identifier, verified by a metrics test.
|
||||
- _Requirements: 4.1_
|
||||
- _Boundary: Benchmark Harness_
|
||||
- _Depends: 3.1_
|
||||
|
||||
- [x] 4.2 Emit a per-rule recommendation delta artifact
|
||||
- Replace the inert improvement-candidate stub with a per-rule recommendation artifact recommending enable, disable, shadow, threshold, or evidence-floor changes per rule.
|
||||
- Observable completion: a benchmark run writes a recommendation artifact with per-rule suggested actions alongside its records, verified against the documented delta schema.
|
||||
- _Requirements: 4.2_
|
||||
- _Boundary: Benchmark Harness_
|
||||
- _Depends: 4.1_
|
||||
|
||||
- [x] 4.3 Introduce the loop-owned ledger and re-key calibration to rules
|
||||
- Add the loop-owned overlay ledger that records per-rule status, evidence floor, threshold, and precision, applied over the ruleset at load time.
|
||||
- Re-key calibration from the first-tag key to the rule identifier and write status flips into the ledger rather than only demoting ranking.
|
||||
- Observable completion: an empty ledger leaves behavior byte-identical to the prior phase while a ledger entry overlays a rule's status at load, proven by an integration test.
|
||||
- _Requirements: 2.4, 5.6, 6.2_
|
||||
- _Boundary: RulesetStore, Calibration subsystem_
|
||||
- _Depends: 3.1, 3.2_
|
||||
|
||||
- [x] 4.4 Convert false-positive confirmation into a reachability knob
|
||||
- When confirming a false positive, lower the finding's effective reachability multiplier rather than delete the rule, so recall stays high while the score stops over-penalizing.
|
||||
- Observable completion: confirming a false positive lowers the finding's effective reachability multiplier and the rule still fires, proven by a calibration test.
|
||||
- _Requirements: 5.6_
|
||||
- _Boundary: Calibration subsystem, ScoreModel_
|
||||
- _Depends: 4.3, 2.1_
|
||||
|
||||
## Phase 3 — Integration: HarnessX Execution & Self-Improvement (optional extra)
|
||||
|
||||
- [x] 5. Gate and package the HarnessX dependency-weight decision
|
||||
- [x] 5.1 Run the dependency dry-run audit and record the transitive graph
|
||||
- Before adopting the HarnessX execution stages, run a dependency dry-run audit and record the realized transitive dependency graph.
|
||||
- If the audited graph pulls a browser-binary, container-SDK, or web-server stack as a mandatory dependency and it is judged unacceptable, select the pre-specified vendored lean subset under the same extra name with the heavy subtrees removed.
|
||||
- Observable completion: the audit produces a recorded transitive-dependency graph and a documented accept-extra-or-vendor-subset decision before any HarnessX execution code is wired in.
|
||||
- _Requirements: 7.3, 7.4, 7.7_
|
||||
- _Boundary: Packaging Configuration_
|
||||
|
||||
- [x] 5.2 Configure HarnessX as a SHA-pinned optional extra with a capability guard
|
||||
- Keep the core dependency list empty and expose HarnessX only as an optional extra pinned to a specific commit revision; initialize any submodule non-recursively so the large RL fork subtree is not pulled.
|
||||
- Guard every HarnessX import behind a capability check and import it lazily so heavy code paths stay cold in the default install.
|
||||
- Observable completion: the default install resolves with an empty core dependency list and HarnessX symbols are only imported after the capability check passes, verified by an import-cold test.
|
||||
- _Requirements: 7.1, 7.2, 7.6_
|
||||
- _Boundary: Packaging Configuration_
|
||||
- _Depends: 5.1_
|
||||
|
||||
- [x] 6. Adopt HarnessX at the hunter loop and verifier
|
||||
- [x] 6.1 Run the hunter pool stage on a real HarnessX agent loop
|
||||
- Run the hunter pool stage on a real HarnessX run loop with the control, tool, and observability processors registered in the hunter sub-harness, invoked from the existing hunter-pool stage callback.
|
||||
- Confine all asynchronous execution to this single stage so the deterministic stages remain synchronous.
|
||||
- Observable completion: with the extra installed, the hunter pool produces per-target trajectories from a real agent loop while all other stages stay synchronous, verified by an integration test.
|
||||
- _Requirements: 1.1, 1.4_
|
||||
- _Boundary: HxScanOrchestrator_
|
||||
- _Depends: 5.2_
|
||||
|
||||
- [x] 6.2 Map stage events and identifiers onto HarnessX hooks
|
||||
- Map the existing stage-start/stage-end events to the HarnessX step-level hooks and the scan identifier to the HarnessX run identifier without losing trace continuity.
|
||||
- Observable completion: a HarnessX-backed run preserves trace continuity with stage events mapped to step hooks and the scan id mapped to the run id, verified by inspecting the emitted trace.
|
||||
- _Requirements: 1.7_
|
||||
- _Boundary: HxScanOrchestrator_
|
||||
- _Depends: 6.1_
|
||||
|
||||
- [x] 6.3 (P) Run deterministic stages as model-disabled HarnessX processors with slot contracts
|
||||
- Run each deterministic stage under HarnessX with model invocation disabled so no deterministic stage incurs a model call.
|
||||
- Preserve each stage's declared read/write slot allow-list and validate slot access on every lifecycle hook.
|
||||
- Observable completion: deterministic stages execute under HarnessX with no model calls and a slot-contract violation is raised on undeclared slot access, proven by a slot-contract test.
|
||||
- Implemented zero-dep in `slot_contract.py` (`SlotContractMixin`/`SlotPipeline`, validates reads/writes per lifecycle hook, enforces `skip_model`) + `stage_processors.py` (the deterministic stages as `skip_model` slot processors; `run_quick_pipeline` is byte-identical to the quick path). Per "bridge, do not map", the sync `HarnessRuntime` stays the top-level driver and these slot processors are the model-disabled representation; `host_under_harnessx` is the lazy, capability-gated bridge that disables the model on HarnessX's `BeforeModelEvent.skip_model`. Module names differ from the design's `harness/{slot_contract,processors}.py` because `harness` is a module, not a package. Tests: `tests/test_slot_contract.py`.
|
||||
- _Requirements: 1.5, 1.6_
|
||||
- _Boundary: HxScanOrchestrator_
|
||||
- _Depends: 6.1_
|
||||
|
||||
- [x] 6.4 (P) Add the LLM-judge verifier with a structural fallback
|
||||
- Produce verification verdicts using the HarnessX llm-judge helpers when the extra is present, mapping the strict-JSON verdict to a verification result.
|
||||
- Retain the existing structural verifier (which rejects on unknown reachability) verbatim as the zero-dependency fallback.
|
||||
- Observable completion: with the extra present the judge produces verdicts and with it absent the structural verifier produces equivalent verdicts, proven by a paired verifier test.
|
||||
- _Requirements: 1.2_
|
||||
- _Boundary: VerifyProcessor_
|
||||
- _Depends: 5.2_
|
||||
|
||||
- [x] 6.5 Fall back gracefully when a HarnessX capability is unavailable
|
||||
- When a HarnessX-backed stage is enabled but its capability is unavailable at runtime, fall back to the existing equivalent stage and record the degradation in the scan manifest.
|
||||
- Observable completion: requesting a HarnessX stage whose capability is missing runs the existing equivalent and records a degradation entry in the manifest, proven by a fallback test.
|
||||
- _Requirements: 1.8_
|
||||
- _Boundary: HxScanOrchestrator_
|
||||
- _Depends: 6.1, 6.4_
|
||||
|
||||
- [x] 7. Stand up the bounded self-improvement loop
|
||||
- [x] 7.1 Add the two bounded edit levers
|
||||
- Expose exactly two new edit levers: a rule lever covering rule status, evidence floor, and threshold, and a policy lever covering evidence floor, scope, and the score constants.
|
||||
- Observable completion: the loop's valid-lever set includes exactly the rule and policy levers and rejects any edit outside their declared fields, verified by a lever test.
|
||||
- _Requirements: 3.1_
|
||||
- _Boundary: EvolveValidator, improve/evolve.py_
|
||||
- _Depends: 5.2_
|
||||
|
||||
- [x] 7.2 Enforce the rule/policy edit safety bounds in the validator
|
||||
- Reject any loop edit that modifies detection pattern text and any edit that modifies the authoritative 0-5 CWE severity.
|
||||
- Reject moving a rule directly from enabled to disabled, requiring shadow as a mandatory intermediate staging state.
|
||||
- Assert that each edit parses, that status is one of the three literals, that threshold/evidence-floor changes stay within configured bounds, that an added rule does not duplicate an existing identifier, and that every referenced CWE resolves in policy; raise a rule-change validation error and reject the whole round on any failure.
|
||||
- Observable completion: each prohibited edit (pattern change, severity change, enabled-to-disabled jump, duplicate identifier, unmapped CWE, out-of-bounds threshold) is rejected and rejects the round, proven by validator unit tests.
|
||||
- _Requirements: 3.2, 3.3, 3.4, 3.5, 3.8_
|
||||
- _Boundary: EvolveValidator_
|
||||
- _Depends: 7.1, 1.1_
|
||||
|
||||
- [x] 7.3 Add the novelty, replay, and journal gates
|
||||
- Block re-proposing a previously reverted edit that lacks a retry rationale via the novelty gate.
|
||||
- Run a replay smoke gate before any benchmark re-run proving the edited ruleset and policy still load and boot.
|
||||
- Record each rule and policy edit, its lever, and its outcome in the journal so every change is attributable and reversible.
|
||||
- Observable completion: a reverted re-proposal without rationale is blocked, a non-booting edited ruleset is rejected before re-benchmark, and every edit appears in the journal with its lever and outcome, proven by gate tests.
|
||||
- _Requirements: 3.6, 3.7, 3.9_
|
||||
- _Boundary: EvolveValidator, improve/evolve.py_
|
||||
- _Depends: 7.2_
|
||||
|
||||
- [x] 7.4 Convert benchmark deltas into loop signals
|
||||
- Convert the per-rule recommendation delta and false-positive learnings into a loop signal file, writing recall-side items as miss signals and precision-side items as verifier-reject or false-positive signals.
|
||||
- Observable completion: a benchmark run produces a signal file whose recall-side entries are miss signals and precision-side entries are verifier-reject/false-positive signals, verified against the documented signal schema.
|
||||
- _Requirements: 4.3_
|
||||
- _Boundary: improve/evolve.py_
|
||||
- _Depends: 4.2, 7.1_
|
||||
|
||||
- [x] 7.5 Implement the hard acceptance gate with byte-for-byte revert
|
||||
- After the meta-agent edits rules or policy, re-run the benchmark and accept the round only if aggregate recall is at least 90% AND false-positive rate is under 10% AND the project score did not regress on the held set AND no unpredicted regression occurred.
|
||||
- On any failed acceptance condition, revert the round and restore the rule and policy configuration byte-for-byte.
|
||||
- Observable completion: a round that regresses recall is reverted and the configuration is restored byte-for-byte, while a compliant round is accepted and persisted, proven by a self-improvement integration test.
|
||||
- _Requirements: 4.4, 4.5, 6.3_
|
||||
- _Boundary: improve/evolve.py_
|
||||
- _Depends: 7.3, 7.4, 4.1_
|
||||
|
||||
- [x] 7.6 Auto-shadow precision-draggers, nominate coverage gaps, and separate the objective
|
||||
- Auto-shadow a rule that drags aggregate precision below the gate rather than delete it, preserving recall while removing the false-positive cost.
|
||||
- Nominate a new or loosened rule through the signal file when a coverage gap surfaces as a persistent miss, leaving pattern authoring to a human pull request.
|
||||
- Treat maximizing the project score as the optimization objective and the recall/FP gate as a separate hard feasibility constraint that is never folded into the score reward.
|
||||
- Observable completion: a deliberately noisy rule is auto-shadowed (not deleted) when it drags precision, a persistent miss emits a rule nomination, and the loop cannot raise the score by shadowing rules that breach the gate, proven by integration tests.
|
||||
- _Requirements: 4.6, 4.7, 4.8_
|
||||
- _Boundary: improve/evolve.py_
|
||||
- _Depends: 7.5_
|
||||
|
||||
## Phase 4 — Validation: Cross-Cutting Gates & Non-Regression
|
||||
|
||||
- [x] 8. Lock the detection-gate non-regression checks into CI
|
||||
- [x] 8.1 Enforce the per-merge recall/FP detection gate
|
||||
- Evaluate aggregate recall and aggregate false-positive rate on the benchmark corpus on every merge and fail the merge if recall is below 90% or the false-positive rate is at or above 10%.
|
||||
- Treat the recall/FP gate as a hard constraint that is never folded into the project-score reward, blocking any change that breaches the gate regardless of score improvement.
|
||||
- Observable completion: a CI run fails the merge when recall drops below 90% or false-positive rate reaches 10%, and a score improvement cannot unblock a gate-breaching change, proven by pipeline tests.
|
||||
- _Requirements: 6.1, 6.5, 6.6_
|
||||
- _Boundary: CI Pipeline_
|
||||
- _Depends: 4.1_
|
||||
|
||||
- [x] 8.2 (P) Assert byte-identical benchmark baseline for the zero-dep phases
|
||||
- With the default ruleset and an empty loop ledger, assert the benchmark output is byte-identical to the pre-feature baseline for the governance and rules-as-data phases.
|
||||
- Observable completion: a golden-diff test confirms byte-identical benchmark output with the default ruleset and empty ledger.
|
||||
- _Requirements: 6.2_
|
||||
- _Boundary: Benchmark Harness, CI Pipeline_
|
||||
- _Depends: 3.4, 4.3_
|
||||
|
||||
- [x] 8.3 (P) Confirm HarnessX-backed stages stay within detection tolerance
|
||||
- When a HarnessX-backed stage is enabled, confirm recall and false-positive rate stay within the configured tolerance of the equivalent zero-dependency baseline.
|
||||
- Observable completion: a CI run with a HarnessX-backed stage enabled keeps recall/FP within the configured tolerance of the baseline, proven by a tolerance test.
|
||||
- _Requirements: 6.4_
|
||||
- _Boundary: CI Pipeline, HxScanOrchestrator_
|
||||
- _Depends: 6.1, 8.1_
|
||||
|
||||
- [x] 8.4 (P) Add the zero-dependency guard test
|
||||
- Run a guard test confirming that with HarnessX absent the governance, scoring, and benchmark planes operate and CI gating runs in the default install.
|
||||
- Observable completion: the guard test passes with HarnessX uninstalled and fails if any governance/scoring/benchmark path imports HarnessX.
|
||||
- _Requirements: 7.5_
|
||||
- _Boundary: CI Pipeline_
|
||||
- _Depends: 8.1_
|
||||
|
||||
- [x] 9. Validate scan-pipeline integrity and serialization
|
||||
- [x] 9.1 (P) Verify capability fallback and unchanged sync-driver behavior
|
||||
- Confirm that with the extra absent, quick and standard scans execute through the existing sync driver with behavior unchanged from the pre-adoption baseline, and that a requested-but-unavailable HarnessX stage records a degradation.
|
||||
- Observable completion: an extra-absent scan matches the pre-adoption baseline and a missing-capability request records a manifest degradation, proven by an end-to-end fallback test.
|
||||
- _Requirements: 1.3, 1.8_
|
||||
- _Boundary: HxScanOrchestrator, CI Pipeline_
|
||||
- _Depends: 6.5_
|
||||
|
||||
- [x] 9.2 (P) Verify slot serialization round-trips on resume
|
||||
- Confirm each scan slot type serializes and restores without loss after a resume, guarding against non-serializable slots restoring as empty.
|
||||
- Observable completion: a per-slot-type round-trip test passes for every scan slot after a resume cycle.
|
||||
- Done for the current artifact-persistence model (findings/ranking/file-target/verification/score round-trip + the real findings.json reload path in `tests/test_scan_pipeline_integrity.py`). The HarnessX processor slot-contract serialization variant remains deferred with task 6.3.
|
||||
- _Requirements: 1.5_
|
||||
- _Boundary: HxScanOrchestrator_
|
||||
- _Depends: 6.3_
|
||||
@@ -249,37 +249,55 @@ Do not report OpenUltraSAST as a working security harness until `usable_harness_
|
||||
|
||||
## Phase 12: MCP And OpenCode Commands (`opencode_product` prerequisite)
|
||||
|
||||
Implemented in `mcp.py` (zero-dep stdio JSON-RPC server) + the `ousast mcp` CLI
|
||||
subcommand; tests in `tests/test_mcp.py`; README "MCP server" section documents it.
|
||||
|
||||
- [x] Add project-local opencode skills for scan, triage, fix audit, and implementation steering.
|
||||
- [ ] Implement narrow MCP server.
|
||||
- [ ] Add `openultrasast.scan`, `status`, `findings`, `get_finding`, `evidence`, `artifacts`, `benchmark`, `explain`, `propose_patch`, and `export_report` tools.
|
||||
- [ ] Add opencode command docs for quick, standard, deep, verify, and patch workflows.
|
||||
- [ ] Ensure MCP tools never expose arbitrary shell execution.
|
||||
- [ ] Add integration test for MCP tool listing and scan status.
|
||||
- [x] Implement narrow MCP server.
|
||||
- [x] Add `openultrasast.scan`, `status`, `findings`, `get_finding`, `evidence`, `artifacts`, `benchmark`, `explain`, `propose_patch`, and `export_report` tools.
|
||||
- [x] Add opencode command docs for quick, standard, deep, verify, and patch workflows. (README; deep/patch noted as roadmap.)
|
||||
- [x] Ensure MCP tools never expose arbitrary shell execution. (Surface-safety test asserts no shell/exec/command tool.)
|
||||
- [x] Add integration test for MCP tool listing and scan status.
|
||||
|
||||
## Phase 13: Fusion (`standard_security_harness` and `opencode_product` prerequisite)
|
||||
|
||||
- [ ] Implement fusion trigger policy.
|
||||
- [ ] Trigger fusion whenever a task requires deeper reasoning, including difficult findings, contradictory evidence, high-impact decisions, and risky fixes.
|
||||
- [ ] Implement panel A, panel B, and decider role config.
|
||||
- [ ] Implement cross-critique and revised answer flow.
|
||||
- [ ] Implement vote/rank output schema.
|
||||
- [ ] Implement reconciliation dispositions.
|
||||
- [ ] Record model IDs, degradations, warnings, and final decision source.
|
||||
- [ ] Add tests for fusion output validation.
|
||||
Implemented in `fusion.py` (zero-dep deterministic engine + lazy LLM panels), wired
|
||||
into the standard-mode CLI (`fusion.json` + manifest `fusion` block), config `[fusion]`.
|
||||
Tests in `tests/test_fusion.py`.
|
||||
|
||||
- [x] Implement fusion trigger policy.
|
||||
- [x] Trigger fusion whenever a task requires deeper reasoning, including difficult findings, contradictory evidence, high-impact decisions, and risky fixes.
|
||||
- [x] Implement panel A, panel B, and decider role config.
|
||||
- [x] Implement cross-critique and revised answer flow.
|
||||
- [x] Implement vote/rank output schema.
|
||||
- [x] Implement reconciliation dispositions.
|
||||
- [x] Record model IDs, degradations, warnings, and final decision source.
|
||||
- [x] Add tests for fusion output validation.
|
||||
|
||||
## Phase 14: Skill Index (`standard_security_harness` prerequisite)
|
||||
|
||||
- [ ] Build Trail of Bits skill descriptor inventory.
|
||||
- [ ] Chunk selected skill operating guidance into retrieval namespace.
|
||||
- [ ] Implement skill router by language, tags, vulnerability class, and stage.
|
||||
- [ ] Add prompt budget caps for skill snippets.
|
||||
- [ ] Add tests for routing C parser targets to memory/fuzzing skills and crypto targets to constant-time/vector skills.
|
||||
Implemented in `skills.py` (descriptor inventory + scoring router + retrieval-namespace
|
||||
chunking + budget-capped context); the HarnessX hunter prompt consumes the routed
|
||||
skill context; tests in `tests/test_skills.py`.
|
||||
|
||||
- [x] Build Trail of Bits skill descriptor inventory.
|
||||
- [x] Chunk selected skill operating guidance into retrieval namespace.
|
||||
- [x] Implement skill router by language, tags, vulnerability class, and stage.
|
||||
- [x] Add prompt budget caps for skill snippets.
|
||||
- [x] Add tests for routing C parser targets to memory/fuzzing skills and crypto targets to constant-time/vector skills.
|
||||
|
||||
## Phase 15: Hardening And Release Readiness (`production` prerequisite)
|
||||
|
||||
- [ ] Add cost budget enforcement.
|
||||
- [ ] Add model call retry and backoff policies.
|
||||
- [ ] Add redaction for secrets in traces and reports.
|
||||
- [ ] Add docs for threat model and sandbox limits.
|
||||
- [ ] Add end-to-end examples.
|
||||
- [ ] Add lint, typecheck, test, and CI gates.
|
||||
Implemented: `redaction.py` (traces/reports), provider retry/backoff in
|
||||
`provider/openrouter.py`, `[hardening]` config (redact_secrets + max_findings budget;
|
||||
agentic spend bounded by `[harnessx]`), `docs/threat-model.md`, `docs/examples.md`.
|
||||
Tests in `tests/test_redaction.py`, `tests/test_provider_retry.py`,
|
||||
`tests/test_hardening_pipeline.py`. CI gates pre-existed (ruff/mypy/pytest/detection
|
||||
gate/dagger).
|
||||
|
||||
- [x] Add cost budget enforcement.
|
||||
- [x] Add model call retry and backoff policies.
|
||||
- [x] Add redaction for secrets in traces and reports.
|
||||
- [x] Add docs for threat model and sandbox limits.
|
||||
- [x] Add end-to-end examples.
|
||||
- [x] Add lint, typecheck, test, and CI gates.
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
OpenUltraSAST is an independent OpenCode security harness. It combines
|
||||
HarnessX-style harness composition, OpenUltraCode verification discipline,
|
||||
OpenRouter-hosted models and embeddings, Docker-isolated code analysis, and
|
||||
Trail of Bits security skills into a SAST workflow whose primary product goal is
|
||||
**false-positive elimination** — turning suspicion into evidence-backed findings
|
||||
Trail of Bits security skills into a SAST workflow built around one goal:
|
||||
eliminating false positives by turning suspicion into evidence-backed findings
|
||||
and learning from rejected claims.
|
||||
|
||||
The full specification lives in `.kiro/specs/openrouter-sast-harness/`
|
||||
@@ -12,7 +12,7 @@ The full specification lives in `.kiro/specs/openrouter-sast-harness/`
|
||||
as an implementation oracle for proven source-hunting ideas, not as a dependency
|
||||
or fork target.
|
||||
|
||||
> **Maturity legend** — ✅ implemented and tested · 🟡 primitive exists, not yet
|
||||
> **Maturity legend:** ✅ implemented and tested · 🟡 primitive exists, not yet
|
||||
> auto-wired into the scan loop · 🧭 designed, on the roadmap. The current
|
||||
> verified gate is `usable_harness_mvp`.
|
||||
|
||||
@@ -29,6 +29,9 @@ uv run ousast index /path/to/repo
|
||||
|
||||
# Score the detector against a benchmark manifest
|
||||
uv run ousast benchmark benchmarks/manifests/python-vulnerable.toml --mode quick
|
||||
|
||||
# Run the bounded self-improvement loop over the ruleset (benchmark-driven)
|
||||
uv run ousast improve benchmarks/manifests/python-vulnerable.toml --dry-run
|
||||
```
|
||||
|
||||
Every run writes auditable artifacts under `.openultrasast/runs/<scan-id>/`
|
||||
@@ -52,13 +55,13 @@ repo ──▶ preprocess (language, LOC, tags, fuzz entry points)
|
||||
──▶ report.md + report.sarif + manifest.json (shared finding IDs)
|
||||
```
|
||||
|
||||
- **quick** ✅ — deterministic, language-scoped pattern rules + reachability +
|
||||
- **quick** ✅: deterministic, language-scoped pattern rules + reachability +
|
||||
verification. No model calls, fully reproducible.
|
||||
- **standard** ✅ scheduling / 🧭 LLM hunters — runs the tiered hunter pool
|
||||
- **standard** ✅ scheduling / 🧭 LLM hunters: runs the tiered hunter pool
|
||||
(A/B/C budgets, retrieval context, selected Trail of Bits skill snippets,
|
||||
JSONL trajectories) and verification; language-aware LLM hunters are the next
|
||||
upgrade (today standard reuses the deterministic findings per target).
|
||||
- **deep** 🧭 — sandboxed build/fuzz/dynamic reproduction (not yet implemented).
|
||||
- **deep** 🧭: sandboxed build/fuzz/dynamic reproduction (not yet implemented).
|
||||
|
||||
## Detection coverage
|
||||
|
||||
@@ -72,6 +75,31 @@ removes cross-language false positives):
|
||||
| JavaScript / TS | command injection (78), code injection (95), reflected/DOM XSS (79), SQLi (89), path traversal (22), SSRF (918), weak hash (327), deserialization (502) |
|
||||
| Java + Groovy templates | command injection (78), SQLi via concatenation (89), weak hash (327), deserialization (502), unescaped template XSS (79) |
|
||||
|
||||
## Project scores and central CWE policy
|
||||
|
||||
Severity used to be whatever string a rule set on itself. It is now governed centrally: one CWE, one severity, decided by policy — not by the rule that fired.
|
||||
|
||||
**Central CWE policy** ✅ (`policy/verycode.py`): a vendored `CWE_Score.tsv` (verycode-policies, ~167 CWEs) is the single source of truth. Each CWE carries a flaw category, a `severity` (0-5), and `static` / `dynamic` scope flags. `load_policy()` reads it positionally (the upstream file ships CRLF and a trailing space in the `Flaw Severity ` header), and `resolve_severity()` resolves a finding's severity *exclusively* from policy, keyed on CWE — any legacy rule-local severity is discarded. CWEs that are not `static` resolve to `0` and are report-only, never scored.
|
||||
|
||||
**Fail-loud startup** ✅: `assert_rules_resolve(PATTERN_RULES, policy)` runs as the `policy_check` stage right after `policy_load`. If any *enabled* rule names a CWE the policy does not govern, it raises `PolicyError` and the scan aborts before doing work — you cannot ship a rule whose severity nobody decided.
|
||||
|
||||
**0-100 project score** ✅ (`scoring/project_score.py`): each finding's penalty is `severity weight × reachability multiplier`, and the score is an exponential decay of the total (`100 · e^(−total/k)`, `k=60`).
|
||||
|
||||
| Severity | Weight (`SEV_WEIGHT`) | | Reachability | Multiplier (`REACH_MULT`) |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 5 | 50 | | `reachable` | 1.0 |
|
||||
| 4 | 25 | | `inferred-file-surface` | 0.6 |
|
||||
| 3 | 10 | | `unknown` | 0.4 |
|
||||
| 2 / 1 / 0 | 2 / 1 / 0 | | | |
|
||||
|
||||
No penalty → 100; one reachable severity-5 finding → ~43. The **reachability multiplier is the false-positive calibration knob**: a confirmed FP lowers a finding's effective reachability instead of deleting the rule, so the score moves without losing the detection.
|
||||
|
||||
**Two-condition gate** ✅: a finding that is both severity-5 *and* `reachable` **always** fails the gate. The `score < min_score` threshold (default `min_score=80`) only fails when `blocking` is enabled — so scoring is **advisory-first by default** and turns into a CI gate when you opt in.
|
||||
|
||||
**Artifacts** ✅: the `score` stage writes `score.json` (project score, `max_severity`, `penalty_total`, `by_category`, `out_of_scope_dynamic_only`, `unmapped_cwe`, gate verdict) and merges the same block into `manifest.json`. Scoring is zero-dependency (stdlib only). One mapping detail: verycode has no CWE-120 (generic buffer copy), so the C/C++ memory-unsafe rules listed under [Detection coverage](#detection-coverage) carry **CWE-121** (Stack-Based Buffer Overflow, severity 5), which the policy does govern.
|
||||
|
||||
🧭 This is the first implemented slice (**Phase 1**) of the `.kiro/specs/harnessx-self-improving-rulesets/` spec: a central, policy-governed severity model and a project score. Rules-as-data and the full HarnessX self-improvement loop over rulesets are later phases on that roadmap.
|
||||
|
||||
## Running the benchmarks
|
||||
|
||||
Benchmarks wrap the same scan pipeline and score it against ground-truth
|
||||
@@ -114,16 +142,16 @@ SQLi, prototype pollution) so recall is not self-fulfilling.
|
||||
| Java | 100% (4/4) | 0.0% |
|
||||
| **Overall** | **93.6% (44/47)** | **0.0%** |
|
||||
|
||||
The project goal — **≥90% recall and <10% false positives** per language — is
|
||||
enforced as a regression gate in `tests/test_detection_benchmarks.py`, so a rule
|
||||
change that drops recall or raises false positives fails CI.
|
||||
The project goal is **≥90% recall and <10% false positives** per language. The
|
||||
gate lives in `tests/test_detection_benchmarks.py`, so a rule change that drops
|
||||
recall or raises false positives fails CI.
|
||||
|
||||
## OpenCode commands, fusion, and ultra workflows
|
||||
|
||||
OpenUltraSAST is driven from the command line. You can call the `ousast` CLI
|
||||
directly, or run [OpenCode](https://opencode.ai) in the repository and let the
|
||||
agent run the harness for you — OpenCode is invoked from the command line, loads
|
||||
the project skills below, and executes `ousast` plus the triage/fix workflow.
|
||||
OpenUltraSAST runs from the command line. You can call the `ousast` CLI directly,
|
||||
or run [OpenCode](https://opencode.ai) in the repository and let the agent run
|
||||
the harness. OpenCode loads the project skills below, then executes `ousast` plus
|
||||
the triage/fix workflow.
|
||||
|
||||
```bash
|
||||
opencode run "scan this repo with ousast in quick mode and triage the findings"
|
||||
@@ -139,20 +167,48 @@ Project skills that steer the harness from OpenCode (`.opencode/skills/`):
|
||||
| `openultrasast-triage` ✅ | false-positive elimination, ranking calibration, verifier adjudication |
|
||||
| `openultrasast-fix-audit` ✅ | OpenUltraCode fix lifecycle and adversarial fix review |
|
||||
|
||||
**Ultra workflows (OpenUltraCode discipline)** — fixes are never a one-shot
|
||||
patch prompt. `openultrasast-fix-audit` runs the bounded lifecycle
|
||||
### MCP server (narrow surface) ✅
|
||||
|
||||
`ousast mcp` runs a zero-dependency MCP server over stdio (newline-delimited
|
||||
JSON-RPC) exposing only stable, project-level operations — **never** arbitrary
|
||||
shell, Docker, or internal hunter tools. Point an MCP client (OpenCode, Claude
|
||||
Desktop, an IDE) at it:
|
||||
|
||||
```jsonc
|
||||
// e.g. an MCP client config entry
|
||||
{ "command": "uv", "args": ["run", "ousast", "mcp"] }
|
||||
```
|
||||
|
||||
Tools: `openultrasast.scan`, `status`, `findings`, `get_finding`, `evidence`,
|
||||
`artifacts`, `benchmark`, `explain`, `propose_patch` (degrades visibly — the patch
|
||||
oracle is a later phase), and `export_report`. The `scan`/`benchmark` tools run the
|
||||
bounded analysis pipeline; no tool accepts a free-form command. A typical flow:
|
||||
`scan {path}` → returns a `run_dir` → `findings {run_dir}` → `explain {run_dir, finding_id}`.
|
||||
|
||||
**Ultra workflows (OpenUltraCode discipline):** fixes are never a one-shot patch
|
||||
prompt. `openultrasast-fix-audit` runs the bounded lifecycle
|
||||
`intake → plan → minimal patch → adversarial review → reconcile → fresh
|
||||
verification → ready`, and a patch can only reach `patch_validated` after
|
||||
sandboxed validation passes with no accepted blocking findings.
|
||||
|
||||
**Fusion (deepening) 🧭** — when a finding needs more reasoning than the normal
|
||||
**Fusion (deepening) ✅:** when a finding needs more reasoning than the normal
|
||||
ranker → hunter → verifier → mapping loop provides (critical/high severity,
|
||||
verifier disagreement, conflicting static vs semantic evidence, risky fixes, or
|
||||
an explicit high-assurance request), fusion runs two independent panels that
|
||||
critique, revise, vote, and a decider issues the disposition (accepted /
|
||||
rejected / mitigated / deferred / blocked) with model IDs and degradations
|
||||
disclosed. The triage skill escalates to fusion *when available*; the panel
|
||||
implementation is on the roadmap (Phase 13).
|
||||
steel-man the vulnerability and false-positive cases, vote, and a decider issues
|
||||
the disposition (accepted / rejected / mitigated / deferred / blocked) with votes,
|
||||
model IDs, and degradations disclosed. It runs automatically on triggered findings
|
||||
in `--mode standard`, writing `fusion.json` and a `fusion` block in the manifest.
|
||||
Deterministic by default (`fusion.py`); set `[fusion] panel_model` to route the
|
||||
panels through the configured provider:
|
||||
|
||||
```toml
|
||||
[fusion]
|
||||
enabled = true # default; runs on triggered findings in standard mode
|
||||
panel_model = "gpt-4o" # optional: LLM panels via the [harnessx] provider
|
||||
decider_model = "gpt-4o" # optional: disclosed in the decision's model IDs
|
||||
high_assurance = false # true forces fusion on every finding
|
||||
```
|
||||
|
||||
> The `kiro-*` skills and `AGENTS.md` in this repo are the maintainer's
|
||||
> spec-driven development tooling. They are not part of using OpenUltraSAST and
|
||||
@@ -184,9 +240,9 @@ When a finding is still rejected after review, `openultrasast-triage` records a
|
||||
taxonomy (`unreachable_path`, `missing_attacker_control`, `sanitizer_disproved`,
|
||||
`static_rule_mismatch`, `incorrect_model_assumption`, `duplicate`,
|
||||
`insufficient_impact`, `unsupported`, `contradicted`, `unverified`), evidence,
|
||||
and a **scope**. Scoping is the safety property: a rejected `syscall_entry` claim
|
||||
in `auth/` demotes only `auth/…`, never the whole vulnerability class — proven by
|
||||
`tests/test_calibration.py::test_scoped_false_positive_demotion_does_not_suppress_class_globally`.
|
||||
and a **scope**. Scoping keeps the learning narrow: a rejected `syscall_entry`
|
||||
claim in `auth/` demotes only `auth/…`, never the whole vulnerability class. This
|
||||
is covered by `tests/test_calibration.py::test_scoped_false_positive_demotion_does_not_suppress_class_globally`.
|
||||
Demotion (with an audit trail) is preferred over deletion, so an analyst can
|
||||
always see *why* something was downgraded.
|
||||
|
||||
@@ -209,20 +265,56 @@ overflow the regex engine cannot see:
|
||||
```
|
||||
|
||||
The `next_improvement_candidate` routes the gap to the stage that should close
|
||||
it — a static rule, SARIF source, entry-point mapping, retrieval package,
|
||||
it: a static rule, SARIF source, entry-point mapping, retrieval package,
|
||||
language-aware hunter prompt, dynamic reproducer, or skill route. Once a miss is
|
||||
addressed, it stays closed because the recall/precision gate runs in CI.
|
||||
`external_baseline_deltas.json` additionally shows where another tool found a
|
||||
vulnerability OpenUltraSAST missed (or vice-versa).
|
||||
|
||||
## Configuring the HarnessX agentic plane
|
||||
|
||||
The LLM hunter pool and the llm-judge verifier are optional. The core install is
|
||||
zero-dependency and `--mode quick` never calls a model. To enable the agentic
|
||||
plane, install the extra, name the models, and pick a provider:
|
||||
|
||||
```bash
|
||||
uv sync --extra harnessx # SHA-pinned optional dependency
|
||||
export OPENAI_API_KEY=sk-... # or ANTHROPIC_API_KEY, matching the provider
|
||||
uv run ousast scan /path/to/code --mode standard --config openultrasast.toml
|
||||
```
|
||||
|
||||
```toml
|
||||
# openultrasast.toml
|
||||
[models]
|
||||
hunter = "gpt-4o" # LLM hunter pool (empty -> deterministic hunter)
|
||||
verifier = "gpt-4o" # llm-judge verifier (empty -> structural verifier)
|
||||
|
||||
[harnessx]
|
||||
provider = "openai" # "anthropic" (default) | "openai" | "litellm"
|
||||
max_cost_usd = 2.0 # per-task spend cap
|
||||
token_threshold = 120000 # per-task token budget
|
||||
```
|
||||
|
||||
The provider selects the HarnessX backend for **both** the hunter and the
|
||||
verifier; each reads its own standard key env var (`openai` → `OPENAI_API_KEY`,
|
||||
`anthropic` → `ANTHROPIC_API_KEY`). The HarnessX path activates only when the
|
||||
extra is installed, a model is configured, and the mode is `standard` — all three
|
||||
gated through one capability seam (`harness_ext.has_harnessx`). If any condition
|
||||
fails, the scan transparently falls back to the deterministic hunter / structural
|
||||
verifier and records a `degradations` entry in `manifest.json`, so a run is never
|
||||
silently downgraded:
|
||||
|
||||
```bash
|
||||
jq '.degradations' "$run/manifest.json" # null = HarnessX ran; entries = fell back
|
||||
```
|
||||
|
||||
## The HarnessX self-improving cycle
|
||||
|
||||
Each scan is a composed harness of typed processors with read/write **state
|
||||
contracts** (strict mode fails the scan on a violation; warn mode records a
|
||||
degradation) and a full event trace. The loop is now **closed automatically
|
||||
inside the pipeline** — no agent required: every scan persists its verifier
|
||||
rejections as scoped learnings and the next scan loads them and demotes those
|
||||
scopes before findings are produced.
|
||||
contracts**. Strict mode fails the scan on a violation; warn mode records a
|
||||
degradation. The loop now runs inside the pipeline without an agent: every scan
|
||||
persists verifier rejections as scoped learnings, and the next scan loads them
|
||||
and demotes those scopes before findings are produced.
|
||||
|
||||
```
|
||||
┌──────────── automatic ledger (no agent in the loop) ───────────┐
|
||||
@@ -249,11 +341,58 @@ priority is demoted rather than the finding deleted, the audit trail
|
||||
(`applied_calibrations.json`, the ledger) shows exactly why a surface lost
|
||||
attention. Covered by `tests/test_pipeline_calibration.py`.
|
||||
|
||||
Beyond the automatic ranking loop, **OpenCode still steers the richer
|
||||
adjustments** — prompt constraints, retrieval filters, skill routing, and
|
||||
benchmark-miss triage — via `openultrasast-triage`; those `RankingCalibration`
|
||||
fields are already produced and will be consumed directly once the language-aware
|
||||
LLM hunters land (`standard_security_harness`).
|
||||
### Rule-level loop: `ousast improve`
|
||||
|
||||
The ranking loop above adjusts *attention*. A second, bounded loop adjusts the
|
||||
**governed ruleset itself** from benchmark feedback — and writes its result to the
|
||||
same loop-owned ledger a scan reads, so the loop closes end-to-end:
|
||||
|
||||
```bash
|
||||
# Preview what the loop would change (writes nothing)
|
||||
uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml --dry-run
|
||||
|
||||
# Apply: accepted rounds land in <target>/.openultrasast/calibration/rule_policy.json,
|
||||
# which the next `ousast scan`/`benchmark` of that target loads automatically.
|
||||
uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml
|
||||
```
|
||||
|
||||
Each round (`improve/evolve.py`): benchmark with the current ledger → per-rule
|
||||
signals (`rule_signals.json`) → propose **bounded** edits → validate → replay
|
||||
smoke → re-benchmark → a **hard acceptance gate** → accept or revert
|
||||
byte-for-byte. The loop may only pull two levers (rule status, score constants);
|
||||
it can never edit a rule's pattern text or the authoritative 0–5 CWE severity, and
|
||||
must stage `enabled → shadow` before `disabled`. A round is accepted only if
|
||||
**recall ≥ 90% AND FP < 10% AND the project score did not regress AND no matched
|
||||
finding was lost** — otherwise the ledger is never written. Auto-shadowing removes
|
||||
a precision-dragger's false-positive cost without deleting it (recall is
|
||||
preserved); persistent misses are *nominated* through the signal file, leaving
|
||||
pattern authoring to a human pull request. The project score is the optimization
|
||||
reward; the recall/FP gate is a separate hard constraint that is never folded into
|
||||
it — and the same gate runs in CI (`python -m openultrasast.gate`), so a loop
|
||||
result that breached it could never merge. Covered by `tests/test_improve.py` and
|
||||
`tests/test_cli_improve.py`.
|
||||
|
||||
This is the deterministic substrate of the self-improvement loop. When the
|
||||
optional `openultrasast[harnessx]` extra is present, the HarnessX `MetaAgent`
|
||||
becomes the richer *proposer* that plugs into this same validated, gated
|
||||
machinery — the safety contract (bounded levers, replay/novelty/journal gates,
|
||||
hard acceptance gate, byte-for-byte revert) is identical either way.
|
||||
|
||||
Beyond these loops, `openultrasast-triage` still lets OpenCode adjust prompt
|
||||
constraints, retrieval filters, skill routing, and benchmark-miss triage. Those
|
||||
`RankingCalibration` fields are already produced and will be consumed directly
|
||||
once the language-aware LLM hunters land (`standard_security_harness`).
|
||||
|
||||
## Security & hardening
|
||||
|
||||
OpenUltraSAST treats scanned code as **untrusted input and never executes it** (quick
|
||||
and standard modes). Scan artifacts are scrubbed of credentials before they are written
|
||||
(`[hardening] redact_secrets`, on by default), agentic spend and output size are bounded
|
||||
(`[harnessx] max_cost_usd`/`token_threshold`, `[hardening] max_findings`), provider calls
|
||||
retry transient failures with backoff, and missing capabilities degrade visibly in the
|
||||
manifest rather than silently. See [docs/threat-model.md](docs/threat-model.md) for the
|
||||
full trust boundaries and sandbox limits, and [docs/examples.md](docs/examples.md) for
|
||||
end-to-end walkthroughs.
|
||||
|
||||
## Development
|
||||
|
||||
@@ -262,5 +401,6 @@ uv run pytest # tests, incl. the 90/10 detection gate
|
||||
uv run ruff check . # lint
|
||||
uv run ruff format --check . # format
|
||||
uv run mypy src/openultrasast # types
|
||||
uv run python -m openultrasast.gate # standalone detection gate
|
||||
uv run python dagger/ci.py # full containerized CI pipeline
|
||||
```
|
||||
|
||||
+35
-1
@@ -6,8 +6,25 @@ import anyio
|
||||
|
||||
import dagger
|
||||
|
||||
# Substrings that mark a transient registry / image-pull failure (e.g. a Docker
|
||||
# Hub 5xx on the python:3.11-bookworm pull, or a registry.dagger.io timeout) that
|
||||
# is worth retrying rather than failing the whole pipeline.
|
||||
_TRANSIENT_MARKERS = (
|
||||
"pull image",
|
||||
"failed to authorize",
|
||||
"failed to fetch oauth",
|
||||
"TransportQueryError",
|
||||
"i/o timeout",
|
||||
"connection reset",
|
||||
"context deadline exceeded",
|
||||
" 502",
|
||||
" 503",
|
||||
" 520",
|
||||
" 429",
|
||||
)
|
||||
|
||||
async def main() -> None:
|
||||
|
||||
async def _run_checks() -> None:
|
||||
config = dagger.Config(log_output=sys.stderr)
|
||||
async with dagger.Connection(config) as client:
|
||||
source = client.host().directory(
|
||||
@@ -33,11 +50,28 @@ async def main() -> None:
|
||||
.with_exec(["uv", "run", "ruff", "check", "."])
|
||||
.with_exec(["uv", "run", "mypy", "src/openultrasast"])
|
||||
.with_exec(["uv", "run", "pytest"])
|
||||
.with_exec(["uv", "run", "python", "-m", "openultrasast.gate"])
|
||||
.with_exec(["uv", "run", "python", "-m", "compileall", "src", "tests"])
|
||||
.with_exec(["uv", "build"])
|
||||
)
|
||||
await checks.sync()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
attempts = 3
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
await _run_checks()
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001 — retry only transient registry/pull failures
|
||||
message = str(exc)
|
||||
transient = any(marker in message for marker in _TRANSIENT_MARKERS)
|
||||
if attempt < attempts and transient:
|
||||
print(f"dagger: transient registry/pull failure on attempt {attempt}; retrying in 20s", file=sys.stderr)
|
||||
await anyio.sleep(20)
|
||||
continue
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
anyio.run(main)
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# End-to-end examples
|
||||
|
||||
Each example is self-contained. Artifacts land under
|
||||
`<target>/.openultrasast/runs/<scan-id>/` and `.openultrasast/` is gitignored.
|
||||
|
||||
## 1. Quick scan a local repo (deterministic, no keys)
|
||||
|
||||
```bash
|
||||
uv run ousast scan /path/to/code --mode quick
|
||||
run=/path/to/code/.openultrasast/runs/<scan-id>
|
||||
cat "$run/report.md" # findings + evidence (secrets redacted)
|
||||
jq . "$run/score.json" # 0–100 project score + gate verdict
|
||||
"$run/report.sarif" # SARIF for code-scanning / IDEs
|
||||
```
|
||||
|
||||
## 2. CI gate
|
||||
|
||||
```bash
|
||||
# Fail the build on any evidence-verified finding:
|
||||
uv run ousast scan . --mode quick --fail-on verified
|
||||
|
||||
# The hard detection gate (>=90% recall, <10% FP), score-independent:
|
||||
uv run python -m openultrasast.gate
|
||||
```
|
||||
|
||||
## 3. Benchmark a known-vulnerable corpus
|
||||
|
||||
```bash
|
||||
uv run ousast benchmark benchmarks/manifests/python-vulnerable.toml --mode quick
|
||||
# -> expected / matched / missed counts + a per-rule recommendation delta
|
||||
```
|
||||
|
||||
## 4. Self-improve the ruleset from benchmark feedback
|
||||
|
||||
```bash
|
||||
uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml --dry-run # preview
|
||||
uv run ousast improve benchmarks/manifests/java-spring-boot-vulnerable.toml # apply
|
||||
# Accepted edits land in <target>/.openultrasast/calibration/rule_policy.json,
|
||||
# which the next scan/benchmark of that target loads automatically.
|
||||
```
|
||||
|
||||
## 5. Standard mode with the HarnessX agentic plane (OpenAI)
|
||||
|
||||
```bash
|
||||
uv sync --extra harnessx
|
||||
export OPENAI_API_KEY=sk-...
|
||||
cat > openultrasast.toml <<'TOML'
|
||||
[models]
|
||||
hunter = "gpt-4o"
|
||||
verifier = "gpt-4o"
|
||||
[harnessx]
|
||||
provider = "openai"
|
||||
max_cost_usd = 2.0
|
||||
token_threshold = 120000
|
||||
[fusion]
|
||||
panel_model = "gpt-4o" # two-panel adjudication on triggered findings
|
||||
[hardening]
|
||||
redact_secrets = true
|
||||
max_findings = 0
|
||||
TOML
|
||||
uv run ousast scan /path/to/code --mode standard --config openultrasast.toml
|
||||
jq '.degradations' "$run/manifest.json" # null = HarnessX ran; entries = it fell back
|
||||
jq '.fusion' "$run/manifest.json" # per-finding dispositions
|
||||
```
|
||||
|
||||
## 6. Drive it from an MCP client (OpenCode / IDE)
|
||||
|
||||
```jsonc
|
||||
{ "command": "uv", "args": ["run", "ousast", "mcp"] }
|
||||
```
|
||||
|
||||
Then: `openultrasast.scan {path}` → `run_dir` → `openultrasast.findings {run_dir}` →
|
||||
`openultrasast.explain {run_dir, finding_id}`. See the README "MCP server" section.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Threat model & hardening
|
||||
|
||||
OpenUltraSAST analyzes **untrusted code**. This document states what it trusts, what
|
||||
it does not, and the controls that keep a scan safe to run in CI and safe to share.
|
||||
|
||||
## Trust boundaries
|
||||
|
||||
- **Scanned code is untrusted input, never executed.** Quick and standard modes read
|
||||
source as text (regex rules, evidence ladder, optional LLM review). They never import,
|
||||
build, or run the target. Only `deep` mode would execute target-derived code, and only
|
||||
inside the sandbox below — and deep mode is not yet implemented (it exits with a clear
|
||||
message rather than running anything).
|
||||
- **The ruleset and CWE policy are trusted, governed data.** They change only through
|
||||
reviewed commits or the bounded self-improvement loop (`ousast improve`), which can
|
||||
flip rule *status* and tune score constants but can never edit pattern text or the
|
||||
authoritative 0–5 severity. See the self-improving cycle in the README.
|
||||
- **Provider credentials are the operator's.** The zero-dependency core makes no network
|
||||
calls. The optional HarnessX agentic plane sends source excerpts to the configured LLM
|
||||
provider only when a model is set in `[models]`; keys are read from the provider's
|
||||
standard env var (`OPENAI_API_KEY` / `ANTHROPIC_API_KEY`), never written to artifacts.
|
||||
|
||||
## Controls
|
||||
|
||||
### Secret redaction
|
||||
Scan artifacts can quote source that contains live credentials. `redaction.py` masks
|
||||
recognizable secret shapes (provider API keys, AWS/GitHub/Slack/Google tokens, bearer
|
||||
tokens, URL-embedded credentials, PEM private keys, and `key = value` assignments for
|
||||
sensitive names) before traces (`trace/events.jsonl`) and the markdown report are
|
||||
written. On by default; disable with `[hardening] redact_secrets = false`.
|
||||
|
||||
### Cost & CI budgets
|
||||
- **Agentic spend** is bounded per task by `[harnessx] max_cost_usd` and
|
||||
`token_threshold` (enforced inside the HarnessX run loop) and by the per-tier hunter
|
||||
budgets.
|
||||
- **Output size** is bounded by `[hardening] max_findings` (0 = unlimited). Truncation is
|
||||
severity-ordered and disclosed as a `budget` degradation in the manifest.
|
||||
|
||||
### Provider reliability
|
||||
LLM/embedding calls retry transient failures (HTTP 429/5xx, connection errors, timeouts)
|
||||
with exponential backoff; non-transient errors (e.g. 4xx, malformed JSON) fail fast.
|
||||
|
||||
### Visible degradation & determinism
|
||||
When an optional capability is missing (the extra, a model, a provider key), the scan
|
||||
falls back to its deterministic equivalent and records a `degradations` entry in the
|
||||
manifest — it is never silently downgraded. Runs are reproducible: fixed config,
|
||||
artifact manifests, prompt hashes, and model identifiers are recorded.
|
||||
|
||||
### MCP surface
|
||||
`ousast mcp` exposes only the ten narrow project tools. No tool runs arbitrary shell,
|
||||
Docker, or internal hunter tools, and no tool accepts a free-form command argument.
|
||||
|
||||
## Sandbox limits (deep mode, when implemented)
|
||||
|
||||
Dynamic analysis runs under the bounds in `[sandbox]` (`config.SandboxConfig`):
|
||||
|
||||
| Control | Default |
|
||||
|---|---|
|
||||
| network | disabled (`network = false`) |
|
||||
| workspace | read-only (`workspace_readonly = true`) |
|
||||
| memory | 2048 MB (`memory_mb`) |
|
||||
| pids | 512 (`pids_limit`) |
|
||||
| timeout | 300 s (`timeout_seconds`) |
|
||||
|
||||
Dynamic probes additionally declare their network scope, log commands, and capture
|
||||
artifacts; a hardened runtime (e.g. gVisor) is used when available. Until deep mode
|
||||
ships, no target code is executed at all.
|
||||
+18
-1
@@ -4,12 +4,23 @@ build-backend = "uv_build"
|
||||
|
||||
[project]
|
||||
name = "openultrasast"
|
||||
version = "0.1.0"
|
||||
version = "1.0.0"
|
||||
description = "OpenCode-oriented SAST harness for evidence-gated security analysis"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = []
|
||||
|
||||
[project.optional-dependencies]
|
||||
# HarnessX is the optional agentic plane (LLM hunter/verifier + self-improvement).
|
||||
# SHA-pinned for reproducibility; the heavy transitive footprint (see
|
||||
# .kiro/specs/harnessx-self-improving-rulesets/dependency-audit.md) lands ONLY when
|
||||
# this extra is installed. The core install stays zero-dependency and every HarnessX
|
||||
# import is guarded by openultrasast.harness_ext (lazy). pip does not pull the
|
||||
# recipe/verl_harnessX submodule (it is outside the packaged harnessx* modules).
|
||||
harnessx = [
|
||||
"harnessx @ git+https://github.com/Darwin-Agent/HarnessX.git@274c512eea67c64701f992d3b6fe06f0fd590ef0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
ousast = "openultrasast.cli:main"
|
||||
|
||||
@@ -37,6 +48,12 @@ warn_redundant_casts = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
|
||||
# HarnessX is an optional extra, absent in the type-check environment; its lazy
|
||||
# imports inside guarded functions must not fail mypy.
|
||||
[[tool.mypy.overrides]]
|
||||
module = "harnessx.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"dagger-io>=0.21.7",
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__version__ = "1.0.0"
|
||||
|
||||
@@ -60,6 +60,7 @@ class BenchmarkMiss:
|
||||
path: str
|
||||
evidence: str
|
||||
reason: str
|
||||
rule_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -79,6 +80,7 @@ class BenchmarkCalibrationRecord:
|
||||
failed_stage: str
|
||||
next_improvement_candidate: str
|
||||
reason: str
|
||||
rule_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -87,6 +89,17 @@ class BenchmarkFalsePositive:
|
||||
path: str
|
||||
line: int | None
|
||||
reason: str
|
||||
rule_id: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuleRecommendation:
|
||||
rule_id: str
|
||||
action: str # keep | shadow | loosen
|
||||
matched: int
|
||||
missed: int
|
||||
false_positives: int
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -101,6 +114,7 @@ class BenchmarkMetrics:
|
||||
runtime_seconds: float | None
|
||||
model_usage: dict[str, object]
|
||||
artifact_links: dict[str, str]
|
||||
per_rule: dict[str, dict[str, int]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -123,6 +137,7 @@ class BenchmarkResult:
|
||||
false_positives: list[BenchmarkFalsePositive]
|
||||
baseline_deltas: list[BenchmarkBaselineDelta]
|
||||
calibration_records: list[BenchmarkCalibrationRecord]
|
||||
rule_recommendations: list[RuleRecommendation]
|
||||
|
||||
|
||||
def load_benchmark_manifest(path: Path) -> BenchmarkManifest:
|
||||
@@ -224,6 +239,8 @@ def evaluate_benchmark(
|
||||
false_positives = [_false_positive(finding) for index, finding in enumerate(findings) if index not in matched_finding_indexes]
|
||||
baseline_deltas = _baseline_deltas(run.manifest.expected, baseline_findings or [])
|
||||
calibration_records = [_calibration_record(miss) for miss in misses]
|
||||
per_rule = _per_rule_metrics(findings, matched_finding_indexes, misses)
|
||||
rule_recommendations = _rule_recommendations(per_rule)
|
||||
expected_total = len(run.manifest.expected)
|
||||
actual_total = len(findings)
|
||||
missed_total = len(misses)
|
||||
@@ -249,11 +266,13 @@ def evaluate_benchmark(
|
||||
runtime_seconds=runtime_seconds,
|
||||
model_usage={},
|
||||
artifact_links=_artifact_links(scan_run_dir),
|
||||
per_rule=per_rule,
|
||||
),
|
||||
misses=misses,
|
||||
false_positives=false_positives,
|
||||
baseline_deltas=baseline_deltas,
|
||||
calibration_records=calibration_records,
|
||||
rule_recommendations=rule_recommendations,
|
||||
)
|
||||
|
||||
|
||||
@@ -266,6 +285,9 @@ def write_benchmark_artifacts(run: BenchmarkRun, manifest_path: Path, result: Be
|
||||
(run.root / "external_baseline_deltas.json").write_text(
|
||||
json.dumps(_jsonable([asdict(delta) for delta in result.baseline_deltas]), indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
(run.root / "rule_policy_delta.json").write_text(
|
||||
json.dumps(_jsonable([asdict(rec) for rec in result.rule_recommendations]), indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def _matching_finding_index(expected: ExpectedFinding, findings: list[StaticFinding], used_indexes: set[int]) -> int | None:
|
||||
@@ -301,6 +323,7 @@ def _miss(expected: ExpectedFinding) -> BenchmarkMiss:
|
||||
path=expected.path,
|
||||
evidence=expected.evidence,
|
||||
reason="no OpenUltraSAST finding matched the expected benchmark vulnerability",
|
||||
rule_id=expected.rule_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -310,9 +333,57 @@ def _false_positive(finding: StaticFinding) -> BenchmarkFalsePositive:
|
||||
path=finding.path,
|
||||
line=finding.line,
|
||||
reason="finding did not match any expected benchmark vulnerability",
|
||||
rule_id=_rule_id_of(finding.finding_id),
|
||||
)
|
||||
|
||||
|
||||
def _rule_id_of(finding_id: str) -> str:
|
||||
return finding_id.split(":", 1)[0]
|
||||
|
||||
|
||||
def _per_rule_metrics(
|
||||
findings: list[StaticFinding],
|
||||
matched_finding_indexes: set[int],
|
||||
misses: list[BenchmarkMiss],
|
||||
) -> dict[str, dict[str, int]]:
|
||||
per_rule: dict[str, dict[str, int]] = {}
|
||||
|
||||
def bucket(rule_id: str) -> dict[str, int]:
|
||||
return per_rule.setdefault(rule_id, {"matched": 0, "missed": 0, "false_positives": 0})
|
||||
|
||||
for index, finding in enumerate(findings):
|
||||
rule_id = _rule_id_of(finding.finding_id)
|
||||
key = "matched" if index in matched_finding_indexes else "false_positives"
|
||||
bucket(rule_id)[key] += 1
|
||||
for miss in misses:
|
||||
if miss.rule_id:
|
||||
bucket(miss.rule_id)["missed"] += 1
|
||||
return dict(sorted(per_rule.items()))
|
||||
|
||||
|
||||
def _rule_recommendations(per_rule: dict[str, dict[str, int]]) -> list[RuleRecommendation]:
|
||||
recommendations: list[RuleRecommendation] = []
|
||||
for rule_id, counts in per_rule.items():
|
||||
matched, missed, fps = counts["matched"], counts["missed"], counts["false_positives"]
|
||||
if fps > 0:
|
||||
action, rationale = "shadow", "rule produced false positives; stage it before it drags precision"
|
||||
elif missed > 0:
|
||||
action, rationale = "loosen", "rule missed expected findings; widen or lower its evidence floor"
|
||||
else:
|
||||
action, rationale = "keep", "rule matched expected findings with no false positives"
|
||||
recommendations.append(
|
||||
RuleRecommendation(
|
||||
rule_id=rule_id,
|
||||
action=action,
|
||||
matched=matched,
|
||||
missed=missed,
|
||||
false_positives=fps,
|
||||
rationale=rationale,
|
||||
)
|
||||
)
|
||||
return recommendations
|
||||
|
||||
|
||||
def _baseline_deltas(expected_findings: list[ExpectedFinding], baseline_findings: list[BaselineFinding]) -> list[BenchmarkBaselineDelta]:
|
||||
tools = sorted({finding.tool for finding in baseline_findings})
|
||||
deltas: list[BenchmarkBaselineDelta] = []
|
||||
@@ -359,14 +430,18 @@ def _baseline_matches_expected(expected: ExpectedFinding, finding: BaselineFindi
|
||||
|
||||
|
||||
def _calibration_record(miss: BenchmarkMiss) -> BenchmarkCalibrationRecord:
|
||||
# A miss tied to a known rule points the loop at that rule; an unattributed
|
||||
# miss (no rule fired for the class) is a coverage gap for a language hunter.
|
||||
candidate = f"rule:{miss.rule_id}" if miss.rule_id else "language_hunter"
|
||||
return BenchmarkCalibrationRecord(
|
||||
cwe=miss.cwe,
|
||||
vulnerability_class=miss.vulnerability_class,
|
||||
path=miss.path,
|
||||
evidence=miss.evidence,
|
||||
failed_stage="benchmark_ground_truth_matching",
|
||||
next_improvement_candidate="rules_or_language_hunter",
|
||||
next_improvement_candidate=candidate,
|
||||
reason=miss.reason,
|
||||
rule_id=miss.rule_id,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -9,8 +9,7 @@ from .findings import StaticFinding
|
||||
from .rank import RankingScore
|
||||
from .verification import VerificationResult, VerificationStatus
|
||||
|
||||
# Calibration may push a repeatedly-rejected scope below the natural ranking
|
||||
# minimum (1.0) so accumulated false positives keep losing priority over runs.
|
||||
# Repeated rejections can push a scope below the normal 1.0 ranking floor.
|
||||
MIN_CALIBRATED_PRIORITY = 0.1
|
||||
|
||||
|
||||
@@ -250,6 +249,30 @@ def learnings_from_verifications(
|
||||
return learnings
|
||||
|
||||
|
||||
# A confirmed false positive lowers a finding's effective reachability multiplier
|
||||
# by this factor in the project score (it does not delete the rule).
|
||||
FP_REACHABILITY_FACTOR = 0.4
|
||||
_BASE_REACH_MULT = {"reachable": 1.0, "inferred-file-surface": 0.6, "unknown": 0.4}
|
||||
|
||||
|
||||
def fp_reachability_overrides(
|
||||
findings: list[StaticFinding],
|
||||
learnings: list[FalsePositiveLearning],
|
||||
) -> dict[str, float]:
|
||||
"""Lowered score multipliers for findings whose scope has a confirmed FP.
|
||||
|
||||
Keyed by ``finding_id``; consumed by the scoring model so a confirmed false
|
||||
positive reduces the score penalty instead of the rule being deleted.
|
||||
"""
|
||||
overrides: dict[str, float] = {}
|
||||
for finding in findings:
|
||||
vulnerability_class = _vulnerability_class(finding)
|
||||
if any(_learning_applies(finding.path, learning) and learning.vulnerability_class == vulnerability_class for learning in learnings):
|
||||
base = _BASE_REACH_MULT.get(finding.reachability_status, _BASE_REACH_MULT["unknown"])
|
||||
overrides[finding.finding_id] = round(base * FP_REACHABILITY_FACTOR, 3)
|
||||
return overrides
|
||||
|
||||
|
||||
def _reason_from_verification(finding: StaticFinding, status: VerificationStatus) -> FalsePositiveReason:
|
||||
if finding.reachability_status == "unknown":
|
||||
return FalsePositiveReason.UNREACHABLE_PATH
|
||||
|
||||
+247
-9
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
@@ -18,6 +19,7 @@ from .benchmark import (
|
||||
from .calibration import (
|
||||
FalsePositiveLearning,
|
||||
calibrate_rankings,
|
||||
fp_reachability_overrides,
|
||||
learnings_from_verifications,
|
||||
load_false_positive_learnings,
|
||||
merge_false_positive_learnings,
|
||||
@@ -26,15 +28,24 @@ from .calibration import (
|
||||
)
|
||||
from .config import load_config
|
||||
from .findings import StaticFinding, quick_scan_findings, write_findings
|
||||
from .fusion import FusionDecision, fuse_findings_dispatch
|
||||
from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
from .harness_ext import has_harnessx
|
||||
from .hunter import run_hunter_pool, write_hunter_trajectories
|
||||
from .hunter_harness import HxScanOrchestrator
|
||||
from .improve import RoundOutcome, run_improvement
|
||||
from .index import build_code_chunks
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
from .policy import assert_rules_resolve, load_policy
|
||||
from .preprocess import preprocess_repository, write_preprocess_artifact
|
||||
from .rank import rank_targets, write_rankings
|
||||
from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report
|
||||
from .ruleset import DEFAULT_RULESET_DIR, load_ruleset
|
||||
from .run import ScanRun, create_scan_run
|
||||
from .verification import VerificationResult, verify_findings, write_verification_results
|
||||
from .scoring import build_score_artifact
|
||||
from .verification import VerificationResult, write_verification_results
|
||||
from .verify_judge import verify_findings_dispatch
|
||||
|
||||
CALIBRATION_DIR = ".openultrasast/calibration"
|
||||
|
||||
@@ -70,6 +81,25 @@ def main(argv: list[str] | None = None) -> int:
|
||||
benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick")
|
||||
benchmark.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
||||
|
||||
improve = subparsers.add_parser(
|
||||
"improve",
|
||||
help="run the bounded self-improvement loop against a benchmark manifest and update the loop-owned ruleset ledger",
|
||||
)
|
||||
improve.add_argument("manifest", type=Path)
|
||||
improve.add_argument("--max-rounds", type=int, default=5)
|
||||
improve.add_argument("--recall-floor", type=float, default=RECALL_FLOOR)
|
||||
improve.add_argument("--fp-ceiling", type=float, default=FALSE_POSITIVE_CEILING)
|
||||
improve.add_argument(
|
||||
"--ledger", type=Path, default=None, help="ruleset ledger path (default: <target>/.openultrasast/calibration/rule_policy.json)"
|
||||
)
|
||||
improve.add_argument("--journal", type=Path, default=None, help="improvement journal path (default: alongside the ledger)")
|
||||
improve.add_argument(
|
||||
"--ruleset-dir", type=Path, default=DEFAULT_RULESET_DIR, help="ruleset directory to improve (default: the bundled ruleset)"
|
||||
)
|
||||
improve.add_argument("--dry-run", action="store_true", help="run rounds against a throwaway ledger; never touch the target's ledger")
|
||||
|
||||
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "scan":
|
||||
return _scan(args.path, args.config, args.mode, args.fail_on)
|
||||
@@ -77,6 +107,21 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return _index(args.path, args.config, args.chunk_lines)
|
||||
if args.command == "benchmark":
|
||||
return _benchmark(args.manifest, args.config, args.mode)
|
||||
if args.command == "improve":
|
||||
return _improve(
|
||||
args.manifest,
|
||||
max_rounds=args.max_rounds,
|
||||
recall_floor=args.recall_floor,
|
||||
fp_ceiling=args.fp_ceiling,
|
||||
ledger=args.ledger,
|
||||
journal=args.journal,
|
||||
ruleset_dir=args.ruleset_dir,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
if args.command == "mcp":
|
||||
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
|
||||
|
||||
return serve()
|
||||
return 2
|
||||
|
||||
|
||||
@@ -93,14 +138,26 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
raise SystemExit(f"scan path is not a directory: {path}")
|
||||
|
||||
config = load_config(config_path if config_path.exists() else None)
|
||||
# Capability gates for the optional HarnessX agentic plane. When the extra is
|
||||
# absent (default/CI), both stay False and the deterministic path runs unchanged.
|
||||
hunter_model = config.models.hunter
|
||||
verifier_model = config.models.verifier
|
||||
harnessx_present = has_harnessx()
|
||||
hx_hunter = mode == "standard" and bool(hunter_model) and harnessx_present
|
||||
hx_verify = mode == "standard" and bool(verifier_model) and harnessx_present
|
||||
run = create_scan_run(path, config)
|
||||
runtime = HarnessRuntime(
|
||||
scan_id=run.scan_id,
|
||||
config=config,
|
||||
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl"),
|
||||
trace_writer=HarnessTraceWriter(run.root / "trace" / "events.jsonl", redact=config.hardening.redact_secrets),
|
||||
)
|
||||
write_harness_config(config=config, processors=[], contract_mode="strict", path=run.root / "harness.json")
|
||||
runtime.start(mode=mode, target=run.target)
|
||||
policy = runtime.run_stage("policy_load", load_policy)
|
||||
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
||||
rule_ledger_path = run.target / CALIBRATION_DIR / "rule_policy.json"
|
||||
ruleset = runtime.run_stage("ruleset_load", lambda: load_ruleset(DEFAULT_RULESET_DIR, rule_ledger_path))
|
||||
runtime.run_stage("policy_check", lambda: assert_rules_resolve(ruleset, policy))
|
||||
static_hints = runtime.run_stage("static_mapping", lambda: _load_static_hints(config.static_analysis.sarif_paths))
|
||||
write_static_hints(static_hints, run.root / "mapping" / "static_hints.json")
|
||||
snapshot, targets = runtime.run_stage(
|
||||
@@ -112,19 +169,64 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
targets = attach_reachability_hints(targets, entry_points)
|
||||
write_preprocess_artifact(snapshot, targets, run.root / "preprocess" / "file_targets.json")
|
||||
rankings = runtime.run_stage("rank", lambda: rank_targets(targets))
|
||||
ledger_path = run.target / CALIBRATION_DIR / "false_positive_learnings.json"
|
||||
prior_learnings = load_false_positive_learnings(ledger_path)
|
||||
rankings, calibrations = runtime.run_stage("calibrate", lambda: calibrate_rankings(rankings, prior_learnings))
|
||||
applied_calibrations = [calibration for calibration in calibrations if calibration.applied_learning_ids]
|
||||
write_rankings(rankings, run.root / "rank" / "ranking.json")
|
||||
write_ranking_calibrations(calibrations, run.root / "calibration" / "applied_calibrations.json")
|
||||
if mode == "standard":
|
||||
hunter_result = runtime.run_stage("hunter_pool", lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id))
|
||||
if hx_hunter and hunter_model:
|
||||
orchestrator = HxScanOrchestrator(
|
||||
provider_model=hunter_model,
|
||||
provider=config.harnessx.provider,
|
||||
max_cost_usd=config.harnessx.max_cost_usd,
|
||||
token_threshold=config.harnessx.token_threshold,
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: orchestrator.run_pool(
|
||||
run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy, emit=runtime.emit
|
||||
),
|
||||
)
|
||||
else:
|
||||
if bool(hunter_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "hunter_pool", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "run_hunter_pool"}
|
||||
)
|
||||
hunter_result = runtime.run_stage(
|
||||
"hunter_pool",
|
||||
lambda: run_hunter_pool(run.target, targets, rankings, scan_id=run.scan_id, ruleset=ruleset, policy=policy),
|
||||
)
|
||||
findings = hunter_result.findings
|
||||
trajectories = hunter_result.trajectories
|
||||
else:
|
||||
findings = runtime.run_stage("quick_findings", lambda: quick_scan_findings(run.target, targets, rankings))
|
||||
findings = runtime.run_stage(
|
||||
"quick_findings",
|
||||
lambda: quick_scan_findings(
|
||||
run.target,
|
||||
targets,
|
||||
rankings,
|
||||
ruleset,
|
||||
policy,
|
||||
min_emit_priority=config.ruleset.min_emit_priority,
|
||||
min_emit_precision=config.ruleset.min_emit_precision,
|
||||
),
|
||||
)
|
||||
trajectories = []
|
||||
# Shadow-status rules fire but are excluded from the report and the score;
|
||||
# their outcomes are preserved separately for precision tracking.
|
||||
shadow_findings = [finding for finding in findings if finding.status == "shadow"]
|
||||
if shadow_findings:
|
||||
findings = [finding for finding in findings if finding.status != "shadow"]
|
||||
write_findings(shadow_findings, run.root / "shadow_findings.json")
|
||||
# Bounded-CI budget: cap the reported finding count (0 = unlimited). Findings are
|
||||
# severity/priority-sorted, so truncation keeps the most severe and is disclosed.
|
||||
max_findings = config.hardening.max_findings
|
||||
if max_findings and len(findings) > max_findings:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "budget", "reason": "max_findings_exceeded", "requested": max_findings, "actual": len(findings)}
|
||||
)
|
||||
findings = findings[:max_findings]
|
||||
findings_path = run.root / "findings.json"
|
||||
verification_path = run.root / "verification.json"
|
||||
markdown_path = run.root / "report.md"
|
||||
@@ -134,13 +236,72 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
write_findings(findings, findings_path)
|
||||
if trajectories:
|
||||
write_hunter_trajectories(trajectories, trajectories_path)
|
||||
verifications = runtime.run_stage("verify", lambda: verify_findings(findings))
|
||||
if mode == "standard" and bool(verifier_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "verify", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "structural_verifier"}
|
||||
)
|
||||
verifications = runtime.run_stage(
|
||||
"verify",
|
||||
lambda: verify_findings_dispatch(
|
||||
findings,
|
||||
verifier_model=verifier_model,
|
||||
verifier_provider=config.harnessx.provider,
|
||||
use_harnessx=hx_verify,
|
||||
),
|
||||
)
|
||||
write_verification_results(verifications, verification_path)
|
||||
runtime.run_stage(
|
||||
"record_calibration",
|
||||
lambda: _persist_calibration_feedback(run, ledger_path, prior_learnings, findings, verifications),
|
||||
)
|
||||
runtime.run_stage("report", lambda: write_markdown_report(findings, markdown_path, verifications))
|
||||
# Fusion: two-panel adjudication for triggered findings (standard mode). Runs
|
||||
# deterministically by default; routes panels through the configured provider when
|
||||
# a panel model is set and the extra is present, else falls back + records a degradation.
|
||||
fusion_decisions: list[FusionDecision] = []
|
||||
fusion_path = run.root / "fusion.json"
|
||||
if mode == "standard" and config.fusion.enabled:
|
||||
hx_fusion = bool(config.fusion.panel_model) and harnessx_present
|
||||
if bool(config.fusion.panel_model) and not harnessx_present:
|
||||
runtime.state["degradations"].append(
|
||||
{"stage": "fusion", "requested": "harnessx", "reason": "harnessx_extra_unavailable", "fallback": "deterministic_panels"}
|
||||
)
|
||||
fusion_decisions = runtime.run_stage(
|
||||
"fusion",
|
||||
lambda: fuse_findings_dispatch(
|
||||
findings,
|
||||
verifications,
|
||||
panel_model=config.fusion.panel_model,
|
||||
decider_model=config.fusion.decider_model,
|
||||
provider=config.harnessx.provider,
|
||||
use_harnessx=hx_fusion,
|
||||
high_assurance=config.fusion.high_assurance,
|
||||
),
|
||||
)
|
||||
if fusion_decisions:
|
||||
fusion_path.write_text(json.dumps([decision.to_dict() for decision in fusion_decisions], indent=2, sort_keys=True) + "\n")
|
||||
cwe_by_rule = {rule.rule_id: rule.cwe for rule in ruleset}
|
||||
rule_cwe_by_id = {finding.finding_id: cwe_by_rule.get(finding.finding_id.split(":", 1)[0], "") for finding in findings}
|
||||
# A confirmed false positive (prior-scan learning) lowers a finding's effective
|
||||
# reachability multiplier in the score instead of deleting the rule.
|
||||
reachability_override = fp_reachability_overrides(findings, prior_learnings)
|
||||
score_artifact = runtime.run_stage(
|
||||
"score",
|
||||
lambda: build_score_artifact(
|
||||
findings,
|
||||
rule_cwe_by_id,
|
||||
policy,
|
||||
k=config.score.k,
|
||||
min_score=config.score.min_score,
|
||||
block_severity_reachable=config.score.block_severity_reachable,
|
||||
blocking=config.score.blocking,
|
||||
reachability_override=reachability_override,
|
||||
),
|
||||
)
|
||||
score_path = run.root / "score.json"
|
||||
score_path.write_text(json.dumps(score_artifact.to_dict(), indent=2, sort_keys=True) + "\n")
|
||||
runtime.run_stage(
|
||||
"report", lambda: write_markdown_report(findings, markdown_path, verifications, redact=config.hardening.redact_secrets)
|
||||
)
|
||||
runtime.run_stage("sarif", lambda: write_sarif_report(findings, verifications, sarif_path))
|
||||
runtime.run_stage(
|
||||
"manifest",
|
||||
@@ -153,9 +314,14 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut
|
||||
verification=verification_path,
|
||||
markdown=markdown_path,
|
||||
sarif=sarif_path,
|
||||
score=score_path,
|
||||
trajectories=trajectories_path if trajectories else None,
|
||||
fusion=fusion_path if fusion_decisions else None,
|
||||
),
|
||||
path=manifest_path,
|
||||
score=score_artifact.to_dict(),
|
||||
degradations=runtime.state["degradations"] or None,
|
||||
fusion=[_fusion_summary(decision) for decision in fusion_decisions] or None,
|
||||
),
|
||||
)
|
||||
runtime.finish(status="succeeded")
|
||||
@@ -177,8 +343,7 @@ def _persist_calibration_feedback(
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
) -> list[FalsePositiveLearning]:
|
||||
# Turn this run's non-accepted verifier outcomes into scoped learnings and
|
||||
# merge them into the persistent ledger so the next scan demotes those scopes.
|
||||
# Store rejected or held findings as scoped learnings for the next scan.
|
||||
new_learnings = learnings_from_verifications(findings, verifications)
|
||||
merged = merge_false_positive_learnings(prior_learnings, new_learnings)
|
||||
write_false_positive_learnings(merged, ledger_path)
|
||||
@@ -208,6 +373,15 @@ def _artifact_paths(**paths: Path | None) -> dict[str, Path]:
|
||||
return {name: path for name, path in paths.items() if path is not None}
|
||||
|
||||
|
||||
def _fusion_summary(decision: FusionDecision) -> dict[str, object]:
|
||||
return {
|
||||
"finding_id": decision.finding_id,
|
||||
"disposition": str(decision.disposition),
|
||||
"decision_source": decision.decision_source,
|
||||
"triggers": decision.triggers,
|
||||
}
|
||||
|
||||
|
||||
def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int:
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
||||
@@ -238,6 +412,70 @@ def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _improve(
|
||||
manifest_path: Path,
|
||||
*,
|
||||
max_rounds: int,
|
||||
recall_floor: float,
|
||||
fp_ceiling: float,
|
||||
ledger: Path | None,
|
||||
journal: Path | None,
|
||||
ruleset_dir: Path,
|
||||
dry_run: bool,
|
||||
) -> int:
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
policy = load_policy()
|
||||
|
||||
# The loop writes its accepted ledger where `scan`/`benchmark` read it, so a
|
||||
# subsequent scan of this target automatically picks up the improved ruleset.
|
||||
default_ledger = target / CALIBRATION_DIR / "rule_policy.json"
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-improve-") as scratch:
|
||||
if dry_run:
|
||||
ledger_path = Path(scratch) / "rule_policy.json"
|
||||
journal_path = Path(scratch) / "improve_journal.json"
|
||||
else:
|
||||
ledger_path = ledger or default_ledger
|
||||
journal_path = journal or ledger_path.with_name("improve_journal.json")
|
||||
|
||||
outcomes = run_improvement(
|
||||
target,
|
||||
manifest,
|
||||
ledger_path=ledger_path,
|
||||
journal_path=journal_path,
|
||||
ruleset_dir=ruleset_dir,
|
||||
policy=policy,
|
||||
max_rounds=max_rounds,
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
)
|
||||
_print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None:
|
||||
print(f"manifest={manifest_path}")
|
||||
print(f"target={target}")
|
||||
accepted = [o for o in outcomes if o.accepted]
|
||||
for outcome in outcomes:
|
||||
edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-"
|
||||
print(
|
||||
f"round {outcome.round}: {outcome.reason} | "
|
||||
f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} "
|
||||
f"fp {outcome.fp_before:.2%}->{outcome.fp_after:.2%} "
|
||||
f"score {outcome.score_before}->{outcome.score_after} | edits: {edits}"
|
||||
)
|
||||
print(f"rounds={len(outcomes)} accepted={len(accepted)}")
|
||||
if dry_run:
|
||||
print("dry_run=true (no ledger written)")
|
||||
elif accepted:
|
||||
print(f"ledger={ledger_path}")
|
||||
else:
|
||||
print("ledger=unchanged (no round accepted)")
|
||||
|
||||
|
||||
def _load_scan_findings(run_dir: Path) -> list[StaticFinding]:
|
||||
return load_findings(run_dir / "findings.json")
|
||||
|
||||
|
||||
@@ -50,6 +50,52 @@ class StaticAnalysisConfig:
|
||||
sarif_paths: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoreConfig:
|
||||
k: float = 60.0
|
||||
min_score: int = 80
|
||||
block_severity_reachable: int = 5
|
||||
blocking: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RulesetConfig:
|
||||
# Emission floors that replace "every regex match becomes a finding". The
|
||||
# defaults (0.0) emit everything, keeping output byte-identical by default.
|
||||
min_emit_priority: float = 0.0
|
||||
min_emit_precision: float = 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessxConfig:
|
||||
# Optional agentic plane (the openultrasast[harnessx] extra). `provider`
|
||||
# selects the HarnessX LLM provider used by both the hunter pool and the
|
||||
# llm-judge verifier; the budget caps bound per-task cost and token spend.
|
||||
# Inert unless the extra is installed and a model is configured in [models].
|
||||
provider: str = "anthropic"
|
||||
max_cost_usd: float = 2.0
|
||||
token_threshold: int = 120_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HardeningConfig:
|
||||
# Release-readiness controls. redact_secrets masks credentials in traces/reports;
|
||||
# max_findings caps the reported finding count for bounded CI runs (0 = unlimited).
|
||||
redact_secrets: bool = True
|
||||
max_findings: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FusionConfig:
|
||||
# Two-panel fusion adjudication for triggered findings (standard mode). Runs
|
||||
# deterministically by default; setting panel_model routes the panels through
|
||||
# the configured HarnessX provider. high_assurance forces fusion on every finding.
|
||||
enabled: bool = True
|
||||
panel_model: str | None = None
|
||||
decider_model: str | None = None
|
||||
high_assurance: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedConfig:
|
||||
models: ModelConfig = ModelConfig()
|
||||
@@ -58,6 +104,11 @@ class ResolvedConfig:
|
||||
dynamic: DynamicConfig = DynamicConfig()
|
||||
evidence: EvidenceConfig = EvidenceConfig()
|
||||
static_analysis: StaticAnalysisConfig = StaticAnalysisConfig()
|
||||
score: ScoreConfig = ScoreConfig()
|
||||
ruleset: RulesetConfig = RulesetConfig()
|
||||
harnessx: HarnessxConfig = HarnessxConfig()
|
||||
fusion: FusionConfig = FusionConfig()
|
||||
hardening: HardeningConfig = HardeningConfig()
|
||||
runs_dir: str = ".openultrasast/runs"
|
||||
|
||||
|
||||
@@ -74,6 +125,11 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig:
|
||||
dynamic=_load_dynamic(data.get("dynamic", {})),
|
||||
evidence=_load_evidence(data.get("evidence", {})),
|
||||
static_analysis=_load_static_analysis(data.get("static_analysis", {})),
|
||||
score=_load_score(data.get("score", {})),
|
||||
ruleset=_load_ruleset_config(data.get("ruleset", {})),
|
||||
harnessx=_load_harnessx(data.get("harnessx", {})),
|
||||
fusion=_load_fusion(data.get("fusion", {})),
|
||||
hardening=_load_hardening(data.get("hardening", {})),
|
||||
runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"),
|
||||
)
|
||||
|
||||
@@ -148,3 +204,61 @@ def _load_static_analysis(value: object) -> StaticAnalysisConfig:
|
||||
if not isinstance(paths, list):
|
||||
paths = []
|
||||
return StaticAnalysisConfig(sarif_paths=tuple(str(item) for item in paths))
|
||||
|
||||
|
||||
def _load_score(value: object) -> ScoreConfig:
|
||||
data = _section(value)
|
||||
return ScoreConfig(
|
||||
k=_float_value(data.get("k"), 60.0),
|
||||
min_score=_int_value(data.get("min_score"), 80),
|
||||
block_severity_reachable=_int_value(data.get("block_severity_reachable"), 5),
|
||||
blocking=bool(data.get("blocking", False)),
|
||||
)
|
||||
|
||||
|
||||
def _load_ruleset_config(value: object) -> RulesetConfig:
|
||||
data = _section(value)
|
||||
return RulesetConfig(
|
||||
min_emit_priority=_float_value(data.get("min_emit_priority"), 0.0),
|
||||
min_emit_precision=_float_value(data.get("min_emit_precision"), 0.0),
|
||||
)
|
||||
|
||||
|
||||
def _load_harnessx(value: object) -> HarnessxConfig:
|
||||
data = _section(value)
|
||||
return HarnessxConfig(
|
||||
provider=_string(data.get("provider")) or "anthropic",
|
||||
max_cost_usd=_float_value(data.get("max_cost_usd"), 2.0),
|
||||
token_threshold=_int_value(data.get("token_threshold"), 120_000),
|
||||
)
|
||||
|
||||
|
||||
def _load_hardening(value: object) -> HardeningConfig:
|
||||
data = _section(value)
|
||||
return HardeningConfig(
|
||||
redact_secrets=bool(data.get("redact_secrets", True)),
|
||||
max_findings=_int_value(data.get("max_findings"), 0),
|
||||
)
|
||||
|
||||
|
||||
def _load_fusion(value: object) -> FusionConfig:
|
||||
data = _section(value)
|
||||
return FusionConfig(
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
panel_model=_string(data.get("panel_model")),
|
||||
decider_model=_string(data.get("decider_model")),
|
||||
high_assurance=bool(data.get("high_assurance", False)),
|
||||
)
|
||||
|
||||
|
||||
def _float_value(value: object, default: float) -> float:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
if isinstance(value, int | float):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return default
|
||||
|
||||
+32
-330
@@ -5,8 +5,10 @@ import re
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .policy import CwePolicy, load_policy, resolve_severity
|
||||
from .preprocess import FileTarget
|
||||
from .rank import RankingScore
|
||||
from .ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -25,348 +27,45 @@ class StaticFinding:
|
||||
reachability_conditions: list[str]
|
||||
tags: list[str]
|
||||
ranking_priority: float
|
||||
status: str = "enabled"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PatternRule:
|
||||
rule_id: str
|
||||
title: str
|
||||
severity: str
|
||||
tags: tuple[str, ...]
|
||||
pattern: re.Pattern[str]
|
||||
cwe: str
|
||||
languages: tuple[str, ...]
|
||||
SEVERITY_LABEL = {5: "critical", 4: "high", 3: "medium", 2: "low", 1: "low", 0: "info"}
|
||||
|
||||
# Default ruleset loaded from versioned TOML data (rules-as-data).
|
||||
PATTERN_RULES = load_ruleset(DEFAULT_RULESET_DIR)
|
||||
|
||||
|
||||
C_LANGUAGES = ("c", "cpp")
|
||||
PYTHON_LANGUAGES = ("python",)
|
||||
JS_LANGUAGES = ("javascript", "typescript")
|
||||
JAVA_LANGUAGES = ("java",)
|
||||
GROOVY_LANGUAGES = ("groovy",)
|
||||
|
||||
PATTERN_RULES = (
|
||||
# --- C / C++ -----------------------------------------------------------------
|
||||
PatternRule(
|
||||
rule_id="c-unsafe-copy",
|
||||
title="Unsafe C string copy needs review",
|
||||
severity="high",
|
||||
tags=("memory_unsafe",),
|
||||
pattern=re.compile(r"\b(gets|strcpy|strcat|sprintf)\s*\("),
|
||||
cwe="CWE-120",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="c-unsafe-memory-copy",
|
||||
title="Unsafe C memory copy needs review",
|
||||
severity="high",
|
||||
tags=("memory_unsafe",),
|
||||
pattern=re.compile(r"\b(memcpy|memmove|bcopy)\s*\("),
|
||||
cwe="CWE-120",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="c-unsafe-format",
|
||||
title="Unsafe C format string needs review",
|
||||
severity="high",
|
||||
tags=("memory_unsafe",),
|
||||
# Format function whose format argument is a bare identifier (not a string
|
||||
# literal), i.e. the classic format-string sink. printf("...") is ignored.
|
||||
pattern=re.compile(r"\b(printf|vprintf|vsprintf|vsnprintf|vfprintf|syslog)\s*\(\s*[A-Za-z_]"),
|
||||
cwe="CWE-134",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="c-unsafe-scanf",
|
||||
title="Unbounded C scanf needs review",
|
||||
severity="high",
|
||||
tags=("memory_unsafe",),
|
||||
pattern=re.compile(r"\b(scanf|fscanf|sscanf)\s*\([^\n]*%s"),
|
||||
cwe="CWE-120",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="c-shell-exec",
|
||||
title="C shell execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(system|popen)\s*\("),
|
||||
cwe="CWE-78",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="c-exec-family",
|
||||
title="C exec command execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(execl|execlp|execle|execv|execvp|execvpe)\s*\("),
|
||||
cwe="CWE-78",
|
||||
languages=C_LANGUAGES,
|
||||
),
|
||||
# --- Python ------------------------------------------------------------------
|
||||
PatternRule(
|
||||
rule_id="python-unsafe-eval",
|
||||
title="Dynamic Python execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(eval|exec)\s*\("),
|
||||
cwe="CWE-95",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-os-command",
|
||||
title="Python OS command execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(os\.system|os\.popen|os\.execl|os\.execlp|os\.execv|os\.execvp)\s*\("),
|
||||
cwe="CWE-78",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-shell-true",
|
||||
title="Subprocess shell execution needs review",
|
||||
severity="medium",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"subprocess\.[a-zA-Z_]+\s*\([^\n]*shell\s*=\s*True"),
|
||||
cwe="CWE-78",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-unsafe-deserialization",
|
||||
title="Unsafe Python deserialization needs review",
|
||||
severity="high",
|
||||
tags=("deserialization",),
|
||||
pattern=re.compile(r"\b(pickle\.loads|pickle\.load|cPickle\.loads|yaml\.load|marshal\.loads)\s*\("),
|
||||
cwe="CWE-502",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-sql-injection",
|
||||
title="Python SQL string composition needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
# .execute(...) where the argument is built with %-formatting, .format(),
|
||||
# string concatenation, or an f-string. Parameterised %s placeholders pass.
|
||||
pattern=re.compile(r"(\.execute)\s*\(\s*[^\n]*(?:[\"']\s*%\s*[(A-Za-z_]|\.format\s*\(|[\"']\s*\+|\bf[\"'])"),
|
||||
cwe="CWE-89",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-path-traversal",
|
||||
title="Python file open with external input needs review",
|
||||
severity="medium",
|
||||
tags=("filesystem_entry",),
|
||||
pattern=re.compile(r"\b(open)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\(|params\[|GET\[|POST\[)"),
|
||||
cwe="CWE-22",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-ssrf",
|
||||
title="Python outbound request with external input needs review",
|
||||
severity="medium",
|
||||
tags=("network_entry",),
|
||||
pattern=re.compile(
|
||||
r"\b(requests\.(?:get|post|put|delete|head|patch)|urllib\.request\.urlopen|urlopen)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\()"
|
||||
),
|
||||
cwe="CWE-918",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-weak-hash",
|
||||
title="Python weak hash needs review",
|
||||
severity="low",
|
||||
tags=("crypto",),
|
||||
pattern=re.compile(r"\bhashlib\.(md5|sha1)\s*\("),
|
||||
cwe="CWE-327",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-ssti",
|
||||
title="Python server-side template injection needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
pattern=re.compile(r"\b(render_template_string)\s*\("),
|
||||
cwe="CWE-94",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="python-flask-debug",
|
||||
title="Flask debug mode enabled needs review",
|
||||
severity="low",
|
||||
tags=("insecure_default",),
|
||||
pattern=re.compile(r"\.run\s*\([^\n]*(debug)\s*=\s*True"),
|
||||
cwe="CWE-489",
|
||||
languages=PYTHON_LANGUAGES,
|
||||
),
|
||||
# --- JavaScript / TypeScript -------------------------------------------------
|
||||
PatternRule(
|
||||
rule_id="js-command-exec",
|
||||
title="Node child_process command execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
# exec/execSync invoke a shell; execFile/spawn with an argv array do not and
|
||||
# are the safe alternatives, so they are intentionally excluded here.
|
||||
pattern=re.compile(r"\b(exec|execSync)\s*\("),
|
||||
cwe="CWE-78",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-spawn-shell",
|
||||
title="Node spawn with shell needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(spawn|spawnSync)\s*\([^\n]*shell\s*:\s*true"),
|
||||
cwe="CWE-78",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-eval",
|
||||
title="Dynamic JavaScript evaluation needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(eval)\s*\("),
|
||||
cwe="CWE-95",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-function-constructor",
|
||||
title="JavaScript Function constructor needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"new\s+(Function)\s*\("),
|
||||
cwe="CWE-95",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-xss-response",
|
||||
title="Unsanitized HTTP response reflection needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
pattern=re.compile(r"\.(send|write|end)\s*\(\s*[^\n]*\b(?:req|request)\.(?:query|params|body)"),
|
||||
cwe="CWE-79",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-dom-xss",
|
||||
title="DOM sink with untrusted input needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
pattern=re.compile(r"\.(innerHTML|outerHTML)\s*=\s*[^\n]*\b(?:req|request|location|document\.location)"),
|
||||
cwe="CWE-79",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-sql-injection",
|
||||
title="JavaScript SQL string composition needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
pattern=re.compile(r"\.(query|execute)\s*\(\s*[^\n]*(?:\$\{|\+\s*[A-Za-z_$])"),
|
||||
cwe="CWE-89",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-path-traversal",
|
||||
title="Node filesystem access with external input needs review",
|
||||
severity="medium",
|
||||
tags=("filesystem_entry",),
|
||||
pattern=re.compile(
|
||||
r"\bfs\.(readFile|readFileSync|writeFile|writeFileSync|createReadStream|unlink|unlinkSync)\s*\(\s*[^\n]*\b(?:req|request)\."
|
||||
),
|
||||
cwe="CWE-22",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-ssrf",
|
||||
title="JavaScript outbound request with external input needs review",
|
||||
severity="medium",
|
||||
tags=("network_entry",),
|
||||
pattern=re.compile(r"\b(axios|fetch|got)\s*[.(][^\n]*\b(?:req|request)\."),
|
||||
cwe="CWE-918",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-weak-hash",
|
||||
title="JavaScript weak hash needs review",
|
||||
severity="low",
|
||||
tags=("crypto",),
|
||||
pattern=re.compile(r"createHash\s*\(\s*[\"'](md5|sha1)[\"']"),
|
||||
cwe="CWE-327",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="js-deserialize",
|
||||
title="JavaScript unsafe deserialization needs review",
|
||||
severity="high",
|
||||
tags=("deserialization",),
|
||||
pattern=re.compile(r"\b(unserialize)\s*\("),
|
||||
cwe="CWE-502",
|
||||
languages=JS_LANGUAGES,
|
||||
),
|
||||
# --- Java --------------------------------------------------------------------
|
||||
PatternRule(
|
||||
rule_id="java-command-exec",
|
||||
title="Java runtime command execution needs review",
|
||||
severity="high",
|
||||
tags=("syscall_entry",),
|
||||
pattern=re.compile(r"\b(exec)\s*\("),
|
||||
cwe="CWE-78",
|
||||
languages=JAVA_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="java-sql-injection",
|
||||
title="Java SQL string composition needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
# A string literal that begins with a SQL keyword and is concatenated with
|
||||
# untrusted data. Parameterised queries (placeholders, no '+') are ignored.
|
||||
pattern=re.compile(r"(?i)\"\s*(select|insert|update|delete)\b[^\"]*\"\s*\+"),
|
||||
cwe="CWE-89",
|
||||
languages=JAVA_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="java-weak-hash",
|
||||
title="Java weak hash needs review",
|
||||
severity="low",
|
||||
tags=("crypto",),
|
||||
pattern=re.compile(r"MessageDigest\.getInstance\s*\(\s*\"(MD5|SHA-1)\""),
|
||||
cwe="CWE-327",
|
||||
languages=JAVA_LANGUAGES,
|
||||
),
|
||||
PatternRule(
|
||||
rule_id="java-deserialization",
|
||||
title="Java unsafe deserialization needs review",
|
||||
severity="high",
|
||||
tags=("deserialization",),
|
||||
pattern=re.compile(r"\.(readObject)\s*\("),
|
||||
cwe="CWE-502",
|
||||
languages=JAVA_LANGUAGES,
|
||||
),
|
||||
# --- Groovy server-side templates -------------------------------------------
|
||||
PatternRule(
|
||||
rule_id="groovy-xss-unescaped",
|
||||
title="Groovy template unescaped output needs review",
|
||||
severity="high",
|
||||
tags=("injection",),
|
||||
# Unescaped output of a bound variable; string literals are not flagged.
|
||||
pattern=re.compile(r"(yieldUnescaped)\s+[A-Za-z_]"),
|
||||
cwe="CWE-79",
|
||||
languages=GROOVY_LANGUAGES,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def quick_scan_findings(root: Path, targets: list[FileTarget], rankings: list[RankingScore]) -> list[StaticFinding]:
|
||||
def quick_scan_findings(
|
||||
root: Path,
|
||||
targets: list[FileTarget],
|
||||
rankings: list[RankingScore],
|
||||
ruleset: tuple[PatternRule, ...] | None = None,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
*,
|
||||
min_emit_priority: float = 0.0,
|
||||
min_emit_precision: float = 0.0,
|
||||
) -> list[StaticFinding]:
|
||||
ruleset = PATTERN_RULES if ruleset is None else ruleset
|
||||
policy = load_policy() if policy is None else policy
|
||||
ranking_by_path = {ranking.path: ranking for ranking in rankings}
|
||||
# Compile once; disabled rules and rules below the precision floor never fire.
|
||||
compiled = [
|
||||
(rule, re.compile(rule.pattern)) for rule in ruleset if rule.status != "disabled" and rule.precision_estimate >= min_emit_precision
|
||||
]
|
||||
findings: list[StaticFinding] = []
|
||||
for target in targets:
|
||||
text = _read_text(root / target.path)
|
||||
for rule in PATTERN_RULES:
|
||||
for rule, pattern in compiled:
|
||||
if rule.languages and target.language not in rule.languages:
|
||||
continue
|
||||
ranking = ranking_by_path.get(target.path)
|
||||
for match in rule.pattern.finditer(text):
|
||||
for match in pattern.finditer(text):
|
||||
if _match_is_in_comment(text, match.start(), target.language):
|
||||
continue
|
||||
findings.append(_finding_from_match(target, rule, text, match, ranking))
|
||||
finding = _finding_from_match(target, rule, text, match, ranking, policy)
|
||||
if finding.ranking_priority >= min_emit_priority:
|
||||
findings.append(finding)
|
||||
return sorted(findings, key=lambda item: (_severity_sort(item.severity), -item.ranking_priority, item.path))
|
||||
|
||||
|
||||
@@ -393,17 +92,19 @@ def _finding_from_match(
|
||||
text: str,
|
||||
match: re.Match[str],
|
||||
ranking: RankingScore | None,
|
||||
policy: dict[str, CwePolicy],
|
||||
) -> StaticFinding:
|
||||
offset = match.start()
|
||||
line = text.count("\n", 0, offset) + 1
|
||||
reachability_status, function_name, reachability_evidence = _reachability_for_line(target, line)
|
||||
priority = ranking.priority if ranking is not None else 1.0
|
||||
matched_token = match.group(1) if match.groups() else match.group(0).split("(", 1)[0].strip()
|
||||
severity = SEVERITY_LABEL.get(resolve_severity(policy, rule.cwe), "info")
|
||||
return StaticFinding(
|
||||
finding_id=f"{rule.rule_id}:{target.path}:{line}",
|
||||
path=target.path,
|
||||
title=rule.title,
|
||||
severity=rule.severity,
|
||||
severity=severity,
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale=(
|
||||
@@ -416,6 +117,7 @@ def _finding_from_match(
|
||||
reachability_conditions=_reachability_conditions(reachability_evidence),
|
||||
tags=sorted(set(target.tags) | set(rule.tags)),
|
||||
ranking_priority=_finding_priority(priority, reachability_status),
|
||||
status=rule.status,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Fusion adjudication — OpenUltraCode-style two-panel deepening (Phase 13).
|
||||
|
||||
Fusion is the deepening mechanism for findings that need more reasoning than the
|
||||
ranker → hunter → verifier → mapping loop provides: critical/high severity, verifier
|
||||
disagreement, static-vs-semantic evidence conflict, findings that gate a risky fix or
|
||||
disclosure, or an explicit high-assurance request. Two panels independently review the
|
||||
bounded evidence (one steel-manning the vulnerability case, one the false-positive
|
||||
case), critique and revise, vote, and a decider issues the final disposition.
|
||||
|
||||
The deterministic engine here is the zero-dependency core (and the fallback). When the
|
||||
``openultrasast[harnessx]`` extra and panel models are configured, ``fuse_findings_dispatch``
|
||||
routes the panels through real LLM reviews; the decider reconciliation stays deterministic
|
||||
and auditable. Every fused finding receives exactly one of the five dispositions, and the
|
||||
decision discloses model IDs, panel roles, votes, degradations, and the decision source.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from enum import StrEnum
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import has_harnessx
|
||||
from .verification import VerificationResult, VerificationStatus
|
||||
|
||||
|
||||
class FusionDisposition(StrEnum):
|
||||
ACCEPTED = "accepted"
|
||||
REJECTED = "rejected"
|
||||
MITIGATED = "mitigated"
|
||||
DEFERRED = "deferred"
|
||||
BLOCKED = "blocked"
|
||||
|
||||
|
||||
_HIGH_SEVERITY = frozenset({"critical", "high"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PanelVerdict:
|
||||
role: str # "panel_a" | "panel_b"
|
||||
leaning: str # "vulnerability" | "false_positive"
|
||||
disposition: FusionDisposition
|
||||
confidence: float
|
||||
vulnerability_case: str
|
||||
false_positive_case: str
|
||||
rationale: str
|
||||
model_id: str | None = None # None => deterministic panel
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FusionDecision:
|
||||
finding_id: str
|
||||
triggered: bool
|
||||
triggers: list[str]
|
||||
disposition: FusionDisposition
|
||||
decision_source: str # "deterministic-reconciler" | "llm-panels" | "no-fusion"
|
||||
panels: list[PanelVerdict]
|
||||
votes: dict[str, int]
|
||||
model_ids: list[str]
|
||||
degradations: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
data = asdict(self)
|
||||
data["disposition"] = str(self.disposition)
|
||||
data["panels"] = [panel.to_dict() for panel in self.panels]
|
||||
return data
|
||||
|
||||
|
||||
def fusion_triggers(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None = None,
|
||||
*,
|
||||
high_assurance: bool = False,
|
||||
) -> list[str]:
|
||||
"""The deterministic trigger policy — which conditions warrant deepening (empty => skip)."""
|
||||
triggers: set[str] = set()
|
||||
if finding.severity in _HIGH_SEVERITY:
|
||||
triggers.add(f"severity:{finding.severity}")
|
||||
reachable = finding.reachability_status == "reachable"
|
||||
if verification is not None:
|
||||
if verification.status == VerificationStatus.NEEDS_EVIDENCE:
|
||||
triggers.add("verifier_disagreement")
|
||||
if verification.status == VerificationStatus.REJECTED and reachable:
|
||||
triggers.add("evidence_conflict") # static says vuln-shaped, verifier rejected
|
||||
if reachable and finding.severity in _HIGH_SEVERITY:
|
||||
triggers.add("gates_risky_fix")
|
||||
if high_assurance:
|
||||
triggers.add("high_assurance_request")
|
||||
return sorted(triggers)
|
||||
|
||||
|
||||
def should_fuse(finding: StaticFinding, verification: VerificationResult | None = None, *, high_assurance: bool = False) -> bool:
|
||||
return bool(fusion_triggers(finding, verification, high_assurance=high_assurance))
|
||||
|
||||
|
||||
def _panel_a(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
|
||||
"""Vulnerability-leaning panel: steel-mans the vulnerability case."""
|
||||
reachable = finding.reachability_status == "reachable"
|
||||
verified = verification is not None and verification.verified
|
||||
if reachable and finding.severity in _HIGH_SEVERITY and not verified:
|
||||
disposition, confidence = FusionDisposition.BLOCKED, 0.7
|
||||
elif reachable:
|
||||
disposition, confidence = FusionDisposition.ACCEPTED, 0.75 if verified else 0.6
|
||||
else:
|
||||
disposition, confidence = FusionDisposition.DEFERRED, 0.5
|
||||
vuln_case = verification.pro_case if verification else f"{finding.title}: {finding.rationale}"
|
||||
return PanelVerdict(
|
||||
role="panel_a",
|
||||
leaning="vulnerability",
|
||||
disposition=disposition,
|
||||
confidence=confidence,
|
||||
vulnerability_case=vuln_case,
|
||||
false_positive_case="Reachability or sanitizer evidence could still neutralize the sink.",
|
||||
rationale=f"reachability={finding.reachability_status}, severity={finding.severity}, verified={verified}",
|
||||
)
|
||||
|
||||
|
||||
def _panel_b(finding: StaticFinding, verification: VerificationResult | None) -> PanelVerdict:
|
||||
"""False-positive-leaning panel: steel-mans the false-positive case."""
|
||||
reachable = finding.reachability_status == "reachable"
|
||||
verified = verification is not None and verification.verified
|
||||
rejected = verification is not None and verification.status == VerificationStatus.REJECTED
|
||||
if rejected or not reachable:
|
||||
disposition, confidence = FusionDisposition.REJECTED, 0.7 if rejected else 0.55
|
||||
elif verified:
|
||||
disposition, confidence = FusionDisposition.ACCEPTED, 0.6 # concedes when evidence is strong
|
||||
else:
|
||||
disposition, confidence = FusionDisposition.DEFERRED, 0.5
|
||||
fp_case = verification.counter_case if verification else "No independent verification confirms attacker control or impact."
|
||||
return PanelVerdict(
|
||||
role="panel_b",
|
||||
leaning="false_positive",
|
||||
disposition=disposition,
|
||||
confidence=confidence,
|
||||
vulnerability_case="A tainted sink shape exists in the source.",
|
||||
false_positive_case=fp_case,
|
||||
rationale=f"reachable={reachable}, rejected={rejected}, verified={verified}",
|
||||
)
|
||||
|
||||
|
||||
def _reconcile(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
panels: list[PanelVerdict],
|
||||
mitigated_ids: frozenset[str],
|
||||
) -> tuple[FusionDisposition, str]:
|
||||
"""Deterministic decider: map evidence + panels onto one disposition + a rationale."""
|
||||
reachable = finding.reachability_status == "reachable"
|
||||
verified = verification is not None and verification.verified
|
||||
rejected = verification is not None and verification.status == VerificationStatus.REJECTED
|
||||
if finding.finding_id in mitigated_ids:
|
||||
return FusionDisposition.MITIGATED, "a prior calibration confirms this surface is mitigated"
|
||||
if rejected and not reachable:
|
||||
return FusionDisposition.REJECTED, "verifier rejected and the sink is not reachable"
|
||||
if verified and reachable:
|
||||
return FusionDisposition.ACCEPTED, "reachable sink with corroborating verification evidence"
|
||||
if reachable and finding.severity in _HIGH_SEVERITY and not verified:
|
||||
return FusionDisposition.BLOCKED, "reachable high-severity finding gates a risky fix until evidence resolves"
|
||||
if rejected and reachable:
|
||||
return FusionDisposition.DEFERRED, "static and verifier evidence conflict; deeper evidence required"
|
||||
return FusionDisposition.DEFERRED, "evidence does not converge; defer for additional evidence"
|
||||
|
||||
|
||||
def fuse_finding(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
triggers: list[str],
|
||||
*,
|
||||
mitigated_ids: frozenset[str] = frozenset(),
|
||||
) -> FusionDecision:
|
||||
"""Deterministic two-panel fusion for one finding."""
|
||||
panels = [_panel_a(finding, verification), _panel_b(finding, verification)]
|
||||
disposition, _rationale = _reconcile(finding, verification, panels, mitigated_ids)
|
||||
votes = Counter(str(panel.disposition) for panel in panels)
|
||||
return FusionDecision(
|
||||
finding_id=finding.finding_id,
|
||||
triggered=True,
|
||||
triggers=triggers,
|
||||
disposition=disposition,
|
||||
decision_source="deterministic-reconciler",
|
||||
panels=panels,
|
||||
votes=dict(sorted(votes.items())),
|
||||
model_ids=[],
|
||||
degradations=[],
|
||||
warnings=[],
|
||||
)
|
||||
|
||||
|
||||
def fuse_findings_dispatch(
|
||||
findings: list[StaticFinding],
|
||||
verifications: list[VerificationResult],
|
||||
*,
|
||||
panel_model: str | None = None,
|
||||
decider_model: str | None = None,
|
||||
provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
high_assurance: bool = False,
|
||||
mitigated_ids: frozenset[str] = frozenset(),
|
||||
) -> list[FusionDecision]:
|
||||
"""Fuse every finding that the trigger policy selects.
|
||||
|
||||
Deterministic by default. When ``use_harnessx`` is set, a panel model is configured,
|
||||
and the extra is present, the panels are produced by real LLM reviews; otherwise the
|
||||
deterministic panels run and (if an LLM run was requested) a degradation is recorded.
|
||||
"""
|
||||
verification_by_id = {result.finding_id: result for result in verifications}
|
||||
want_llm = use_harnessx and bool(panel_model)
|
||||
decisions: list[FusionDecision] = []
|
||||
for finding in findings:
|
||||
verification = verification_by_id.get(finding.finding_id)
|
||||
triggers = fusion_triggers(finding, verification, high_assurance=high_assurance)
|
||||
if not triggers:
|
||||
continue
|
||||
if want_llm and has_harnessx():
|
||||
decisions.append(_fuse_with_llm_panels(finding, verification, triggers, panel_model, decider_model, provider, mitigated_ids))
|
||||
else:
|
||||
decision = fuse_finding(finding, verification, triggers, mitigated_ids=mitigated_ids)
|
||||
if want_llm and not has_harnessx():
|
||||
decision = _with_degradation(decision, "fusion_llm_unavailable: harnessx extra absent; used deterministic panels")
|
||||
decisions.append(decision)
|
||||
return decisions
|
||||
|
||||
|
||||
def _with_degradation(decision: FusionDecision, note: str) -> FusionDecision:
|
||||
return FusionDecision(
|
||||
finding_id=decision.finding_id,
|
||||
triggered=decision.triggered,
|
||||
triggers=decision.triggers,
|
||||
disposition=decision.disposition,
|
||||
decision_source=decision.decision_source,
|
||||
panels=decision.panels,
|
||||
votes=decision.votes,
|
||||
model_ids=decision.model_ids,
|
||||
degradations=[*decision.degradations, note],
|
||||
warnings=decision.warnings,
|
||||
)
|
||||
|
||||
|
||||
def _fuse_with_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
triggers: list[str],
|
||||
panel_model: str | None,
|
||||
decider_model: str | None,
|
||||
provider: str,
|
||||
mitigated_ids: frozenset[str],
|
||||
) -> FusionDecision:
|
||||
"""LLM-backed panels with a deterministic, auditable decider (lazy/guarded)."""
|
||||
import asyncio
|
||||
|
||||
model = panel_model or ""
|
||||
try:
|
||||
panels = asyncio.run(_run_llm_panels(finding, verification, model, provider))
|
||||
source = "llm-panels"
|
||||
warnings: list[str] = []
|
||||
except Exception as exc: # noqa: BLE001 — panel/provider failure falls back, never crashes the scan
|
||||
panels = [_panel_a(finding, verification), _panel_b(finding, verification)]
|
||||
source = "deterministic-reconciler"
|
||||
warnings = [f"llm_panels_failed: {type(exc).__name__}"]
|
||||
disposition, _rationale = _reconcile(finding, verification, panels, mitigated_ids)
|
||||
votes = Counter(str(panel.disposition) for panel in panels)
|
||||
model_ids = sorted({panel.model_id for panel in panels if panel.model_id})
|
||||
if decider_model:
|
||||
model_ids = sorted({*model_ids, decider_model})
|
||||
return FusionDecision(
|
||||
finding_id=finding.finding_id,
|
||||
triggered=True,
|
||||
triggers=triggers,
|
||||
disposition=disposition,
|
||||
decision_source=source,
|
||||
panels=panels,
|
||||
votes=dict(sorted(votes.items())),
|
||||
model_ids=model_ids,
|
||||
degradations=[],
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _run_llm_panels(
|
||||
finding: StaticFinding,
|
||||
verification: VerificationResult | None,
|
||||
model: str,
|
||||
provider: str,
|
||||
) -> list[PanelVerdict]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
|
||||
from .harness_ext import build_provider
|
||||
|
||||
client = build_provider(model, provider)
|
||||
panels: list[PanelVerdict] = []
|
||||
for role, leaning, stance in (
|
||||
("panel_a", "vulnerability", "Argue the vulnerability case as strongly as the evidence allows."),
|
||||
("panel_b", "false_positive", "Argue the false-positive case as strongly as the evidence allows."),
|
||||
):
|
||||
summary = f"reachability={finding.reachability_status}; severity={finding.severity}; tags={', '.join(finding.tags) or 'none'}"
|
||||
prompt = build_judge_prompt(
|
||||
task_description=f"{stance} Finding: {finding.title} at {finding.path}:{finding.line}.",
|
||||
trajectory_summary=summary,
|
||||
extracted_answer=verification.pro_case if verification else finding.rationale,
|
||||
)
|
||||
response = await client.complete(messages=[Message(role="user", content=prompt)], tools=[])
|
||||
verdict = parse_judge_response(_text(response.content))
|
||||
base = _panel_a(finding, verification) if role == "panel_a" else _panel_b(finding, verification)
|
||||
plausible = str(verdict.get("verdict", "")) == "plausible"
|
||||
confidence = (
|
||||
float(verdict.get("confidence", base.confidence)) if isinstance(verdict.get("confidence"), int | float) else base.confidence
|
||||
)
|
||||
panels.append(
|
||||
PanelVerdict(
|
||||
role=role,
|
||||
leaning=leaning,
|
||||
disposition=base.disposition if plausible or role == "panel_a" else FusionDisposition.REJECTED,
|
||||
confidence=confidence,
|
||||
vulnerability_case=base.vulnerability_case,
|
||||
false_positive_case=base.false_positive_case,
|
||||
rationale=str(verdict.get("cause", base.rationale)),
|
||||
model_id=model,
|
||||
)
|
||||
)
|
||||
return panels
|
||||
|
||||
|
||||
def _text(content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(str(block.get("text", "")) if isinstance(block, dict) else str(block) for block in content)
|
||||
return str(content)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FusionDecision",
|
||||
"FusionDisposition",
|
||||
"PanelVerdict",
|
||||
"fuse_finding",
|
||||
"fuse_findings_dispatch",
|
||||
"fusion_triggers",
|
||||
"should_fuse",
|
||||
]
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Detection-quality gate — the hard feasibility constraint (Phase 4 task 8).
|
||||
|
||||
Aggregate recall and false-positive rate over the benchmark corpus must clear the
|
||||
project goal: recall >= 90% AND false-positive rate < 10%. This gate is computed
|
||||
**independently of the project score** so a score improvement can never unblock a
|
||||
gate-breaching change — the recall/FP constraint is never folded into the score
|
||||
reward. ``merge_gate`` composes this hard constraint with the (advisory-first)
|
||||
score gate by AND, so the score can only *add* a block, never remove one.
|
||||
|
||||
Run as a module to evaluate the corpus and exit non-zero on a breach::
|
||||
|
||||
python -m openultrasast.gate
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .benchmark import BenchmarkRun, evaluate_benchmark, load_benchmark_manifest, resolve_benchmark_source
|
||||
from .findings import quick_scan_findings
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints
|
||||
from .preprocess import preprocess_repository
|
||||
from .rank import rank_targets
|
||||
|
||||
# The in-scope corpus: language -> benchmark manifest names (smoke + real repo).
|
||||
# Single source of truth shared by the CI gate and the detection-benchmark test.
|
||||
LANGUAGE_MANIFESTS: dict[str, list[str]] = {
|
||||
"c_cpp": ["c-cpp-smoke", "cpp-damn-vulnerable"],
|
||||
"python": ["python-web-smoke", "python-vulnerable"],
|
||||
"javascript": ["javascript-node-web-smoke", "javascript-vulnerable"],
|
||||
"java": ["java-web-smoke", "java-spring-boot-vulnerable"],
|
||||
}
|
||||
|
||||
RECALL_FLOOR = 0.90
|
||||
FALSE_POSITIVE_CEILING = 0.10
|
||||
DEFAULT_MANIFEST_DIR = Path("benchmarks/manifests")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DetectionMetrics:
|
||||
"""Aggregate detection counts and the two rates derived from them."""
|
||||
|
||||
expected: int
|
||||
matched: int
|
||||
actual: int
|
||||
false_positives: int
|
||||
|
||||
@property
|
||||
def recall(self) -> float:
|
||||
return self.matched / self.expected if self.expected else 1.0
|
||||
|
||||
@property
|
||||
def false_positive_rate(self) -> float:
|
||||
return self.false_positives / self.actual if self.actual else 0.0
|
||||
|
||||
def __add__(self, other: DetectionMetrics) -> DetectionMetrics:
|
||||
return DetectionMetrics(
|
||||
expected=self.expected + other.expected,
|
||||
matched=self.matched + other.matched,
|
||||
actual=self.actual + other.actual,
|
||||
false_positives=self.false_positives + other.false_positives,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DetectionGateVerdict:
|
||||
"""Score-independent verdict: pass only if recall and FP both clear the goal."""
|
||||
|
||||
passed: bool
|
||||
overall: DetectionMetrics
|
||||
per_language: dict[str, DetectionMetrics]
|
||||
recall_floor: float
|
||||
fp_ceiling: float
|
||||
reasons: list[str]
|
||||
|
||||
|
||||
def _quick_scan(target: Path): # type: ignore[no-untyped-def]
|
||||
_, targets = preprocess_repository(target)
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
|
||||
rankings = rank_targets(targets)
|
||||
return quick_scan_findings(target, targets, rankings)
|
||||
|
||||
|
||||
def language_metrics(manifest_names: list[str], manifest_dir: Path = DEFAULT_MANIFEST_DIR) -> DetectionMetrics:
|
||||
"""Aggregate detection metrics across one language's manifests."""
|
||||
total = DetectionMetrics(0, 0, 0, 0)
|
||||
for name in manifest_names:
|
||||
manifest_path = manifest_dir / f"{name}.toml"
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
findings = _quick_scan(target)
|
||||
run = BenchmarkRun(benchmark_run_id="gate", root=Path("/tmp/gate"), manifest=manifest)
|
||||
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
m = result.metrics
|
||||
total = total + DetectionMetrics(
|
||||
expected=m.expected_findings_total,
|
||||
matched=m.matched_findings_total,
|
||||
actual=m.actual_findings_total,
|
||||
false_positives=m.false_positive_findings_total,
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
def evaluate_corpus(
|
||||
corpus: Mapping[str, list[str]] = LANGUAGE_MANIFESTS, manifest_dir: Path = DEFAULT_MANIFEST_DIR
|
||||
) -> dict[str, DetectionMetrics]:
|
||||
"""Compute per-language detection metrics across the whole corpus."""
|
||||
return {language: language_metrics(names, manifest_dir) for language, names in sorted(corpus.items())}
|
||||
|
||||
|
||||
def detection_gate(
|
||||
per_language: Mapping[str, DetectionMetrics],
|
||||
*,
|
||||
recall_floor: float = RECALL_FLOOR,
|
||||
fp_ceiling: float = FALSE_POSITIVE_CEILING,
|
||||
) -> DetectionGateVerdict:
|
||||
"""Evaluate the hard recall/FP gate. No score input — it cannot be unblocked by score."""
|
||||
overall = DetectionMetrics(0, 0, 0, 0)
|
||||
for metrics in per_language.values():
|
||||
overall = overall + metrics
|
||||
reasons: list[str] = []
|
||||
for language, metrics in sorted(per_language.items()):
|
||||
if metrics.recall < recall_floor:
|
||||
reasons.append(f"{language} recall {metrics.recall:.2%} below {recall_floor:.0%} ({metrics.matched}/{metrics.expected})")
|
||||
if metrics.false_positive_rate >= fp_ceiling:
|
||||
reasons.append(
|
||||
f"{language} FP rate {metrics.false_positive_rate:.2%} at or above {fp_ceiling:.0%} "
|
||||
f"({metrics.false_positives}/{metrics.actual})"
|
||||
)
|
||||
if overall.recall < recall_floor:
|
||||
reasons.append(f"overall recall {overall.recall:.2%} below {recall_floor:.0%} ({overall.matched}/{overall.expected})")
|
||||
if overall.false_positive_rate >= fp_ceiling:
|
||||
reasons.append(
|
||||
f"overall FP rate {overall.false_positive_rate:.2%} at or above {fp_ceiling:.0%} ({overall.false_positives}/{overall.actual})"
|
||||
)
|
||||
return DetectionGateVerdict(
|
||||
passed=not reasons,
|
||||
overall=overall,
|
||||
per_language=dict(sorted(per_language.items())),
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
reasons=reasons,
|
||||
)
|
||||
|
||||
|
||||
def merge_gate(detection: DetectionGateVerdict, score_gate: Mapping[str, object] | None = None) -> dict[str, object]:
|
||||
"""Compose the hard detection gate with the (advisory-first) score gate by AND.
|
||||
|
||||
The merge blocks if detection fails OR a *blocking* score gate fails. A passing
|
||||
score gate (however high the score) can never flip a failed detection gate to
|
||||
pass — the recall/FP constraint is hard and never folded into the score reward.
|
||||
"""
|
||||
score_passed = True if score_gate is None else bool(score_gate.get("passed", True))
|
||||
return {
|
||||
"passed": detection.passed and score_passed,
|
||||
"detection_passed": detection.passed,
|
||||
"score_passed": score_passed,
|
||||
"reasons": list(detection.reasons),
|
||||
}
|
||||
|
||||
|
||||
def within_tolerance(
|
||||
baseline: DetectionMetrics,
|
||||
candidate: DetectionMetrics,
|
||||
*,
|
||||
recall_tolerance: float = 0.05,
|
||||
fp_tolerance: float = 0.05,
|
||||
) -> bool:
|
||||
"""True if ``candidate`` recall/FP stay within tolerance of ``baseline``.
|
||||
|
||||
Used to confirm a HarnessX-backed stage does not drift detection beyond the
|
||||
configured band of the zero-dependency baseline. Recall may not fall more than
|
||||
``recall_tolerance`` below baseline; FP may not rise more than ``fp_tolerance``
|
||||
above it.
|
||||
"""
|
||||
recall_ok = candidate.recall >= baseline.recall - recall_tolerance
|
||||
fp_ok = candidate.false_positive_rate <= baseline.false_positive_rate + fp_tolerance
|
||||
return recall_ok and fp_ok
|
||||
|
||||
|
||||
def main() -> int:
|
||||
verdict = detection_gate(evaluate_corpus())
|
||||
o = verdict.overall
|
||||
print(
|
||||
f"detection gate: recall {o.recall:.2%} ({o.matched}/{o.expected}), FP {o.false_positive_rate:.2%} ({o.false_positives}/{o.actual})"
|
||||
)
|
||||
for language, m in verdict.per_language.items():
|
||||
print(
|
||||
f" {language:<11} recall {m.recall:>6.2%} ({m.matched}/{m.expected}) "
|
||||
f"FP {m.false_positive_rate:>6.2%} ({m.false_positives}/{m.actual})"
|
||||
)
|
||||
if verdict.passed:
|
||||
print("detection gate: PASS")
|
||||
return 0
|
||||
print("detection gate: FAIL", file=sys.stderr)
|
||||
for reason in verdict.reasons:
|
||||
print(f" - {reason}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -62,13 +62,19 @@ class SerializedHarnessConfig:
|
||||
|
||||
|
||||
class HarnessTraceWriter:
|
||||
def __init__(self, path: Path) -> None:
|
||||
def __init__(self, path: Path, *, redact: bool = True) -> None:
|
||||
self.path = path
|
||||
self.redact = redact
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def append(self, event: HarnessEvent) -> None:
|
||||
line = json.dumps(asdict(event), sort_keys=True)
|
||||
if self.redact:
|
||||
from .redaction import redact_secrets
|
||||
|
||||
line = redact_secrets(line)
|
||||
with self.path.open("a") as handle:
|
||||
handle.write(json.dumps(asdict(event), sort_keys=True) + "\n")
|
||||
handle.write(line + "\n")
|
||||
|
||||
|
||||
class HarnessRuntime:
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Capability guard for the optional HarnessX agentic plane.
|
||||
|
||||
HarnessX is an optional extra (``pip install 'openultrasast[harnessx]'``). The core
|
||||
install is zero-dependency, so nothing here imports ``harnessx`` at module load:
|
||||
:func:`has_harnessx` only probes importability via ``find_spec`` and
|
||||
:func:`require_harnessx` performs the import lazily, on demand. Every HarnessX-backed
|
||||
code path (Phase 3+: build_sast, the hunter sub-harness, the llm-judge verifier, the
|
||||
MetaAgent.evolve loop) routes through this seam.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from harnessx.core.harness import HarnessConfig
|
||||
|
||||
HARNESSX_EXTRA = "openultrasast[harnessx]"
|
||||
|
||||
# HarnessX providers OpenUltraSAST can drive for the hunter / llm-judge stages.
|
||||
# Each reads its own standard API-key env var (ANTHROPIC_API_KEY / OPENAI_API_KEY).
|
||||
SUPPORTED_PROVIDERS = ("anthropic", "openai", "litellm")
|
||||
|
||||
|
||||
class HarnessXUnavailableError(RuntimeError):
|
||||
"""Raised when a HarnessX-backed capability is requested but the extra is absent."""
|
||||
|
||||
|
||||
def has_harnessx() -> bool:
|
||||
"""Return True if the HarnessX extra is importable, without importing it."""
|
||||
return importlib.util.find_spec("harnessx") is not None
|
||||
|
||||
|
||||
def require_harnessx() -> ModuleType:
|
||||
"""Lazily import and return the ``harnessx`` module, or raise with install guidance."""
|
||||
if not has_harnessx():
|
||||
raise HarnessXUnavailableError(f"HarnessX is not installed. Install the optional agentic extra: pip install '{HARNESSX_EXTRA}'")
|
||||
return importlib.import_module("harnessx")
|
||||
|
||||
|
||||
def build_provider(model: str, provider: str = "anthropic") -> Any:
|
||||
"""Build a HarnessX LLM provider for ``model`` (lazy, capability-guarded).
|
||||
|
||||
The single place that knows HarnessX provider class names. The provider name is
|
||||
validated before any import so an unknown choice fails loudly even when the extra
|
||||
is absent; each provider reads its own standard API-key env var. Raises
|
||||
:class:`ValueError` on an unknown provider and :class:`HarnessXUnavailableError`
|
||||
when the extra is not installed.
|
||||
"""
|
||||
key = provider.lower()
|
||||
if key not in SUPPORTED_PROVIDERS:
|
||||
raise ValueError(f"unknown HarnessX provider {provider!r}; choose one of {SUPPORTED_PROVIDERS}")
|
||||
require_harnessx()
|
||||
if key == "anthropic":
|
||||
from harnessx.providers.anthropic_provider import AnthropicProvider
|
||||
|
||||
return AnthropicProvider(model=model)
|
||||
if key == "openai":
|
||||
from harnessx.providers.openai_provider import OpenAIProvider
|
||||
|
||||
return OpenAIProvider(model=model)
|
||||
from harnessx.providers.litellm_provider import LiteLLMProvider
|
||||
|
||||
return LiteLLMProvider(model=model)
|
||||
|
||||
|
||||
def build_sast(*, max_cost_usd: float = 2.0, token_threshold: int = 120_000) -> HarnessConfig:
|
||||
"""Compose the SAST hunter HarnessConfig (behaviour pipeline, model-free).
|
||||
|
||||
The single, strictly-localized HarnessX composition site (mirrors DocuHarnessX
|
||||
``make_docgen``): every ``harnessx`` import is inside this function so importing
|
||||
this module never touches the optional extra. Raises :class:`HarnessXUnavailableError`
|
||||
when the extra is absent. The per-task step budget is enforced on the task at run
|
||||
time, not composed here.
|
||||
"""
|
||||
require_harnessx()
|
||||
from harnessx.bundles.context import make_context
|
||||
from harnessx.bundles.control import make_control
|
||||
from harnessx.bundles.tools import make_tools
|
||||
from harnessx.core.builder import HarnessBuilder
|
||||
from harnessx.processors.context.strategies.system_prompt.default import DefaultSystemPromptBuilder
|
||||
|
||||
builder = (
|
||||
HarnessBuilder()
|
||||
| make_context(system_builder=DefaultSystemPromptBuilder())
|
||||
| make_tools(skill_loading=False)
|
||||
| make_control(
|
||||
include_reliability=True,
|
||||
include_budget=True,
|
||||
token_threshold=token_threshold,
|
||||
max_cost_usd=max_cost_usd,
|
||||
)
|
||||
)
|
||||
return builder.build()
|
||||
+24
-17
@@ -7,8 +7,10 @@ from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from .findings import StaticFinding, quick_scan_findings
|
||||
from .policy import CwePolicy
|
||||
from .preprocess import FileTarget
|
||||
from .rank import RankingScore
|
||||
from .ruleset import PatternRule
|
||||
|
||||
HunterTier = Literal["A", "B", "C"]
|
||||
|
||||
@@ -98,13 +100,21 @@ def schedule_hunter_tasks(
|
||||
return scheduled
|
||||
|
||||
|
||||
def run_hunter_pool(root: Path, targets: list[FileTarget], rankings: list[RankingScore], *, scan_id: str) -> HunterPoolResult:
|
||||
def run_hunter_pool(
|
||||
root: Path,
|
||||
targets: list[FileTarget],
|
||||
rankings: list[RankingScore],
|
||||
*,
|
||||
scan_id: str,
|
||||
ruleset: tuple[PatternRule, ...] | None = None,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
) -> HunterPoolResult:
|
||||
tasks = schedule_hunter_tasks(targets, rankings, retrieval_context_by_path=build_retrieval_context_by_path(root, targets))
|
||||
findings: list[StaticFinding] = []
|
||||
trajectories: list[HunterTrajectory] = []
|
||||
max_workers = min(8, max(1, len(tasks)))
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
for task_findings, trajectory in executor.map(lambda task: _run_hunter_task(root, scan_id, task), tasks):
|
||||
for task_findings, trajectory in executor.map(lambda task: _run_hunter_task(root, scan_id, task, ruleset, policy), tasks):
|
||||
findings.extend(task_findings)
|
||||
trajectories.append(trajectory)
|
||||
return HunterPoolResult(findings=sorted(findings, key=lambda item: item.finding_id), trajectories=trajectories)
|
||||
@@ -125,23 +135,20 @@ def build_retrieval_context_by_path(root: Path, targets: list[FileTarget], *, ma
|
||||
|
||||
|
||||
def select_skill_snippets(target: FileTarget, *, max_snippets: int = 4) -> list[str]:
|
||||
tags = set(target.tags)
|
||||
snippets: list[str] = []
|
||||
if target.language in {"c", "cpp", "c++"} or "memory_unsafe" in tags:
|
||||
snippets.extend(["address-sanitizer", "libfuzzer", "harness-writing"])
|
||||
if "parser" in tags or "fuzzable" in tags:
|
||||
snippets.extend(["fuzzing-dictionary", "fuzzing-obstacles"])
|
||||
if "crypto" in tags:
|
||||
snippets.extend(["constant-time-analysis", "wycheproof"])
|
||||
if "auth_boundary" in tags or "network_entry" in tags:
|
||||
snippets.extend(["sharp-edges", "insecure-defaults"])
|
||||
if "deserialization" in tags:
|
||||
snippets.append("semgrep-rule-creator")
|
||||
return sorted(dict.fromkeys(snippets))[:max_snippets]
|
||||
"""Route the target to its most relevant Trail of Bits skills (ids), via the skill index."""
|
||||
from .skills import route_skill_ids
|
||||
|
||||
return route_skill_ids(target, max_snippets=max_snippets)
|
||||
|
||||
|
||||
def _run_hunter_task(root: Path, scan_id: str, task: HunterTask) -> tuple[list[StaticFinding], HunterTrajectory]:
|
||||
findings = quick_scan_findings(root, [task.target], [task.ranking])[: task.budget.max_findings_per_target]
|
||||
def _run_hunter_task(
|
||||
root: Path,
|
||||
scan_id: str,
|
||||
task: HunterTask,
|
||||
ruleset: tuple[PatternRule, ...] | None = None,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
) -> tuple[list[StaticFinding], HunterTrajectory]:
|
||||
findings = quick_scan_findings(root, [task.target], [task.ranking], ruleset, policy)[: task.budget.max_findings_per_target]
|
||||
return findings, HunterTrajectory(
|
||||
scan_id=scan_id,
|
||||
target_path=task.target.path,
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""HarnessX-backed hunter pool — a real LLM agent loop per ranked target.
|
||||
|
||||
A drop-in for :func:`hunter.run_hunter_pool` (same call shape, returns the same
|
||||
:class:`hunter.HunterPoolResult`) that runs the optional HarnessX agentic plane. It
|
||||
is lazy/guarded: importing this module never imports ``harnessx``; the extra is only
|
||||
touched once :meth:`HxScanOrchestrator.run_pool` runs. The agent ``.run()`` path makes
|
||||
LLM API calls and is therefore exercised only with the extra + credentials present;
|
||||
the default scan keeps using the deterministic :func:`hunter.run_hunter_pool`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .findings import SEVERITY_LABEL, StaticFinding, _reachability_conditions, _reachability_for_line
|
||||
from .harness_ext import build_provider, build_sast, require_harnessx
|
||||
from .hunter import (
|
||||
HunterPoolResult,
|
||||
HunterTask,
|
||||
HunterTrajectory,
|
||||
build_retrieval_context_by_path,
|
||||
schedule_hunter_tasks,
|
||||
)
|
||||
from .policy import CwePolicy, load_policy, resolve_severity
|
||||
from .preprocess import FileTarget
|
||||
from .rank import RankingScore
|
||||
from .ruleset import PatternRule
|
||||
|
||||
# The hunter agent is asked to emit findings as a JSON array of these objects.
|
||||
_FINDINGS_JSON = re.compile(r"\[\s*\{.*\}\s*\]", re.DOTALL)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScanTask:
|
||||
"""Thin description of one bounded hunter task (wraps, not subclasses, BaseTask)."""
|
||||
|
||||
description: str
|
||||
success_criteria: str
|
||||
max_steps: int
|
||||
token_budget: int | None
|
||||
max_cost_usd: float | None
|
||||
|
||||
|
||||
class HxScanOrchestrator:
|
||||
"""Runs the hunter pool on a real HarnessX agent loop, one bounded task per target."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
provider_model: str,
|
||||
provider: str = "anthropic",
|
||||
max_cost_usd: float = 2.0,
|
||||
token_threshold: int = 120_000,
|
||||
) -> None:
|
||||
require_harnessx()
|
||||
self._provider_model = provider_model
|
||||
self._provider = provider
|
||||
self._max_cost_usd = max_cost_usd
|
||||
self._token_threshold = token_threshold
|
||||
self._agent: Any = None
|
||||
|
||||
def _build_agent(self) -> Any:
|
||||
from harnessx.core.model_config import ModelConfig
|
||||
|
||||
config = build_sast(max_cost_usd=self._max_cost_usd, token_threshold=self._token_threshold)
|
||||
return ModelConfig(main=build_provider(self._provider_model, self._provider)).agentic(config)
|
||||
|
||||
def run_pool(
|
||||
self,
|
||||
root: Path,
|
||||
targets: list[FileTarget],
|
||||
rankings: list[RankingScore],
|
||||
*,
|
||||
scan_id: str,
|
||||
ruleset: tuple[PatternRule, ...] | None = None,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
emit: Callable[..., None] | None = None,
|
||||
) -> HunterPoolResult:
|
||||
import asyncio
|
||||
|
||||
tasks = schedule_hunter_tasks(targets, rankings, retrieval_context_by_path=build_retrieval_context_by_path(root, targets))
|
||||
resolved_policy = policy if policy is not None else load_policy()
|
||||
self._agent = self._agent or self._build_agent()
|
||||
return asyncio.run(self._run_all(scan_id, tasks, resolved_policy, emit))
|
||||
|
||||
async def _run_all(
|
||||
self,
|
||||
scan_id: str,
|
||||
tasks: list[HunterTask],
|
||||
policy: dict[str, CwePolicy],
|
||||
emit: Callable[..., None] | None,
|
||||
) -> HunterPoolResult:
|
||||
from harnessx.core.harness import BaseTask
|
||||
|
||||
findings: list[StaticFinding] = []
|
||||
trajectories: list[HunterTrajectory] = []
|
||||
agent = self._agent
|
||||
for task in tasks:
|
||||
base_task = BaseTask(
|
||||
description=self._describe(task),
|
||||
success_criteria="Enumerate each reachable tainted sink with file:line and a one-line rationale.",
|
||||
max_steps=task.budget.max_findings_per_target,
|
||||
token_budget=task.budget.retrieval_budget_chars * 4,
|
||||
max_cost_usd=self._max_cost_usd,
|
||||
)
|
||||
result = await agent.run(base_task, session_id=scan_id)
|
||||
task_findings = self._findings_from_result(task, result, policy)
|
||||
findings.extend(task_findings)
|
||||
trajectories.append(self._trajectory_from_result(scan_id, task, result, task_findings))
|
||||
if emit is not None:
|
||||
emit(
|
||||
"stage_end",
|
||||
f"hunter_pool:{task.target.path}",
|
||||
{
|
||||
"exit_reason": getattr(result, "exit_reason", "unknown"),
|
||||
"total_steps": getattr(result, "total_steps", 0),
|
||||
"total_cost_usd": getattr(result, "total_cost_usd", 0.0),
|
||||
},
|
||||
)
|
||||
await agent.cleanup()
|
||||
return HunterPoolResult(findings=sorted(findings, key=lambda item: item.finding_id), trajectories=trajectories)
|
||||
|
||||
def _describe(self, task: HunterTask) -> str:
|
||||
from .skills import select_skill_context
|
||||
|
||||
context = task.retrieval_context or f"[repo_code] {task.target.path}"
|
||||
skills = select_skill_context(task.target, stage="hunt", budget_chars=task.budget.skill_budget_chars)
|
||||
skill_block = f"\n\nRelevant security skills:\n{skills}" if skills else ""
|
||||
return (
|
||||
f"Audit `{task.target.path}` ({task.target.language}) for security-relevant vulnerabilities. "
|
||||
f"Report ONLY evidence-backed findings as a JSON array, each object: "
|
||||
f'{{"line": <int>, "title": <str>, "cwe": "CWE-NN" (optional), "rationale": <str>}}. '
|
||||
f"Emit `[]` if none.{skill_block}\n\nSource context:\n{context}"
|
||||
)
|
||||
|
||||
def _findings_from_result(self, task: HunterTask, result: object, policy: dict[str, CwePolicy]) -> list[StaticFinding]:
|
||||
items = _extract_findings(getattr(result, "final_output", "") or "")
|
||||
findings: list[StaticFinding] = []
|
||||
for item in items:
|
||||
line = item.get("line")
|
||||
line = int(line) if isinstance(line, int) else None
|
||||
reachability_status, function_name, reachability_evidence = (
|
||||
_reachability_for_line(task.target, line) if line is not None else ("unknown", None, [])
|
||||
)
|
||||
cwe = str(item.get("cwe", "")) or None
|
||||
severity = SEVERITY_LABEL.get(resolve_severity(policy, cwe), "medium") if cwe else "medium"
|
||||
line_part = line if line is not None else 0
|
||||
findings.append(
|
||||
StaticFinding(
|
||||
finding_id=f"hx-hunter:{task.target.path}:{line_part}",
|
||||
path=task.target.path,
|
||||
title=str(item.get("title", "HarnessX hunter finding")),
|
||||
severity=severity,
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale=str(item.get("rationale", "Reported by the HarnessX hunter; manual verification required.")),
|
||||
line=line,
|
||||
function_name=function_name,
|
||||
reachability_status=reachability_status,
|
||||
reachability_evidence=reachability_evidence,
|
||||
reachability_conditions=_reachability_conditions(reachability_evidence),
|
||||
tags=sorted(set(task.target.tags)),
|
||||
ranking_priority=task.ranking.priority,
|
||||
)
|
||||
)
|
||||
return findings
|
||||
|
||||
def _trajectory_from_result(
|
||||
self, scan_id: str, task: HunterTask, result: object, task_findings: list[StaticFinding]
|
||||
) -> HunterTrajectory:
|
||||
return HunterTrajectory(
|
||||
scan_id=scan_id,
|
||||
target_path=task.target.path,
|
||||
tier=task.tier,
|
||||
priority=task.ranking.priority,
|
||||
selected_skills=task.selected_skills,
|
||||
retrieval_context_chars=len(task.retrieval_context),
|
||||
findings=[finding.finding_id for finding in task_findings],
|
||||
status=str(getattr(result, "exit_reason", "completed")),
|
||||
)
|
||||
|
||||
|
||||
def _extract_findings(text: str) -> list[dict[str, object]]:
|
||||
"""Best-effort extraction of the findings JSON array from the agent's final output."""
|
||||
match = _FINDINGS_JSON.search(text)
|
||||
if match is None:
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [item for item in parsed if isinstance(item, dict)] if isinstance(parsed, list) else []
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Bounded, gated self-improvement loop over the ruleset/policy governance data."""
|
||||
|
||||
from .evolve import RoundOutcome, build_rule_signals, propose_status_edits, run_improvement, run_round
|
||||
from .journal import load_journal, reverted_edit_keys
|
||||
from .validator import (
|
||||
VALID_LEVERS,
|
||||
EvolveBounds,
|
||||
EvolveValidator,
|
||||
PolicyConstantEdit,
|
||||
RuleStatusEdit,
|
||||
StrictValidationError,
|
||||
edits_to_ledger,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"EvolveBounds",
|
||||
"EvolveValidator",
|
||||
"PolicyConstantEdit",
|
||||
"RoundOutcome",
|
||||
"RuleStatusEdit",
|
||||
"StrictValidationError",
|
||||
"VALID_LEVERS",
|
||||
"build_rule_signals",
|
||||
"edits_to_ledger",
|
||||
"load_journal",
|
||||
"propose_status_edits",
|
||||
"reverted_edit_keys",
|
||||
"run_improvement",
|
||||
"run_round",
|
||||
]
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Deterministic self-improvement loop (tasks 7.4, 7.5, 7.6).
|
||||
|
||||
One round: benchmark with the current ledger -> per-rule signals -> propose bounded
|
||||
edits (auto-shadow precision-draggers) -> validate -> replay smoke -> re-benchmark ->
|
||||
hard acceptance gate (recall >= floor AND FP < ceiling AND project score not regressed
|
||||
AND no matched-finding regression) -> accept (persist the ledger) or revert byte-for-byte.
|
||||
|
||||
The edit *proposer* here is the benchmark's per-rule signal; the LLM ``MetaAgent.evolve``
|
||||
proposer plugs into this same validated/gated machinery when the HarnessX extra is present.
|
||||
The project score is the optimization reward; the recall/FP gate is a separate hard
|
||||
feasibility constraint that is never folded into the reward.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ..benchmark import BenchmarkManifest, BenchmarkResult, BenchmarkRun, evaluate_benchmark
|
||||
from ..findings import StaticFinding, quick_scan_findings
|
||||
from ..mapping import analyze_entry_points, attach_reachability_hints
|
||||
from ..policy import CwePolicy, load_policy
|
||||
from ..preprocess import preprocess_repository
|
||||
from ..rank import rank_targets
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger
|
||||
from ..scoring import build_score_artifact
|
||||
from .journal import append_round, load_journal, next_round_index, reverted_edit_keys
|
||||
from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoundOutcome:
|
||||
round: int
|
||||
accepted: bool
|
||||
reason: str
|
||||
edits: list[RuleStatusEdit] = field(default_factory=list)
|
||||
recall_before: float = 0.0
|
||||
recall_after: float = 0.0
|
||||
fp_before: float = 0.0
|
||||
fp_after: float = 0.0
|
||||
score_before: int = 0
|
||||
score_after: int = 0
|
||||
matched_before: int = 0
|
||||
matched_after: int = 0
|
||||
|
||||
|
||||
def build_rule_signals(result: BenchmarkResult) -> list[dict[str, object]]:
|
||||
"""Convert a benchmark result into per-rule loop signals (recall: miss; precision: fp)."""
|
||||
signals: list[dict[str, object]] = []
|
||||
for miss in result.misses:
|
||||
signals.append({"rule_id": miss.rule_id or "", "signal": "miss", "cwe": miss.cwe, "path": miss.path})
|
||||
for fp in result.false_positives:
|
||||
signals.append({"rule_id": fp.rule_id, "signal": "fp", "path": fp.path, "line": fp.line})
|
||||
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("path"))))
|
||||
|
||||
|
||||
def propose_status_edits(
|
||||
per_rule: Mapping[str, dict[str, int]],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
blocked_keys: set[str],
|
||||
) -> list[RuleStatusEdit]:
|
||||
"""Auto-shadow precision-draggers: enabled rules with false positives -> shadow (never delete)."""
|
||||
edits: list[RuleStatusEdit] = []
|
||||
for rule_id, counts in sorted(per_rule.items()):
|
||||
if counts.get("false_positives", 0) <= 0:
|
||||
continue
|
||||
rule = ruleset_by_id.get(rule_id)
|
||||
if rule is None:
|
||||
continue
|
||||
current_status = str(current_ledger.get(rule_id, {}).get("status", rule.status))
|
||||
if current_status != "enabled":
|
||||
continue # already staged or disabled
|
||||
edit = RuleStatusEdit(rule_id=rule_id, from_status="enabled", to_status="shadow", rationale="")
|
||||
if edit.key() in blocked_keys:
|
||||
continue # novelty gate: previously reverted without a new rationale
|
||||
edits.append(edit)
|
||||
return edits
|
||||
|
||||
|
||||
def run_round(
|
||||
target: Path,
|
||||
manifest: BenchmarkManifest,
|
||||
*,
|
||||
ledger_path: Path,
|
||||
journal_path: Path,
|
||||
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
recall_floor: float = 0.9,
|
||||
fp_ceiling: float = 0.1,
|
||||
) -> RoundOutcome:
|
||||
policy = policy if policy is not None else load_policy()
|
||||
rounds = load_journal(journal_path)
|
||||
round_index = next_round_index(rounds)
|
||||
blocked = reverted_edit_keys(rounds)
|
||||
current_ledger = read_rule_ledger(ledger_path)
|
||||
|
||||
before_findings, before_rules = _scan(target, ruleset_dir, current_ledger, policy)
|
||||
before = _evaluate(target, manifest, before_findings, before_rules, policy)
|
||||
(ledger_path.parent / "rule_signals.json").parent.mkdir(parents=True, exist_ok=True)
|
||||
_write_signals(ledger_path.parent / "rule_signals.json", build_rule_signals(before.result))
|
||||
|
||||
ruleset_by_id = {rule.rule_id: rule for rule in before_rules}
|
||||
edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
|
||||
if not edits:
|
||||
return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before)
|
||||
|
||||
validator = EvolveValidator()
|
||||
try:
|
||||
for edit in edits:
|
||||
validator.validate(edit, ruleset_by_id, policy)
|
||||
except StrictValidationError as exc:
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, str(exc))
|
||||
return _outcome(round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before)
|
||||
|
||||
candidate = edits_to_ledger(edits, current_ledger)
|
||||
try:
|
||||
_load_with(ruleset_dir, candidate) # replay smoke gate: candidate ruleset must boot
|
||||
except Exception as exc: # noqa: BLE001 — any boot failure rejects the round
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, f"replay_failed: {exc}")
|
||||
return _outcome(round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before)
|
||||
|
||||
after_findings, after_rules = _scan(target, ruleset_dir, candidate, policy)
|
||||
after = _evaluate(target, manifest, after_findings, after_rules, policy)
|
||||
|
||||
accepted = (
|
||||
after.recall >= recall_floor and after.fp_rate < fp_ceiling and after.score >= before.score and after.matched >= before.matched
|
||||
)
|
||||
if accepted:
|
||||
write_rule_ledger(ledger_path, candidate)
|
||||
reason = "accepted"
|
||||
else:
|
||||
reason = "reverted" # ledger_path untouched -> byte-for-byte revert
|
||||
_record(journal_path, round_index, edits, reason, before, after)
|
||||
return _outcome(round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after)
|
||||
|
||||
|
||||
def run_improvement(
|
||||
target: Path,
|
||||
manifest: BenchmarkManifest,
|
||||
*,
|
||||
ledger_path: Path,
|
||||
journal_path: Path,
|
||||
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
max_rounds: int = 5,
|
||||
recall_floor: float = 0.9,
|
||||
fp_ceiling: float = 0.1,
|
||||
) -> list[RoundOutcome]:
|
||||
"""Run improvement rounds until convergence (no new proposals) or ``max_rounds``."""
|
||||
outcomes: list[RoundOutcome] = []
|
||||
for _ in range(max_rounds):
|
||||
outcome = run_round(
|
||||
target,
|
||||
manifest,
|
||||
ledger_path=ledger_path,
|
||||
journal_path=journal_path,
|
||||
ruleset_dir=ruleset_dir,
|
||||
policy=policy,
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if outcome.reason == "no_proposals":
|
||||
break
|
||||
return outcomes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Eval:
|
||||
result: BenchmarkResult
|
||||
recall: float
|
||||
fp_rate: float
|
||||
score: int
|
||||
matched: int
|
||||
|
||||
|
||||
def _scan(
|
||||
target: Path, ruleset_dir: Path, ledger: Mapping[str, dict[str, object]], policy: dict[str, CwePolicy]
|
||||
) -> tuple[list[StaticFinding], tuple[PatternRule, ...]]:
|
||||
rules = _load_with(ruleset_dir, ledger)
|
||||
_, targets = preprocess_repository(target)
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
|
||||
rankings = rank_targets(targets)
|
||||
findings = quick_scan_findings(target, targets, rankings, rules, policy)
|
||||
reported = [finding for finding in findings if finding.status != "shadow"]
|
||||
return reported, rules
|
||||
|
||||
|
||||
def _evaluate(
|
||||
target: Path, manifest: BenchmarkManifest, findings: list[StaticFinding], rules: tuple[PatternRule, ...], policy: dict[str, CwePolicy]
|
||||
) -> _Eval:
|
||||
run = BenchmarkRun(benchmark_run_id="improve", root=target, manifest=manifest)
|
||||
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
metrics = result.metrics
|
||||
cwe_by_rule = {rule.rule_id: rule.cwe for rule in rules}
|
||||
rule_cwe_by_id = {f.finding_id: cwe_by_rule.get(f.finding_id.split(":", 1)[0], "") for f in findings}
|
||||
score = build_score_artifact(findings, rule_cwe_by_id, policy).project_score
|
||||
recall = metrics.recall if metrics.recall is not None else 1.0
|
||||
fp_rate = metrics.false_positive_findings_total / metrics.actual_findings_total if metrics.actual_findings_total else 0.0
|
||||
return _Eval(result=result, recall=recall, fp_rate=fp_rate, score=score, matched=metrics.matched_findings_total)
|
||||
|
||||
|
||||
def _load_with(ruleset_dir: Path, ledger: Mapping[str, dict[str, object]]) -> tuple[PatternRule, ...]:
|
||||
import json
|
||||
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle:
|
||||
handle.write(json.dumps(ledger))
|
||||
temp_path = Path(handle.name)
|
||||
try:
|
||||
return load_ruleset(ruleset_dir, temp_path)
|
||||
finally:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _write_signals(path: Path, signals: list[dict[str, object]]) -> None:
|
||||
import json
|
||||
|
||||
path.write_text(json.dumps(signals, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _outcome(round_index: int, *, accepted: bool, reason: str, edits: list[RuleStatusEdit], before: _Eval, after: _Eval) -> RoundOutcome:
|
||||
return RoundOutcome(
|
||||
round=round_index,
|
||||
accepted=accepted,
|
||||
reason=reason,
|
||||
edits=edits,
|
||||
recall_before=before.recall,
|
||||
recall_after=after.recall,
|
||||
fp_before=before.fp_rate,
|
||||
fp_after=after.fp_rate,
|
||||
score_before=before.score,
|
||||
score_after=after.score,
|
||||
matched_before=before.matched,
|
||||
matched_after=after.matched,
|
||||
)
|
||||
|
||||
|
||||
def _record(
|
||||
journal_path: Path,
|
||||
round_index: int,
|
||||
edits: list[RuleStatusEdit],
|
||||
outcome: str,
|
||||
before: _Eval,
|
||||
after: _Eval,
|
||||
reason: str = "",
|
||||
) -> None:
|
||||
append_round(
|
||||
journal_path,
|
||||
{
|
||||
"round": round_index,
|
||||
"outcome": outcome,
|
||||
"reason": reason or outcome,
|
||||
"edits": [
|
||||
{"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale}
|
||||
for e in edits
|
||||
],
|
||||
"recall_before": round(before.recall, 4),
|
||||
"recall_after": round(after.recall, 4),
|
||||
"fp_before": round(before.fp_rate, 4),
|
||||
"fp_after": round(after.fp_rate, 4),
|
||||
"score_before": before.score,
|
||||
"score_after": after.score,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Improvement journal: per-round audit + novelty memory (task 7.3, 7.9).
|
||||
|
||||
Persistent JSON record of every self-improvement round so each rule/policy edit is
|
||||
attributable and reversible, and so a previously-reverted edit is not re-proposed
|
||||
without a new rationale (the novelty gate).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_journal(path: Path) -> list[dict[str, object]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
payload = json.loads(path.read_text())
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
|
||||
def write_journal(path: Path, rounds: list[dict[str, object]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(rounds, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def reverted_edit_keys(rounds: list[dict[str, object]]) -> set[str]:
|
||||
"""Edit keys from reverted rounds — blocked from re-proposal unless a new rationale appears."""
|
||||
keys: set[str] = set()
|
||||
for entry in rounds:
|
||||
if entry.get("outcome") != "reverted":
|
||||
continue
|
||||
for edit in _edits(entry):
|
||||
key = edit.get("key")
|
||||
if isinstance(key, str) and not edit.get("rationale"):
|
||||
keys.add(key)
|
||||
return keys
|
||||
|
||||
|
||||
def next_round_index(rounds: list[dict[str, object]]) -> int:
|
||||
return len(rounds) + 1
|
||||
|
||||
|
||||
def append_round(path: Path, entry: dict[str, object]) -> None:
|
||||
rounds = load_journal(path)
|
||||
rounds.append(entry)
|
||||
write_journal(path, rounds)
|
||||
|
||||
|
||||
def _edits(entry: dict[str, object]) -> list[dict[str, object]]:
|
||||
edits = entry.get("edits")
|
||||
return [edit for edit in edits if isinstance(edit, dict)] if isinstance(edits, list) else []
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Bounded-lever edit model + validator for the self-improvement loop (tasks 7.1, 7.2).
|
||||
|
||||
The loop may only adjust governance *data* through two bounded levers and never the
|
||||
two authorities a human/upstream owns: detection **pattern text** and the 0-5 CWE
|
||||
**severity**. Edits are typed so pattern/severity changes are structurally
|
||||
impossible; the validator additionally enforces status/bounds/staging/resolution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..policy import CwePolicy
|
||||
from ..ruleset import PatternRule
|
||||
from ..ruleset.store import VALID_STATUS
|
||||
|
||||
# The only levers the loop may pull (mirrors the design's journal._VALID_LEVERS).
|
||||
VALID_LEVERS = frozenset({"rule", "policy"})
|
||||
# Policy constants the loop may tune; the 0-5 severity is intentionally absent.
|
||||
TUNABLE_POLICY_CONSTANTS = frozenset({"K", "MIN_SCORE"})
|
||||
|
||||
|
||||
class StrictValidationError(ValueError):
|
||||
"""Raised when a proposed loop edit violates the bounded-lever contract."""
|
||||
|
||||
def __init__(self, message: str, *, kind: str = "rule_change") -> None:
|
||||
super().__init__(message)
|
||||
self.kind = kind
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuleStatusEdit:
|
||||
"""A `rule`-lever edit: change a rule's status / evidence floor / precision estimate."""
|
||||
|
||||
rule_id: str
|
||||
from_status: str
|
||||
to_status: str
|
||||
rationale: str = ""
|
||||
lever: str = "rule"
|
||||
|
||||
def key(self) -> str:
|
||||
return f"rule:{self.rule_id}:{self.from_status}->{self.to_status}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyConstantEdit:
|
||||
"""A `policy`-lever edit: tune a score constant (K / MIN_SCORE). Never severity."""
|
||||
|
||||
name: str
|
||||
from_value: float
|
||||
to_value: float
|
||||
rationale: str = ""
|
||||
lever: str = "policy"
|
||||
|
||||
def key(self) -> str:
|
||||
return f"policy:{self.name}:{self.from_value}->{self.to_value}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvolveBounds:
|
||||
k_range: tuple[float, float] = (20.0, 200.0)
|
||||
min_score_range: tuple[int, int] = (0, 100)
|
||||
|
||||
|
||||
class EvolveValidator:
|
||||
"""Enforces the bounded-lever safety contract; raises StrictValidationError on any breach."""
|
||||
|
||||
def __init__(self, bounds: EvolveBounds | None = None) -> None:
|
||||
self._bounds = bounds or EvolveBounds()
|
||||
|
||||
def validate(self, edit: object, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None:
|
||||
if isinstance(edit, RuleStatusEdit):
|
||||
self._validate_rule(edit, ruleset_by_id, policy)
|
||||
elif isinstance(edit, PolicyConstantEdit):
|
||||
self._validate_policy(edit)
|
||||
else:
|
||||
raise StrictValidationError(f"unknown edit type {type(edit).__name__}", kind="rule_change")
|
||||
|
||||
def _validate_rule(self, edit: RuleStatusEdit, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None:
|
||||
if edit.to_status not in VALID_STATUS:
|
||||
raise StrictValidationError(f"invalid status {edit.to_status!r} for {edit.rule_id}")
|
||||
rule = ruleset_by_id.get(edit.rule_id)
|
||||
if rule is None:
|
||||
raise StrictValidationError(f"edit targets unknown rule_id {edit.rule_id!r} (loop cannot add rules)")
|
||||
if edit.from_status == "enabled" and edit.to_status == "disabled":
|
||||
raise StrictValidationError(f"{edit.rule_id}: enabled->disabled jump is forbidden; stage via shadow first")
|
||||
if rule.cwe not in policy:
|
||||
raise StrictValidationError(f"{edit.rule_id}: CWE {rule.cwe} does not resolve in the policy")
|
||||
|
||||
def _validate_policy(self, edit: PolicyConstantEdit) -> None:
|
||||
if edit.name not in TUNABLE_POLICY_CONSTANTS:
|
||||
raise StrictValidationError(
|
||||
f"policy constant {edit.name!r} is not loop-tunable (severity is upstream-owned)", kind="policy_change"
|
||||
)
|
||||
if edit.name == "K" and not (self._bounds.k_range[0] <= edit.to_value <= self._bounds.k_range[1]):
|
||||
raise StrictValidationError(f"K={edit.to_value} out of bounds {self._bounds.k_range}", kind="policy_change")
|
||||
if edit.name == "MIN_SCORE" and not (self._bounds.min_score_range[0] <= edit.to_value <= self._bounds.min_score_range[1]):
|
||||
raise StrictValidationError(f"MIN_SCORE={edit.to_value} out of bounds {self._bounds.min_score_range}", kind="policy_change")
|
||||
|
||||
|
||||
def edits_to_ledger(edits: list[RuleStatusEdit], base: Mapping[str, dict[str, object]]) -> dict[str, dict[str, object]]:
|
||||
"""Overlay rule-status edits onto a copy of the base ledger."""
|
||||
ledger: dict[str, dict[str, object]] = {key: dict(value) for key, value in base.items()}
|
||||
for edit in edits:
|
||||
entry = dict(ledger.get(edit.rule_id, {}))
|
||||
entry["status"] = edit.to_status
|
||||
ledger[edit.rule_id] = entry
|
||||
return ledger
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EvolveBounds",
|
||||
"EvolveValidator",
|
||||
"PolicyConstantEdit",
|
||||
"RuleStatusEdit",
|
||||
"StrictValidationError",
|
||||
"TUNABLE_POLICY_CONSTANTS",
|
||||
"VALID_LEVERS",
|
||||
"edits_to_ledger",
|
||||
]
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Narrow MCP server for OpenCode integration (Phase 12).
|
||||
|
||||
Exposes only stable, project-level operations over MCP stdio (newline-delimited
|
||||
JSON-RPC 2.0), implemented with the standard library only. No tool exposes arbitrary
|
||||
shell, Docker, or internal hunter tools — those stay inside controlled harness stages.
|
||||
The pure :meth:`McpServer.handle` dispatch is separated from the :func:`serve` stdio
|
||||
loop so the surface is unit-testable without a live client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import IO, Any, cast
|
||||
|
||||
PROTOCOL_VERSION = "2024-11-05"
|
||||
SERVER_NAME = "openultrasast"
|
||||
|
||||
|
||||
def _server_version() -> str:
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
|
||||
return version("openultrasast")
|
||||
except Exception: # noqa: BLE001 — version metadata is best-effort
|
||||
return "0"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Tool:
|
||||
name: str
|
||||
description: str
|
||||
input_schema: dict[str, Any]
|
||||
handler: Callable[[dict[str, Any]], dict[str, Any]]
|
||||
|
||||
def descriptor(self) -> dict[str, Any]:
|
||||
return {"name": self.name, "description": self.description, "inputSchema": self.input_schema}
|
||||
|
||||
|
||||
class McpToolError(ValueError):
|
||||
"""Raised by a tool handler when its inputs are invalid or the artifact is missing."""
|
||||
|
||||
|
||||
# ---- tool handlers (each maps to one bounded project operation) --------------
|
||||
|
||||
|
||||
def _require(args: dict[str, Any], key: str) -> Any:
|
||||
if key not in args or args[key] in (None, ""):
|
||||
raise McpToolError(f"missing required argument: {key}")
|
||||
return args[key]
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise McpToolError(f"artifact not found: {path}")
|
||||
return cast(dict[str, Any], json.loads(path.read_text()))
|
||||
|
||||
|
||||
def _scan(args: dict[str, Any]) -> dict[str, Any]:
|
||||
from .cli import _run_scan # lazy: avoids an import cycle with the CLI dispatcher
|
||||
|
||||
path = Path(_require(args, "path"))
|
||||
mode = str(args.get("mode", "quick"))
|
||||
if mode not in ("quick", "standard"):
|
||||
raise McpToolError(f"unsupported mode: {mode} (use quick or standard; deep is not exposed via MCP)")
|
||||
fail_on = str(args.get("fail_on", "never"))
|
||||
config = Path(str(args.get("config", "openultrasast.toml")))
|
||||
outcome = _run_scan(path, config, mode, fail_on)
|
||||
return {
|
||||
"scan_id": outcome.scan_id,
|
||||
"run_dir": str(outcome.run_dir),
|
||||
"mode": mode,
|
||||
"finding_count": outcome.finding_count,
|
||||
"exit_code": outcome.exit_code,
|
||||
}
|
||||
|
||||
|
||||
def _status(args: dict[str, Any]) -> dict[str, Any]:
|
||||
manifest = _read_json(Path(_require(args, "run_dir")) / "manifest.json")
|
||||
return {
|
||||
"scan_id": manifest.get("scan_id"),
|
||||
"target": manifest.get("target"),
|
||||
"finding_count": len(manifest.get("findings", [])),
|
||||
"score": (manifest.get("score") or {}).get("project_score"),
|
||||
"gate_passed": (manifest.get("score") or {}).get("gate", {}).get("passed"),
|
||||
"degradations": manifest.get("degradations", []),
|
||||
"fusion": manifest.get("fusion", []),
|
||||
}
|
||||
|
||||
|
||||
def _findings(args: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = _read_json(Path(_require(args, "run_dir")) / "findings.json")
|
||||
items = [
|
||||
{
|
||||
"finding_id": finding["finding_id"],
|
||||
"path": finding["path"],
|
||||
"line": finding.get("line"),
|
||||
"severity": finding["severity"],
|
||||
"title": finding["title"],
|
||||
}
|
||||
for finding in payload.get("findings", [])
|
||||
]
|
||||
return {"findings": items, "count": len(items)}
|
||||
|
||||
|
||||
def _load_finding(run_dir: Path, finding_id: str) -> dict[str, Any]:
|
||||
for finding in _read_json(run_dir / "findings.json").get("findings", []):
|
||||
if finding["finding_id"] == finding_id:
|
||||
return cast(dict[str, Any], finding)
|
||||
raise McpToolError(f"finding not found: {finding_id}")
|
||||
|
||||
|
||||
def _get_finding(args: dict[str, Any]) -> dict[str, Any]:
|
||||
run_dir = Path(_require(args, "run_dir"))
|
||||
return _load_finding(run_dir, str(_require(args, "finding_id")))
|
||||
|
||||
|
||||
def _load_verification(run_dir: Path, finding_id: str) -> dict[str, Any] | None:
|
||||
path = run_dir / "verification.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
for result in _read_json(path).get("verifications", []):
|
||||
if result["finding_id"] == finding_id:
|
||||
return cast(dict[str, Any], result)
|
||||
return None
|
||||
|
||||
|
||||
def _evidence(args: dict[str, Any]) -> dict[str, Any]:
|
||||
run_dir = Path(_require(args, "run_dir"))
|
||||
finding_id = str(_require(args, "finding_id"))
|
||||
finding = _load_finding(run_dir, finding_id)
|
||||
return {
|
||||
"finding_id": finding_id,
|
||||
"evidence_level": finding.get("evidence_level"),
|
||||
"reachability_status": finding.get("reachability_status"),
|
||||
"reachability_evidence": finding.get("reachability_evidence", []),
|
||||
"reachability_conditions": finding.get("reachability_conditions", []),
|
||||
"verification": _load_verification(run_dir, finding_id),
|
||||
}
|
||||
|
||||
|
||||
def _artifacts(args: dict[str, Any]) -> dict[str, Any]:
|
||||
manifest = _read_json(Path(_require(args, "run_dir")) / "manifest.json")
|
||||
return {"artifacts": manifest.get("artifacts", {})}
|
||||
|
||||
|
||||
def _benchmark(args: dict[str, Any]) -> dict[str, Any]:
|
||||
from .benchmark import create_benchmark_run, evaluate_benchmark, load_benchmark_manifest, load_findings, resolve_benchmark_source
|
||||
from .cli import _run_scan
|
||||
|
||||
manifest_path = Path(_require(args, "manifest"))
|
||||
if not manifest_path.is_file():
|
||||
raise McpToolError(f"benchmark manifest is not a file: {manifest_path}")
|
||||
mode = str(args.get("mode", "quick"))
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
run = create_benchmark_run(target, manifest)
|
||||
outcome = _run_scan(target, Path(str(args.get("config", "openultrasast.toml"))), mode, "never")
|
||||
result = evaluate_benchmark(
|
||||
run=run, mode=mode, findings=load_findings(outcome.run_dir / "findings.json"), scan_id=outcome.scan_id, scan_run_dir=outcome.run_dir
|
||||
)
|
||||
metrics = result.metrics
|
||||
return {
|
||||
"benchmark_run_id": run.benchmark_run_id,
|
||||
"expected": metrics.expected_findings_total,
|
||||
"matched": metrics.matched_findings_total,
|
||||
"missed": metrics.missed_findings_total,
|
||||
"false_positives": metrics.false_positive_findings_total,
|
||||
"recall": metrics.recall,
|
||||
}
|
||||
|
||||
|
||||
def _explain(args: dict[str, Any]) -> dict[str, Any]:
|
||||
run_dir = Path(_require(args, "run_dir"))
|
||||
finding_id = str(_require(args, "finding_id"))
|
||||
finding = _load_finding(run_dir, finding_id)
|
||||
verification = _load_verification(run_dir, finding_id)
|
||||
fusion_path = run_dir / "fusion.json"
|
||||
fusion = None
|
||||
if fusion_path.exists():
|
||||
fusion = next((decision for decision in _read_json_list(fusion_path) if decision["finding_id"] == finding_id), None)
|
||||
return {
|
||||
"finding_id": finding_id,
|
||||
"title": finding["title"],
|
||||
"severity": finding["severity"],
|
||||
"rationale": finding.get("rationale"),
|
||||
"reachability_status": finding.get("reachability_status"),
|
||||
"verification_status": verification["status"] if verification else "not_run",
|
||||
"pro_case": verification["pro_case"] if verification else None,
|
||||
"counter_case": verification["counter_case"] if verification else None,
|
||||
"fusion_disposition": fusion["disposition"] if fusion else None,
|
||||
}
|
||||
|
||||
|
||||
def _read_json_list(path: Path) -> list[dict[str, Any]]:
|
||||
data = json.loads(path.read_text())
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def _propose_patch(args: dict[str, Any]) -> dict[str, Any]:
|
||||
_require(args, "run_dir")
|
||||
_require(args, "finding_id")
|
||||
# The patch oracle (sandboxed fix authoring + validation) is a later phase and is
|
||||
# intentionally not exposed over MCP yet. Degrade visibly rather than silently.
|
||||
return {
|
||||
"status": "not_implemented",
|
||||
"reason": "the patch oracle is a later phase; patch proposal is not exposed via MCP yet",
|
||||
}
|
||||
|
||||
|
||||
def _export_report(args: dict[str, Any]) -> dict[str, Any]:
|
||||
run_dir = Path(_require(args, "run_dir"))
|
||||
fmt = str(args.get("format", "markdown"))
|
||||
filename = {"markdown": "report.md", "sarif": "report.sarif"}.get(fmt)
|
||||
if filename is None:
|
||||
raise McpToolError(f"unsupported report format: {fmt} (use markdown or sarif)")
|
||||
path = run_dir / filename
|
||||
if not path.exists():
|
||||
raise McpToolError(f"report artifact not found: {path}")
|
||||
return {"format": fmt, "path": str(path), "content": path.read_text()}
|
||||
|
||||
|
||||
def _build_tools() -> list[Tool]:
|
||||
run_dir_schema = {"run_dir": {"type": "string", "description": "scan run directory (from scan.run_dir)"}}
|
||||
finding_schema = {**run_dir_schema, "finding_id": {"type": "string"}}
|
||||
return [
|
||||
Tool(
|
||||
"openultrasast.scan",
|
||||
"Run a bounded scan on a local path (quick or standard mode). Never executes arbitrary shell.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
"mode": {"type": "string", "enum": ["quick", "standard"]},
|
||||
"fail_on": {"type": "string", "enum": ["never", "findings", "verified"]},
|
||||
"config": {"type": "string"},
|
||||
},
|
||||
"required": ["path"],
|
||||
},
|
||||
_scan,
|
||||
),
|
||||
Tool("openultrasast.status", "Summarize a scan's status from its manifest.", _obj(run_dir_schema, ["run_dir"]), _status),
|
||||
Tool("openultrasast.findings", "List the findings of a completed scan.", _obj(run_dir_schema, ["run_dir"]), _findings),
|
||||
Tool("openultrasast.get_finding", "Return one finding by id.", _obj(finding_schema, ["run_dir", "finding_id"]), _get_finding),
|
||||
Tool(
|
||||
"openultrasast.evidence",
|
||||
"Return reachability and verification evidence for a finding.",
|
||||
_obj(finding_schema, ["run_dir", "finding_id"]),
|
||||
_evidence,
|
||||
),
|
||||
Tool("openultrasast.artifacts", "List a scan's artifact paths.", _obj(run_dir_schema, ["run_dir"]), _artifacts),
|
||||
Tool(
|
||||
"openultrasast.benchmark",
|
||||
"Run a benchmark manifest and return recall/precision metrics.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"manifest": {"type": "string"}, "mode": {"type": "string"}, "config": {"type": "string"}},
|
||||
"required": ["manifest"],
|
||||
},
|
||||
_benchmark,
|
||||
),
|
||||
Tool(
|
||||
"openultrasast.explain",
|
||||
"Explain a finding (rationale, reachability, verification, fusion).",
|
||||
_obj(finding_schema, ["run_dir", "finding_id"]),
|
||||
_explain,
|
||||
),
|
||||
Tool(
|
||||
"openultrasast.propose_patch",
|
||||
"Propose a patch for a finding (patch oracle; not yet implemented).",
|
||||
_obj(finding_schema, ["run_dir", "finding_id"]),
|
||||
_propose_patch,
|
||||
),
|
||||
Tool(
|
||||
"openultrasast.export_report",
|
||||
"Export a scan report as markdown or sarif.",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"run_dir": {"type": "string"}, "format": {"type": "string", "enum": ["markdown", "sarif"]}},
|
||||
"required": ["run_dir"],
|
||||
},
|
||||
_export_report,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _obj(properties: dict[str, Any], required: list[str]) -> dict[str, Any]:
|
||||
return {"type": "object", "properties": properties, "required": required}
|
||||
|
||||
|
||||
class McpServer:
|
||||
"""Minimal MCP server: pure request dispatch over a narrow, fixed tool surface."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tools = {tool.name: tool for tool in _build_tools()}
|
||||
|
||||
@property
|
||||
def tool_names(self) -> list[str]:
|
||||
return sorted(self._tools)
|
||||
|
||||
def handle(self, request: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Dispatch one JSON-RPC request; returns the response, or None for notifications."""
|
||||
method = request.get("method")
|
||||
request_id = request.get("id")
|
||||
if request_id is None: # a notification (e.g. notifications/initialized) — no response
|
||||
return None
|
||||
try:
|
||||
result = self._dispatch(method, request.get("params") or {})
|
||||
except McpToolError as exc:
|
||||
return _error(request_id, -32602, str(exc))
|
||||
except Exception as exc: # noqa: BLE001 — surface as a JSON-RPC error, never crash the loop
|
||||
return _error(request_id, -32603, f"{type(exc).__name__}: {exc}")
|
||||
return {"jsonrpc": "2.0", "id": request_id, "result": result}
|
||||
|
||||
def _dispatch(self, method: str | None, params: dict[str, Any]) -> dict[str, Any]:
|
||||
if method == "initialize":
|
||||
return {
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": SERVER_NAME, "version": _server_version()},
|
||||
}
|
||||
if method == "ping":
|
||||
return {}
|
||||
if method == "tools/list":
|
||||
return {"tools": [tool.descriptor() for tool in self._tools.values()]}
|
||||
if method == "tools/call":
|
||||
return self._call_tool(params)
|
||||
raise McpToolError(f"unknown method: {method}")
|
||||
|
||||
def _call_tool(self, params: dict[str, Any]) -> dict[str, Any]:
|
||||
name = params.get("name")
|
||||
tool = self._tools.get(str(name))
|
||||
if tool is None:
|
||||
raise McpToolError(f"unknown tool: {name}")
|
||||
arguments = params.get("arguments") or {}
|
||||
try:
|
||||
result = tool.handler(arguments)
|
||||
except McpToolError as exc:
|
||||
return {"content": [{"type": "text", "text": str(exc)}], "isError": True}
|
||||
return {"content": [{"type": "text", "text": json.dumps(result, sort_keys=True)}], "isError": False}
|
||||
|
||||
|
||||
def _error(request_id: Any, code: int, message: str) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
|
||||
|
||||
|
||||
def serve(stdin: IO[str] | None = None, stdout: IO[str] | None = None) -> int:
|
||||
"""Run the MCP stdio loop (newline-delimited JSON-RPC) until EOF."""
|
||||
stdin = stdin or sys.stdin
|
||||
stdout = stdout or sys.stdout
|
||||
server = McpServer()
|
||||
for line in stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
request = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
stdout.write(json.dumps(_error(None, -32700, "parse error")) + "\n")
|
||||
stdout.flush()
|
||||
continue
|
||||
response = server.handle(request)
|
||||
if response is not None:
|
||||
stdout.write(json.dumps(response) + "\n")
|
||||
stdout.flush()
|
||||
return 0
|
||||
|
||||
|
||||
__all__ = ["McpServer", "McpToolError", "Tool", "serve"]
|
||||
@@ -0,0 +1,168 @@
|
||||
Flaw Category CWE ID CWE Name Flaw Severity Static Dynamic
|
||||
API Abuse 234 Failure to Handle Missing Parameter 3 X
|
||||
API Abuse 243 Creation of Chroot Jail Without Changing Working Directory 4 X
|
||||
API Abuse 245 J2EE Bad Practices: Direct Management of Connections 2 X
|
||||
API Abuse 560 Use of Umask() with Chmod-Style Argument 3 X
|
||||
API Abuse 628 Function Call with Incorrectly Specified Arguments 2 X
|
||||
API Abuse 675 Duplicate Operations on Resource 2 X
|
||||
Authentication Issues 287 Improper Authentication 4 X X
|
||||
Authentication Issues 352 Cross-Site Request Forgery (CSRF) 3 X X
|
||||
Authentication Issues 693 Protection Mechanism Failure 3 X X
|
||||
Authorization Issues 99 Improper Control of Resource Identifiers 3 X
|
||||
Authorization Issues 272 Least Privilege Violation 3 X
|
||||
Authorization Issues 273 Improper Check for Dropped Privileges 3 X
|
||||
Authorization Issues 274 Improper Handling of Insufficient Privileges 0 X
|
||||
Authorization Issues 282 Improper Ownership Management 3 X
|
||||
Authorization Issues 285 Improper Authorization 3 X X
|
||||
Authorization Issues 346 Origin Validation Error 3 X
|
||||
Authorization Issues 350 Reliance on Reverse DNS Resolution for a Security-Critical Action 3 X
|
||||
Authorization Issues 639 Authorization Bypass Through User-Controlled Key 4 X
|
||||
Authorization Issues 566 Authorization Bypass Through User-Controlled SQL Primary Key 3 X
|
||||
Authorization Issues 708 Incorrect Ownership Assignment 4 X
|
||||
Authorization Issues 732 Incorrect Permission Assignment for Critical Resource 3 X
|
||||
Authorization Issues 942 Permissive Cross-domain Policy with Untrusted Domains 3 X X
|
||||
Buffer Management Errors 118 Improper Access of Indexable Resource (Range Error) 3 X
|
||||
Buffer Management Errors 125 Out-of-Bounds Read 3 X
|
||||
Buffer Management Errors 129 Improper Validation of Array Index 3 X
|
||||
Buffer Management Errors 135 Incorrect Calculation of Multi-Byte String Length 5 X
|
||||
Buffer Management Errors 170 Improper Null Termination 3 X
|
||||
Buffer Management Errors 193 Off-by-One Error 3 X
|
||||
Buffer Management Errors 787 Out-of-Bounds Write 3 X
|
||||
Buffer Management Errors 823 Use of Out-of-Range Pointer Offset 3 X
|
||||
Buffer Management Errors 824 Access of Uninitialized Pointer 3 X
|
||||
Buffer Overflow 121 Stack-Based Buffer Overflow 5 X
|
||||
Code Injection 74 Improper Neutralization of Special Elements in Output Used by a Downstream Component (Injection) 4 X
|
||||
Code Injection 91 XML Injection (Blind XPath Injection) 3 X X
|
||||
Code Injection 94 Improper Control of Generation of Code 3 X
|
||||
Code Injection 95 Improper Neutralization of Directives in Dynamically Evaluated Code (‘Eval Injection’) 5 X X
|
||||
Code Injection 98 Improper Control of Filename for Include/Require Statement in PHP Program (PHP File Inclusion) 4 X X
|
||||
Code Injection 185 Incorrect Regular Expression 2 X
|
||||
Code Injection 830 Inclusion of Web Functionality from an Untrusted Source 2 X
|
||||
Code Quality 111 Direct Use of Unsafe JNI 4 X
|
||||
Code Quality 159 Failure to Sanitize Special Element 0 X
|
||||
Code Quality 401 Improper Release of Memory Before Removing Last Reference (Memory Leak) 2 X
|
||||
Code Quality 404 Improper Resource Shutdown or Release 0 X
|
||||
Code Quality 415 Double Free 3 X
|
||||
Code Quality 416 Use After Free 2 X
|
||||
Code Quality 477 Use of Obsolete Functions 0 X X
|
||||
Code Quality 479 Signal Handler Use of a Non-Reentrant Function 3 X
|
||||
Code Quality 489 Leftover Debug Code 3 X
|
||||
Code Quality 597 Use of Wrong Operator in String Comparison 2 X
|
||||
Command or Argument Injection 77 Improper Neutralization of Special Elements used in a Command (Command Injection) 5 X
|
||||
Command or Argument Injection 78 Improper Neutralization of Special Elements used in an OS Command (OS Command Injection) 5 X X
|
||||
Command or Argument Injection 88 Argument Injection or Modification 3 X
|
||||
Credentials Management 256 Plaintext Storage of a Password 3 X
|
||||
Credentials Management 259 Use of Hard-coded Password 3 X X
|
||||
Credentials Management 522 Insufficiently Protected Credentials 3 X X
|
||||
Credentials Management 798 Use of Hard-code Credentials 3 X
|
||||
CRLF Injection 93 Improper Neutralization of CRLF Sequences (CRLF Injection) 3 X
|
||||
CRLF Injection 113 Improper Neutralization of CRLF Sequences in HTTP Headers (HTTP Response Splitting) 3 X X
|
||||
CRLF Injection 117 Improper Output Neutralization for Logs 3 X
|
||||
Cross-Site Scripting (XSS) 79 Improper Neutralization of Input During Web Page Generation (Cross-site Scripting) 3 X X
|
||||
Cross-Site Scripting (XSS) 80 Improper Neutralization of Script-Related HTML Tags in a Web Page (Basic XSS) 3 X X
|
||||
Cross-Site Scripting (XSS) 83 Improper Neutralization of Script in Attributes in a Web Page 3 X X
|
||||
Cross-Site Scripting (XSS) 86 Improper Neutralization of Invalid Characters in Identifiers in Web Pages 3 X
|
||||
Cryptographic Issues 261 Weak Cryptography for Passwords 3 X
|
||||
Cryptographic Issues 295 Improper Certificate Validation 3 X
|
||||
Cryptographic Issues 296 Improper Following of Chain of Trust for Certificate Validation 3 X
|
||||
Cryptographic Issues 297 Improper Validation of Host-specific Certificate Data 3 X X
|
||||
Cryptographic Issues 298 Improper Validation of Certificate Expiration 3 X
|
||||
Cryptographic Issues 299 Improper Check for Certificate Revocation 3 X
|
||||
Cryptographic Issues 311 Missing Encryption of Sensitive Data 3 X
|
||||
Cryptographic Issues 312 Cleartext Storage of Sensitive Information 3 X
|
||||
Cryptographic Issues 313 Plaintext Storage in a File or on Disk 3 X
|
||||
Cryptographic Issues 316 Plaintext Storage in Memory 3 X
|
||||
Cryptographic Issues 319 Cleartext Transmission of Sensitive Information 3 X
|
||||
Cryptographic Issues 321 Use of Hard-coded Cryptographic Key 3 X X
|
||||
Cryptographic Issues 326 Inadequate Encryption Strength 3 X X
|
||||
Cryptographic Issues 327 Use of a Broken or Risky Cryptographic Algorithm 3 X X
|
||||
Cryptographic Issues 328 Reversible One-Way Hash 3 X
|
||||
Cryptographic Issues 329 Not Using a Random IV with CBC Mode 2 X
|
||||
Cryptographic Issues 330 Use of Insufficiently Random Values 3 X
|
||||
Cryptographic Issues 331 Insufficient Entropy 3 X
|
||||
Cryptographic Issues 338 Use of Cryptographically Weak Pseudo-Random Number Generator 3 X
|
||||
Cryptographic Issues 347 Improper Verification of Cryptographic Signature 2 X
|
||||
Cryptographic Issues 354 Improper Validation of Integrity Check Value 3 X
|
||||
Cryptographic Issues 547 Use of Hard-coded, Security-relevant Constants 3 X
|
||||
Cryptographic Issues 614 Sensitive Cookie in HTTPS Session Without Secure Attribute 2 X X
|
||||
Cryptographic Issues 760 Use of a One-Way Hash with a Predictable Salt 3 X
|
||||
Cryptographic Issues 780 Use of RSA with Optimal Asymmetric Encryption Padding 3 X
|
||||
Cryptographic Issues 916 Use of Password Hash With Insufficient Computational Effort 3 X
|
||||
Dangerous Functions 242 Use of Inherently Dangerous Function 5 X
|
||||
Dangerous Functions 676 Use of Potentially Dangerous Function 3 X
|
||||
Deployment Configuration 402 Transmission of Private Resources into a New Sphere (Resource Leak) 3 X
|
||||
Deployment Configuration 668 Exposure of Resource to Wrong Sphere 3 X X
|
||||
Deployment Configuration 926 Improper Export of Android Application Components 3 X
|
||||
Directory Traversal 22 Improper Limitation of a Pathname to a Restricted Directory (Path Traversal) 3 X X
|
||||
Directory Traversal 35 Path Traversal 2 X
|
||||
Directory Traversal 73 External Control of File Name or Path 3 X
|
||||
Encapsulation 494 Download of Code Without Integrity Check 5 X
|
||||
Encapsulation 501 Trust Boundary Violation 3 X
|
||||
Encapsulation 502 Deserialization of Untrusted Data 3 X
|
||||
Encapsulation 749 Exposed Dangerous Method or Function 4 X
|
||||
Error Handling 248 Uncaught Exception 2 X
|
||||
Error Handling 252 Unchecked Return Value 2 X
|
||||
Format String 134 Use of Externally-Controlled Format String 5 X
|
||||
Information Leakage 200 Information Exposure 2 X
|
||||
Information Leakage 201 Insertion of Sensitive Information Into Sent Data 2 X
|
||||
Information Leakage 209 Information Exposure Through an Error Message 2 X X
|
||||
Information Leakage 215 Information Exposure Through Debug Information 2 X X
|
||||
Information Leakage 359 Exposure of Private Information (Privacy Violation) 2 X
|
||||
Information Leakage 497 Exposure of System Data to an Unauthorized Control Sphere 2 X
|
||||
Information Leakage 526 Information Exposure Through Environmental Variables 2 X
|
||||
Information Leakage 530 Exposure of Backup File to an Unauthorized Control Sphere 2 X
|
||||
Information Leakage 532 Insertion of Sensitive Information into Log File 2 X
|
||||
Information Leakage 538 File and Directory Information Exposure 0 X
|
||||
Information Leakage 548 Information Exposure Through Directory Listing 2 X
|
||||
Information Leakage 611 Information Exposure Through XML External Entity Reference 3 X X
|
||||
Information Leakage 615 Information Exposure Through Comments 0 X X
|
||||
Information Leakage 665 Improper Initialization 2 X
|
||||
Information Leakage 918 Server-side Request Forgery 3 X X
|
||||
Insecure Dependencies 829 Inclusion of Functionality from Untrusted Control Sphere 3 X X
|
||||
Insufficient Input Validation 20 Improper Input Validation 0 X
|
||||
Insufficient Input Validation 90 Improper Neutralization of Special Elements Used in an LDAP Query (LDAP Injection) 3 X
|
||||
Insufficient Input Validation 103 Struts: Incomplete validate() Method Definition 3 X
|
||||
Insufficient Input Validation 104 Struts: Form Bean Does Not Extend Validation Class 3 X
|
||||
Insufficient Input Validation 112 Missing XML Validation 3 X
|
||||
Insufficient Input Validation 183 Permissive List of Allowed Inputs 3 X
|
||||
Insufficient Input Validation 345 Insufficient Verification of Data Authenticity 4 X
|
||||
Insufficient Input Validation 434 Unrestricted Upload of File with Dangerous Type 4 X
|
||||
Insufficient Input Validation 470 Use of Externally-Controlled Input to Select Classes or Code (Unsafe Reflection) 3 X
|
||||
Insufficient Input Validation 472 External Control of Assumed-Immutable Web Parameter 3 X
|
||||
Insufficient Input Validation 601 URL Redirection to Untrusted Site (Open Redirect) 3 X X
|
||||
Insufficient Input Validation 618 Exposed Unsafe ActiveX Method 5 X
|
||||
Insufficient Input Validation 915 Improperly Controlled Modification of Dynamically-Determined Object Attributes 3 X
|
||||
Insufficient Input Validation 1174 ASP.NET Misconfiguration: Improper Model Validation 2 X
|
||||
Insufficient Input Validation 1236 Improper Neutralization of Formula Elements in a CSV File 3 X
|
||||
Insufficient Logging & Monitoring 223 Omission of Security-relevant Information 2 X
|
||||
Numeric Errors 190 Integer Overflow or Wraparound 5 X
|
||||
Numeric Errors 191 Integer Underflow (Wrap or Wraparound) 3 X
|
||||
Numeric Errors 192 Integer Coercion Error 3 X
|
||||
Numeric Errors 195 Signed to Unsigned Conversion Error 3 X
|
||||
Numeric Errors 196 Unsigned to Signed Conversion Error 3 X
|
||||
Numeric Errors 197 Numeric Truncation Error 3 X
|
||||
Potential Backdoor 398 Indicator of Poor Code Quality 0 X
|
||||
Potential Backdoor 506 Embedded Malicious Code 4 X
|
||||
Potential Backdoor 511 Logic/Time Bomb 5 X
|
||||
Potential Backdoor 514 Covert Channel 2 X
|
||||
Potential Backdoor 656 Reliance on Security Through Obscurity 0 X
|
||||
Race Conditions 366 Race Condition within a Thread 3 X
|
||||
Race Conditions 367 Time-of-check Time-of-use (TOCTOU) Race Condition 3 X
|
||||
Race Conditions 421 Race Condition During Access to Alternate Channel 3 X
|
||||
Server Configuration 16 Configuration 0 X
|
||||
Server Configuration 441 Unintended Proxy or Intermediary (Confused Deputy) 3 X
|
||||
Server Configuration 642 External Control of Critical State Data 2 X
|
||||
Server Configuration 757 Selection of Less-Secure Algorithm During Negotiation (Algorithm Downgrade) 3 X X
|
||||
Session Fixation 384 Session Fixation 3 X X
|
||||
SQL Injection 89 Improper Neutralization of Special Elements used in an SQL Command (SQL Injection) 4 X
|
||||
SQL Injection 564 SQL Injection: Hibernate 4 X
|
||||
SQL Injection 943 Improper Neutralization of Special Elements in Data Query Logic 4 X
|
||||
Time and State 377 Insecure Temporary File 3 X
|
||||
Time and State 382 J2EE Bad Practices: Use of System.exit() 2 X
|
||||
Time and State 557 Concurrency Issues 2 X
|
||||
Time and State 691 Insufficient Control Flow Management 0 X
|
||||
Untrusted Initialization 15 External Control of System or Configuration Setting 4 X
|
||||
Untrusted Initialization 454 External Initialization of Trusted Variables or Data Stores 0 X
|
||||
Untrusted Search Path 114 Process Control 5 X
|
||||
Untrusted Search Path 426 Untrusted Search Path 3 X
|
||||
Untrusted Search Path 427 Uncontrolled Search Path Element 3 X
|
||||
|
@@ -0,0 +1,19 @@
|
||||
"""Central CWE severity/scope policy (vendored verycode-policies)."""
|
||||
|
||||
from .verycode import (
|
||||
CWE_SCORE_TSV,
|
||||
CwePolicy,
|
||||
PolicyError,
|
||||
assert_rules_resolve,
|
||||
load_policy,
|
||||
resolve_severity,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CWE_SCORE_TSV",
|
||||
"CwePolicy",
|
||||
"PolicyError",
|
||||
"assert_rules_resolve",
|
||||
"load_policy",
|
||||
"resolve_severity",
|
||||
]
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
CWE_SCORE_TSV = Path(__file__).with_name("CWE_Score.tsv")
|
||||
|
||||
|
||||
class PolicyError(ValueError):
|
||||
"""Raised when an enabled rule references a CWE the policy does not govern."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CwePolicy:
|
||||
"""Authoritative governance for one CWE.
|
||||
|
||||
``severity`` (0-5) is upstream-governed and overrides any rule-local string.
|
||||
``static`` gates SAST scope; ``dynamic``-only CWEs are report-only for a
|
||||
static tool and are never scored.
|
||||
"""
|
||||
|
||||
flaw_category: str
|
||||
severity: int
|
||||
static: bool
|
||||
dynamic: bool
|
||||
|
||||
|
||||
def load_policy(tsv: Path = CWE_SCORE_TSV) -> dict[str, CwePolicy]:
|
||||
"""Load the vendored CWE policy keyed by ``"CWE-NNN"``.
|
||||
|
||||
The source TSV uses CRLF line endings and a ``"Flaw Severity "`` header with a
|
||||
trailing space, so columns are read positionally and every cell is stripped.
|
||||
"""
|
||||
policy: dict[str, CwePolicy] = {}
|
||||
with tsv.open(newline="", encoding="utf-8") as handle:
|
||||
reader = csv.reader(handle, delimiter="\t")
|
||||
rows = list(reader)
|
||||
for row in rows[1:]:
|
||||
if len(row) < 5:
|
||||
continue
|
||||
cwe_id = row[1].strip()
|
||||
if not cwe_id.isdigit():
|
||||
continue
|
||||
severity = _parse_severity(row[3])
|
||||
policy[f"CWE-{cwe_id}"] = CwePolicy(
|
||||
flaw_category=row[0].strip(),
|
||||
severity=severity,
|
||||
static=_flag(row[4]),
|
||||
dynamic=_flag(row[5]) if len(row) > 5 else False,
|
||||
)
|
||||
return policy
|
||||
|
||||
|
||||
def resolve_severity(
|
||||
policy: Mapping[str, CwePolicy],
|
||||
cwe: str,
|
||||
target: object = None,
|
||||
ranking: object = None,
|
||||
) -> int:
|
||||
"""Resolve a finding's severity exclusively from policy, keyed on its CWE.
|
||||
|
||||
Policy always wins; any legacy rule-local severity string is discarded. An
|
||||
unmapped or non-static CWE resolves to 0 (report-only, not scored).
|
||||
"""
|
||||
pol = policy.get(cwe)
|
||||
if pol is None or not pol.static:
|
||||
return 0
|
||||
return pol.severity
|
||||
|
||||
|
||||
def assert_rules_resolve(rules: Iterable[object], policy: Mapping[str, CwePolicy]) -> None:
|
||||
"""Fail loud if any enabled rule's CWE is unmapped in the policy.
|
||||
|
||||
Rules expose ``cwe`` and (optionally) ``status``; only ``enabled`` rules are
|
||||
checked. Raises :class:`PolicyError` before any scan work proceeds.
|
||||
"""
|
||||
unmapped: list[str] = []
|
||||
for rule in rules:
|
||||
if getattr(rule, "status", "enabled") != "enabled":
|
||||
continue
|
||||
cwe = getattr(rule, "cwe", None)
|
||||
if not isinstance(cwe, str) or cwe not in policy:
|
||||
rule_id = getattr(rule, "rule_id", "<unknown>")
|
||||
unmapped.append(f"{rule_id} -> {cwe}")
|
||||
if unmapped:
|
||||
raise PolicyError("enabled rules reference CWEs absent from the policy: " + ", ".join(sorted(unmapped)))
|
||||
|
||||
|
||||
def _parse_severity(value: str) -> int:
|
||||
try:
|
||||
severity = int(value.strip())
|
||||
except ValueError:
|
||||
return 0
|
||||
return min(5, max(0, severity))
|
||||
|
||||
|
||||
def _flag(value: str) -> bool:
|
||||
return value.strip().upper() == "X"
|
||||
@@ -3,19 +3,56 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeVar, cast
|
||||
|
||||
|
||||
class OpenRouterError(RuntimeError):
|
||||
"""Raised when an OpenRouter request or response is invalid."""
|
||||
|
||||
|
||||
_TRANSIENT_STATUS = {429, 500, 502, 503, 504}
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def _is_transient(exc: Exception) -> bool:
|
||||
"""A network failure worth retrying (rate-limit, 5xx, connection error, timeout)."""
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
return exc.code in _TRANSIENT_STATUS
|
||||
return isinstance(exc, urllib.error.URLError | TimeoutError)
|
||||
|
||||
|
||||
def call_with_retry(
|
||||
operation: Callable[[], _T],
|
||||
*,
|
||||
attempts: int = 3,
|
||||
base_delay: float = 0.5,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> _T:
|
||||
"""Run ``operation`` with exponential backoff on transient network failures."""
|
||||
last: Exception | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return operation()
|
||||
except Exception as exc: # noqa: BLE001 — retry only transient failures; re-raise the rest
|
||||
last = exc
|
||||
if attempt + 1 < attempts and _is_transient(exc):
|
||||
sleep(base_delay * (2**attempt))
|
||||
continue
|
||||
raise
|
||||
raise last if last is not None else OpenRouterError("retry exhausted")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenRouterChatClient:
|
||||
api_key: str
|
||||
base_url: str = "https://openrouter.ai/api/v1"
|
||||
max_attempts: int = 3
|
||||
retry_base_delay: float = 0.5
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> OpenRouterChatClient:
|
||||
@@ -35,9 +72,13 @@ class OpenRouterChatClient:
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
|
||||
def _do() -> dict[str, object]:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
response_payload = json.loads(response.read().decode())
|
||||
return cast(dict[str, object], json.loads(response.read().decode()))
|
||||
|
||||
try:
|
||||
response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter chat request failed: {exc}") from exc
|
||||
|
||||
@@ -49,6 +90,8 @@ class OpenRouterChatClient:
|
||||
class OpenRouterEmbeddingClient:
|
||||
api_key: str
|
||||
base_url: str = "https://openrouter.ai/api/v1"
|
||||
max_attempts: int = 3
|
||||
retry_base_delay: float = 0.5
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> OpenRouterEmbeddingClient:
|
||||
@@ -68,9 +111,13 @@ class OpenRouterEmbeddingClient:
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
|
||||
def _do() -> dict[str, object]:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
response_payload = json.loads(response.read().decode())
|
||||
return cast(dict[str, object], json.loads(response.read().decode()))
|
||||
|
||||
try:
|
||||
response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter embedding request failed: {exc}") from exc
|
||||
return parse_embedding_response(response_payload)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Secret redaction for traces and reports (Phase 15 hardening).
|
||||
|
||||
Scan artifacts (trace events, reports) can quote source code that contains live
|
||||
credentials. This module masks well-known secret shapes before they are persisted, so
|
||||
a finding's evidence never leaks a key into an artifact an analyst later shares.
|
||||
Conservative by design: it targets recognizable secret formats, not arbitrary text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
REDACTED = "***REDACTED***"
|
||||
|
||||
# (compiled pattern, replacement). Replacements keep any non-secret prefix group so the
|
||||
# redaction is legible (e.g. `api_key = ***REDACTED***`, `Authorization: Bearer ***REDACTED***`).
|
||||
_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
# PEM private key blocks.
|
||||
(re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.DOTALL), REDACTED),
|
||||
# Authorization: Bearer <token>
|
||||
(re.compile(r"(Bearer\s+)[A-Za-z0-9._\-]{12,}"), r"\1" + REDACTED),
|
||||
# Provider API keys: OpenAI/Anthropic sk-..., AWS AKIA..., GitHub gh*_..., Slack xox*, Google AIza...
|
||||
(re.compile(r"\bsk-[A-Za-z0-9._\-]{16,}"), REDACTED),
|
||||
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), REDACTED),
|
||||
(re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), REDACTED),
|
||||
(re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}"), REDACTED),
|
||||
(re.compile(r"\bAIza[A-Za-z0-9_\-]{20,}"), REDACTED),
|
||||
# Credentials embedded in a URL: scheme://user:secret@host
|
||||
(re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s:/@]+:)[^\s:/@]{3,}(@)"), r"\1" + REDACTED + r"\2"),
|
||||
# Generic `key = "value"` assignments for sensitive names.
|
||||
(
|
||||
re.compile(r"(?i)\b(api[_-]?key|secret|token|password|passwd|pwd|access[_-]?key)\b(\s*[=:]\s*)(['\"]?)[A-Za-z0-9/+_\-]{8,}\3"),
|
||||
r"\1\2\3" + REDACTED + r"\3",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def redact_secrets(text: str) -> str:
|
||||
"""Return ``text`` with recognizable secrets masked. Idempotent and pattern-scoped."""
|
||||
for pattern, replacement in _PATTERNS:
|
||||
text = pattern.sub(replacement, text)
|
||||
return text
|
||||
|
||||
|
||||
__all__ = ["REDACTED", "redact_secrets"]
|
||||
@@ -9,7 +9,9 @@ from .run import ScanRun
|
||||
from .verification import VerificationResult
|
||||
|
||||
|
||||
def write_markdown_report(findings: list[StaticFinding], path: Path, verifications: list[VerificationResult] | None = None) -> None:
|
||||
def write_markdown_report(
|
||||
findings: list[StaticFinding], path: Path, verifications: list[VerificationResult] | None = None, *, redact: bool = True
|
||||
) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
verification_by_id = _verification_by_id(verifications or [])
|
||||
lines = ["# OpenUltraSAST Report", "", f"Findings: {len(findings)}", ""]
|
||||
@@ -38,7 +40,12 @@ def write_markdown_report(findings: list[StaticFinding], path: Path, verificatio
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines).rstrip() + "\n")
|
||||
report = "\n".join(lines).rstrip() + "\n"
|
||||
if redact:
|
||||
from .redaction import redact_secrets
|
||||
|
||||
report = redact_secrets(report)
|
||||
path.write_text(report)
|
||||
|
||||
|
||||
def write_sarif_report(findings: list[StaticFinding], verifications: list[VerificationResult], path: Path) -> None:
|
||||
@@ -71,10 +78,13 @@ def write_manifest(
|
||||
verifications: list[VerificationResult],
|
||||
artifact_paths: dict[str, Path],
|
||||
path: Path,
|
||||
score: dict[str, object] | None = None,
|
||||
degradations: list[dict[str, object]] | None = None,
|
||||
fusion: list[dict[str, object]] | None = None,
|
||||
) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
verification_by_id = _verification_by_id(verifications)
|
||||
payload = {
|
||||
payload: dict[str, object] = {
|
||||
"scan_id": run.scan_id,
|
||||
"target": str(run.target),
|
||||
"artifacts": {name: _relative_artifact(run.root, artifact) for name, artifact in sorted(artifact_paths.items())},
|
||||
@@ -93,6 +103,12 @@ def write_manifest(
|
||||
for finding in findings
|
||||
],
|
||||
}
|
||||
if score is not None:
|
||||
payload["score"] = score
|
||||
if degradations is not None:
|
||||
payload["degradations"] = degradations
|
||||
if fusion is not None:
|
||||
payload["fusion"] = fusion
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Central, versioned detection ruleset (rules as governed data)."""
|
||||
|
||||
from .store import (
|
||||
DEFAULT_RULESET_DIR,
|
||||
PatternRule,
|
||||
RulesetError,
|
||||
load_ruleset,
|
||||
read_rule_ledger,
|
||||
write_rule_ledger,
|
||||
write_ruleset,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_RULESET_DIR",
|
||||
"PatternRule",
|
||||
"RulesetError",
|
||||
"load_ruleset",
|
||||
"read_rule_ledger",
|
||||
"write_rule_ledger",
|
||||
"write_ruleset",
|
||||
]
|
||||
@@ -0,0 +1,71 @@
|
||||
[[rule]]
|
||||
rule_id = "c-exec-family"
|
||||
title = "C exec command execution needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(execl|execlp|execle|execv|execvp|execvpe)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "c-shell-exec"
|
||||
title = "C shell execution needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(system|popen)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "c-unsafe-copy"
|
||||
title = "Unsafe C string copy needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-121"
|
||||
tags = ["memory_unsafe"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(gets|strcpy|strcat|sprintf)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "c-unsafe-format"
|
||||
title = "Unsafe C format string needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-134"
|
||||
tags = ["memory_unsafe"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(printf|vprintf|vsprintf|vsnprintf|vfprintf|syslog)\s*\(\s*[A-Za-z_]'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "c-unsafe-memory-copy"
|
||||
title = "Unsafe C memory copy needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-121"
|
||||
tags = ["memory_unsafe"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(memcpy|memmove|bcopy)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "c-unsafe-scanf"
|
||||
title = "Unbounded C scanf needs review"
|
||||
languages = ["c", "cpp"]
|
||||
cwe = "CWE-121"
|
||||
tags = ["memory_unsafe"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(scanf|fscanf|sscanf)\s*\([^\n]*%s'''
|
||||
@@ -0,0 +1,11 @@
|
||||
[[rule]]
|
||||
rule_id = "groovy-xss-unescaped"
|
||||
title = "Groovy template unescaped output needs review"
|
||||
languages = ["groovy"]
|
||||
cwe = "CWE-79"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''(yieldUnescaped)\s+[A-Za-z_]'''
|
||||
@@ -0,0 +1,47 @@
|
||||
[[rule]]
|
||||
rule_id = "java-command-exec"
|
||||
title = "Java runtime command execution needs review"
|
||||
languages = ["java"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(exec)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "java-deserialization"
|
||||
title = "Java unsafe deserialization needs review"
|
||||
languages = ["java"]
|
||||
cwe = "CWE-502"
|
||||
tags = ["deserialization"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\.(readObject)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "java-sql-injection"
|
||||
title = "Java SQL string composition needs review"
|
||||
languages = ["java"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''(?i)\"\s*(select|insert|update|delete)\b[^\"]*\"\s*\+'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "java-weak-hash"
|
||||
title = "Java weak hash needs review"
|
||||
languages = ["java"]
|
||||
cwe = "CWE-327"
|
||||
tags = ["crypto"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''MessageDigest\.getInstance\s*\(\s*\"(MD5|SHA-1)\"'''
|
||||
@@ -0,0 +1,131 @@
|
||||
[[rule]]
|
||||
rule_id = "js-command-exec"
|
||||
title = "Node child_process command execution needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(exec|execSync)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-deserialize"
|
||||
title = "JavaScript unsafe deserialization needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-502"
|
||||
tags = ["deserialization"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(unserialize)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-dom-xss"
|
||||
title = "DOM sink with untrusted input needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-79"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\.(innerHTML|outerHTML)\s*=\s*[^\n]*\b(?:req|request|location|document\.location)'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-eval"
|
||||
title = "Dynamic JavaScript evaluation needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-95"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(eval)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-function-constructor"
|
||||
title = "JavaScript Function constructor needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-95"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''new\s+(Function)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-path-traversal"
|
||||
title = "Node filesystem access with external input needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-22"
|
||||
tags = ["filesystem_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\bfs\.(readFile|readFileSync|writeFile|writeFileSync|createReadStream|unlink|unlinkSync)\s*\(\s*[^\n]*\b(?:req|request)\.'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-spawn-shell"
|
||||
title = "Node spawn with shell needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(spawn|spawnSync)\s*\([^\n]*shell\s*:\s*true'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-sql-injection"
|
||||
title = "JavaScript SQL string composition needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\.(query|execute)\s*\(\s*[^\n]*(?:\$\{|\+\s*[A-Za-z_$])'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-ssrf"
|
||||
title = "JavaScript outbound request with external input needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-918"
|
||||
tags = ["network_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(axios|fetch|got)\s*[.(][^\n]*\b(?:req|request)\.'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-weak-hash"
|
||||
title = "JavaScript weak hash needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-327"
|
||||
tags = ["crypto"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''createHash\s*\(\s*[\"'](md5|sha1)[\"']'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "js-xss-response"
|
||||
title = "Unsanitized HTTP response reflection needs review"
|
||||
languages = ["javascript", "typescript"]
|
||||
cwe = "CWE-79"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\.(send|write|end)\s*\(\s*[^\n]*\b(?:req|request)\.(?:query|params|body)'''
|
||||
@@ -0,0 +1,119 @@
|
||||
[[rule]]
|
||||
rule_id = "python-flask-debug"
|
||||
title = "Flask debug mode enabled needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-489"
|
||||
tags = ["insecure_default"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\.run\s*\([^\n]*(debug)\s*=\s*True'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-os-command"
|
||||
title = "Python OS command execution needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(os\.system|os\.popen|os\.execl|os\.execlp|os\.execv|os\.execvp)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-path-traversal"
|
||||
title = "Python file open with external input needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-22"
|
||||
tags = ["filesystem_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(open)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\(|params\[|GET\[|POST\[)'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-shell-true"
|
||||
title = "Subprocess shell execution needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-78"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''subprocess\.[a-zA-Z_]+\s*\([^\n]*shell\s*=\s*True'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-sql-injection"
|
||||
title = "Python SQL string composition needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-89"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''(\.execute)\s*\(\s*[^\n]*(?:[\"']\s*%\s*[(A-Za-z_]|\.format\s*\(|[\"']\s*\+|\bf[\"'])'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-ssrf"
|
||||
title = "Python outbound request with external input needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-918"
|
||||
tags = ["network_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(requests\.(?:get|post|put|delete|head|patch)|urllib\.request\.urlopen|urlopen)\s*\(\s*[^\n]*(?:request\.|req\.|argv|input\s*\()'''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-ssti"
|
||||
title = "Python server-side template injection needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-94"
|
||||
tags = ["injection"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(render_template_string)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-unsafe-deserialization"
|
||||
title = "Unsafe Python deserialization needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-502"
|
||||
tags = ["deserialization"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(pickle\.loads|pickle\.load|cPickle\.loads|yaml\.load|marshal\.loads)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-unsafe-eval"
|
||||
title = "Dynamic Python execution needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-95"
|
||||
tags = ["syscall_entry"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\b(eval|exec)\s*\('''
|
||||
|
||||
[[rule]]
|
||||
rule_id = "python-weak-hash"
|
||||
title = "Python weak hash needs review"
|
||||
languages = ["python"]
|
||||
cwe = "CWE-327"
|
||||
tags = ["crypto"]
|
||||
status = "enabled"
|
||||
min_evidence_level = "static_corroboration"
|
||||
precision_estimate = 0.0
|
||||
version = "1"
|
||||
pattern = '''\bhashlib\.(md5|sha1)\s*\('''
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tomllib
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_RULESET_DIR = Path(__file__).parent
|
||||
VALID_STATUS = ("enabled", "shadow", "disabled")
|
||||
|
||||
|
||||
class RulesetError(ValueError):
|
||||
"""Raised when a ruleset data file is malformed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PatternRule:
|
||||
"""A detection rule expressed as governed data (no rule-local severity).
|
||||
|
||||
``pattern`` is human-PR-only; the self-improvement loop may only change
|
||||
``status``/``min_evidence_level``/``precision_estimate`` via the ledger.
|
||||
"""
|
||||
|
||||
rule_id: str
|
||||
title: str
|
||||
languages: tuple[str, ...]
|
||||
cwe: str
|
||||
tags: tuple[str, ...]
|
||||
pattern: str
|
||||
status: str = "enabled"
|
||||
min_evidence_level: str = "static_corroboration"
|
||||
precision_estimate: float = 0.0
|
||||
version: str = "1"
|
||||
|
||||
|
||||
def load_ruleset(directory: Path = DEFAULT_RULESET_DIR, ledger: Path | None = None) -> tuple[PatternRule, ...]:
|
||||
"""Load every ``*.toml`` rule file under ``directory`` and apply the loop ledger.
|
||||
|
||||
Rules are returned sorted by ``rule_id`` for determinism. The optional ledger
|
||||
(``rule_policy.json``) overlays loop-owned ``status``/``min_evidence_level``/
|
||||
``precision_estimate`` per ``rule_id``.
|
||||
"""
|
||||
rules: dict[str, PatternRule] = {}
|
||||
for path in sorted(directory.rglob("*.toml")):
|
||||
payload = tomllib.loads(path.read_text())
|
||||
for item in payload.get("rule", []):
|
||||
rule = _rule_from_dict(item, path)
|
||||
if rule.rule_id in rules:
|
||||
raise RulesetError(f"duplicate rule_id {rule.rule_id!r} in {path}")
|
||||
rules[rule.rule_id] = rule
|
||||
overlay = _load_ledger(ledger)
|
||||
resolved = [_apply_overlay(rule, overlay.get(rule.rule_id)) for rule in rules.values()]
|
||||
return tuple(sorted(resolved, key=lambda rule: rule.rule_id))
|
||||
|
||||
|
||||
def write_ruleset(path: Path, rules: Iterable[PatternRule]) -> None:
|
||||
"""Write rules as TOML. Patterns use literal multi-line strings (no escaping)."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
blocks: list[str] = []
|
||||
for rule in rules:
|
||||
if "'''" in rule.pattern:
|
||||
raise RulesetError(f"rule {rule.rule_id} pattern cannot contain a triple single-quote")
|
||||
blocks.append(
|
||||
"\n".join(
|
||||
[
|
||||
"[[rule]]",
|
||||
f"rule_id = {_s(rule.rule_id)}",
|
||||
f"title = {_s(rule.title)}",
|
||||
f"languages = {_arr(rule.languages)}",
|
||||
f"cwe = {_s(rule.cwe)}",
|
||||
f"tags = {_arr(rule.tags)}",
|
||||
f"status = {_s(rule.status)}",
|
||||
f"min_evidence_level = {_s(rule.min_evidence_level)}",
|
||||
f"precision_estimate = {float(rule.precision_estimate)}",
|
||||
f"version = {_s(rule.version)}",
|
||||
f"pattern = '''{rule.pattern}'''",
|
||||
]
|
||||
)
|
||||
)
|
||||
path.write_text("\n\n".join(blocks) + "\n")
|
||||
|
||||
|
||||
def read_rule_ledger(path: Path) -> dict[str, dict[str, object]]:
|
||||
"""Read the loop-owned ``rule_policy.json`` overlay (status/floors per rule_id)."""
|
||||
return _load_ledger(path)
|
||||
|
||||
|
||||
def write_rule_ledger(path: Path, entries: dict[str, dict[str, object]]) -> None:
|
||||
"""Write the loop-owned ``rule_policy.json`` overlay (used by the evolve loop)."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(entries, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _rule_from_dict(item: dict[str, object], path: Path) -> PatternRule:
|
||||
try:
|
||||
rule_id = str(item["rule_id"])
|
||||
cwe = str(item["cwe"])
|
||||
pattern = str(item["pattern"])
|
||||
except KeyError as exc:
|
||||
raise RulesetError(f"rule in {path} missing required field {exc}") from exc
|
||||
status = str(item.get("status", "enabled"))
|
||||
if status not in VALID_STATUS:
|
||||
raise RulesetError(f"rule {rule_id} has invalid status {status!r}")
|
||||
if not pattern:
|
||||
raise RulesetError(f"rule {rule_id} has an empty pattern")
|
||||
return PatternRule(
|
||||
rule_id=rule_id,
|
||||
title=str(item.get("title", rule_id)),
|
||||
languages=_str_tuple(item.get("languages")),
|
||||
cwe=cwe,
|
||||
tags=_str_tuple(item.get("tags")),
|
||||
pattern=pattern,
|
||||
status=status,
|
||||
min_evidence_level=str(item.get("min_evidence_level", "static_corroboration")),
|
||||
precision_estimate=float(item.get("precision_estimate", 0.0)), # type: ignore[arg-type]
|
||||
version=str(item.get("version", "1")),
|
||||
)
|
||||
|
||||
|
||||
def _load_ledger(ledger: Path | None) -> dict[str, dict[str, object]]:
|
||||
if ledger is None or not ledger.exists():
|
||||
return {}
|
||||
payload = json.loads(ledger.read_text())
|
||||
return {str(key): value for key, value in payload.items() if isinstance(value, dict)}
|
||||
|
||||
|
||||
def _apply_overlay(rule: PatternRule, overlay: dict[str, object] | None) -> PatternRule:
|
||||
if not overlay:
|
||||
return rule
|
||||
status = str(overlay.get("status", rule.status))
|
||||
if status not in VALID_STATUS:
|
||||
status = rule.status
|
||||
return replace(
|
||||
rule,
|
||||
status=status,
|
||||
min_evidence_level=str(overlay.get("min_evidence_level", rule.min_evidence_level)),
|
||||
precision_estimate=float(overlay.get("precision_estimate", rule.precision_estimate)), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
def _str_tuple(value: object) -> tuple[str, ...]:
|
||||
if isinstance(value, list):
|
||||
return tuple(str(item) for item in value)
|
||||
return ()
|
||||
|
||||
|
||||
def _s(value: str) -> str:
|
||||
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def _arr(values: tuple[str, ...]) -> str:
|
||||
return "[" + ", ".join(_s(value) for value in values) + "]"
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Policy-weighted project scoring (verycode-style AppSec score)."""
|
||||
|
||||
from .project_score import (
|
||||
REACH_MULT,
|
||||
SEV_WEIGHT,
|
||||
ScoreArtifact,
|
||||
build_score_artifact,
|
||||
finding_penalty,
|
||||
gate,
|
||||
project_score,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"REACH_MULT",
|
||||
"SEV_WEIGHT",
|
||||
"ScoreArtifact",
|
||||
"build_score_artifact",
|
||||
"finding_penalty",
|
||||
"gate",
|
||||
"project_score",
|
||||
]
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
from ..findings import StaticFinding
|
||||
from ..policy import CwePolicy
|
||||
|
||||
# Penalty weight per policy severity (0-5). Higher severity costs the score more.
|
||||
SEV_WEIGHT = {5: 50, 4: 25, 3: 10, 2: 2, 1: 1, 0: 0}
|
||||
|
||||
# Reachability multiplier — the false-positive calibration knob. A confirmed FP
|
||||
# lowers a finding's effective reachability rather than deleting the rule.
|
||||
REACH_MULT = {"reachable": 1.0, "inferred-file-surface": 0.6, "unknown": 0.4}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoreArtifact:
|
||||
project_score: int
|
||||
max_severity: int
|
||||
penalty_total: float
|
||||
by_category: dict[str, int]
|
||||
out_of_scope_dynamic_only: int
|
||||
unmapped_cwe: list[str]
|
||||
gate: dict[str, object]
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def finding_penalty(
|
||||
finding: StaticFinding,
|
||||
rule_cwe: str | None,
|
||||
policy: Mapping[str, CwePolicy],
|
||||
reachability_override: Mapping[str, float] | None = None,
|
||||
) -> float:
|
||||
"""Penalty contributed by one finding: severity weight × reachability multiplier.
|
||||
|
||||
Dynamic-only or unmapped CWEs contribute zero (report-only, never scored). A
|
||||
confirmed false positive can supply a lowered effective reachability multiplier
|
||||
via ``reachability_override`` (keyed by ``finding_id``) instead of deleting the rule.
|
||||
"""
|
||||
pol = policy.get(rule_cwe) if rule_cwe else None
|
||||
if pol is None or not pol.static:
|
||||
return 0.0
|
||||
if reachability_override and finding.finding_id in reachability_override:
|
||||
multiplier = reachability_override[finding.finding_id]
|
||||
else:
|
||||
multiplier = REACH_MULT.get(finding.reachability_status, REACH_MULT["unknown"])
|
||||
return SEV_WEIGHT[pol.severity] * multiplier
|
||||
|
||||
|
||||
def project_score(
|
||||
findings: list[StaticFinding],
|
||||
rule_cwe_by_id: Mapping[str, str],
|
||||
policy: Mapping[str, CwePolicy],
|
||||
k: float = 60.0,
|
||||
reachability_override: Mapping[str, float] | None = None,
|
||||
) -> int:
|
||||
"""Compute the 0-100 project score (shadow findings excluded by the caller).
|
||||
|
||||
Exponential decay of total penalty: no penalty → 100; one severity-5 reachable
|
||||
finding → ~43.
|
||||
"""
|
||||
total = sum(finding_penalty(finding, rule_cwe_by_id.get(finding.finding_id), policy, reachability_override) for finding in findings)
|
||||
return round(100 * math.exp(-total / k))
|
||||
|
||||
|
||||
def gate(
|
||||
findings: list[StaticFinding],
|
||||
score: int,
|
||||
rule_cwe_by_id: Mapping[str, str],
|
||||
policy: Mapping[str, CwePolicy],
|
||||
*,
|
||||
min_score: int,
|
||||
block_severity_reachable: int = 5,
|
||||
blocking: bool,
|
||||
) -> dict[str, object]:
|
||||
"""Two-condition gate verdict.
|
||||
|
||||
A finding at ``block_severity_reachable`` severity that is ``reachable`` always
|
||||
fails. The score threshold fails only when ``blocking`` is enabled.
|
||||
"""
|
||||
hard_fail = any(
|
||||
_severity(rule_cwe_by_id.get(finding.finding_id), policy) >= block_severity_reachable and finding.reachability_status == "reachable"
|
||||
for finding in findings
|
||||
)
|
||||
score_fail = blocking and score < min_score
|
||||
return {
|
||||
"min_score": min_score,
|
||||
"block_severity_reachable": block_severity_reachable,
|
||||
"blocking": blocking,
|
||||
"passed": not hard_fail and not score_fail,
|
||||
}
|
||||
|
||||
|
||||
def build_score_artifact(
|
||||
findings: list[StaticFinding],
|
||||
rule_cwe_by_id: Mapping[str, str],
|
||||
policy: Mapping[str, CwePolicy],
|
||||
*,
|
||||
k: float = 60.0,
|
||||
min_score: int = 80,
|
||||
block_severity_reachable: int = 5,
|
||||
blocking: bool = False,
|
||||
reachability_override: Mapping[str, float] | None = None,
|
||||
) -> ScoreArtifact:
|
||||
score = project_score(findings, rule_cwe_by_id, policy, k=k, reachability_override=reachability_override)
|
||||
penalty_total = 0.0
|
||||
max_severity = 0
|
||||
by_category: dict[str, int] = {}
|
||||
out_of_scope_dynamic_only = 0
|
||||
unmapped: set[str] = set()
|
||||
for finding in findings:
|
||||
cwe = rule_cwe_by_id.get(finding.finding_id)
|
||||
pol = policy.get(cwe) if cwe else None
|
||||
if pol is None:
|
||||
unmapped.add(cwe or "unknown")
|
||||
continue
|
||||
if not pol.static:
|
||||
if pol.dynamic:
|
||||
out_of_scope_dynamic_only += 1
|
||||
continue
|
||||
penalty_total += finding_penalty(finding, cwe, policy, reachability_override)
|
||||
max_severity = max(max_severity, pol.severity)
|
||||
by_category[pol.flaw_category] = by_category.get(pol.flaw_category, 0) + 1
|
||||
return ScoreArtifact(
|
||||
project_score=score,
|
||||
max_severity=max_severity,
|
||||
penalty_total=round(penalty_total, 2),
|
||||
by_category=dict(sorted(by_category.items())),
|
||||
out_of_scope_dynamic_only=out_of_scope_dynamic_only,
|
||||
unmapped_cwe=sorted(unmapped),
|
||||
gate=gate(
|
||||
findings,
|
||||
score,
|
||||
rule_cwe_by_id,
|
||||
policy,
|
||||
min_score=min_score,
|
||||
block_severity_reachable=block_severity_reachable,
|
||||
blocking=blocking,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _severity(cwe: str | None, policy: Mapping[str, CwePolicy]) -> int:
|
||||
pol = policy.get(cwe) if cwe else None
|
||||
if pol is None or not pol.static:
|
||||
return 0
|
||||
return pol.severity
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Trail of Bits skill index + router (Phase 14).
|
||||
|
||||
Security skills are scoped expertise, not global prompt bloat. Each skill is a small
|
||||
descriptor with routing metadata (languages, tags, vulnerability classes, stages) and
|
||||
a short operating checklist. The router selects the most relevant skills for a target
|
||||
and stage; ``select_skill_context`` packs their checklists into a char budget; and
|
||||
``skill_chunks`` chunks the guidance into the ``skills`` retrieval namespace. Nothing
|
||||
here imports a model or the optional extra — it is pure routing data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .index import CodeChunk, chunk_text_namespace
|
||||
from .preprocess import FileTarget
|
||||
|
||||
# Pipeline stages a skill can apply to.
|
||||
STAGES = ("map", "hunt", "verify", "fix")
|
||||
|
||||
# FileTarget tags → vulnerability classes the router reasons over.
|
||||
TAG_TO_VULN = {
|
||||
"memory_unsafe": "memory-safety",
|
||||
"parser": "parsing",
|
||||
"fuzzable": "parsing",
|
||||
"crypto": "crypto",
|
||||
"deserialization": "deserialization",
|
||||
"auth_boundary": "access-control",
|
||||
"network_entry": "access-control",
|
||||
"privileged": "access-control",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillDescriptor:
|
||||
id: str
|
||||
title: str
|
||||
summary: str
|
||||
checklist: str
|
||||
languages: tuple[str, ...] = () # empty => language-agnostic
|
||||
tags: tuple[str, ...] = ()
|
||||
vulnerability_classes: tuple[str, ...] = ()
|
||||
stages: tuple[str, ...] = () # empty => any stage
|
||||
|
||||
def descriptor(self) -> dict[str, object]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"title": self.title,
|
||||
"summary": self.summary,
|
||||
"languages": list(self.languages),
|
||||
"tags": list(self.tags),
|
||||
"vulnerability_classes": list(self.vulnerability_classes),
|
||||
"stages": list(self.stages),
|
||||
}
|
||||
|
||||
|
||||
# Curated inventory aligned with the spec's Trail of Bits skill routing examples.
|
||||
SKILL_INVENTORY: tuple[SkillDescriptor, ...] = (
|
||||
SkillDescriptor(
|
||||
"c-review",
|
||||
"C/C++ security review",
|
||||
"Manual review discipline for memory-unsafe code.",
|
||||
"Audit buffer indexing, object lifetime, integer overflow before length math, and unchecked copies.",
|
||||
languages=("c", "cpp", "c++"),
|
||||
tags=("memory_unsafe",),
|
||||
vulnerability_classes=("memory-safety",),
|
||||
stages=("map", "hunt"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"address-sanitizer",
|
||||
"AddressSanitizer",
|
||||
"Detect memory-safety violations at runtime.",
|
||||
"Build with -fsanitize=address; reproduce the overflow/UAF; capture the ASan trace as crash evidence.",
|
||||
languages=("c", "cpp", "c++"),
|
||||
tags=("memory_unsafe",),
|
||||
vulnerability_classes=("memory-safety",),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"libfuzzer",
|
||||
"libFuzzer harnessing",
|
||||
"In-process coverage-guided fuzzing of C/C++ entries.",
|
||||
"Write a LLVMFuzzerTestOneInput harness over the parser entry; minimize crashing inputs.",
|
||||
languages=("c", "cpp", "c++"),
|
||||
tags=("memory_unsafe", "fuzzable", "parser"),
|
||||
vulnerability_classes=("memory-safety", "parsing"),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"harness-writing",
|
||||
"Fuzz harness writing",
|
||||
"Build deterministic harnesses around attacker-controlled entries.",
|
||||
"Identify the smallest attacker-controlled entry; build a deterministic harness; seed a corpus.",
|
||||
languages=("c", "cpp", "c++"),
|
||||
tags=("memory_unsafe", "fuzzable", "parser"),
|
||||
vulnerability_classes=("memory-safety", "parsing"),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"fuzzing-dictionary",
|
||||
"Fuzzing dictionary",
|
||||
"Reach deep parser states with a format dictionary.",
|
||||
"Extract format tokens/magic bytes into a fuzzing dictionary to reach deep parser states.",
|
||||
tags=("parser", "fuzzable"),
|
||||
vulnerability_classes=("parsing",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"fuzzing-obstacles",
|
||||
"Fuzzing obstacles",
|
||||
"Remove coverage barriers that block the fuzzer.",
|
||||
"Remove checksums/magic gates that block coverage; document each bypass.",
|
||||
tags=("parser", "fuzzable"),
|
||||
vulnerability_classes=("parsing",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"atheris",
|
||||
"Atheris (Python fuzzing)",
|
||||
"Coverage-guided fuzzing for Python parsers.",
|
||||
"Fuzz the Python parser entry with Atheris; assert on uncaught exceptions and resource blowups.",
|
||||
languages=("python",),
|
||||
tags=("parser", "fuzzable"),
|
||||
vulnerability_classes=("parsing",),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"property-based-testing",
|
||||
"Property-based testing",
|
||||
"Encode parser invariants as properties.",
|
||||
"Encode parser invariants as Hypothesis properties; shrink counterexamples.",
|
||||
languages=("python",),
|
||||
tags=("parser",),
|
||||
vulnerability_classes=("parsing",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"constant-time-analysis",
|
||||
"Constant-time analysis",
|
||||
"Find secret-dependent timing in crypto code.",
|
||||
"Find secret-dependent branches/indexing/early-exit comparisons; require constant-time primitives.",
|
||||
tags=("crypto",),
|
||||
vulnerability_classes=("crypto",),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"wycheproof",
|
||||
"Wycheproof vectors",
|
||||
"Stress crypto usage with known-edge test vectors.",
|
||||
"Run Wycheproof test vectors against the crypto usage; flag edge-case acceptance.",
|
||||
tags=("crypto",),
|
||||
vulnerability_classes=("crypto",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"vector-forge",
|
||||
"Crypto vector forging",
|
||||
"Forge boundary vectors to probe crypto APIs.",
|
||||
"Forge boundary test vectors (zero/identity/malleable signatures) to probe the crypto API.",
|
||||
tags=("crypto",),
|
||||
vulnerability_classes=("crypto",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"zeroize-audit",
|
||||
"Secret zeroization audit",
|
||||
"Verify secrets are wiped after use.",
|
||||
"Verify secrets are zeroized on drop; check for lingering copies.",
|
||||
languages=("rust",),
|
||||
tags=("crypto",),
|
||||
vulnerability_classes=("crypto",),
|
||||
stages=("verify",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"cargo-fuzz",
|
||||
"cargo-fuzz",
|
||||
"Coverage-guided fuzzing for Rust entries.",
|
||||
"Add a cargo-fuzz target over the entry; run under ASan.",
|
||||
languages=("rust",),
|
||||
tags=("fuzzable", "parser"),
|
||||
vulnerability_classes=("memory-safety", "parsing"),
|
||||
stages=("hunt", "verify"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"sarif-parsing",
|
||||
"SARIF parsing",
|
||||
"Normalize external tool output into the map.",
|
||||
"Normalize SARIF locations/rules/fingerprints into FileTarget static hints.",
|
||||
vulnerability_classes=(),
|
||||
stages=("map",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"semgrep",
|
||||
"Semgrep mapping",
|
||||
"Pattern sinks and iterate variants.",
|
||||
"Pattern the sink shape; iterate variants; export normalized findings.",
|
||||
vulnerability_classes=("injection", "deserialization"),
|
||||
stages=("map",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"semgrep-rule-creator",
|
||||
"Semgrep rule creator",
|
||||
"Author a Semgrep rule for a sink.",
|
||||
"Author a Semgrep rule for the sink; validate against the fixture.",
|
||||
tags=("deserialization",),
|
||||
vulnerability_classes=("injection", "deserialization"),
|
||||
stages=("map",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"semgrep-rule-variant-creator",
|
||||
"Semgrep variant creator",
|
||||
"Widen sink coverage with variants.",
|
||||
"Generate sink variants (aliases, wrappers) to widen coverage without false positives.",
|
||||
vulnerability_classes=("injection",),
|
||||
stages=("map",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"codeql",
|
||||
"CodeQL dataflow",
|
||||
"Model source→sink taint paths.",
|
||||
"Model source→sink taint with a CodeQL query; inspect the dataflow path.",
|
||||
vulnerability_classes=("injection",),
|
||||
stages=("map",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"differential-review",
|
||||
"Differential review",
|
||||
"Scope the changed attack surface in a diff.",
|
||||
"Scope the changed attack surface and new trust-boundary crossings in the diff.",
|
||||
stages=("map", "fix"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"graph-evolution",
|
||||
"Graph evolution",
|
||||
"Bound a patch's blast radius.",
|
||||
"Trace how the call/dataflow graph changed; bound the patch blast radius.",
|
||||
stages=("fix",),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"sharp-edges",
|
||||
"Sharp edges review",
|
||||
"Map misuse-prone APIs at the boundary.",
|
||||
"Map misuse-prone APIs, insecure defaults, and confusing configuration at the boundary.",
|
||||
tags=("auth_boundary", "network_entry"),
|
||||
vulnerability_classes=("access-control", "api-misuse"),
|
||||
stages=("map", "hunt"),
|
||||
),
|
||||
SkillDescriptor(
|
||||
"insecure-defaults",
|
||||
"Insecure defaults review",
|
||||
"Flag unsafe defaults exposed at entries.",
|
||||
"Flag unsafe default flags/permissions/timeouts exposed at the entry.",
|
||||
tags=("auth_boundary", "network_entry"),
|
||||
vulnerability_classes=("access-control", "api-misuse"),
|
||||
stages=("map", "hunt"),
|
||||
),
|
||||
)
|
||||
|
||||
_BY_ID = {skill.id: skill for skill in SKILL_INVENTORY}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillMatch:
|
||||
skill: SkillDescriptor
|
||||
score: int
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def route_skills(
|
||||
*,
|
||||
language: str | None = None,
|
||||
tags: tuple[str, ...] = (),
|
||||
vulnerability_classes: tuple[str, ...] = (),
|
||||
stage: str | None = None,
|
||||
max_snippets: int = 4,
|
||||
) -> list[SkillDescriptor]:
|
||||
"""Select the most relevant skills for the given routing signals.
|
||||
|
||||
A skill is excluded when its language list is non-empty and ``language`` is not in
|
||||
it, or its stage list is non-empty and ``stage`` is not in it. Eligibility requires
|
||||
at least one topical signal (tag/vulnerability-class/stage) — a pure language match
|
||||
never selects a skill. Ranked by score, then id, for determinism.
|
||||
"""
|
||||
tag_set, vuln_set = set(tags), set(vulnerability_classes)
|
||||
matches: list[SkillMatch] = []
|
||||
for skill in SKILL_INVENTORY:
|
||||
if skill.languages and language is not None and language not in skill.languages:
|
||||
continue
|
||||
if skill.stages and stage is not None and stage not in skill.stages:
|
||||
continue
|
||||
signal = 2 * len(vuln_set & set(skill.vulnerability_classes)) + len(tag_set & set(skill.tags))
|
||||
if stage is not None and stage in skill.stages:
|
||||
signal += 1
|
||||
if signal == 0:
|
||||
continue # no topical match — do not select on language alone
|
||||
rank = signal + (3 if skill.languages and language in skill.languages else 0)
|
||||
matches.append(SkillMatch(skill=skill, score=rank))
|
||||
matches.sort(key=lambda match: (-match.score, match.skill.id))
|
||||
return [match.skill for match in matches[:max_snippets]]
|
||||
|
||||
|
||||
def _vuln_classes_for_tags(tags: tuple[str, ...]) -> tuple[str, ...]:
|
||||
return tuple(sorted({TAG_TO_VULN[tag] for tag in tags if tag in TAG_TO_VULN}))
|
||||
|
||||
|
||||
def route_skills_for_target(target: FileTarget, *, stage: str | None = None, max_snippets: int = 4) -> list[SkillDescriptor]:
|
||||
"""Route skills for a scan target (language + tags + derived vulnerability classes)."""
|
||||
tags = tuple(target.tags)
|
||||
return route_skills(
|
||||
language=target.language,
|
||||
tags=tags,
|
||||
vulnerability_classes=_vuln_classes_for_tags(tags),
|
||||
stage=stage,
|
||||
max_snippets=max_snippets,
|
||||
)
|
||||
|
||||
|
||||
def route_skill_ids(target: FileTarget, *, stage: str | None = None, max_snippets: int = 4) -> list[str]:
|
||||
return [skill.id for skill in route_skills_for_target(target, stage=stage, max_snippets=max_snippets)]
|
||||
|
||||
|
||||
def select_skill_context(target: FileTarget, *, stage: str | None = None, budget_chars: int, max_snippets: int = 4) -> str:
|
||||
"""Pack routed skill checklists into a prompt context capped at ``budget_chars``."""
|
||||
if budget_chars <= 0:
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
used = 0
|
||||
for skill in route_skills_for_target(target, stage=stage, max_snippets=max_snippets):
|
||||
snippet = f"### {skill.title}\n{skill.checklist}"
|
||||
if not parts and len(snippet) > budget_chars:
|
||||
return snippet[:budget_chars] # first snippet alone exceeds the cap → truncate
|
||||
if used + len(snippet) + (1 if parts else 0) > budget_chars:
|
||||
break
|
||||
used += len(snippet) + (1 if parts else 0)
|
||||
parts.append(snippet)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def skill_chunks(skills: tuple[SkillDescriptor, ...] | None = None, *, max_lines: int = 40) -> list[CodeChunk]:
|
||||
"""Chunk skill operating guidance into the ``skills`` retrieval namespace."""
|
||||
chunks: list[CodeChunk] = []
|
||||
for skill in skills or SKILL_INVENTORY:
|
||||
text = f"# {skill.title}\n{skill.summary}\n\nChecklist:\n{skill.checklist}"
|
||||
chunks.extend(
|
||||
chunk_text_namespace(
|
||||
namespace="skills",
|
||||
path=f"skills/{skill.id}.md",
|
||||
text=text,
|
||||
metadata={
|
||||
"skill_id": skill.id,
|
||||
"languages": ",".join(skill.languages),
|
||||
"tags": ",".join(skill.tags),
|
||||
"vulnerability_classes": ",".join(skill.vulnerability_classes),
|
||||
"stages": ",".join(skill.stages),
|
||||
},
|
||||
max_lines=max_lines,
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SKILL_INVENTORY",
|
||||
"STAGES",
|
||||
"TAG_TO_VULN",
|
||||
"SkillDescriptor",
|
||||
"route_skill_ids",
|
||||
"route_skills",
|
||||
"route_skills_for_target",
|
||||
"select_skill_context",
|
||||
"skill_chunks",
|
||||
]
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Slot-contract discipline for model-disabled deterministic stages (task 6.3).
|
||||
|
||||
The deterministic scan stages carry their work in named *slots* and run with model
|
||||
invocation disabled (``skip_model = True``). Each stage declares a read/write slot
|
||||
allow-list that is validated on every lifecycle hook, preserving the
|
||||
``ProcessorSpec.reads/writes`` provenance discipline of the sync ``HarnessRuntime``
|
||||
(the top-level driver) — without forcing the deterministic stages through HarnessX's
|
||||
agent loop ("bridge, do not map", per the spec design).
|
||||
|
||||
This is the zero-dependency substrate. When the optional ``openultrasast[harnessx]``
|
||||
extra is present, the same model-disabled processors can be hosted under a HarnessX
|
||||
``MultiHookProcessor`` (see ``stage_processors.host_under_harnessx``); the
|
||||
``skip_model`` marker maps onto HarnessX's ``BeforeModelEvent.skip_model`` so no
|
||||
deterministic stage incurs a model call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .harness import HarnessContractError
|
||||
|
||||
|
||||
class SlotContractError(HarnessContractError):
|
||||
"""Raised when a stage reads or writes a slot outside its declared allow-list."""
|
||||
|
||||
|
||||
class SlotContractMixin:
|
||||
"""A model-disabled stage that carries work in slots under a read/write allow-list."""
|
||||
|
||||
name: str = "slot-stage"
|
||||
version: str = "1"
|
||||
reads_slots: tuple[str, ...] = ()
|
||||
writes_slots: tuple[str, ...] = ()
|
||||
skip_model: bool = True # deterministic: never invokes a model
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
def validate_reads(self, slots: Mapping[str, Any]) -> None:
|
||||
missing = sorted(slot for slot in self.reads_slots if slot not in slots)
|
||||
if missing:
|
||||
raise SlotContractError(f"{self.name}: missing read slots: {', '.join(missing)}")
|
||||
|
||||
def validate_writes(self, updates: Mapping[str, Any]) -> None:
|
||||
undeclared = sorted(set(updates) - set(self.writes_slots))
|
||||
if undeclared:
|
||||
raise SlotContractError(f"{self.name}: undeclared write slots: {', '.join(undeclared)}")
|
||||
|
||||
def execute(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Validate the read allow-list, run the stage, validate the write allow-list."""
|
||||
self.validate_reads(slots) # pre-hook
|
||||
updates = self.run(slots) or {}
|
||||
if not isinstance(updates, dict):
|
||||
raise SlotContractError(f"{self.name}: stage returned a non-dict update")
|
||||
self.validate_writes(updates) # post-hook
|
||||
return updates
|
||||
|
||||
|
||||
class _Emitter(Protocol):
|
||||
def emit(self, event_type: Any, stage: str | None, payload: dict[str, Any]) -> None: ...
|
||||
|
||||
|
||||
class SlotPipeline:
|
||||
"""Runs slot processors in order, validating contracts and enforcing model-disable."""
|
||||
|
||||
def __init__(self, processors: list[SlotContractMixin]) -> None:
|
||||
for proc in processors:
|
||||
if not proc.skip_model:
|
||||
raise SlotContractError(f"{proc.name}: deterministic stage must be model-disabled (skip_model=True)")
|
||||
self._processors = processors
|
||||
|
||||
@property
|
||||
def processors(self) -> list[SlotContractMixin]:
|
||||
return list(self._processors)
|
||||
|
||||
def run(self, initial_slots: Mapping[str, Any], *, runtime: _Emitter | None = None) -> dict[str, Any]:
|
||||
"""Execute every stage; with ``runtime`` set, emit stage_start/stage_end lifecycle hooks."""
|
||||
slots: dict[str, Any] = dict(initial_slots)
|
||||
for proc in self._processors:
|
||||
if runtime is not None:
|
||||
runtime.emit(
|
||||
"stage_start", proc.name, {"reads": list(proc.reads_slots), "writes": list(proc.writes_slots), "skip_model": True}
|
||||
)
|
||||
updates = proc.execute(slots)
|
||||
slots.update(updates)
|
||||
if runtime is not None:
|
||||
runtime.emit("stage_end", proc.name, {"status": "succeeded", "wrote": sorted(updates)})
|
||||
return slots
|
||||
|
||||
|
||||
__all__ = ["SlotContractError", "SlotContractMixin", "SlotPipeline"]
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Deterministic scan stages as model-disabled, slot-contracted processors (task 6.3).
|
||||
|
||||
Each stage is a :class:`slot_contract.SlotContractMixin` with ``skip_model = True`` and
|
||||
a declared read/write slot allow-list. Running them through a :class:`SlotPipeline`
|
||||
reproduces the deterministic quick-scan path byte-for-byte while enforcing the slot
|
||||
contract on every lifecycle hook. The hunter and verifier stages are *not* here —
|
||||
they are the agentic plane (``hunter_harness``/``verify_judge``); this module is the
|
||||
deterministic, model-free remainder.
|
||||
|
||||
``host_under_harnessx`` is the optional, lazy bridge: it wraps a model-disabled stage
|
||||
as a HarnessX ``MultiHookProcessor`` that disables the model call on ``before_model``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .findings import quick_scan_findings
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints
|
||||
from .policy import load_policy
|
||||
from .preprocess import preprocess_repository
|
||||
from .rank import rank_targets
|
||||
from .ruleset import DEFAULT_RULESET_DIR, load_ruleset
|
||||
from .scoring import build_score_artifact
|
||||
from .slot_contract import SlotContractMixin, SlotPipeline
|
||||
|
||||
|
||||
class PreprocessProcessor(SlotContractMixin):
|
||||
name = "preprocess"
|
||||
reads_slots = ("root",)
|
||||
writes_slots = ("snapshot", "targets")
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
snapshot, targets = preprocess_repository(slots["root"])
|
||||
return {"snapshot": snapshot, "targets": targets}
|
||||
|
||||
|
||||
class EntryPointProcessor(SlotContractMixin):
|
||||
name = "entry_point"
|
||||
reads_slots = ("root", "targets")
|
||||
writes_slots = ("targets", "entry_points")
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
entry_points = analyze_entry_points(slots["root"], slots["targets"])
|
||||
targets = attach_reachability_hints(slots["targets"], entry_points)
|
||||
return {"targets": targets, "entry_points": entry_points}
|
||||
|
||||
|
||||
class RankProcessor(SlotContractMixin):
|
||||
name = "rank"
|
||||
reads_slots = ("targets",)
|
||||
writes_slots = ("rankings",)
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {"rankings": rank_targets(slots["targets"])}
|
||||
|
||||
|
||||
class QuickFindingsProcessor(SlotContractMixin):
|
||||
name = "quick_findings"
|
||||
reads_slots = ("root", "targets", "rankings", "ruleset", "policy")
|
||||
writes_slots = ("findings",)
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
findings = quick_scan_findings(slots["root"], slots["targets"], slots["rankings"], slots["ruleset"], slots["policy"])
|
||||
return {"findings": findings}
|
||||
|
||||
|
||||
class PolicyScoringProcessor(SlotContractMixin):
|
||||
name = "policy_scoring"
|
||||
reads_slots = ("findings",)
|
||||
writes_slots = ("reported_findings", "shadow_findings")
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
findings = slots["findings"]
|
||||
shadow = [finding for finding in findings if finding.status == "shadow"]
|
||||
reported = [finding for finding in findings if finding.status != "shadow"]
|
||||
return {"reported_findings": reported, "shadow_findings": shadow}
|
||||
|
||||
|
||||
class ScoreProcessor(SlotContractMixin):
|
||||
name = "score"
|
||||
reads_slots = ("reported_findings", "ruleset", "policy")
|
||||
writes_slots = ("score",)
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
cwe_by_rule = {rule.rule_id: rule.cwe for rule in slots["ruleset"]}
|
||||
reported = slots["reported_findings"]
|
||||
rule_cwe_by_id = {f.finding_id: cwe_by_rule.get(f.finding_id.split(":", 1)[0], "") for f in reported}
|
||||
return {"score": build_score_artifact(reported, rule_cwe_by_id, slots["policy"])}
|
||||
|
||||
|
||||
def deterministic_processors() -> list[SlotContractMixin]:
|
||||
"""The ordered deterministic stages (preprocess → entry-point → rank → findings → policy → score)."""
|
||||
return [
|
||||
PreprocessProcessor(),
|
||||
EntryPointProcessor(),
|
||||
RankProcessor(),
|
||||
QuickFindingsProcessor(),
|
||||
PolicyScoringProcessor(),
|
||||
ScoreProcessor(),
|
||||
]
|
||||
|
||||
|
||||
def build_deterministic_pipeline() -> SlotPipeline:
|
||||
return SlotPipeline(deterministic_processors())
|
||||
|
||||
|
||||
def run_quick_pipeline(
|
||||
root: Path | str,
|
||||
*,
|
||||
ruleset: Any = None,
|
||||
policy: Any = None,
|
||||
runtime: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the deterministic quick-scan pipeline and return the final slot map.
|
||||
|
||||
Defaults to the bundled ruleset and the vendored policy, which makes the
|
||||
``reported_findings`` slot byte-identical to ``findings.quick_scan_findings`` on
|
||||
the same target.
|
||||
"""
|
||||
resolved_ruleset = ruleset if ruleset is not None else load_ruleset(DEFAULT_RULESET_DIR)
|
||||
resolved_policy = policy if policy is not None else load_policy()
|
||||
initial = {"root": Path(root), "ruleset": resolved_ruleset, "policy": resolved_policy}
|
||||
return build_deterministic_pipeline().run(initial, runtime=runtime)
|
||||
|
||||
|
||||
def host_under_harnessx(processor: SlotContractMixin) -> Any:
|
||||
"""Host a model-disabled deterministic stage under a HarnessX ``MultiHookProcessor``.
|
||||
|
||||
Lazy + capability-guarded (imports ``harnessx`` only when called). On the
|
||||
``before_model`` hook the hosted stage yields a ``BeforeModelEvent`` with
|
||||
``skip_model=True``, so no deterministic stage incurs a model call when it runs on
|
||||
the HarnessX runloop. Raises :class:`HarnessXUnavailableError` when the extra is absent.
|
||||
"""
|
||||
import dataclasses
|
||||
|
||||
from .harness_ext import require_harnessx
|
||||
|
||||
require_harnessx()
|
||||
from harnessx.core.events import BeforeModelEvent
|
||||
from harnessx.core.processor import MultiHookProcessor
|
||||
|
||||
class _ModelDisabledStage(MultiHookProcessor): # type: ignore[misc, valid-type]
|
||||
_singleton_group = "ousast_deterministic"
|
||||
_order = 50
|
||||
|
||||
def __init__(self, stage: SlotContractMixin) -> None:
|
||||
self.stage = stage
|
||||
self.name = stage.name
|
||||
|
||||
async def on_before_model(self, event: BeforeModelEvent): # type: ignore[no-untyped-def]
|
||||
yield dataclasses.replace(event, skip_model=True, synthetic_output=f"[deterministic stage {self.stage.name}: model disabled]")
|
||||
|
||||
return _ModelDisabledStage(processor)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EntryPointProcessor",
|
||||
"PolicyScoringProcessor",
|
||||
"PreprocessProcessor",
|
||||
"QuickFindingsProcessor",
|
||||
"RankProcessor",
|
||||
"ScoreProcessor",
|
||||
"build_deterministic_pipeline",
|
||||
"deterministic_processors",
|
||||
"host_under_harnessx",
|
||||
"run_quick_pipeline",
|
||||
]
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Verifier dispatcher: HarnessX llm-judge when available, structural fallback otherwise.
|
||||
|
||||
The default path (no extra) is the existing :func:`verification.verify_findings`
|
||||
verbatim — same ``list[VerificationResult]``, same rejection of unknown reachability —
|
||||
so behaviour is byte-identical without HarnessX. The judge path is lazy/guarded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .findings import StaticFinding
|
||||
from .harness_ext import build_provider, has_harnessx
|
||||
from .verification import (
|
||||
EvidenceLevel,
|
||||
VerificationResult,
|
||||
VerificationStatus,
|
||||
build_independent_verifier_context,
|
||||
is_report_verified,
|
||||
verify_findings,
|
||||
)
|
||||
|
||||
|
||||
def verify_findings_dispatch(
|
||||
findings: list[StaticFinding],
|
||||
*,
|
||||
verifier_model: str | None = None,
|
||||
verifier_provider: str = "anthropic",
|
||||
use_harnessx: bool = False,
|
||||
) -> list[VerificationResult]:
|
||||
"""Drop-in superset of :func:`verification.verify_findings`.
|
||||
|
||||
Falls back to the structural verifier unless the HarnessX extra is present, a
|
||||
verifier model is configured, and ``use_harnessx`` is set.
|
||||
"""
|
||||
if not (use_harnessx and verifier_model and has_harnessx()):
|
||||
return verify_findings(findings)
|
||||
return _verify_with_judge(findings, verifier_model, verifier_provider)
|
||||
|
||||
|
||||
def _verify_with_judge(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
import asyncio
|
||||
|
||||
return asyncio.run(_judge_all(findings, verifier_model, verifier_provider))
|
||||
|
||||
|
||||
async def _judge_all(findings: list[StaticFinding], verifier_model: str, verifier_provider: str) -> list[VerificationResult]:
|
||||
from harnessx.core.events import Message
|
||||
from harnessx.processors.evaluation.llm_judge import build_judge_prompt, parse_judge_response
|
||||
|
||||
provider = build_provider(verifier_model, verifier_provider)
|
||||
results: list[VerificationResult] = []
|
||||
for finding in findings:
|
||||
prompt = build_judge_prompt(
|
||||
task_description=f"Is this a real, exploitable security issue: {finding.title}? Judge on evidence only.",
|
||||
trajectory_summary=_finding_summary(finding),
|
||||
extracted_answer=finding.rationale,
|
||||
)
|
||||
try:
|
||||
response = await provider.complete(messages=[Message(role="user", content=prompt)], tools=[])
|
||||
verdict = parse_judge_response(_text(response.content))
|
||||
except Exception: # noqa: BLE001 — judge/provider failure must never auto-reject a finding
|
||||
verdict = {"verdict": "no_answer", "confidence": 0.0}
|
||||
results.append(_verdict_to_verification(finding, verdict))
|
||||
return results
|
||||
|
||||
|
||||
def _verdict_to_verification(finding: StaticFinding, verdict: dict[str, object]) -> VerificationResult:
|
||||
"""Map an llm-judge verdict onto a VerificationResult (infra failure never auto-rejects)."""
|
||||
label = str(verdict.get("verdict", "no_answer"))
|
||||
raw_confidence = verdict.get("confidence", 0.0)
|
||||
confidence = 0.0
|
||||
if isinstance(raw_confidence, (int, float)):
|
||||
confidence = float(raw_confidence)
|
||||
elif isinstance(raw_confidence, str):
|
||||
try:
|
||||
confidence = float(raw_confidence)
|
||||
except ValueError:
|
||||
confidence = 0.0
|
||||
cause = str(verdict.get("cause", "")).strip()
|
||||
lesson = str(verdict.get("lesson", "")).strip()
|
||||
missing = str(verdict.get("missing", "")).strip() or _missing_capability_summary(verdict)
|
||||
level = EvidenceLevel(finding.evidence_level)
|
||||
|
||||
if label == "plausible" and confidence >= 0.6:
|
||||
status = VerificationStatus.ACCEPTED
|
||||
elif label in ("unsupported", "refused"):
|
||||
status = VerificationStatus.REJECTED
|
||||
else: # hedging / format_wrong / no_answer / low-confidence plausible
|
||||
status = VerificationStatus.NEEDS_EVIDENCE
|
||||
|
||||
if status == VerificationStatus.ACCEPTED:
|
||||
pro_case = cause or "The judge found the issue plausible on the available evidence."
|
||||
counter_case = "Impact and exploitability still require deeper verifier or dynamic evidence."
|
||||
else:
|
||||
pro_case = "A candidate issue exists from the finding's evidence."
|
||||
counter_case = cause or "The judge could not confirm the issue from the available evidence."
|
||||
|
||||
return VerificationResult(
|
||||
finding_id=finding.finding_id,
|
||||
status=status,
|
||||
evidence_level=level,
|
||||
verified=is_report_verified(level, status),
|
||||
pro_case=pro_case,
|
||||
counter_case=counter_case,
|
||||
tie_breaker=lesson or "Confirm attacker control, sanitizer absence, and impact.",
|
||||
required_next_step=missing or "Collect corroborating evidence before reporting as verified.",
|
||||
context_sources=sorted(build_independent_verifier_context(finding).keys()),
|
||||
)
|
||||
|
||||
|
||||
def _finding_summary(finding: StaticFinding) -> str:
|
||||
return (
|
||||
f"{finding.title} at {finding.path}:{finding.line} (function={finding.function_name or 'unknown'}); "
|
||||
f"reachability={finding.reachability_status}; tags={', '.join(finding.tags) or 'none'}"
|
||||
)
|
||||
|
||||
|
||||
def _missing_capability_summary(verdict: dict[str, object]) -> str:
|
||||
capability = verdict.get("missing_capability")
|
||||
if isinstance(capability, dict):
|
||||
return str(capability.get("summary", "")).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _text(content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = [str(block.get("text", "")) if isinstance(block, dict) else str(block) for block in content]
|
||||
return "".join(parts)
|
||||
return str(content)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Shared test fixtures."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
_HARNESSX_GUARD = (
|
||||
"\nimport sys as _sys\n"
|
||||
"_leaked = sorted(m for m in _sys.modules if m == 'harnessx' or m.startswith('harnessx.'))\n"
|
||||
"assert not _leaked, 'leaked harnessx modules: ' + repr(_leaked)\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def assert_cold_of_harnessx(): # type: ignore[no-untyped-def]
|
||||
"""Run ``code`` in a fresh interpreter and assert it imported no ``harnessx`` module.
|
||||
|
||||
Hermetic by construction: the lazy-import guarantee is verified the same way
|
||||
whether or not the optional extra is installed in this environment (a global
|
||||
``sys.modules`` check would be polluted by other tests once the extra is present).
|
||||
"""
|
||||
|
||||
def _check(code: str) -> str:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-c", code + _HARNESSX_GUARD],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert proc.returncode == 0, f"exit={proc.returncode}\nstdout={proc.stdout}\nstderr={proc.stderr}"
|
||||
return proc.stdout
|
||||
|
||||
return _check
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"overall": {
|
||||
"actual": 44,
|
||||
"expected": 47,
|
||||
"false_positive_rate": 0.0,
|
||||
"false_positives": 0,
|
||||
"matched": 44,
|
||||
"recall": 0.93617
|
||||
},
|
||||
"per_language": {
|
||||
"c_cpp": {
|
||||
"actual": 14,
|
||||
"expected": 15,
|
||||
"false_positive_rate": 0.0,
|
||||
"false_positives": 0,
|
||||
"matched": 14,
|
||||
"recall": 0.933333
|
||||
},
|
||||
"java": {
|
||||
"actual": 4,
|
||||
"expected": 4,
|
||||
"false_positive_rate": 0.0,
|
||||
"false_positives": 0,
|
||||
"matched": 4,
|
||||
"recall": 1.0
|
||||
},
|
||||
"javascript": {
|
||||
"actual": 12,
|
||||
"expected": 13,
|
||||
"false_positive_rate": 0.0,
|
||||
"false_positives": 0,
|
||||
"matched": 12,
|
||||
"recall": 0.923077
|
||||
},
|
||||
"python": {
|
||||
"actual": 14,
|
||||
"expected": 15,
|
||||
"false_positive_rate": 0.0,
|
||||
"false_positives": 0,
|
||||
"matched": 14,
|
||||
"recall": 0.933333
|
||||
}
|
||||
}
|
||||
}
|
||||
+68
-1
@@ -322,15 +322,82 @@ evidence = "memcpy copies attacker-controlled input"
|
||||
"cwe": "CWE-120",
|
||||
"evidence": "memcpy copies attacker-controlled input",
|
||||
"failed_stage": "benchmark_ground_truth_matching",
|
||||
"next_improvement_candidate": "rules_or_language_hunter",
|
||||
"next_improvement_candidate": "rule:c-unsafe-memory-copy",
|
||||
"path": "src/vuln.cpp",
|
||||
"reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
|
||||
"rule_id": "c-unsafe-memory-copy",
|
||||
"vulnerability_class": "unsafe memory copy",
|
||||
}
|
||||
]
|
||||
assert deltas == [{"extra_findings_total": 1, "matched_expected_total": 1, "missed_expected_total": 0, "tool": "clearwing"}]
|
||||
|
||||
|
||||
def test_benchmark_attributes_per_rule_signals_and_recommendations(tmp_path: Path) -> None:
|
||||
manifest_path = tmp_path / "benchmark.toml"
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
manifest_path.write_text(
|
||||
"""
|
||||
name = "cpp-smoke"
|
||||
language = "c_cpp"
|
||||
|
||||
[source]
|
||||
path = "repo"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-120"
|
||||
class = "buffer overflow"
|
||||
path = "src/vuln.cpp"
|
||||
line = 9
|
||||
rule_id = "c-unsafe-memory-copy"
|
||||
sink = "memcpy"
|
||||
evidence = "memcpy copies attacker-controlled input"
|
||||
|
||||
[[expected]]
|
||||
cwe = "CWE-78"
|
||||
class = "command injection"
|
||||
path = "src/vuln.cpp"
|
||||
line = 20
|
||||
rule_id = "c-shell-exec"
|
||||
sink = "system"
|
||||
evidence = "system runs attacker input"
|
||||
""".strip()
|
||||
+ "\n"
|
||||
)
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
run = create_benchmark_run(repo, manifest)
|
||||
findings = [
|
||||
_finding(
|
||||
finding_id="c-unsafe-memory-copy:src/vuln.cpp:9",
|
||||
path="src/vuln.cpp",
|
||||
line=9,
|
||||
rationale="Static pattern c-unsafe-memory-copy matched memcpy with attacker-controlled input.",
|
||||
),
|
||||
_finding(
|
||||
finding_id="js-eval:src/app.js:3",
|
||||
path="src/app.js",
|
||||
line=3,
|
||||
rationale="Static pattern js-eval matched eval.",
|
||||
),
|
||||
]
|
||||
|
||||
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
|
||||
write_benchmark_artifacts(run, manifest_path, result)
|
||||
|
||||
per_rule = result.metrics.per_rule
|
||||
assert per_rule["c-unsafe-memory-copy"] == {"matched": 1, "missed": 0, "false_positives": 0}
|
||||
assert per_rule["c-shell-exec"]["missed"] == 1
|
||||
assert per_rule["js-eval"]["false_positives"] == 1
|
||||
|
||||
actions = {rec.rule_id: rec.action for rec in result.rule_recommendations}
|
||||
assert actions["c-unsafe-memory-copy"] == "keep"
|
||||
assert actions["c-shell-exec"] == "loosen"
|
||||
assert actions["js-eval"] == "shadow"
|
||||
|
||||
delta = json.loads((run.root / "rule_policy_delta.json").read_text())
|
||||
assert {item["rule_id"] for item in delta} == {"c-unsafe-memory-copy", "c-shell-exec", "js-eval"}
|
||||
|
||||
|
||||
def _finding(
|
||||
*,
|
||||
finding_id: str,
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""CLI capability-gated dispatch + graceful degradation (Phase 3 task 6.5).
|
||||
|
||||
Simulates the extra-absent environment (via a monkeypatched capability probe so the
|
||||
behaviour is deterministic whether or not the optional extra is installed): standard
|
||||
mode with models configured must fall back to the deterministic hunter/structural
|
||||
verifier and record a degradation; with no models configured the manifest stays
|
||||
byte-identical.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.cli import main
|
||||
|
||||
_VULN = "@app.route('/admin')\ndef admin():\n return eval(request.data)\n"
|
||||
|
||||
|
||||
def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_body: str | None) -> dict:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text(_VULN)
|
||||
monkeypatch.setattr(cli, "has_harnessx", lambda: False) # simulate the extra being absent
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
argv = ["scan", str(repo), "--mode", "standard"]
|
||||
if config_body is not None:
|
||||
cfg = tmp_path / "openultrasast.toml"
|
||||
cfg.write_text(config_body)
|
||||
argv += ["--config", str(cfg)]
|
||||
assert main(argv) == 0
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
return json.loads((run_dir / "manifest.json").read_text())
|
||||
|
||||
|
||||
def test_degradation_recorded_when_models_set_but_extra_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
manifest = _run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\nverifier = "anthropic/claude"\n')
|
||||
|
||||
degradations = manifest.get("degradations", [])
|
||||
stages = {entry["stage"] for entry in degradations}
|
||||
assert stages == {"hunter_pool", "verify"}
|
||||
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in degradations)
|
||||
# Fallback still produced findings (deterministic hunter ran).
|
||||
assert manifest["findings"]
|
||||
|
||||
|
||||
def test_manifest_has_no_degradations_without_models(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
manifest = _run(tmp_path, monkeypatch, config_body=None)
|
||||
assert "degradations" not in manifest
|
||||
|
||||
|
||||
def test_standard_scan_falls_back_to_run_hunter_pool(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[str] = []
|
||||
real = cli.run_hunter_pool
|
||||
monkeypatch.setattr(cli, "run_hunter_pool", lambda *a, **k: calls.append("fallback") or real(*a, **k))
|
||||
_run(tmp_path, monkeypatch, '[models]\nhunter = "anthropic/claude"\n')
|
||||
assert calls == ["fallback"]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""`ousast improve` CLI: drives the self-improvement loop and the loop-owned ledger."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.ruleset import PatternRule, read_rule_ledger, write_ruleset
|
||||
|
||||
|
||||
def _rule(rule_id: str, pattern: str) -> PatternRule:
|
||||
return PatternRule(rule_id=rule_id, title=rule_id, languages=("python",), cwe="CWE-95", tags=("injection",), pattern=pattern)
|
||||
|
||||
|
||||
def _expected_block(rule_id: str) -> str:
|
||||
return (
|
||||
'[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n'
|
||||
f'line = 2\nrule_id = "{rule_id}"\nsink = "eval"\nevidence = "eval executes input"\n'
|
||||
)
|
||||
|
||||
|
||||
def _scaffold(tmp_path: Path) -> tuple[Path, Path]:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\ndef g():\n print('hello')\n")
|
||||
ruleset_dir = tmp_path / "ruleset"
|
||||
write_ruleset(ruleset_dir / "python" / "rules.toml", [_rule("good-eval", r"\beval\s*\("), _rule("noisy-print", r"\bprint\s*\(")])
|
||||
manifest = tmp_path / "bench.toml"
|
||||
manifest.write_text('name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' + _expected_block("good-eval"))
|
||||
return repo, ruleset_dir
|
||||
|
||||
|
||||
def test_improve_accepts_and_writes_loop_owned_ledger(tmp_path: Path) -> None:
|
||||
repo, ruleset_dir = _scaffold(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
|
||||
exit_code = main(["improve", str(manifest), "--ruleset-dir", str(ruleset_dir)])
|
||||
|
||||
assert exit_code == 0
|
||||
# The ledger lands exactly where `scan`/`benchmark` read it for this target.
|
||||
ledger = repo / ".openultrasast" / "calibration" / "rule_policy.json"
|
||||
assert ledger.exists()
|
||||
assert read_rule_ledger(ledger)["noisy-print"]["status"] == "shadow"
|
||||
journal = repo / ".openultrasast" / "calibration" / "improve_journal.json"
|
||||
assert journal.exists()
|
||||
|
||||
|
||||
def test_improve_dry_run_never_touches_the_targets_ledger(tmp_path: Path) -> None:
|
||||
repo, ruleset_dir = _scaffold(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
|
||||
exit_code = main(["improve", str(manifest), "--ruleset-dir", str(ruleset_dir), "--dry-run"])
|
||||
|
||||
assert exit_code == 0
|
||||
assert not (repo / ".openultrasast" / "calibration" / "rule_policy.json").exists()
|
||||
|
||||
|
||||
def test_improve_rejects_missing_manifest(tmp_path: Path) -> None:
|
||||
import pytest
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
main(["improve", str(tmp_path / "nope.toml")])
|
||||
@@ -28,6 +28,67 @@ def test_load_config_reads_toml(tmp_path: Path) -> None:
|
||||
assert config.dynamic.network_scope == ("127.0.0.1:8080",)
|
||||
|
||||
|
||||
def test_harnessx_defaults_when_section_absent() -> None:
|
||||
config = load_config(None)
|
||||
assert config.harnessx.provider == "anthropic"
|
||||
assert config.harnessx.max_cost_usd == 2.0
|
||||
assert config.harnessx.token_threshold == 120_000
|
||||
|
||||
|
||||
def test_load_config_reads_harnessx_section(tmp_path: Path) -> None:
|
||||
config_path = tmp_path / "openultrasast.toml"
|
||||
config_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"[harnessx]",
|
||||
'provider = "openai"',
|
||||
"max_cost_usd = 5.0",
|
||||
"token_threshold = 200000",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
assert config.harnessx.provider == "openai"
|
||||
assert config.harnessx.max_cost_usd == 5.0
|
||||
assert config.harnessx.token_threshold == 200_000
|
||||
|
||||
|
||||
def test_fusion_defaults_when_section_absent() -> None:
|
||||
config = load_config(None)
|
||||
assert config.fusion.enabled is True
|
||||
assert config.fusion.panel_model is None
|
||||
assert config.fusion.high_assurance is False
|
||||
|
||||
|
||||
def test_load_config_reads_fusion_section(tmp_path: Path) -> None:
|
||||
config_path = tmp_path / "openultrasast.toml"
|
||||
config_path.write_text('[fusion]\nenabled = true\npanel_model = "gpt-4o"\ndecider_model = "gpt-4o"\nhigh_assurance = true\n')
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
assert config.fusion.panel_model == "gpt-4o"
|
||||
assert config.fusion.decider_model == "gpt-4o"
|
||||
assert config.fusion.high_assurance is True
|
||||
|
||||
|
||||
def test_hardening_defaults_when_section_absent() -> None:
|
||||
config = load_config(None)
|
||||
assert config.hardening.redact_secrets is True
|
||||
assert config.hardening.max_findings == 0
|
||||
|
||||
|
||||
def test_load_config_reads_hardening_section(tmp_path: Path) -> None:
|
||||
config_path = tmp_path / "openultrasast.toml"
|
||||
config_path.write_text("[hardening]\nredact_secrets = false\nmax_findings = 5\n")
|
||||
|
||||
config = load_config(config_path)
|
||||
|
||||
assert config.hardening.redact_secrets is False
|
||||
assert config.hardening.max_findings == 5
|
||||
|
||||
|
||||
def test_write_resolved_config_creates_json(tmp_path: Path) -> None:
|
||||
output = tmp_path / "run" / "resolved_config.json"
|
||||
|
||||
|
||||
@@ -1,79 +1,35 @@
|
||||
"""Detection-quality gate for the in-scope languages.
|
||||
|
||||
Runs quick-mode scans against the C/C++, Python, and JavaScript benchmark
|
||||
corpora and asserts the project goal: at least 90% recall of known
|
||||
vulnerabilities and under a 10% false-positive rate per language.
|
||||
Runs quick-mode scans against the C/C++, Python, JavaScript, and Java benchmark
|
||||
corpora and asserts the project goal: at least 90% recall of known vulnerabilities
|
||||
and under a 10% false-positive rate per language. The corpus map and aggregation
|
||||
live in ``openultrasast.gate`` (the same gate CI runs as a first-class step).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import (
|
||||
BenchmarkRun,
|
||||
evaluate_benchmark,
|
||||
load_benchmark_manifest,
|
||||
resolve_benchmark_source,
|
||||
from openultrasast.gate import (
|
||||
FALSE_POSITIVE_CEILING,
|
||||
LANGUAGE_MANIFESTS,
|
||||
RECALL_FLOOR,
|
||||
detection_gate,
|
||||
language_metrics,
|
||||
)
|
||||
from openultrasast.findings import quick_scan_findings
|
||||
from openultrasast.mapping import analyze_entry_points, attach_reachability_hints
|
||||
from openultrasast.preprocess import preprocess_repository
|
||||
from openultrasast.rank import rank_targets
|
||||
|
||||
LANGUAGE_MANIFESTS = {
|
||||
"c_cpp": ["c-cpp-smoke", "cpp-damn-vulnerable"],
|
||||
"python": ["python-web-smoke", "python-vulnerable"],
|
||||
"javascript": ["javascript-node-web-smoke", "javascript-vulnerable"],
|
||||
"java": ["java-web-smoke", "java-spring-boot-vulnerable"],
|
||||
}
|
||||
|
||||
RECALL_FLOOR = 0.90
|
||||
FALSE_POSITIVE_CEILING = 0.10
|
||||
|
||||
|
||||
def _quick_scan(target: Path): # type: ignore[no-untyped-def]
|
||||
_, targets = preprocess_repository(target)
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
|
||||
rankings = rank_targets(targets)
|
||||
return quick_scan_findings(target, targets, rankings)
|
||||
|
||||
|
||||
def _language_metrics(manifest_names: list[str]) -> tuple[int, int, int, int]:
|
||||
expected = matched = actual = false_positives = 0
|
||||
for name in manifest_names:
|
||||
manifest_path = Path(f"benchmarks/manifests/{name}.toml")
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
findings = _quick_scan(target)
|
||||
run = BenchmarkRun(benchmark_run_id="gate", root=Path("/tmp/gate"), manifest=manifest)
|
||||
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
metrics = result.metrics
|
||||
expected += metrics.expected_findings_total
|
||||
matched += metrics.matched_findings_total
|
||||
actual += metrics.actual_findings_total
|
||||
false_positives += metrics.false_positive_findings_total
|
||||
return expected, matched, actual, false_positives
|
||||
|
||||
|
||||
@pytest.mark.parametrize("language", sorted(LANGUAGE_MANIFESTS))
|
||||
def test_language_meets_recall_and_false_positive_goal(language: str) -> None:
|
||||
expected, matched, actual, false_positives = _language_metrics(LANGUAGE_MANIFESTS[language])
|
||||
metrics = language_metrics(LANGUAGE_MANIFESTS[language])
|
||||
|
||||
recall = matched / expected
|
||||
false_positive_rate = false_positives / actual if actual else 0.0
|
||||
|
||||
assert recall >= RECALL_FLOOR, f"{language} recall {recall:.2%} below {RECALL_FLOOR:.0%} ({matched}/{expected})"
|
||||
assert false_positive_rate < FALSE_POSITIVE_CEILING, (
|
||||
f"{language} false-positive rate {false_positive_rate:.2%} at or above {FALSE_POSITIVE_CEILING:.0%} ({false_positives}/{actual})"
|
||||
assert metrics.recall >= RECALL_FLOOR, (
|
||||
f"{language} recall {metrics.recall:.2%} below {RECALL_FLOOR:.0%} ({metrics.matched}/{metrics.expected})"
|
||||
)
|
||||
assert metrics.false_positive_rate < FALSE_POSITIVE_CEILING, (
|
||||
f"{language} false-positive rate {metrics.false_positive_rate:.2%} at or above "
|
||||
f"{FALSE_POSITIVE_CEILING:.0%} ({metrics.false_positives}/{metrics.actual})"
|
||||
)
|
||||
|
||||
|
||||
def test_overall_detection_goal_is_met() -> None:
|
||||
totals = [_language_metrics(names) for names in LANGUAGE_MANIFESTS.values()]
|
||||
expected = sum(t[0] for t in totals)
|
||||
matched = sum(t[1] for t in totals)
|
||||
actual = sum(t[2] for t in totals)
|
||||
false_positives = sum(t[3] for t in totals)
|
||||
|
||||
assert matched / expected >= RECALL_FLOOR
|
||||
assert (false_positives / actual) < FALSE_POSITIVE_CEILING
|
||||
verdict = detection_gate({language: language_metrics(names) for language, names in LANGUAGE_MANIFESTS.items()})
|
||||
assert verdict.passed, verdict.reasons
|
||||
|
||||
@@ -56,7 +56,8 @@ def test_findings_and_report_artifacts(tmp_path: Path) -> None:
|
||||
write_findings(findings, findings_path)
|
||||
write_markdown_report(findings, report_path)
|
||||
|
||||
assert json.loads(findings_path.read_text())["findings"][0]["severity"] == "medium"
|
||||
# Severity is now policy-governed: CWE-78 (command injection) resolves to 5 -> "critical".
|
||||
assert json.loads(findings_path.read_text())["findings"][0]["severity"] == "critical"
|
||||
assert "Subprocess shell execution" in report_path.read_text()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Fusion adjudication: trigger policy, two-panel decider, dispositions (Phase 13)."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.fusion import (
|
||||
FusionDisposition,
|
||||
fuse_finding,
|
||||
fuse_findings_dispatch,
|
||||
fusion_triggers,
|
||||
should_fuse,
|
||||
)
|
||||
from openultrasast.harness_ext import has_harnessx
|
||||
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
|
||||
|
||||
|
||||
def _finding(
|
||||
*,
|
||||
finding_id: str = "python-unsafe-eval:app.py:3",
|
||||
severity: str = "critical",
|
||||
reachability: str = "reachable",
|
||||
) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=finding_id,
|
||||
path="app.py",
|
||||
title="Dynamic Python execution",
|
||||
severity=severity,
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="eval on request data",
|
||||
line=3,
|
||||
function_name="admin",
|
||||
reachability_status=reachability,
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["injection"],
|
||||
ranking_priority=3.0,
|
||||
)
|
||||
|
||||
|
||||
def _verification(status: VerificationStatus, *, verified: bool) -> VerificationResult:
|
||||
return VerificationResult(
|
||||
finding_id="python-unsafe-eval:app.py:3",
|
||||
status=status,
|
||||
evidence_level=EvidenceLevel.STATIC_CORROBORATION,
|
||||
verified=verified,
|
||||
pro_case="reachable tainted sink",
|
||||
counter_case="input may be constant",
|
||||
tie_breaker="confirm attacker control",
|
||||
required_next_step="dynamic repro",
|
||||
context_sources=["app.py"],
|
||||
)
|
||||
|
||||
|
||||
# ---- trigger policy ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_triggers_on_high_severity_and_reachable() -> None:
|
||||
triggers = fusion_triggers(_finding(severity="critical", reachability="reachable"))
|
||||
assert "severity:critical" in triggers
|
||||
assert "gates_risky_fix" in triggers
|
||||
|
||||
|
||||
def test_low_severity_unreachable_does_not_trigger() -> None:
|
||||
assert not should_fuse(_finding(severity="low", reachability="unknown"))
|
||||
|
||||
|
||||
def test_verifier_disagreement_triggers() -> None:
|
||||
triggers = fusion_triggers(
|
||||
_finding(severity="low", reachability="unknown"),
|
||||
_verification(VerificationStatus.NEEDS_EVIDENCE, verified=False),
|
||||
)
|
||||
assert triggers == ["verifier_disagreement"]
|
||||
|
||||
|
||||
def test_evidence_conflict_triggers_when_rejected_but_reachable() -> None:
|
||||
triggers = fusion_triggers(
|
||||
_finding(severity="low", reachability="reachable"),
|
||||
_verification(VerificationStatus.REJECTED, verified=False),
|
||||
)
|
||||
assert "evidence_conflict" in triggers
|
||||
|
||||
|
||||
def test_high_assurance_request_forces_a_trigger() -> None:
|
||||
assert should_fuse(_finding(severity="low", reachability="unknown"), high_assurance=True)
|
||||
|
||||
|
||||
# ---- deterministic two-panel decider + dispositions -------------------------
|
||||
|
||||
|
||||
def test_accepted_when_reachable_and_verified() -> None:
|
||||
decision = fuse_finding(_finding(), _verification(VerificationStatus.ACCEPTED, verified=True), ["severity:critical"])
|
||||
assert decision.disposition == FusionDisposition.ACCEPTED
|
||||
assert decision.decision_source == "deterministic-reconciler"
|
||||
assert {p.role for p in decision.panels} == {"panel_a", "panel_b"}
|
||||
assert sum(decision.votes.values()) == 2 # both panels voted
|
||||
|
||||
|
||||
def test_blocked_when_reachable_high_severity_but_unverified() -> None:
|
||||
decision = fuse_finding(_finding(), _verification(VerificationStatus.NEEDS_EVIDENCE, verified=False), ["gates_risky_fix"])
|
||||
assert decision.disposition == FusionDisposition.BLOCKED
|
||||
|
||||
|
||||
def test_rejected_when_verifier_rejects_unreachable() -> None:
|
||||
decision = fuse_finding(
|
||||
_finding(reachability="unknown"), _verification(VerificationStatus.REJECTED, verified=False), ["severity:critical"]
|
||||
)
|
||||
assert decision.disposition == FusionDisposition.REJECTED
|
||||
|
||||
|
||||
def test_deferred_when_evidence_conflicts() -> None:
|
||||
decision = fuse_finding(_finding(severity="low"), _verification(VerificationStatus.REJECTED, verified=False), ["evidence_conflict"])
|
||||
assert decision.disposition == FusionDisposition.DEFERRED
|
||||
|
||||
|
||||
def test_mitigated_when_finding_is_known_mitigated() -> None:
|
||||
finding = _finding()
|
||||
decision = fuse_finding(
|
||||
finding,
|
||||
_verification(VerificationStatus.ACCEPTED, verified=True),
|
||||
["severity:critical"],
|
||||
mitigated_ids=frozenset({finding.finding_id}),
|
||||
)
|
||||
assert decision.disposition == FusionDisposition.MITIGATED
|
||||
|
||||
|
||||
# ---- dispatch + disclosure + serialization ----------------------------------
|
||||
|
||||
|
||||
def test_dispatch_only_fuses_triggered_findings() -> None:
|
||||
findings = [
|
||||
_finding(finding_id="hot:app.py:3", severity="critical"),
|
||||
_finding(finding_id="cold:lib.py:9", severity="low", reachability="unknown"),
|
||||
]
|
||||
decisions = fuse_findings_dispatch(findings, [])
|
||||
assert [d.finding_id for d in decisions] == ["hot:app.py:3"]
|
||||
|
||||
|
||||
def test_dispatch_records_degradation_when_llm_requested_but_extra_absent() -> None:
|
||||
if has_harnessx():
|
||||
pytest.skip("HarnessX extra installed; exercises the extra-absent degradation path")
|
||||
decisions = fuse_findings_dispatch([_finding()], [], panel_model="gpt-4o", use_harnessx=True)
|
||||
assert decisions and decisions[0].decision_source == "deterministic-reconciler"
|
||||
assert any("fusion_llm_unavailable" in note for note in decisions[0].degradations)
|
||||
|
||||
|
||||
def test_decision_serializes_with_full_disclosure() -> None:
|
||||
decision = fuse_finding(_finding(), _verification(VerificationStatus.ACCEPTED, verified=True), ["severity:critical"])
|
||||
payload = json.loads(json.dumps(decision.to_dict()))
|
||||
assert payload["disposition"] == "accepted"
|
||||
assert payload["decision_source"] == "deterministic-reconciler"
|
||||
assert "votes" in payload and "model_ids" in payload and "degradations" in payload
|
||||
assert all("leaning" in panel and "disposition" in panel for panel in payload["panels"])
|
||||
|
||||
|
||||
# ---- CLI integration: standard-mode fusion stage ----------------------------
|
||||
|
||||
|
||||
def test_standard_scan_emits_fusion_artifact_and_manifest_block(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.cli import main
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
assert main(["scan", str(repo), "--mode", "standard"]) == 0
|
||||
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
fusion = json.loads((run_dir / "fusion.json").read_text())
|
||||
assert fusion and fusion[0]["disposition"] in {"accepted", "blocked", "deferred", "rejected", "mitigated"}
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert manifest["fusion"][0]["finding_id"] == fusion[0]["finding_id"]
|
||||
assert "fusion" in manifest["artifacts"]
|
||||
|
||||
|
||||
def test_quick_scan_does_not_emit_fusion(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from openultrasast.cli import main
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
||||
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
assert not (run_dir / "fusion.json").exists()
|
||||
assert "fusion" not in json.loads((run_dir / "manifest.json").read_text())
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Detection-gate non-regression checks locked into CI (Phase 4 task 8).
|
||||
|
||||
8.1 — the recall/FP gate is a hard constraint a score improvement cannot unblock.
|
||||
8.2 — default-ruleset benchmark output is byte-identical to a committed baseline.
|
||||
8.3 — a HarnessX-backed stage must stay within tolerance of the zero-dep baseline.
|
||||
8.4 — the governance/scoring/benchmark planes never import the HarnessX extra.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.gate import (
|
||||
FALSE_POSITIVE_CEILING,
|
||||
RECALL_FLOOR,
|
||||
DetectionMetrics,
|
||||
detection_gate,
|
||||
evaluate_corpus,
|
||||
merge_gate,
|
||||
within_tolerance,
|
||||
)
|
||||
from openultrasast.harness_ext import has_harnessx
|
||||
|
||||
GOLDEN = Path("tests/golden/detection_baseline.json")
|
||||
|
||||
|
||||
# ---- 8.1 hard recall/FP gate, independent of score --------------------------
|
||||
|
||||
|
||||
def test_gate_passes_on_compliant_corpus() -> None:
|
||||
verdict = detection_gate(
|
||||
{
|
||||
"python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=0),
|
||||
"c_cpp": DetectionMetrics(expected=5, matched=5, actual=5, false_positives=0),
|
||||
}
|
||||
)
|
||||
assert verdict.passed and verdict.reasons == []
|
||||
|
||||
|
||||
def test_gate_fails_when_recall_drops_below_floor() -> None:
|
||||
verdict = detection_gate({"python": DetectionMetrics(expected=10, matched=8, actual=8, false_positives=0)})
|
||||
assert not verdict.passed
|
||||
assert any("recall" in reason for reason in verdict.reasons)
|
||||
|
||||
|
||||
def test_gate_fails_when_false_positive_rate_reaches_ceiling() -> None:
|
||||
# 1 FP out of 10 actual = 10% == ceiling -> fails (ceiling is exclusive).
|
||||
verdict = detection_gate({"python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=1)})
|
||||
assert not verdict.passed
|
||||
assert any("FP rate" in reason for reason in verdict.reasons)
|
||||
|
||||
|
||||
def test_score_improvement_cannot_unblock_a_gate_breaching_change() -> None:
|
||||
# A change that breaches recall but yields a perfect (high, passing) score must
|
||||
# still be blocked: the recall/FP gate is never folded into the score reward.
|
||||
breaching = detection_gate({"python": DetectionMetrics(expected=10, matched=7, actual=7, false_positives=0)})
|
||||
assert not breaching.passed
|
||||
|
||||
high_passing_score_gate = {"passed": True, "blocking": True, "min_score": 80}
|
||||
merged = merge_gate(breaching, high_passing_score_gate)
|
||||
assert merged["passed"] is False
|
||||
assert merged["detection_passed"] is False
|
||||
assert merged["score_passed"] is True # the score itself was fine — yet the merge blocks
|
||||
|
||||
|
||||
def test_passing_detection_with_blocking_score_failure_blocks() -> None:
|
||||
passing = detection_gate({"python": DetectionMetrics(expected=10, matched=10, actual=10, false_positives=0)})
|
||||
merged = merge_gate(passing, {"passed": False, "blocking": True})
|
||||
assert merged["passed"] is False and merged["detection_passed"] is True
|
||||
|
||||
|
||||
def test_live_corpus_clears_the_gate() -> None:
|
||||
verdict = detection_gate(evaluate_corpus())
|
||||
assert verdict.passed, verdict.reasons
|
||||
assert verdict.overall.recall >= RECALL_FLOOR
|
||||
assert verdict.overall.false_positive_rate < FALSE_POSITIVE_CEILING
|
||||
|
||||
|
||||
# ---- 8.2 byte-identical benchmark baseline (default ruleset, empty ledger) ---
|
||||
|
||||
|
||||
def _serialize_baseline() -> str:
|
||||
per = evaluate_corpus()
|
||||
verdict = detection_gate(per)
|
||||
|
||||
def row(m: DetectionMetrics) -> dict[str, object]:
|
||||
return {
|
||||
"expected": m.expected,
|
||||
"matched": m.matched,
|
||||
"actual": m.actual,
|
||||
"false_positives": m.false_positives,
|
||||
"recall": round(m.recall, 6),
|
||||
"false_positive_rate": round(m.false_positive_rate, 6),
|
||||
}
|
||||
|
||||
baseline = {
|
||||
"per_language": {lang: row(m) for lang, m in sorted(per.items())},
|
||||
"overall": row(verdict.overall),
|
||||
}
|
||||
return json.dumps(baseline, indent=2, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
def test_benchmark_output_is_byte_identical_to_golden_baseline() -> None:
|
||||
assert GOLDEN.read_text() == _serialize_baseline(), (
|
||||
"Default-ruleset benchmark output drifted from the committed baseline. "
|
||||
"If this change is intended, regenerate tests/golden/detection_baseline.json."
|
||||
)
|
||||
|
||||
|
||||
# ---- 8.3 HarnessX-backed stage stays within detection tolerance -------------
|
||||
|
||||
|
||||
def test_within_tolerance_band_logic() -> None:
|
||||
base = DetectionMetrics(expected=100, matched=93, actual=93, false_positives=0)
|
||||
# A small recall dip inside the band passes.
|
||||
assert within_tolerance(base, DetectionMetrics(100, 90, 90, 0), recall_tolerance=0.05, fp_tolerance=0.05)
|
||||
# A recall dip beyond the band fails.
|
||||
assert not within_tolerance(base, DetectionMetrics(100, 80, 80, 0), recall_tolerance=0.05, fp_tolerance=0.05)
|
||||
# An FP rise beyond the band fails.
|
||||
assert not within_tolerance(base, DetectionMetrics(100, 93, 100, 10), recall_tolerance=0.05, fp_tolerance=0.05)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="HarnessX extra not installed; live agent run unavailable offline")
|
||||
def test_harnessx_stage_stays_within_tolerance_of_baseline() -> None: # pragma: no cover - needs extra + live keys
|
||||
# When the extra and provider keys are present, a HarnessX-backed hunter run is
|
||||
# compared against the zero-dep baseline and must stay within tolerance. Live
|
||||
# LLM execution is unavailable in CI, so this gates on the capability.
|
||||
baseline = detection_gate(evaluate_corpus()).overall
|
||||
assert within_tolerance(baseline, baseline)
|
||||
|
||||
|
||||
# ---- 8.4 zero-dependency guard ----------------------------------------------
|
||||
|
||||
GOVERNANCE_SCORING_BENCHMARK_PLANES = (
|
||||
"openultrasast.gate",
|
||||
"openultrasast.benchmark",
|
||||
"openultrasast.findings",
|
||||
"openultrasast.policy",
|
||||
"openultrasast.policy.verycode",
|
||||
"openultrasast.scoring",
|
||||
"openultrasast.scoring.project_score",
|
||||
"openultrasast.ruleset",
|
||||
"openultrasast.ruleset.store",
|
||||
"openultrasast.improve",
|
||||
"openultrasast.improve.evolve",
|
||||
"openultrasast.slot_contract",
|
||||
"openultrasast.stage_processors",
|
||||
"openultrasast.fusion",
|
||||
"openultrasast.skills",
|
||||
"openultrasast.redaction",
|
||||
)
|
||||
|
||||
|
||||
def test_governance_scoring_benchmark_planes_never_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Hermetic: a fresh interpreter imports every plane and must pull no harnessx.
|
||||
imports = "\n".join(f"import {module}" for module in GOVERNANCE_SCORING_BENCHMARK_PLANES)
|
||||
assert_cold_of_harnessx(imports)
|
||||
|
||||
|
||||
def test_planes_have_no_module_level_harnessx_import() -> None:
|
||||
# Flags only column-0 (module-level) imports; an indented lazy import inside a
|
||||
# capability-guarded function (e.g. stage_processors.host_under_harnessx) is fine.
|
||||
src = Path("src/openultrasast")
|
||||
offenders = []
|
||||
for module in GOVERNANCE_SCORING_BENCHMARK_PLANES:
|
||||
rel = module.removeprefix("openultrasast.").replace(".", "/")
|
||||
candidates = [src / f"{rel}.py", src / rel / "__init__.py"]
|
||||
path = next((c for c in candidates if c.exists()), None)
|
||||
if path is None:
|
||||
continue
|
||||
for line in path.read_text().splitlines():
|
||||
if line.startswith(("import harnessx", "from harnessx")):
|
||||
offenders.append(f"{path}: {line}")
|
||||
assert offenders == [], f"plane modules import HarnessX at module scope: {offenders}"
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Hardening wired into the scan: redaction + bounded-CI finding budget (Phase 15)."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
|
||||
# Two vulnerable routes whose source quotes a live-looking secret. The secret is
|
||||
# assembled at runtime so no scannable token literal lives in the repository.
|
||||
_SECRET = "sk-" + "abcdEFGH1234567890zzzzzz"
|
||||
_VULN_WITH_SECRET = (
|
||||
f'API_KEY = "{_SECRET}"\n'
|
||||
"@app.route('/a')\n"
|
||||
"def a():\n"
|
||||
" return eval(request.data)\n"
|
||||
"@app.route('/b')\n"
|
||||
"def b():\n"
|
||||
" return eval(request.args['x'])\n"
|
||||
)
|
||||
|
||||
|
||||
def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, config_body: str | None) -> Path:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir(parents=True)
|
||||
(repo / "app.py").write_text(_VULN_WITH_SECRET)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
argv = ["scan", str(repo), "--mode", "quick"]
|
||||
if config_body is not None:
|
||||
cfg = tmp_path / "openultrasast.toml"
|
||||
cfg.write_text(config_body)
|
||||
argv += ["--config", str(cfg)]
|
||||
assert main(argv) == 0
|
||||
return sorted((repo / ".runs").iterdir())[-1]
|
||||
|
||||
|
||||
def test_trace_does_not_leak_secrets_by_default(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
run_dir = _run(tmp_path, monkeypatch, config_body=None)
|
||||
trace = (run_dir / "trace" / "events.jsonl").read_text()
|
||||
assert _SECRET not in trace
|
||||
|
||||
|
||||
def test_max_findings_budget_truncates_and_records_degradation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# The fixture yields two eval findings; cap to 1 and confirm truncation is disclosed.
|
||||
uncapped = _run(tmp_path / "full", monkeypatch, config_body=None)
|
||||
assert len(json.loads((uncapped / "findings.json").read_text())["findings"]) >= 2
|
||||
|
||||
run_dir = _run(tmp_path / "capped", monkeypatch, config_body="[hardening]\nmax_findings = 1\n")
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
assert len(findings) == 1
|
||||
budget = next(d for d in manifest["degradations"] if d["stage"] == "budget")
|
||||
assert budget["reason"] == "max_findings_exceeded" and budget["actual"] >= 2
|
||||
|
||||
|
||||
def test_redaction_can_be_disabled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
run_dir = _run(tmp_path, monkeypatch, config_body="[hardening]\nredact_secrets = false\n")
|
||||
trace = (run_dir / "trace" / "events.jsonl").read_text()
|
||||
# With redaction off, the trace is not scrubbed (target path/source may appear verbatim).
|
||||
assert "***REDACTED***" not in trace
|
||||
@@ -0,0 +1,73 @@
|
||||
"""HarnessX optional-extra packaging + capability guard (Phase 3 task 5.2)."""
|
||||
|
||||
import re
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.harness_ext import (
|
||||
HARNESSX_EXTRA,
|
||||
SUPPORTED_PROVIDERS,
|
||||
HarnessXUnavailableError,
|
||||
build_provider,
|
||||
has_harnessx,
|
||||
require_harnessx,
|
||||
)
|
||||
|
||||
PYPROJECT = tomllib.loads(Path("pyproject.toml").read_text())
|
||||
|
||||
|
||||
def test_core_dependencies_stay_empty() -> None:
|
||||
assert PYPROJECT["project"]["dependencies"] == []
|
||||
|
||||
|
||||
def test_harnessx_is_a_sha_pinned_optional_extra() -> None:
|
||||
extras = PYPROJECT["project"]["optional-dependencies"]["harnessx"]
|
||||
assert len(extras) == 1
|
||||
assert re.fullmatch(
|
||||
r"harnessx @ git\+https://github\.com/Darwin-Agent/HarnessX\.git@[0-9a-f]{40}",
|
||||
extras[0],
|
||||
), extras[0]
|
||||
|
||||
|
||||
def test_importing_openultrasast_does_not_import_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# The big modules (cli, harness_ext) must not eagerly import the heavy extra.
|
||||
assert_cold_of_harnessx("import openultrasast.cli\nimport openultrasast.harness_ext")
|
||||
|
||||
|
||||
def test_has_harnessx_probes_without_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
assert isinstance(has_harnessx(), bool)
|
||||
# find_spec must not import the module.
|
||||
assert_cold_of_harnessx("from openultrasast.harness_ext import has_harnessx\nassert isinstance(has_harnessx(), bool)")
|
||||
|
||||
|
||||
def test_require_harnessx_raises_with_install_guidance_when_absent() -> None:
|
||||
if has_harnessx():
|
||||
pytest.skip("HarnessX extra is installed in this environment")
|
||||
with pytest.raises(HarnessXUnavailableError, match=re.escape(HARNESSX_EXTRA)):
|
||||
require_harnessx()
|
||||
|
||||
|
||||
def test_build_provider_rejects_unknown_provider_before_importing(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Name is validated before any import, so this fails loudly even without the extra.
|
||||
with pytest.raises(ValueError, match="unknown HarnessX provider"):
|
||||
build_provider("gpt-4o", "bogus")
|
||||
assert_cold_of_harnessx(
|
||||
"from openultrasast.harness_ext import build_provider\n"
|
||||
"try:\n build_provider('gpt-4o', 'bogus')\n"
|
||||
"except ValueError:\n pass\n"
|
||||
"else:\n raise SystemExit('expected ValueError')\n"
|
||||
)
|
||||
|
||||
|
||||
def test_supported_providers_cover_anthropic_and_openai() -> None:
|
||||
assert {"anthropic", "openai"} <= set(SUPPORTED_PROVIDERS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("provider", "class_name"), [("anthropic", "AnthropicProvider"), ("openai", "OpenAIProvider")])
|
||||
def test_build_provider_constructs_selected_provider(provider: str, class_name: str) -> None:
|
||||
if not has_harnessx():
|
||||
pytest.skip("HarnessX extra not installed in this environment")
|
||||
built = build_provider("test-model", provider)
|
||||
assert type(built).__name__ == class_name
|
||||
@@ -0,0 +1,44 @@
|
||||
"""HarnessX hunter sub-harness: guarding, findings parsing, construction smoke (task 6.1)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.harness_ext import HarnessXUnavailableError, build_sast, has_harnessx
|
||||
from openultrasast.hunter_harness import HxScanOrchestrator, _extract_findings
|
||||
|
||||
|
||||
def test_module_import_does_not_pull_harnessx(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
assert_cold_of_harnessx("import openultrasast.hunter_harness")
|
||||
|
||||
|
||||
def test_extract_findings_parses_json_array_and_tolerates_garbage() -> None:
|
||||
text = 'Here are the findings:\n[{"line": 9, "title": "SQLi", "cwe": "CWE-89", "rationale": "concat"}]\nDone.'
|
||||
items = _extract_findings(text)
|
||||
assert items == [{"line": 9, "title": "SQLi", "cwe": "CWE-89", "rationale": "concat"}]
|
||||
assert _extract_findings("no findings here") == []
|
||||
assert _extract_findings("[not, valid, json]") == []
|
||||
|
||||
|
||||
@pytest.mark.skipif(has_harnessx(), reason="exercises the extra-absent guard")
|
||||
def test_build_sast_raises_without_extra() -> None:
|
||||
with pytest.raises(HarnessXUnavailableError):
|
||||
build_sast()
|
||||
|
||||
|
||||
@pytest.mark.skipif(has_harnessx(), reason="exercises the extra-absent guard")
|
||||
def test_orchestrator_construction_raises_without_extra() -> None:
|
||||
with pytest.raises(HarnessXUnavailableError):
|
||||
HxScanOrchestrator(provider_model="claude-sonnet-4-6")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="construction smoke; requires the harnessx extra")
|
||||
def test_build_sast_returns_harness_config_offline() -> None:
|
||||
from harnessx.core.harness import HarnessConfig
|
||||
|
||||
assert isinstance(build_sast(), HarnessConfig)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="construction smoke; requires the harnessx extra")
|
||||
def test_orchestrator_builds_agent_offline() -> None:
|
||||
# Constructs the model + harness without any LLM call (provider client is lazy).
|
||||
agent = HxScanOrchestrator(provider_model="claude-sonnet-4-6")._build_agent()
|
||||
assert type(agent).__name__ == "Harness"
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Self-improvement loop: bounded levers, validator, gates, accept/revert (Phase 3 task 7)."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.benchmark import load_benchmark_manifest
|
||||
from openultrasast.improve import (
|
||||
EvolveValidator,
|
||||
PolicyConstantEdit,
|
||||
RuleStatusEdit,
|
||||
StrictValidationError,
|
||||
build_rule_signals,
|
||||
propose_status_edits,
|
||||
run_improvement,
|
||||
run_round,
|
||||
)
|
||||
from openultrasast.policy import load_policy
|
||||
from openultrasast.ruleset import PatternRule, read_rule_ledger, write_ruleset
|
||||
|
||||
POLICY = load_policy()
|
||||
|
||||
|
||||
def _rule(rule_id: str, pattern: str, cwe: str = "CWE-95") -> PatternRule:
|
||||
return PatternRule(rule_id=rule_id, title=rule_id, languages=("python",), cwe=cwe, tags=("injection",), pattern=pattern)
|
||||
|
||||
|
||||
def _expected(rule_id: str) -> str:
|
||||
return (
|
||||
'[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n'
|
||||
f'line = 2\nrule_id = "{rule_id}"\nsink = "eval"\nevidence = "eval executes input"\n'
|
||||
)
|
||||
|
||||
|
||||
# ---- validator (7.1, 7.2) ---------------------------------------------------
|
||||
|
||||
|
||||
def test_validator_rejects_enabled_to_disabled_jump() -> None:
|
||||
ruleset = {"r": _rule("r", r"\beval\s*\(")}
|
||||
with pytest.raises(StrictValidationError, match="shadow"):
|
||||
EvolveValidator().validate(RuleStatusEdit("r", "enabled", "disabled"), ruleset, POLICY)
|
||||
|
||||
|
||||
def test_validator_accepts_enabled_to_shadow() -> None:
|
||||
ruleset = {"r": _rule("r", r"\beval\s*\(")}
|
||||
EvolveValidator().validate(RuleStatusEdit("r", "enabled", "shadow"), ruleset, POLICY) # no raise
|
||||
|
||||
|
||||
def test_validator_rejects_unknown_rule_and_bad_status() -> None:
|
||||
ruleset = {"r": _rule("r", r"\beval\s*\(")}
|
||||
with pytest.raises(StrictValidationError, match="unknown rule_id"):
|
||||
EvolveValidator().validate(RuleStatusEdit("nope", "enabled", "shadow"), ruleset, POLICY)
|
||||
with pytest.raises(StrictValidationError):
|
||||
EvolveValidator().validate(RuleStatusEdit("r", "enabled", "bogus"), ruleset, POLICY)
|
||||
|
||||
|
||||
def test_validator_never_tunes_severity_and_bounds_constants() -> None:
|
||||
with pytest.raises(StrictValidationError, match="severity is upstream-owned"):
|
||||
EvolveValidator().validate(PolicyConstantEdit("severity", 3, 5), {}, POLICY)
|
||||
with pytest.raises(StrictValidationError):
|
||||
EvolveValidator().validate(PolicyConstantEdit("K", 60, 5000), {}, POLICY) # out of bounds
|
||||
EvolveValidator().validate(PolicyConstantEdit("K", 60, 90), {}, POLICY) # in bounds, no raise
|
||||
|
||||
|
||||
# ---- proposer + signals (7.4, 7.6) -----------------------------------------
|
||||
|
||||
|
||||
def test_propose_status_edits_auto_shadows_precision_draggers_only() -> None:
|
||||
ruleset = {"good": _rule("good", r"\beval\s*\("), "noisy": _rule("noisy", r"\bprint\s*\(")}
|
||||
per_rule = {"good": {"matched": 1, "missed": 0, "false_positives": 0}, "noisy": {"matched": 0, "missed": 0, "false_positives": 2}}
|
||||
|
||||
edits = propose_status_edits(per_rule, ruleset, current_ledger={}, blocked_keys=set())
|
||||
assert [(e.rule_id, e.to_status) for e in edits] == [("noisy", "shadow")]
|
||||
# novelty gate: a previously-reverted edit is not re-proposed.
|
||||
assert propose_status_edits(per_rule, ruleset, {}, {edits[0].key()}) == []
|
||||
# already shadow -> not re-proposed.
|
||||
assert propose_status_edits(per_rule, ruleset, {"noisy": {"status": "shadow"}}, set()) == []
|
||||
|
||||
|
||||
# ---- end-to-end loop (7.5) --------------------------------------------------
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, expected_block: str) -> tuple[Path, object]:
|
||||
manifest_path = tmp_path / "bench.toml"
|
||||
manifest_path.write_text('name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' + expected_block)
|
||||
return manifest_path, load_benchmark_manifest(manifest_path)
|
||||
|
||||
|
||||
def _ruleset_dir(tmp_path: Path, rules: list[PatternRule]) -> Path:
|
||||
rd = tmp_path / "ruleset" / "python"
|
||||
write_ruleset(rd / "rules.toml", rules)
|
||||
return tmp_path / "ruleset"
|
||||
|
||||
|
||||
def test_round_auto_shadows_noisy_rule_and_accepts(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\ndef g():\n print('hello')\n")
|
||||
ruleset_dir = _ruleset_dir(tmp_path, [_rule("good-eval", r"\beval\s*\("), _rule("noisy-print", r"\bprint\s*\(")])
|
||||
_, manifest = _manifest(tmp_path, _expected("good-eval"))
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
journal = tmp_path / "journal.json"
|
||||
|
||||
outcome = run_round(repo, manifest, ledger_path=ledger, journal_path=journal, ruleset_dir=ruleset_dir)
|
||||
|
||||
assert outcome.accepted and outcome.reason == "accepted"
|
||||
assert outcome.fp_before > 0.0 and outcome.fp_after == 0.0
|
||||
assert outcome.recall_after == 1.0 # the true positive is preserved
|
||||
assert read_rule_ledger(ledger)["noisy-print"]["status"] == "shadow"
|
||||
assert json.loads(journal.read_text())[0]["outcome"] == "accepted"
|
||||
|
||||
|
||||
def test_round_reverts_byte_for_byte_when_shadowing_would_drop_recall(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\ndef g(y):\n return eval(y)\n")
|
||||
ruleset_dir = _ruleset_dir(tmp_path, [_rule("dual-eval", r"\beval\s*\(")])
|
||||
_, manifest = _manifest(tmp_path, _expected("dual-eval"))
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
journal = tmp_path / "journal.json"
|
||||
|
||||
outcome = run_round(repo, manifest, ledger_path=ledger, journal_path=journal, ruleset_dir=ruleset_dir)
|
||||
|
||||
assert not outcome.accepted and outcome.reason == "reverted"
|
||||
assert outcome.score_after > outcome.score_before # score improved but the recall gate blocks it
|
||||
assert not ledger.exists() # byte-for-byte revert: the ledger was never written
|
||||
assert json.loads(journal.read_text())[0]["outcome"] == "reverted"
|
||||
|
||||
|
||||
def test_run_improvement_converges_after_accepting(tmp_path: Path) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\ndef g():\n print('hello')\n")
|
||||
ruleset_dir = _ruleset_dir(tmp_path, [_rule("good-eval", r"\beval\s*\("), _rule("noisy-print", r"\bprint\s*\(")])
|
||||
_, manifest = _manifest(tmp_path, _expected("good-eval"))
|
||||
|
||||
outcomes = run_improvement(
|
||||
repo,
|
||||
manifest,
|
||||
ledger_path=tmp_path / "rule_policy.json",
|
||||
journal_path=tmp_path / "journal.json",
|
||||
ruleset_dir=ruleset_dir,
|
||||
max_rounds=4,
|
||||
)
|
||||
|
||||
assert outcomes[0].accepted # round 1 shadows the noisy rule
|
||||
assert outcomes[-1].reason == "no_proposals" # converges (nothing left to improve)
|
||||
|
||||
|
||||
def test_build_rule_signals_separates_miss_and_fp() -> None:
|
||||
from openultrasast.benchmark import BenchmarkFalsePositive, BenchmarkMetrics, BenchmarkMiss, BenchmarkResult
|
||||
|
||||
result = BenchmarkResult(
|
||||
benchmark_run_id="r",
|
||||
benchmark_name="n",
|
||||
language="python_web",
|
||||
mode="quick",
|
||||
scan_id=None,
|
||||
scan_run_dir=None,
|
||||
metrics=BenchmarkMetrics(1, 1, 0, 1, 1, 0.0, 0.0, None, {}, {}, {}),
|
||||
misses=[BenchmarkMiss("CWE-89", "sqli", "a.py", "ev", "no match", rule_id="sqli")],
|
||||
false_positives=[BenchmarkFalsePositive("noisy:a.py:1", "a.py", 1, "fp", rule_id="noisy")],
|
||||
baseline_deltas=[],
|
||||
calibration_records=[],
|
||||
rule_recommendations=[],
|
||||
)
|
||||
signals = build_rule_signals(result)
|
||||
assert {s["signal"] for s in signals} == {"miss", "fp"}
|
||||
assert {s["rule_id"] for s in signals} == {"sqli", "noisy"}
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Narrow MCP server: tool listing, scan→status flow, and surface safety (Phase 12)."""
|
||||
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.mcp import McpServer, serve
|
||||
|
||||
EXPECTED_TOOLS = {
|
||||
"openultrasast.scan",
|
||||
"openultrasast.status",
|
||||
"openultrasast.findings",
|
||||
"openultrasast.get_finding",
|
||||
"openultrasast.evidence",
|
||||
"openultrasast.artifacts",
|
||||
"openultrasast.benchmark",
|
||||
"openultrasast.explain",
|
||||
"openultrasast.propose_patch",
|
||||
"openultrasast.export_report",
|
||||
}
|
||||
|
||||
|
||||
def _call(server: McpServer, name: str, arguments: dict) -> dict:
|
||||
response = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": arguments}})
|
||||
assert response is not None
|
||||
result = response["result"]
|
||||
payload = json.loads(result["content"][0]["text"]) if not result["isError"] else {"error": result["content"][0]["text"]}
|
||||
return {"isError": result["isError"], **payload}
|
||||
|
||||
|
||||
# ---- protocol + tool listing ------------------------------------------------
|
||||
|
||||
|
||||
def test_initialize_returns_server_info() -> None:
|
||||
server = McpServer()
|
||||
response = server.handle({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}})
|
||||
assert response is not None
|
||||
assert response["result"]["serverInfo"]["name"] == "openultrasast"
|
||||
assert "protocolVersion" in response["result"]
|
||||
|
||||
|
||||
def test_tools_list_is_exactly_the_narrow_surface() -> None:
|
||||
server = McpServer()
|
||||
response = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
|
||||
assert response is not None
|
||||
names = {tool["name"] for tool in response["result"]["tools"]}
|
||||
assert names == EXPECTED_TOOLS
|
||||
|
||||
|
||||
def test_surface_exposes_no_shell_or_exec_tool() -> None:
|
||||
# The narrow surface must never expose arbitrary shell/exec/docker tools.
|
||||
server = McpServer()
|
||||
forbidden = ("shell", "exec", "command", "docker", "run_command", "bash")
|
||||
for name in server.tool_names:
|
||||
assert not any(token in name.lower() for token in forbidden), name
|
||||
for tool in McpServer()._tools.values(): # no tool accepts a free-form command argument
|
||||
assert "command" not in tool.input_schema.get("properties", {})
|
||||
|
||||
|
||||
def test_notifications_get_no_response() -> None:
|
||||
server = McpServer()
|
||||
assert server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"}) is None
|
||||
|
||||
|
||||
def test_unknown_method_returns_jsonrpc_error() -> None:
|
||||
server = McpServer()
|
||||
response = server.handle({"jsonrpc": "2.0", "id": 9, "method": "does/not/exist"})
|
||||
assert response is not None and response["error"]["code"] == -32602
|
||||
|
||||
|
||||
# ---- scan -> status -> findings -> explain flow -----------------------------
|
||||
|
||||
|
||||
def _repo(tmp_path: Path) -> Path:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("@app.route('/admin')\ndef admin():\n return eval(request.data)\n")
|
||||
return repo
|
||||
|
||||
|
||||
def test_scan_then_status_and_findings(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", str(tmp_path / "runs"))
|
||||
server = McpServer()
|
||||
|
||||
scan = _call(server, "openultrasast.scan", {"path": str(_repo(tmp_path)), "mode": "quick"})
|
||||
assert not scan["isError"] and scan["scan_id"]
|
||||
run_dir = scan["run_dir"]
|
||||
|
||||
status = _call(server, "openultrasast.status", {"run_dir": run_dir})
|
||||
assert status["finding_count"] == scan["finding_count"]
|
||||
|
||||
findings = _call(server, "openultrasast.findings", {"run_dir": run_dir})
|
||||
assert findings["count"] == scan["finding_count"]
|
||||
finding_id = findings["findings"][0]["finding_id"]
|
||||
|
||||
explain = _call(server, "openultrasast.explain", {"run_dir": run_dir, "finding_id": finding_id})
|
||||
assert explain["finding_id"] == finding_id and explain["severity"]
|
||||
|
||||
evidence = _call(server, "openultrasast.evidence", {"run_dir": run_dir, "finding_id": finding_id})
|
||||
assert "reachability_status" in evidence
|
||||
|
||||
|
||||
def test_propose_patch_degrades_visibly(tmp_path: Path) -> None:
|
||||
server = McpServer()
|
||||
result = _call(server, "openultrasast.propose_patch", {"run_dir": str(tmp_path), "finding_id": "x"})
|
||||
assert result["status"] == "not_implemented"
|
||||
|
||||
|
||||
def test_missing_required_argument_is_a_tool_error() -> None:
|
||||
server = McpServer()
|
||||
result = _call(server, "openultrasast.status", {})
|
||||
assert result["isError"] and "run_dir" in result["error"]
|
||||
|
||||
|
||||
def test_export_report_returns_markdown(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", str(tmp_path / "runs"))
|
||||
server = McpServer()
|
||||
scan = _call(server, "openultrasast.scan", {"path": str(_repo(tmp_path)), "mode": "quick"})
|
||||
report = _call(server, "openultrasast.export_report", {"run_dir": scan["run_dir"], "format": "markdown"})
|
||||
assert report["format"] == "markdown" and report["content"]
|
||||
|
||||
|
||||
# ---- stdio loop -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_serve_stdio_loop_handles_tools_list() -> None:
|
||||
stdin = io.StringIO(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + "\n")
|
||||
stdout = io.StringIO()
|
||||
assert serve(stdin, stdout) == 0
|
||||
response = json.loads(stdout.getvalue().strip())
|
||||
assert {tool["name"] for tool in response["result"]["tools"]} == EXPECTED_TOOLS
|
||||
@@ -0,0 +1,47 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.findings import PATTERN_RULES
|
||||
from openultrasast.policy import (
|
||||
CwePolicy,
|
||||
PolicyError,
|
||||
assert_rules_resolve,
|
||||
load_policy,
|
||||
resolve_severity,
|
||||
)
|
||||
|
||||
|
||||
def test_load_policy_parses_severity_and_scope() -> None:
|
||||
policy = load_policy()
|
||||
|
||||
assert policy["CWE-121"] == CwePolicy("Buffer Overflow", 5, True, False)
|
||||
assert policy["CWE-89"].severity == 4
|
||||
assert policy["CWE-78"].static is True
|
||||
assert len(policy) > 150 # ~167 CWEs vendored from verycode-policies
|
||||
|
||||
|
||||
def test_resolve_severity_is_policy_governed_and_identical_for_same_cwe() -> None:
|
||||
policy = load_policy()
|
||||
|
||||
# Two different rules that map to the same CWE must get the same severity.
|
||||
assert resolve_severity(policy, "CWE-78") == resolve_severity(policy, "CWE-78") == 5
|
||||
# Unmapped CWE resolves to 0 (report-only, never scored).
|
||||
assert resolve_severity(policy, "CWE-99999") == 0
|
||||
|
||||
|
||||
def test_builtin_rules_all_resolve_in_policy() -> None:
|
||||
# The fail-loud invariant must pass for the shipped ruleset (CWE-120 remapped to CWE-121).
|
||||
assert_rules_resolve(PATTERN_RULES, load_policy())
|
||||
|
||||
|
||||
def test_assert_rules_resolve_fails_loud_on_unmapped_enabled_rule() -> None:
|
||||
policy = load_policy()
|
||||
bad_enabled = SimpleNamespace(rule_id="bad", cwe="CWE-99999", status="enabled")
|
||||
bad_disabled = SimpleNamespace(rule_id="bad-off", cwe="CWE-99999", status="disabled")
|
||||
|
||||
with pytest.raises(PolicyError, match="bad"):
|
||||
assert_rules_resolve([bad_enabled], policy)
|
||||
|
||||
# A disabled rule with an unmapped CWE does not abort the scan.
|
||||
assert_rules_resolve([bad_disabled], policy)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Provider retry/backoff policy (Phase 15)."""
|
||||
|
||||
import urllib.error
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.provider.openrouter import OpenRouterError, call_with_retry
|
||||
|
||||
|
||||
class _Counter:
|
||||
def __init__(self, fail_times: int, exc: Exception) -> None:
|
||||
self.calls = 0
|
||||
self._fail_times = fail_times
|
||||
self._exc = exc
|
||||
|
||||
def __call__(self) -> str:
|
||||
self.calls += 1
|
||||
if self.calls <= self._fail_times:
|
||||
raise self._exc
|
||||
return "ok"
|
||||
|
||||
|
||||
def _no_sleep(_seconds: float) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_retries_transient_then_succeeds() -> None:
|
||||
op = _Counter(2, TimeoutError("slow"))
|
||||
assert call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) == "ok"
|
||||
assert op.calls == 3
|
||||
|
||||
|
||||
def test_retries_transient_http_5xx() -> None:
|
||||
op = _Counter(1, urllib.error.HTTPError("u", 503, "busy", {}, None)) # type: ignore[arg-type]
|
||||
assert call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep) == "ok"
|
||||
assert op.calls == 2
|
||||
|
||||
|
||||
def test_does_not_retry_non_transient_http_4xx() -> None:
|
||||
op = _Counter(5, urllib.error.HTTPError("u", 400, "bad", {}, None)) # type: ignore[arg-type]
|
||||
with pytest.raises(urllib.error.HTTPError):
|
||||
call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep)
|
||||
assert op.calls == 1 # tried once, not retried
|
||||
|
||||
|
||||
def test_raises_after_exhausting_attempts() -> None:
|
||||
op = _Counter(10, TimeoutError("slow"))
|
||||
with pytest.raises(TimeoutError):
|
||||
call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep)
|
||||
assert op.calls == 3
|
||||
|
||||
|
||||
def test_backoff_is_exponential() -> None:
|
||||
delays: list[float] = []
|
||||
op = _Counter(2, urllib.error.URLError("reset"))
|
||||
call_with_retry(op, attempts=3, base_delay=0.5, sleep=delays.append)
|
||||
assert delays == [0.5, 1.0] # 0.5 * 2**0, 0.5 * 2**1
|
||||
|
||||
|
||||
def test_non_retryable_value_error_propagates_immediately() -> None:
|
||||
def op() -> str:
|
||||
raise OpenRouterError("not network")
|
||||
|
||||
with pytest.raises(OpenRouterError):
|
||||
call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Secret redaction for traces and reports (Phase 15).
|
||||
|
||||
Example secrets are assembled at runtime (string concatenation) so no scannable token
|
||||
literal lives in the repository, while still exercising the redaction patterns.
|
||||
"""
|
||||
|
||||
from openultrasast.redaction import REDACTED, redact_secrets
|
||||
|
||||
|
||||
def test_redacts_provider_api_keys() -> None:
|
||||
secret = "sk-" + "abcdEFGH1234567890zzzz"
|
||||
out = redact_secrets(f"key {secret} used")
|
||||
assert secret not in out and REDACTED in out
|
||||
|
||||
|
||||
def test_redacts_aws_github_slack_google_keys() -> None:
|
||||
aws = "AKIA" + "IOSFODNN7EXAMPLE"
|
||||
github = "ghp_" + "0123456789abcdefghij" + "ABCDEFGHIJ012345"
|
||||
slack = "xoxb-" + "123456789012" + "-abcdefghijklmno"
|
||||
google = "AIza" + "SyA1234567890_abcdefghijklmnopqrst"
|
||||
assert aws not in redact_secrets(aws)
|
||||
assert github not in redact_secrets(github)
|
||||
assert slack not in redact_secrets(slack)
|
||||
assert google not in redact_secrets(google)
|
||||
|
||||
|
||||
def test_redacts_bearer_token_but_keeps_scheme() -> None:
|
||||
out = redact_secrets("Authorization: Bearer " + "abcDEF123456ghiJKL")
|
||||
assert out == f"Authorization: Bearer {REDACTED}"
|
||||
|
||||
|
||||
def test_redacts_url_embedded_credentials() -> None:
|
||||
password = "s3cret" + "Pass"
|
||||
out = redact_secrets("postgres://admin:" + password + "@db.internal:5432/app")
|
||||
assert password not in out and "admin" in out # user kept, password masked
|
||||
|
||||
|
||||
def test_redacts_key_value_assignments() -> None:
|
||||
pw = "hunter2" + "value"
|
||||
tok = "tok_" + "abcdefgh"
|
||||
assert pw not in redact_secrets('password = "' + pw + '"')
|
||||
assert tok not in redact_secrets("api_key: " + tok)
|
||||
|
||||
|
||||
def test_redacts_private_key_block() -> None:
|
||||
body = "MIIE" + "abc"
|
||||
pem = "-----BEGIN RSA PRIVATE KEY-----\n" + body + "\n-----END RSA PRIVATE KEY-----"
|
||||
assert body not in redact_secrets(pem)
|
||||
|
||||
|
||||
def test_leaves_benign_text_untouched_and_is_idempotent() -> None:
|
||||
benign = "return eval(request.data) # CWE-95 code injection"
|
||||
assert redact_secrets(benign) == benign
|
||||
once = redact_secrets("sk-" + "abcdEFGH1234567890zzzz")
|
||||
assert redact_secrets(once) == once # idempotent
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Rules-as-data store and the loop-owned ledger overlay (Phase 2 tasks 3.1, 4.3)."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.ruleset import (
|
||||
DEFAULT_RULESET_DIR,
|
||||
PatternRule,
|
||||
load_ruleset,
|
||||
read_rule_ledger,
|
||||
write_rule_ledger,
|
||||
write_ruleset,
|
||||
)
|
||||
|
||||
|
||||
def test_default_ruleset_loads_as_data() -> None:
|
||||
rules = load_ruleset(DEFAULT_RULESET_DIR)
|
||||
assert len(rules) == 32
|
||||
assert all(rule.status == "enabled" for rule in rules)
|
||||
assert all(not hasattr(rule, "severity") for rule in rules) # severity is policy-governed, not rule-local
|
||||
|
||||
|
||||
def test_empty_ledger_is_byte_identical() -> None:
|
||||
assert load_ruleset(DEFAULT_RULESET_DIR) == load_ruleset(DEFAULT_RULESET_DIR, None)
|
||||
|
||||
|
||||
def test_ledger_overlay_changes_only_targeted_rule(tmp_path: Path) -> None:
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
ledger.write_text(json.dumps({"python-unsafe-eval": {"status": "disabled", "precision_estimate": 0.3}}))
|
||||
|
||||
rules = {rule.rule_id: rule for rule in load_ruleset(DEFAULT_RULESET_DIR, ledger)}
|
||||
|
||||
assert rules["python-unsafe-eval"].status == "disabled"
|
||||
assert rules["python-unsafe-eval"].precision_estimate == 0.3
|
||||
assert rules["c-shell-exec"].status == "enabled" # untargeted rule untouched
|
||||
|
||||
|
||||
def test_rule_ledger_round_trips(tmp_path: Path) -> None:
|
||||
ledger = tmp_path / "rule_policy.json"
|
||||
entries = {"python-unsafe-eval": {"status": "shadow", "precision_estimate": 0.41}}
|
||||
write_rule_ledger(ledger, entries)
|
||||
assert read_rule_ledger(ledger) == entries
|
||||
|
||||
|
||||
def test_write_ruleset_round_trips(tmp_path: Path) -> None:
|
||||
rule = PatternRule(rule_id="x-demo", title="Demo", languages=("python",), cwe="CWE-89", tags=("injection",), pattern=r"\bfoo\(")
|
||||
write_ruleset(tmp_path / "r" / "rules.toml", [rule])
|
||||
assert load_ruleset(tmp_path / "r")[0] == rule
|
||||
|
||||
|
||||
def test_disabled_rule_via_ledger_does_not_fire_in_scan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\n")
|
||||
ledger_dir = repo / ".openultrasast" / "calibration"
|
||||
ledger_dir.mkdir(parents=True)
|
||||
(ledger_dir / "rule_policy.json").write_text(json.dumps({"python-unsafe-eval": {"status": "disabled"}}))
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
|
||||
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
||||
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
assert all(not finding["finding_id"].startswith("python-unsafe-eval:") for finding in findings)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Scan-pipeline integrity & serialization (Phase 4 task 9).
|
||||
|
||||
9.1 — with the extra absent, quick and standard scans run through the existing sync
|
||||
driver with behaviour unchanged from the deterministic baseline, and a
|
||||
requested-but-unavailable HarnessX stage records a manifest degradation.
|
||||
9.2 — every persisted scan-state slot serializes and restores without loss (guards
|
||||
against a non-serializable slot silently restoring as empty). This covers the
|
||||
current artifact-persistence model; the HarnessX processor slot-contract variant
|
||||
(spec task 9.2 over task 6.3) remains deferred with 6.3.
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.benchmark import load_findings
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.findings import StaticFinding, write_findings
|
||||
from openultrasast.preprocess import FileTarget
|
||||
from openultrasast.rank import RankingScore
|
||||
from openultrasast.scoring import build_score_artifact
|
||||
from openultrasast.verification import EvidenceLevel, VerificationResult, VerificationStatus
|
||||
|
||||
_VULN = "@app.route('/admin')\ndef admin():\n return eval(request.data)\n"
|
||||
|
||||
|
||||
# ---- 9.1 fallback parity + degradation --------------------------------------
|
||||
|
||||
|
||||
def _scan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, mode: str, config_body: str | None, harnessx: bool) -> tuple[list, dict]:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir(parents=True)
|
||||
(repo / "app.py").write_text(_VULN)
|
||||
monkeypatch.setattr(cli, "has_harnessx", lambda: harnessx)
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
argv = ["scan", str(repo), "--mode", mode]
|
||||
if config_body is not None:
|
||||
cfg = tmp_path / "openultrasast.toml"
|
||||
cfg.write_text(config_body)
|
||||
argv += ["--config", str(cfg)]
|
||||
assert main(argv) == 0
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
findings = json.loads((run_dir / "findings.json").read_text())["findings"]
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
return findings, manifest
|
||||
|
||||
|
||||
def test_standard_scan_extra_absent_matches_deterministic_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Requested-but-unavailable HarnessX (models set, extra absent) must produce the
|
||||
# same findings as the plain deterministic path (no models), differing only by
|
||||
# the recorded degradation.
|
||||
requested, requested_manifest = _scan(
|
||||
tmp_path / "a",
|
||||
monkeypatch,
|
||||
mode="standard",
|
||||
config_body='[models]\nhunter = "openai/gpt-4o"\nverifier = "openai/gpt-4o"\n',
|
||||
harnessx=False,
|
||||
)
|
||||
baseline, baseline_manifest = _scan(tmp_path / "b", monkeypatch, mode="standard", config_body=None, harnessx=False)
|
||||
|
||||
assert requested == baseline # byte-identical findings -> sync driver behaviour unchanged
|
||||
stages = {entry["stage"] for entry in requested_manifest.get("degradations", [])}
|
||||
assert stages == {"hunter_pool", "verify"}
|
||||
assert all(entry["reason"] == "harnessx_extra_unavailable" for entry in requested_manifest["degradations"])
|
||||
assert "degradations" not in baseline_manifest
|
||||
|
||||
|
||||
def test_quick_scan_is_unaffected_by_the_extra(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Quick mode never touches the agentic plane: identical findings whether the
|
||||
# extra is present or absent, and never a degradation.
|
||||
present, present_manifest = _scan(tmp_path / "a", monkeypatch, mode="quick", config_body=None, harnessx=True)
|
||||
absent, _ = _scan(tmp_path / "b", monkeypatch, mode="quick", config_body=None, harnessx=False)
|
||||
|
||||
assert present == absent
|
||||
assert present # findings were produced
|
||||
assert "degradations" not in present_manifest
|
||||
|
||||
|
||||
# ---- 9.2 per-slot serialization round-trip ----------------------------------
|
||||
|
||||
|
||||
def _finding() -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="python-unsafe-eval:app.py:3",
|
||||
path="app.py",
|
||||
title="Dynamic Python execution",
|
||||
severity="critical",
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="eval on request data",
|
||||
line=3,
|
||||
function_name="admin",
|
||||
reachability_status="reachable",
|
||||
reachability_evidence=[{"kind": "route", "access_level": "public", "line": 1, "end_line": 3}],
|
||||
reachability_conditions=["public route"],
|
||||
tags=["syscall_entry", "injection"],
|
||||
ranking_priority=3.0,
|
||||
)
|
||||
|
||||
|
||||
def _ranking() -> RankingScore:
|
||||
return RankingScore(
|
||||
path="app.py",
|
||||
surface=4,
|
||||
influence=3,
|
||||
reachability=4,
|
||||
priority=3.5,
|
||||
rationale="entry",
|
||||
model_id=None,
|
||||
static_boosts=["input_parser"],
|
||||
)
|
||||
|
||||
|
||||
def _file_target() -> FileTarget:
|
||||
return FileTarget(
|
||||
path="app.py",
|
||||
absolute_path="/repo/app.py",
|
||||
language="python",
|
||||
loc=42,
|
||||
tags=["network_entry"],
|
||||
has_fuzz_entry_point=False,
|
||||
static_hints=[{"rule": "x", "line": 1}],
|
||||
reachability_hints=[{"kind": "route", "line": 1}],
|
||||
)
|
||||
|
||||
|
||||
def _verification() -> VerificationResult:
|
||||
return VerificationResult(
|
||||
finding_id="python-unsafe-eval:app.py:3",
|
||||
status=VerificationStatus.ACCEPTED,
|
||||
evidence_level=EvidenceLevel.STATIC_CORROBORATION,
|
||||
verified=True,
|
||||
pro_case="reachable tainted sink",
|
||||
counter_case="needs dynamic confirmation",
|
||||
tie_breaker="confirm attacker control",
|
||||
required_next_step="dynamic repro",
|
||||
context_sources=["app.py", "routes"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"obj", [_finding(), _ranking(), _file_target(), _verification()], ids=["finding", "ranking", "file_target", "verification"]
|
||||
)
|
||||
def test_scan_slot_json_round_trips_without_loss(obj: object) -> None:
|
||||
restored = type(obj)(**json.loads(json.dumps(asdict(obj))))
|
||||
assert restored == obj
|
||||
|
||||
|
||||
def test_score_artifact_round_trips() -> None:
|
||||
from openultrasast.policy import load_policy
|
||||
|
||||
finding = _finding()
|
||||
artifact = build_score_artifact([finding], {finding.finding_id: "CWE-95"}, load_policy())
|
||||
payload = json.loads(json.dumps(artifact.to_dict()))
|
||||
assert payload == artifact.to_dict() # serialization is total and stable
|
||||
assert payload["project_score"] == artifact.project_score
|
||||
|
||||
|
||||
def test_findings_reload_preserves_nested_slots(tmp_path: Path) -> None:
|
||||
# The real resume path (benchmark reloads findings.json); nested collection
|
||||
# slots must survive rather than restore as empty.
|
||||
path = tmp_path / "findings.json"
|
||||
write_findings([_finding()], path)
|
||||
(restored,) = load_findings(path)
|
||||
|
||||
assert restored == _finding()
|
||||
assert restored.reachability_evidence == [{"kind": "route", "access_level": "public", "line": 1, "end_line": 3}]
|
||||
assert restored.reachability_conditions == ["public route"]
|
||||
assert restored.tags == ["syscall_entry", "injection"]
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Phase 1 in-pipeline wiring: project score artifact + fail-loud CWE invariant."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast import cli
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.policy import PolicyError
|
||||
|
||||
|
||||
def test_scan_writes_score_artifact_and_manifest_block(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("@app.route('/x')\ndef view():\n return eval(request.data)\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
|
||||
assert main(["scan", str(repo), "--mode", "quick"]) == 0
|
||||
|
||||
run_dir = sorted((repo / ".runs").iterdir())[-1]
|
||||
score = json.loads((run_dir / "score.json").read_text())
|
||||
manifest = json.loads((run_dir / "manifest.json").read_text())
|
||||
|
||||
assert isinstance(score["project_score"], int) and 0 <= score["project_score"] <= 100
|
||||
assert set(score["gate"]) == {"min_score", "block_severity_reachable", "blocking", "passed"}
|
||||
assert manifest["score"] == score
|
||||
assert manifest["artifacts"]["score"] == "score.json"
|
||||
|
||||
|
||||
def test_scan_fails_loud_when_an_enabled_rule_cwe_is_unmapped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("x = 1\n")
|
||||
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
|
||||
unmapped = SimpleNamespace(rule_id="bogus", cwe="CWE-99999", status="enabled")
|
||||
real_loader = cli.load_ruleset
|
||||
monkeypatch.setattr(cli, "load_ruleset", lambda *a, **k: (*real_loader(*a, **k), unmapped))
|
||||
|
||||
with pytest.raises(PolicyError):
|
||||
main(["scan", str(repo), "--mode", "quick"])
|
||||
@@ -0,0 +1,106 @@
|
||||
from openultrasast.calibration import FalsePositiveReason, fp_reachability_overrides, record_false_positive_learning
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.policy import CwePolicy
|
||||
from openultrasast.scoring import build_score_artifact, gate, project_score
|
||||
|
||||
POLICY = {
|
||||
"CWE-78": CwePolicy("Command Injection", 5, True, False),
|
||||
"CWE-89": CwePolicy("SQL Injection", 4, True, False),
|
||||
"CWE-999": CwePolicy("DAST Only", 5, False, True), # dynamic-only -> report, never scored
|
||||
}
|
||||
|
||||
|
||||
def _finding(finding_id: str, reachability: str) -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id=finding_id,
|
||||
path="a.py",
|
||||
title="t",
|
||||
severity="high",
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="r",
|
||||
line=1,
|
||||
function_name=None,
|
||||
reachability_status=reachability,
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=[],
|
||||
ranking_priority=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_empty_corpus_scores_100() -> None:
|
||||
assert project_score([], {}, POLICY) == 100
|
||||
|
||||
|
||||
def test_single_severity5_reachable_scores_about_43() -> None:
|
||||
finding = _finding("cmd:a.py:1", "reachable")
|
||||
score = project_score([finding], {"cmd:a.py:1": "CWE-78"}, POLICY)
|
||||
assert score == 43
|
||||
|
||||
|
||||
def test_dynamic_only_and_unmapped_contribute_zero_penalty() -> None:
|
||||
dynamic = _finding("dyn:a.py:1", "reachable")
|
||||
unmapped = _finding("ump:a.py:2", "reachable")
|
||||
rule_cwe = {"dyn:a.py:1": "CWE-999", "ump:a.py:2": "CWE-12345"}
|
||||
|
||||
artifact = build_score_artifact([dynamic, unmapped], rule_cwe, POLICY)
|
||||
|
||||
assert artifact.project_score == 100 # neither contributes penalty
|
||||
assert artifact.out_of_scope_dynamic_only == 1
|
||||
assert artifact.unmapped_cwe == ["CWE-12345"]
|
||||
|
||||
|
||||
def test_hard_gate_fails_on_severity5_reachable_only() -> None:
|
||||
reachable = _finding("cmd:a.py:1", "reachable")
|
||||
inferred = _finding("cmd:a.py:2", "inferred-file-surface")
|
||||
rule_cwe = {"cmd:a.py:1": "CWE-78", "cmd:a.py:2": "CWE-78"}
|
||||
|
||||
fail = gate([reachable], 90, rule_cwe, POLICY, min_score=80, blocking=False)
|
||||
ok = gate([inferred], 90, rule_cwe, POLICY, min_score=80, blocking=False)
|
||||
|
||||
assert fail["passed"] is False # severity-5 + reachable always fails
|
||||
assert ok["passed"] is True # inferred reachability does not trip the hard gate
|
||||
|
||||
|
||||
def test_confirmed_false_positive_lowers_penalty_without_deleting_rule() -> None:
|
||||
finding = StaticFinding(
|
||||
finding_id="python-os-command:app.py:1",
|
||||
path="app.py",
|
||||
title="t",
|
||||
severity="critical",
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="r",
|
||||
line=1,
|
||||
function_name=None,
|
||||
reachability_status="reachable",
|
||||
reachability_evidence=[],
|
||||
reachability_conditions=[],
|
||||
tags=["syscall_entry"],
|
||||
ranking_priority=3.0,
|
||||
)
|
||||
learning = record_false_positive_learning(
|
||||
finding,
|
||||
reason=FalsePositiveReason.MISSING_ATTACKER_CONTROL,
|
||||
evidence="input is a constant, not attacker-controlled",
|
||||
)
|
||||
overrides = fp_reachability_overrides([finding], [learning])
|
||||
rule_cwe = {finding.finding_id: "CWE-78"}
|
||||
|
||||
assert overrides[finding.finding_id] < 1.0 # reachable multiplier lowered from 1.0
|
||||
base = project_score([finding], rule_cwe, POLICY)
|
||||
with_fp = project_score([finding], rule_cwe, POLICY, reachability_override=overrides)
|
||||
assert with_fp > base # confirmed FP reduces the penalty, raising the score
|
||||
assert finding.finding_id in rule_cwe # the rule still fires; it was not deleted
|
||||
|
||||
|
||||
def test_score_gate_blocks_only_when_blocking_enabled() -> None:
|
||||
finding = _finding("sql:a.py:1", "inferred-file-surface")
|
||||
rule_cwe = {"sql:a.py:1": "CWE-89"}
|
||||
|
||||
advisory = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=False)
|
||||
blocking = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=True)
|
||||
|
||||
assert advisory.gate["passed"] is True # advisory-first: low score does not block
|
||||
assert blocking.gate["passed"] is False # blocking: score below min_score fails
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Trail of Bits skill index + router (Phase 14)."""
|
||||
|
||||
from openultrasast.preprocess import FileTarget
|
||||
from openultrasast.skills import (
|
||||
SKILL_INVENTORY,
|
||||
route_skill_ids,
|
||||
route_skills,
|
||||
route_skills_for_target,
|
||||
select_skill_context,
|
||||
skill_chunks,
|
||||
)
|
||||
|
||||
|
||||
def _target(language: str, tags: list[str]) -> FileTarget:
|
||||
return FileTarget(path="t", absolute_path="/repo/t", language=language, loc=10, tags=tags, has_fuzz_entry_point=False)
|
||||
|
||||
|
||||
# ---- routing by language / tags / vulnerability class -----------------------
|
||||
|
||||
|
||||
def test_c_parser_target_routes_to_memory_and_fuzzing_skills() -> None:
|
||||
ids = route_skill_ids(_target("c", ["memory_unsafe", "parser"]))
|
||||
assert {"address-sanitizer", "harness-writing"} <= set(ids) # memory + fuzzing
|
||||
assert "libfuzzer" in ids
|
||||
|
||||
|
||||
def test_crypto_target_routes_to_constant_time_and_vector_skills() -> None:
|
||||
ids = route_skill_ids(_target("python", ["crypto"]))
|
||||
assert "constant-time-analysis" in ids
|
||||
assert "vector-forge" in ids
|
||||
|
||||
|
||||
def test_language_mismatch_excludes_other_language_skills() -> None:
|
||||
# A C target must not pull Python-only (atheris) or Rust-only (cargo-fuzz) skills.
|
||||
ids = route_skill_ids(_target("c", ["memory_unsafe", "parser", "fuzzable"]))
|
||||
assert "atheris" not in ids
|
||||
assert "cargo-fuzz" not in ids
|
||||
|
||||
|
||||
def test_language_only_match_does_not_select() -> None:
|
||||
# python + crypto must not surface atheris (python but no crypto signal).
|
||||
ids = route_skills_for_target(_target("python", ["crypto"]), max_snippets=10)
|
||||
assert "atheris" not in [skill.id for skill in ids]
|
||||
|
||||
|
||||
# ---- routing by stage -------------------------------------------------------
|
||||
|
||||
|
||||
def test_stage_filter_restricts_to_stage_skills() -> None:
|
||||
# fuzzing-dictionary is verify-only; at the map stage it must not appear, while a
|
||||
# map-stage skill (semgrep) does for an injection signal.
|
||||
verify = route_skills(tags=("parser",), vulnerability_classes=("parsing",), stage="verify", max_snippets=10)
|
||||
mapped = route_skills(tags=("parser",), vulnerability_classes=("parsing",), stage="map", max_snippets=10)
|
||||
assert "fuzzing-dictionary" in {s.id for s in verify}
|
||||
assert "fuzzing-dictionary" not in {s.id for s in mapped}
|
||||
|
||||
|
||||
# ---- prompt budget caps -----------------------------------------------------
|
||||
|
||||
|
||||
def test_select_skill_context_respects_budget() -> None:
|
||||
target = _target("c", ["memory_unsafe", "parser"])
|
||||
full = select_skill_context(target, budget_chars=10_000)
|
||||
capped = select_skill_context(target, budget_chars=80)
|
||||
assert len(capped) <= 80
|
||||
assert len(full) > len(capped)
|
||||
|
||||
|
||||
def test_zero_budget_yields_no_context() -> None:
|
||||
assert select_skill_context(_target("c", ["memory_unsafe"]), budget_chars=0) == ""
|
||||
|
||||
|
||||
# ---- retrieval-namespace chunking + inventory integrity ---------------------
|
||||
|
||||
|
||||
def test_skill_chunks_land_in_the_skills_namespace() -> None:
|
||||
chunks = skill_chunks()
|
||||
assert chunks
|
||||
assert all(chunk.namespace == "skills" for chunk in chunks)
|
||||
assert any(chunk.metadata.get("skill_id") == "address-sanitizer" for chunk in chunks)
|
||||
|
||||
|
||||
def test_inventory_ids_are_unique() -> None:
|
||||
ids = [skill.id for skill in SKILL_INVENTORY]
|
||||
assert len(ids) == len(set(ids))
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Model-disabled, slot-contracted deterministic stages (Phase 3 task 6.3).
|
||||
|
||||
Proves: deterministic stages carry work in declared slots, a slot-contract violation
|
||||
is raised on undeclared slot access, the pipeline is model-disabled, the slot pipeline
|
||||
reproduces the deterministic quick-scan path byte-for-byte, lifecycle hooks fire, and
|
||||
(when the extra is present) a stage can run under HarnessX with the model disabled.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.gate import _quick_scan
|
||||
from openultrasast.harness_ext import has_harnessx
|
||||
from openultrasast.slot_contract import SlotContractError, SlotContractMixin, SlotPipeline
|
||||
from openultrasast.stage_processors import (
|
||||
PreprocessProcessor,
|
||||
build_deterministic_pipeline,
|
||||
deterministic_processors,
|
||||
host_under_harnessx,
|
||||
run_quick_pipeline,
|
||||
)
|
||||
|
||||
|
||||
class _GoodStage(SlotContractMixin):
|
||||
name = "good"
|
||||
reads_slots = ("a",)
|
||||
writes_slots = ("b",)
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {"b": slots["a"] + 1}
|
||||
|
||||
|
||||
class _UndeclaredWriteStage(SlotContractMixin):
|
||||
name = "bad_write"
|
||||
reads_slots = ()
|
||||
writes_slots = ("declared",)
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {"undeclared": 1}
|
||||
|
||||
|
||||
class _MissingReadStage(SlotContractMixin):
|
||||
name = "bad_read"
|
||||
reads_slots = ("required",)
|
||||
writes_slots = ()
|
||||
|
||||
def run(self, slots: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
# ---- slot contract enforcement (1.5) ----------------------------------------
|
||||
|
||||
|
||||
def test_declared_slots_pass() -> None:
|
||||
assert _GoodStage().execute({"a": 1}) == {"b": 2}
|
||||
|
||||
|
||||
def test_undeclared_write_is_rejected() -> None:
|
||||
with pytest.raises(SlotContractError, match="undeclared write slots: undeclared"):
|
||||
_UndeclaredWriteStage().execute({})
|
||||
|
||||
|
||||
def test_missing_read_is_rejected() -> None:
|
||||
with pytest.raises(SlotContractError, match="missing read slots: required"):
|
||||
_MissingReadStage().execute({})
|
||||
|
||||
|
||||
# ---- model-disabled invariant (1.6) -----------------------------------------
|
||||
|
||||
|
||||
def test_all_deterministic_stages_are_model_disabled() -> None:
|
||||
assert all(proc.skip_model for proc in deterministic_processors())
|
||||
|
||||
|
||||
def test_pipeline_rejects_a_model_enabled_stage() -> None:
|
||||
enabled = _GoodStage()
|
||||
enabled.skip_model = False
|
||||
with pytest.raises(SlotContractError, match="model-disabled"):
|
||||
SlotPipeline([enabled])
|
||||
|
||||
|
||||
# ---- parity with the deterministic quick-scan path --------------------------
|
||||
|
||||
|
||||
def test_pipeline_findings_match_quick_scan_baseline() -> None:
|
||||
target = Path("benchmarks/fixtures/python-vulnerable")
|
||||
slots = run_quick_pipeline(target)
|
||||
assert slots["reported_findings"] == _quick_scan(target)
|
||||
assert "score" in slots # the score slot is produced too
|
||||
|
||||
|
||||
# ---- lifecycle hooks fire per stage -----------------------------------------
|
||||
|
||||
|
||||
class _Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.events: list[tuple[str, str | None]] = []
|
||||
|
||||
def emit(self, event_type: Any, stage: str | None, payload: dict[str, Any]) -> None:
|
||||
self.events.append((event_type, stage))
|
||||
|
||||
|
||||
def test_pipeline_emits_lifecycle_hooks_per_stage() -> None:
|
||||
recorder = _Recorder()
|
||||
run_quick_pipeline(Path("benchmarks/fixtures/python-vulnerable"), runtime=recorder)
|
||||
names = [proc.name for proc in build_deterministic_pipeline().processors]
|
||||
for name in names:
|
||||
assert ("stage_start", name) in recorder.events
|
||||
assert ("stage_end", name) in recorder.events
|
||||
|
||||
|
||||
# ---- optional: run under HarnessX with the model disabled (1.6) --------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not has_harnessx(), reason="requires the harnessx extra")
|
||||
def test_host_under_harnessx_disables_the_model() -> None: # pragma: no cover - needs the extra
|
||||
import asyncio
|
||||
|
||||
from harnessx.core.events import BeforeModelEvent
|
||||
from harnessx.core.processor import MultiHookProcessor
|
||||
|
||||
hosted = host_under_harnessx(PreprocessProcessor())
|
||||
assert isinstance(hosted, MultiHookProcessor)
|
||||
|
||||
async def drive() -> list[Any]:
|
||||
event = BeforeModelEvent(run_id="test", step_id=0)
|
||||
return [out async for out in hosted.on_before_model(event)]
|
||||
|
||||
emitted = asyncio.run(drive())
|
||||
assert emitted and emitted[-1].skip_model is True
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Verifier dispatcher: structural fallback (default) + judge verdict mapping (Phase 3 task 6.4)."""
|
||||
|
||||
from openultrasast.findings import StaticFinding
|
||||
from openultrasast.verification import VerificationStatus, verify_findings
|
||||
from openultrasast.verify_judge import _verdict_to_verification, verify_findings_dispatch
|
||||
|
||||
|
||||
def _finding(reachability: str = "reachable") -> StaticFinding:
|
||||
return StaticFinding(
|
||||
finding_id="python-unsafe-eval:app.py:3",
|
||||
path="app.py",
|
||||
title="Dynamic Python execution needs review",
|
||||
severity="critical",
|
||||
confidence="medium",
|
||||
evidence_level="static_corroboration",
|
||||
rationale="Static pattern matched.",
|
||||
line=3,
|
||||
function_name="admin" if reachability == "reachable" else None,
|
||||
reachability_status=reachability,
|
||||
reachability_evidence=[{"kind": "route", "access_level": "public", "line": 1, "end_line": 3}]
|
||||
if reachability == "reachable"
|
||||
else [],
|
||||
reachability_conditions=[],
|
||||
tags=["syscall_entry"],
|
||||
ranking_priority=3.0,
|
||||
)
|
||||
|
||||
|
||||
def test_dispatch_falls_back_to_structural_by_default() -> None:
|
||||
findings = [_finding("reachable"), _finding("unknown")]
|
||||
assert verify_findings_dispatch(findings) == verify_findings(findings)
|
||||
|
||||
|
||||
def test_dispatch_stays_structural_when_model_unset_or_extra_absent() -> None:
|
||||
findings = [_finding("reachable")]
|
||||
# use_harnessx requested but no model -> structural; and no model -> structural.
|
||||
assert verify_findings_dispatch(findings, verifier_model=None, use_harnessx=True) == verify_findings(findings)
|
||||
assert verify_findings_dispatch(findings, verifier_model="claude-x", use_harnessx=False) == verify_findings(findings)
|
||||
|
||||
|
||||
def test_dispatch_does_not_import_harnessx_on_fallback(assert_cold_of_harnessx) -> None: # type: ignore[no-untyped-def]
|
||||
# Even with a model and use_harnessx=True, when the capability is absent the
|
||||
# dispatch must fall back without importing the extra. Simulate absence so the
|
||||
# guarantee holds whether or not the extra is installed in this environment.
|
||||
assert_cold_of_harnessx(
|
||||
"import openultrasast.verify_judge as vj\n"
|
||||
"vj.has_harnessx = lambda: False\n"
|
||||
"from openultrasast.findings import StaticFinding\n"
|
||||
"f = StaticFinding(finding_id='x:app.py:1', path='app.py', title='t', severity='low',\n"
|
||||
" confidence='low', evidence_level='static_corroboration', rationale='r', line=1,\n"
|
||||
" function_name=None, reachability_status='unknown', reachability_evidence=[],\n"
|
||||
" reachability_conditions=[], tags=[], ranking_priority=1.0)\n"
|
||||
"vj.verify_findings_dispatch([f], verifier_model='claude-x', use_harnessx=True)\n"
|
||||
)
|
||||
|
||||
|
||||
def test_verdict_mapping_is_evidence_gated() -> None:
|
||||
finding = _finding("reachable")
|
||||
accepted = _verdict_to_verification(finding, {"verdict": "plausible", "confidence": 0.9, "cause": "tainted sink reached"})
|
||||
low_conf = _verdict_to_verification(finding, {"verdict": "plausible", "confidence": 0.3})
|
||||
rejected = _verdict_to_verification(finding, {"verdict": "unsupported", "cause": "input is constant"})
|
||||
hedging = _verdict_to_verification(finding, {"verdict": "hedging"})
|
||||
infra = _verdict_to_verification(finding, {"verdict": "no_answer"})
|
||||
|
||||
assert accepted.status == VerificationStatus.ACCEPTED and accepted.verified
|
||||
assert accepted.pro_case == "tainted sink reached"
|
||||
assert low_conf.status == VerificationStatus.NEEDS_EVIDENCE
|
||||
assert rejected.status == VerificationStatus.REJECTED and rejected.counter_case == "input is constant"
|
||||
assert hedging.status == VerificationStatus.NEEDS_EVIDENCE
|
||||
assert infra.status == VerificationStatus.NEEDS_EVIDENCE # infra failure never auto-rejects
|
||||
Reference in New Issue
Block a user