feat(governance): Phase 1 — CWE policy store + project score (zero-dep)

First implementation slice of the approved harnessx-self-improving-rulesets spec
(Foundation: governance + scoring planes). Additive and zero-dependency; findings
stay byte-identical and the >=90% recall / <10% FP detection gate is unchanged.

- PolicyStore (policy/verycode.py): vendors verycode-policies CWE_Score.tsv (167
  CWEs, CRLF + trailing-space header parsed defensively); CwePolicy(severity 0-5,
  static/dynamic); load_policy, resolve_severity (severity is policy-governed,
  keyed on CWE — one CWE = one severity), assert_rules_resolve (fail-loud if an
  enabled rule's CWE is unmapped). Remap the C/C++ buffer rules CWE-120 -> CWE-121
  (CWE-120 is absent from verycode; 121 = Stack-Based Buffer Overflow, severity 5).
- ScoreModel (scoring/project_score.py): SEV_WEIGHT, REACH_MULT, finding_penalty,
  project_score (0-100 exponential decay; one sev-5 reachable -> ~43), two-condition
  gate (sev-5+reachable always fails; score<min_score fails only when blocking),
  build_score_artifact.
- Wire into _run_scan: policy_load + fail-loud policy_check stages; score stage
  writes score.json and merges a score block into the manifest (advisory-first).
- ScoreConfig (k/min_score/block_severity_reachable/blocking) in config.py.
- pyproject uv_build ships the vendored TSV in the wheel (verified).
- Tests: test_policy, test_scoring, test_score_pipeline (artifact + fail-loud).
  85 passed; ruff + mypy clean. Spec phases 1.1-2.4 checked off; spec approved.

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-06-22 12:31:35 +02:00
co-authored by Claude Opus 4.8
parent 9b399fdc60
commit ae365f6cb7
14 changed files with 685 additions and 20 deletions
+47
View File
@@ -0,0 +1,47 @@
from types import SimpleNamespace
import pytest
from openultrasast.findings import PATTERN_RULES
from openultrasast.policy import (
CwePolicy,
PolicyError,
assert_rules_resolve,
load_policy,
resolve_severity,
)
def test_load_policy_parses_severity_and_scope() -> None:
policy = load_policy()
assert policy["CWE-121"] == CwePolicy("Buffer Overflow", 5, True, False)
assert policy["CWE-89"].severity == 4
assert policy["CWE-78"].static is True
assert len(policy) > 150 # ~167 CWEs vendored from verycode-policies
def test_resolve_severity_is_policy_governed_and_identical_for_same_cwe() -> None:
policy = load_policy()
# Two different rules that map to the same CWE must get the same severity.
assert resolve_severity(policy, "CWE-78") == resolve_severity(policy, "CWE-78") == 5
# Unmapped CWE resolves to 0 (report-only, never scored).
assert resolve_severity(policy, "CWE-99999") == 0
def test_builtin_rules_all_resolve_in_policy() -> None:
# The fail-loud invariant must pass for the shipped ruleset (CWE-120 remapped to CWE-121).
assert_rules_resolve(PATTERN_RULES, load_policy())
def test_assert_rules_resolve_fails_loud_on_unmapped_enabled_rule() -> None:
policy = load_policy()
bad_enabled = SimpleNamespace(rule_id="bad", cwe="CWE-99999", status="enabled")
bad_disabled = SimpleNamespace(rule_id="bad-off", cwe="CWE-99999", status="disabled")
with pytest.raises(PolicyError, match="bad"):
assert_rules_resolve([bad_enabled], policy)
# A disabled rule with an unmapped CWE does not abort the scan.
assert_rules_resolve([bad_disabled], policy)
+41
View File
@@ -0,0 +1,41 @@
"""Phase 1 in-pipeline wiring: project score artifact + fail-loud CWE invariant."""
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from openultrasast import cli
from openultrasast.cli import main
from openultrasast.policy import PolicyError
def test_scan_writes_score_artifact_and_manifest_block(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "app.py").write_text("@app.route('/x')\ndef view():\n return eval(request.data)\n")
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
assert main(["scan", str(repo), "--mode", "quick"]) == 0
run_dir = sorted((repo / ".runs").iterdir())[-1]
score = json.loads((run_dir / "score.json").read_text())
manifest = json.loads((run_dir / "manifest.json").read_text())
assert isinstance(score["project_score"], int) and 0 <= score["project_score"] <= 100
assert set(score["gate"]) == {"min_score", "block_severity_reachable", "blocking", "passed"}
assert manifest["score"] == score
assert manifest["artifacts"]["score"] == "score.json"
def test_scan_fails_loud_when_an_enabled_rule_cwe_is_unmapped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
repo = tmp_path / "repo"
repo.mkdir()
(repo / "app.py").write_text("x = 1\n")
monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
unmapped = SimpleNamespace(rule_id="bogus", cwe="CWE-99999", status="enabled")
monkeypatch.setattr(cli, "PATTERN_RULES", (*cli.PATTERN_RULES, unmapped))
with pytest.raises(PolicyError):
main(["scan", str(repo), "--mode", "quick"])
+73
View File
@@ -0,0 +1,73 @@
from openultrasast.findings import StaticFinding
from openultrasast.policy import CwePolicy
from openultrasast.scoring import build_score_artifact, gate, project_score
POLICY = {
"CWE-78": CwePolicy("Command Injection", 5, True, False),
"CWE-89": CwePolicy("SQL Injection", 4, True, False),
"CWE-999": CwePolicy("DAST Only", 5, False, True), # dynamic-only -> report, never scored
}
def _finding(finding_id: str, reachability: str) -> StaticFinding:
return StaticFinding(
finding_id=finding_id,
path="a.py",
title="t",
severity="high",
confidence="medium",
evidence_level="static_corroboration",
rationale="r",
line=1,
function_name=None,
reachability_status=reachability,
reachability_evidence=[],
reachability_conditions=[],
tags=[],
ranking_priority=1.0,
)
def test_empty_corpus_scores_100() -> None:
assert project_score([], {}, POLICY) == 100
def test_single_severity5_reachable_scores_about_43() -> None:
finding = _finding("cmd:a.py:1", "reachable")
score = project_score([finding], {"cmd:a.py:1": "CWE-78"}, POLICY)
assert score == 43
def test_dynamic_only_and_unmapped_contribute_zero_penalty() -> None:
dynamic = _finding("dyn:a.py:1", "reachable")
unmapped = _finding("ump:a.py:2", "reachable")
rule_cwe = {"dyn:a.py:1": "CWE-999", "ump:a.py:2": "CWE-12345"}
artifact = build_score_artifact([dynamic, unmapped], rule_cwe, POLICY)
assert artifact.project_score == 100 # neither contributes penalty
assert artifact.out_of_scope_dynamic_only == 1
assert artifact.unmapped_cwe == ["CWE-12345"]
def test_hard_gate_fails_on_severity5_reachable_only() -> None:
reachable = _finding("cmd:a.py:1", "reachable")
inferred = _finding("cmd:a.py:2", "inferred-file-surface")
rule_cwe = {"cmd:a.py:1": "CWE-78", "cmd:a.py:2": "CWE-78"}
fail = gate([reachable], 90, rule_cwe, POLICY, min_score=80, blocking=False)
ok = gate([inferred], 90, rule_cwe, POLICY, min_score=80, blocking=False)
assert fail["passed"] is False # severity-5 + reachable always fails
assert ok["passed"] is True # inferred reachability does not trip the hard gate
def test_score_gate_blocks_only_when_blocking_enabled() -> None:
finding = _finding("sql:a.py:1", "inferred-file-surface")
rule_cwe = {"sql:a.py:1": "CWE-89"}
advisory = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=False)
blocking = build_score_artifact([finding], rule_cwe, POLICY, min_score=99, blocking=True)
assert advisory.gate["passed"] is True # advisory-first: low score does not block
assert blocking.gate["passed"] is False # blocking: score below min_score fails