feat(benchmark): Phase 2 (task 4) — actionable per-rule signals, ledger, FP knob

Complete Phase 2: make benchmark output drive per-rule improvement, wire the
loop-owned ledger, and turn a confirmed false positive into a score knob. Zero
new dependencies; detection stays byte-identical (recall 44/47 = 93.6%, FP 0%)
and the 90/10 gate holds.

- Per-rule attribution (4.1): BenchmarkMiss/BenchmarkFalsePositive carry rule_id;
  BenchmarkMetrics.per_rule aggregates matched/missed/false_positives per rule.
- Recommendation delta (4.2): evaluate_benchmark emits RuleRecommendation
  (keep/shadow/loosen) written to rule_policy_delta.json; the inert
  next_improvement_candidate stub is now rule-aware ("rule:<id>" or
  "language_hunter").
- Loop ledger (4.3): cli loads the ruleset with the rule_policy.json overlay
  (empty -> byte-identical; an entry overlays a rule's status/floors at load);
  read_rule_ledger/write_rule_ledger added for the evolve loop. A disabled rule
  in the ledger no longer fires (covered by an integration test).
- FP -> reachability knob (4.4): a confirmed false-positive learning lowers a
  finding's effective reachability multiplier in the score
  (fp_reachability_overrides + scoring reachability_override) instead of deleting
  the rule, so recall stays high while the score stops over-penalizing.

93 tests pass; ruff + mypy clean. Spec tasks 4.1-4.4 checked off — Phase 2
(Core: rules-as-data + benchmark signals) is complete. The loop-driven decision
to apply ledger flips remains the gated evolve loop (Phase 3/4).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

tests/test_benchmark.py

@@ -322,15 +322,82 @@ evidence = "memcpy copies attacker-controlled input"
             "cwe": "CWE-120",
             "evidence": "memcpy copies attacker-controlled input",
             "failed_stage": "benchmark_ground_truth_matching",
-            "next_improvement_candidate": "rules_or_language_hunter",
+            "next_improvement_candidate": "rule:c-unsafe-memory-copy",
             "path": "src/vuln.cpp",
             "reason": "no OpenUltraSAST finding matched the expected benchmark vulnerability",
+            "rule_id": "c-unsafe-memory-copy",
             "vulnerability_class": "unsafe memory copy",
         }
     ]
     assert deltas == [{"extra_findings_total": 1, "matched_expected_total": 1, "missed_expected_total": 0, "tool": "clearwing"}]
 
 
+def test_benchmark_attributes_per_rule_signals_and_recommendations(tmp_path: Path) -> None:
+    manifest_path = tmp_path / "benchmark.toml"
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    manifest_path.write_text(
+        """
+name = "cpp-smoke"
+language = "c_cpp"
+
+[source]
+path = "repo"
+
+[[expected]]
+cwe = "CWE-120"
+class = "buffer overflow"
+path = "src/vuln.cpp"
+line = 9
+rule_id = "c-unsafe-memory-copy"
+sink = "memcpy"
+evidence = "memcpy copies attacker-controlled input"
+
+[[expected]]
+cwe = "CWE-78"
+class = "command injection"
+path = "src/vuln.cpp"
+line = 20
+rule_id = "c-shell-exec"
+sink = "system"
+evidence = "system runs attacker input"
+""".strip()
+        + "\n"
+    )
+    manifest = load_benchmark_manifest(manifest_path)
+    run = create_benchmark_run(repo, manifest)
+    findings = [
+        _finding(
+            finding_id="c-unsafe-memory-copy:src/vuln.cpp:9",
+            path="src/vuln.cpp",
+            line=9,
+            rationale="Static pattern c-unsafe-memory-copy matched memcpy with attacker-controlled input.",
+        ),
+        _finding(
+            finding_id="js-eval:src/app.js:3",
+            path="src/app.js",
+            line=3,
+            rationale="Static pattern js-eval matched eval.",
+        ),
+    ]
+
+    result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id="scan-1", scan_run_dir=repo / ".runs" / "scan-1")
+    write_benchmark_artifacts(run, manifest_path, result)
+
+    per_rule = result.metrics.per_rule
+    assert per_rule["c-unsafe-memory-copy"] == {"matched": 1, "missed": 0, "false_positives": 0}
+    assert per_rule["c-shell-exec"]["missed"] == 1
+    assert per_rule["js-eval"]["false_positives"] == 1
+
+    actions = {rec.rule_id: rec.action for rec in result.rule_recommendations}
+    assert actions["c-unsafe-memory-copy"] == "keep"
+    assert actions["c-shell-exec"] == "loosen"
+    assert actions["js-eval"] == "shadow"
+
+    delta = json.loads((run.root / "rule_policy_delta.json").read_text())
+    assert {item["rule_id"] for item in delta} == {"c-unsafe-memory-copy", "c-shell-exec", "js-eval"}
+
+
 def _finding(
     *,
     finding_id: str,

tests/test_ruleset.py

@@ -0,0 +1,67 @@
+"""Rules-as-data store and the loop-owned ledger overlay (Phase 2 tasks 3.1, 4.3)."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.cli import main
+from openultrasast.ruleset import (
+    DEFAULT_RULESET_DIR,
+    PatternRule,
+    load_ruleset,
+    read_rule_ledger,
+    write_rule_ledger,
+    write_ruleset,
+)
+
+
+def test_default_ruleset_loads_as_data() -> None:
+    rules = load_ruleset(DEFAULT_RULESET_DIR)
+    assert len(rules) == 32
+    assert all(rule.status == "enabled" for rule in rules)
+    assert all(not hasattr(rule, "severity") for rule in rules)  # severity is policy-governed, not rule-local
+
+
+def test_empty_ledger_is_byte_identical() -> None:
+    assert load_ruleset(DEFAULT_RULESET_DIR) == load_ruleset(DEFAULT_RULESET_DIR, None)
+
+
+def test_ledger_overlay_changes_only_targeted_rule(tmp_path: Path) -> None:
+    ledger = tmp_path / "rule_policy.json"
+    ledger.write_text(json.dumps({"python-unsafe-eval": {"status": "disabled", "precision_estimate": 0.3}}))
+
+    rules = {rule.rule_id: rule for rule in load_ruleset(DEFAULT_RULESET_DIR, ledger)}
+
+    assert rules["python-unsafe-eval"].status == "disabled"
+    assert rules["python-unsafe-eval"].precision_estimate == 0.3
+    assert rules["c-shell-exec"].status == "enabled"  # untargeted rule untouched
+
+
+def test_rule_ledger_round_trips(tmp_path: Path) -> None:
+    ledger = tmp_path / "rule_policy.json"
+    entries = {"python-unsafe-eval": {"status": "shadow", "precision_estimate": 0.41}}
+    write_rule_ledger(ledger, entries)
+    assert read_rule_ledger(ledger) == entries
+
+
+def test_write_ruleset_round_trips(tmp_path: Path) -> None:
+    rule = PatternRule(rule_id="x-demo", title="Demo", languages=("python",), cwe="CWE-89", tags=("injection",), pattern=r"\bfoo\(")
+    write_ruleset(tmp_path / "r" / "rules.toml", [rule])
+    assert load_ruleset(tmp_path / "r")[0] == rule
+
+
+def test_disabled_rule_via_ledger_does_not_fire_in_scan(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "app.py").write_text("def f(x):\n    return eval(x)\n")
+    ledger_dir = repo / ".openultrasast" / "calibration"
+    ledger_dir.mkdir(parents=True)
+    (ledger_dir / "rule_policy.json").write_text(json.dumps({"python-unsafe-eval": {"status": "disabled"}}))
+    monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs")
+
+    assert main(["scan", str(repo), "--mode", "quick"]) == 0
+
+    run_dir = sorted((repo / ".runs").iterdir())[-1]
+    findings = json.loads((run_dir / "findings.json").read_text())["findings"]
+    assert all(not finding["finding_id"].startswith("python-unsafe-eval:") for finding in findings)

tests/test_scoring.py

@@ -1,3 +1,4 @@
+from openultrasast.calibration import FalsePositiveReason, fp_reachability_overrides, record_false_positive_learning
 from openultrasast.findings import StaticFinding
 from openultrasast.policy import CwePolicy
 from openultrasast.scoring import build_score_artifact, gate, project_score
@@ -62,6 +63,38 @@ def test_hard_gate_fails_on_severity5_reachable_only() -> None:
     assert ok["passed"] is True  # inferred reachability does not trip the hard gate
 
 
+def test_confirmed_false_positive_lowers_penalty_without_deleting_rule() -> None:
+    finding = StaticFinding(
+        finding_id="python-os-command:app.py:1",
+        path="app.py",
+        title="t",
+        severity="critical",
+        confidence="medium",
+        evidence_level="static_corroboration",
+        rationale="r",
+        line=1,
+        function_name=None,
+        reachability_status="reachable",
+        reachability_evidence=[],
+        reachability_conditions=[],
+        tags=["syscall_entry"],
+        ranking_priority=3.0,
+    )
+    learning = record_false_positive_learning(
+        finding,
+        reason=FalsePositiveReason.MISSING_ATTACKER_CONTROL,
+        evidence="input is a constant, not attacker-controlled",
+    )
+    overrides = fp_reachability_overrides([finding], [learning])
+    rule_cwe = {finding.finding_id: "CWE-78"}
+
+    assert overrides[finding.finding_id] < 1.0  # reachable multiplier lowered from 1.0
+    base = project_score([finding], rule_cwe, POLICY)
+    with_fp = project_score([finding], rule_cwe, POLICY, reachability_override=overrides)
+    assert with_fp > base  # confirmed FP reduces the penalty, raising the score
+    assert finding.finding_id in rule_cwe  # the rule still fires; it was not deleted
+
+
 def test_score_gate_blocks_only_when_blocking_enabled() -> None:
     finding = _finding("sql:a.py:1", "inferred-file-surface")
     rule_cwe = {"sql:a.py:1": "CWE-89"}