mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
268 lines
11 KiB
Python
268 lines
11 KiB
Python
|
"""Deterministic self-improvement loop (tasks 7.4, 7.5, 7.6).
|
||
|
|
|
||
|
|
One round: benchmark with the current ledger -> per-rule signals -> propose bounded
|
||
|
|
edits (auto-shadow precision-draggers) -> validate -> replay smoke -> re-benchmark ->
|
||
|
|
hard acceptance gate (recall >= floor AND FP < ceiling AND project score not regressed
|
||
|
|
AND no matched-finding regression) -> accept (persist the ledger) or revert byte-for-byte.
|
||
|
|
|
||
|
|
The edit *proposer* here is the benchmark's per-rule signal; the LLM ``MetaAgent.evolve``
|
||
|
|
proposer plugs into this same validated/gated machinery when the HarnessX extra is present.
|
||
|
|
The project score is the optimization reward; the recall/FP gate is a separate hard
|
||
|
|
feasibility constraint that is never folded into the reward.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import tempfile
|
||
|
|
from collections.abc import Mapping
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from ..benchmark import BenchmarkManifest, BenchmarkResult, BenchmarkRun, evaluate_benchmark
|
||
|
|
from ..findings import StaticFinding, quick_scan_findings
|
||
|
|
from ..mapping import analyze_entry_points, attach_reachability_hints
|
||
|
|
from ..policy import CwePolicy, load_policy
|
||
|
|
from ..preprocess import preprocess_repository
|
||
|
|
from ..rank import rank_targets
|
||
|
|
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger
|
||
|
|
from ..scoring import build_score_artifact
|
||
|
|
from .journal import append_round, load_journal, next_round_index, reverted_edit_keys
|
||
|
|
from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class RoundOutcome:
|
||
|
|
round: int
|
||
|
|
accepted: bool
|
||
|
|
reason: str
|
||
|
|
edits: list[RuleStatusEdit] = field(default_factory=list)
|
||
|
|
recall_before: float = 0.0
|
||
|
|
recall_after: float = 0.0
|
||
|
|
fp_before: float = 0.0
|
||
|
|
fp_after: float = 0.0
|
||
|
|
score_before: int = 0
|
||
|
|
score_after: int = 0
|
||
|
|
matched_before: int = 0
|
||
|
|
matched_after: int = 0
|
||
|
|
|
||
|
|
|
||
|
|
def build_rule_signals(result: BenchmarkResult) -> list[dict[str, object]]:
|
||
|
|
"""Convert a benchmark result into per-rule loop signals (recall: miss; precision: fp)."""
|
||
|
|
signals: list[dict[str, object]] = []
|
||
|
|
for miss in result.misses:
|
||
|
|
signals.append({"rule_id": miss.rule_id or "", "signal": "miss", "cwe": miss.cwe, "path": miss.path})
|
||
|
|
for fp in result.false_positives:
|
||
|
|
signals.append({"rule_id": fp.rule_id, "signal": "fp", "path": fp.path, "line": fp.line})
|
||
|
|
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("path"))))
|
||
|
|
|
||
|
|
|
||
|
|
def propose_status_edits(
|
||
|
|
per_rule: Mapping[str, dict[str, int]],
|
||
|
|
ruleset_by_id: Mapping[str, PatternRule],
|
||
|
|
current_ledger: Mapping[str, dict[str, object]],
|
||
|
|
blocked_keys: set[str],
|
||
|
|
) -> list[RuleStatusEdit]:
|
||
|
|
"""Auto-shadow precision-draggers: enabled rules with false positives -> shadow (never delete)."""
|
||
|
|
edits: list[RuleStatusEdit] = []
|
||
|
|
for rule_id, counts in sorted(per_rule.items()):
|
||
|
|
if counts.get("false_positives", 0) <= 0:
|
||
|
|
continue
|
||
|
|
rule = ruleset_by_id.get(rule_id)
|
||
|
|
if rule is None:
|
||
|
|
continue
|
||
|
|
current_status = str(current_ledger.get(rule_id, {}).get("status", rule.status))
|
||
|
|
if current_status != "enabled":
|
||
|
|
continue # already staged or disabled
|
||
|
|
edit = RuleStatusEdit(rule_id=rule_id, from_status="enabled", to_status="shadow", rationale="")
|
||
|
|
if edit.key() in blocked_keys:
|
||
|
|
continue # novelty gate: previously reverted without a new rationale
|
||
|
|
edits.append(edit)
|
||
|
|
return edits
|
||
|
|
|
||
|
|
|
||
|
|
def run_round(
|
||
|
|
target: Path,
|
||
|
|
manifest: BenchmarkManifest,
|
||
|
|
*,
|
||
|
|
ledger_path: Path,
|
||
|
|
journal_path: Path,
|
||
|
|
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||
|
|
policy: dict[str, CwePolicy] | None = None,
|
||
|
|
recall_floor: float = 0.9,
|
||
|
|
fp_ceiling: float = 0.1,
|
||
|
|
) -> RoundOutcome:
|
||
|
|
policy = policy if policy is not None else load_policy()
|
||
|
|
rounds = load_journal(journal_path)
|
||
|
|
round_index = next_round_index(rounds)
|
||
|
|
blocked = reverted_edit_keys(rounds)
|
||
|
|
current_ledger = read_rule_ledger(ledger_path)
|
||
|
|
|
||
|
|
before_findings, before_rules = _scan(target, ruleset_dir, current_ledger, policy)
|
||
|
|
before = _evaluate(target, manifest, before_findings, before_rules, policy)
|
||
|
|
(ledger_path.parent / "rule_signals.json").parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
_write_signals(ledger_path.parent / "rule_signals.json", build_rule_signals(before.result))
|
||
|
|
|
||
|
|
ruleset_by_id = {rule.rule_id: rule for rule in before_rules}
|
||
|
|
edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
|
||
|
|
if not edits:
|
||
|
|
return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before)
|
||
|
|
|
||
|
|
validator = EvolveValidator()
|
||
|
|
try:
|
||
|
|
for edit in edits:
|
||
|
|
validator.validate(edit, ruleset_by_id, policy)
|
||
|
|
except StrictValidationError as exc:
|
||
|
|
_record(journal_path, round_index, edits, "rejected", before, before, str(exc))
|
||
|
|
return _outcome(round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before)
|
||
|
|
|
||
|
|
candidate = edits_to_ledger(edits, current_ledger)
|
||
|
|
try:
|
||
|
|
_load_with(ruleset_dir, candidate) # replay smoke gate: candidate ruleset must boot
|
||
|
|
except Exception as exc: # noqa: BLE001 — any boot failure rejects the round
|
||
|
|
_record(journal_path, round_index, edits, "rejected", before, before, f"replay_failed: {exc}")
|
||
|
|
return _outcome(round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before)
|
||
|
|
|
||
|
|
after_findings, after_rules = _scan(target, ruleset_dir, candidate, policy)
|
||
|
|
after = _evaluate(target, manifest, after_findings, after_rules, policy)
|
||
|
|
|
||
|
|
accepted = (
|
||
|
|
after.recall >= recall_floor and after.fp_rate < fp_ceiling and after.score >= before.score and after.matched >= before.matched
|
||
|
|
)
|
||
|
|
if accepted:
|
||
|
|
write_rule_ledger(ledger_path, candidate)
|
||
|
|
reason = "accepted"
|
||
|
|
else:
|
||
|
|
reason = "reverted" # ledger_path untouched -> byte-for-byte revert
|
||
|
|
_record(journal_path, round_index, edits, reason, before, after)
|
||
|
|
return _outcome(round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after)
|
||
|
|
|
||
|
|
|
||
|
|
def run_improvement(
|
||
|
|
target: Path,
|
||
|
|
manifest: BenchmarkManifest,
|
||
|
|
*,
|
||
|
|
ledger_path: Path,
|
||
|
|
journal_path: Path,
|
||
|
|
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||
|
|
policy: dict[str, CwePolicy] | None = None,
|
||
|
|
max_rounds: int = 5,
|
||
|
|
recall_floor: float = 0.9,
|
||
|
|
fp_ceiling: float = 0.1,
|
||
|
|
) -> list[RoundOutcome]:
|
||
|
|
"""Run improvement rounds until convergence (no new proposals) or ``max_rounds``."""
|
||
|
|
outcomes: list[RoundOutcome] = []
|
||
|
|
for _ in range(max_rounds):
|
||
|
|
outcome = run_round(
|
||
|
|
target,
|
||
|
|
manifest,
|
||
|
|
ledger_path=ledger_path,
|
||
|
|
journal_path=journal_path,
|
||
|
|
ruleset_dir=ruleset_dir,
|
||
|
|
policy=policy,
|
||
|
|
recall_floor=recall_floor,
|
||
|
|
fp_ceiling=fp_ceiling,
|
||
|
|
)
|
||
|
|
outcomes.append(outcome)
|
||
|
|
if outcome.reason == "no_proposals":
|
||
|
|
break
|
||
|
|
return outcomes
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class _Eval:
|
||
|
|
result: BenchmarkResult
|
||
|
|
recall: float
|
||
|
|
fp_rate: float
|
||
|
|
score: int
|
||
|
|
matched: int
|
||
|
|
|
||
|
|
|
||
|
|
def _scan(
|
||
|
|
target: Path, ruleset_dir: Path, ledger: Mapping[str, dict[str, object]], policy: dict[str, CwePolicy]
|
||
|
|
) -> tuple[list[StaticFinding], tuple[PatternRule, ...]]:
|
||
|
|
rules = _load_with(ruleset_dir, ledger)
|
||
|
|
_, targets = preprocess_repository(target)
|
||
|
|
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
|
||
|
|
rankings = rank_targets(targets)
|
||
|
|
findings = quick_scan_findings(target, targets, rankings, rules, policy)
|
||
|
|
reported = [finding for finding in findings if finding.status != "shadow"]
|
||
|
|
return reported, rules
|
||
|
|
|
||
|
|
|
||
|
|
def _evaluate(
|
||
|
|
target: Path, manifest: BenchmarkManifest, findings: list[StaticFinding], rules: tuple[PatternRule, ...], policy: dict[str, CwePolicy]
|
||
|
|
) -> _Eval:
|
||
|
|
run = BenchmarkRun(benchmark_run_id="improve", root=target, manifest=manifest)
|
||
|
|
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||
|
|
metrics = result.metrics
|
||
|
|
cwe_by_rule = {rule.rule_id: rule.cwe for rule in rules}
|
||
|
|
rule_cwe_by_id = {f.finding_id: cwe_by_rule.get(f.finding_id.split(":", 1)[0], "") for f in findings}
|
||
|
|
score = build_score_artifact(findings, rule_cwe_by_id, policy).project_score
|
||
|
|
recall = metrics.recall if metrics.recall is not None else 1.0
|
||
|
|
fp_rate = metrics.false_positive_findings_total / metrics.actual_findings_total if metrics.actual_findings_total else 0.0
|
||
|
|
return _Eval(result=result, recall=recall, fp_rate=fp_rate, score=score, matched=metrics.matched_findings_total)
|
||
|
|
|
||
|
|
|
||
|
|
def _load_with(ruleset_dir: Path, ledger: Mapping[str, dict[str, object]]) -> tuple[PatternRule, ...]:
|
||
|
|
import json
|
||
|
|
|
||
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle:
|
||
|
|
handle.write(json.dumps(ledger))
|
||
|
|
temp_path = Path(handle.name)
|
||
|
|
try:
|
||
|
|
return load_ruleset(ruleset_dir, temp_path)
|
||
|
|
finally:
|
||
|
|
temp_path.unlink(missing_ok=True)
|
||
|
|
|
||
|
|
|
||
|
|
def _write_signals(path: Path, signals: list[dict[str, object]]) -> None:
|
||
|
|
import json
|
||
|
|
|
||
|
|
path.write_text(json.dumps(signals, indent=2, sort_keys=True) + "\n")
|
||
|
|
|
||
|
|
|
||
|
|
def _outcome(round_index: int, *, accepted: bool, reason: str, edits: list[RuleStatusEdit], before: _Eval, after: _Eval) -> RoundOutcome:
|
||
|
|
return RoundOutcome(
|
||
|
|
round=round_index,
|
||
|
|
accepted=accepted,
|
||
|
|
reason=reason,
|
||
|
|
edits=edits,
|
||
|
|
recall_before=before.recall,
|
||
|
|
recall_after=after.recall,
|
||
|
|
fp_before=before.fp_rate,
|
||
|
|
fp_after=after.fp_rate,
|
||
|
|
score_before=before.score,
|
||
|
|
score_after=after.score,
|
||
|
|
matched_before=before.matched,
|
||
|
|
matched_after=after.matched,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _record(
|
||
|
|
journal_path: Path,
|
||
|
|
round_index: int,
|
||
|
|
edits: list[RuleStatusEdit],
|
||
|
|
outcome: str,
|
||
|
|
before: _Eval,
|
||
|
|
after: _Eval,
|
||
|
|
reason: str = "",
|
||
|
|
) -> None:
|
||
|
|
append_round(
|
||
|
|
journal_path,
|
||
|
|
{
|
||
|
|
"round": round_index,
|
||
|
|
"outcome": outcome,
|
||
|
|
"reason": reason or outcome,
|
||
|
|
"edits": [
|
||
|
|
{"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale}
|
||
|
|
for e in edits
|
||
|
|
],
|
||
|
|
"recall_before": round(before.recall, 4),
|
||
|
|
"recall_after": round(after.recall, 4),
|
||
|
|
"fp_before": round(before.fp_rate, 4),
|
||
|
|
"fp_after": round(after.fp_rate, 4),
|
||
|
|
"score_before": before.score,
|
||
|
|
"score_after": after.score,
|
||
|
|
},
|
||
|
|
)
|