"""Deterministic self-improvement loop (tasks 7.4, 7.5, 7.6). One round: benchmark with the current ledger -> per-rule signals -> propose bounded edits (auto-shadow precision-draggers) -> validate -> replay smoke -> re-benchmark -> hard acceptance gate (recall >= floor AND FP < ceiling AND project score not regressed AND no matched-finding regression) -> accept (persist the ledger) or revert byte-for-byte. The edit *proposer* here is the benchmark's per-rule signal; the LLM ``MetaAgent.evolve`` proposer plugs into this same validated/gated machinery when the HarnessX extra is present. The project score is the optimization reward; the recall/FP gate is a separate hard feasibility constraint that is never folded into the reward. """ from __future__ import annotations import tempfile from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from ..benchmark import BenchmarkManifest, BenchmarkResult, BenchmarkRun, evaluate_benchmark from ..findings import StaticFinding, quick_scan_findings from ..mapping import analyze_entry_points, attach_reachability_hints from ..policy import CwePolicy, load_policy from ..preprocess import preprocess_repository from ..rank import rank_targets from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger from ..scoring import build_score_artifact from ..semantic.facts import SemanticFacts from .journal import append_round, load_journal, next_round_index, reverted_edit_keys from .validator import EvolveValidator, MechanismEdit, RuleStatusEdit, StrictValidationError, apply_mechanism_edits, edits_to_ledger @dataclass(frozen=True) class RoundOutcome: round: int accepted: bool reason: str edits: list[RuleStatusEdit] = field(default_factory=list) recall_before: float = 0.0 recall_after: float = 0.0 fp_before: float = 0.0 fp_after: float = 0.0 score_before: int = 0 score_after: int = 0 matched_before: int = 0 matched_after: int = 0 profile_regressions: list[str] = field(default_factory=list) mechanism_edits: list[MechanismEdit] = field(default_factory=list) mechanisms_before: dict[str, dict[str, float]] = field(default_factory=dict) mechanisms_after: dict[str, dict[str, float]] = field(default_factory=dict) profiles_under_minimum: list[str] = field(default_factory=list) per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict) per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict) MAX_PROFILES = 4 def evaluate_profiles( pair_cases: Sequence[object], rules: tuple[PatternRule, ...], ) -> dict[str, dict[str, float]]: """Pair-corpus metrics per provenance profile under ``rules`` (holdout split, no inventory sidecar). Only ``seeded`` and ``reviewed`` pairs gate (Req 9.3); ``advisory`` and ``title`` pairs are scored by ``pairs``, never here. """ from ..pairs import GATING_TIERS, PairCase, evaluate_catalog, select_tier, select_vendored # Gates score vendored pairs only (Req 10.4); pointer pairs are for `pairs --pointers`, never for the improve gate. cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) if not cases: return {} result = evaluate_catalog(cases, ruleset=rules, inventory_sidecar=False) return { profile: { "pairs": float(metrics.pairs), "pair_correct": float(metrics.pair_correct), "pair_pass_rate": metrics.pair_pass_rate, "youden": metrics.youden, } for profile, metrics in result.per_profile.items() } def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]: """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers).""" from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored from ..semantic.loo import score_pair_with_store from ..semantic.mechanisms import MechanismStore cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) store = MechanismStore(store_path) by_profile: dict[str, list[tuple[bool, bool]]] = {} for case in cases: outcome = score_pair_with_store(case, store, _facts()) by_profile.setdefault(case.provenance, []).append((outcome.detected, outcome.silent)) result: dict[str, dict[str, float]] = {} for profile, items in sorted(by_profile.items()): pairs = len(items) correct = sum(1 for detected, silent in items if detected and silent) recall = sum(1 for detected, _ in items if detected) / pairs silence = sum(1 for _, silent in items if silent) / pairs result[profile] = { "pairs": float(pairs), "pair_correct": float(correct), "pair_pass_rate": correct / pairs, "youden": recall + silence - 1.0, } return result def propose_mechanism_edits(pair_cases: Sequence[object], candidates: object, store_path: Path) -> list[MechanismEdit]: """Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker.""" from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored from ..semantic.loo import score_pair_with_store from ..semantic.mechanisms import MechanismStore, corpus_mechanisms cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) if not cases: return [] facts = _facts() scan = MechanismStore(store_path) admitted = {record.id: record for record in corpus_mechanisms(scan.load())} current = {case.name: score_pair_with_store(case, scan, facts) for case in cases} edits: list[MechanismEdit] = [] leakers = sorted({mechanism_id for outcome in current.values() for mechanism_id in outcome.leaked_by if mechanism_id in admitted}) for mechanism_id in leakers: leaked = sorted(name for name, outcome in current.items() if mechanism_id in outcome.leaked_by) edits.append( MechanismEdit( action="retract", mechanism_id=mechanism_id, source="loo", rationale=f"leaks on the fixed side of {', '.join(leaked)}" ) ) missed = [case for case in cases if not current[case.name].detected] for record in corpus_mechanisms(candidates.load()): # type: ignore[attr-defined] if record.id in admitted or not missed: continue with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch: trial = MechanismStore(Path(scratch) / "trial.jsonl") trial.append(record) recovered = [case.name for case in missed if score_pair_with_store(case, trial, facts).detected] leaks = [case.name for case in cases if not score_pair_with_store(case, trial, facts).silent] if recovered and not leaks: edits.append( MechanismEdit( action="admit", mechanism_id=record.id, source="loo", rationale=f"recovers holdout {', '.join(recovered)}; taught by {', '.join(record.pairs)}", ) ) return edits def _facts() -> SemanticFacts: from ..semantic.facts import FactLoadError, load_facts try: return load_facts() except FactLoadError: return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) def _leaking_admissions(edits: Sequence[MechanismEdit], pair_cases: Sequence[object], store_path: Path) -> list[str]: """Admitted record ids that hit the fixed side of any holdout pair with the store as it stands after the edits.""" from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored from ..semantic.loo import score_pair_with_store from ..semantic.mechanisms import MechanismStore admitted = {edit.mechanism_id for edit in edits if edit.action == "admit"} if not admitted: return [] cases = select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS) store = MechanismStore(store_path) facts = _facts() leaking: set[str] = set() for case in cases: leaking |= set(score_pair_with_store(case, store, facts).leaked_by) & admitted return sorted(leaking) def profile_regressions( before: dict[str, dict[str, float]], after: dict[str, dict[str, float]], *, tolerance: float = 0.0, min_pairs: int = 5, ) -> tuple[list[str], list[str]]: """(regressed profiles, profiles under the minimum). A profile regresses when its pair pass rate or Youden drops by more than ``tolerance`` (both are rates in [-1, 1], so one tolerance applies to both). """ regressed: list[str] = [] small: list[str] = [] for profile in sorted(set(before) | set(after)): old = before.get(profile, {"pairs": 0.0, "pair_correct": 0.0, "pair_pass_rate": 0.0, "youden": 0.0}) new = after.get(profile, old) if old["pairs"] < min_pairs: small.append(profile) continue old_rate = old.get("pair_pass_rate", old["pair_correct"] / old["pairs"] if old["pairs"] else 0.0) new_rate = new.get("pair_pass_rate", new["pair_correct"] / new["pairs"] if new["pairs"] else 0.0) if new_rate < old_rate - tolerance or new["youden"] < old["youden"] - tolerance: regressed.append(profile) return regressed, small def build_rule_signals(result: BenchmarkResult) -> list[dict[str, object]]: """Convert a benchmark result into per-rule loop signals (recall: miss; precision: fp).""" signals: list[dict[str, object]] = [] for miss in result.misses: signals.append({"rule_id": miss.rule_id or "", "signal": "miss", "cwe": miss.cwe, "path": miss.path}) for fp in result.false_positives: signals.append({"rule_id": fp.rule_id, "signal": "fp", "path": fp.path, "line": fp.line}) return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("path")))) def propose_status_edits( per_rule: Mapping[str, dict[str, int]], ruleset_by_id: Mapping[str, PatternRule], current_ledger: Mapping[str, dict[str, object]], blocked_keys: set[str], ) -> list[RuleStatusEdit]: """Auto-shadow precision-draggers: enabled rules with false positives -> shadow (never delete).""" edits: list[RuleStatusEdit] = [] for rule_id, counts in sorted(per_rule.items()): if counts.get("false_positives", 0) <= 0: continue rule = ruleset_by_id.get(rule_id) if rule is None: continue current_status = str(current_ledger.get(rule_id, {}).get("status", rule.status)) if current_status != "enabled": continue # already staged or disabled edit = RuleStatusEdit(rule_id=rule_id, from_status="enabled", to_status="shadow", rationale="") if edit.key() in blocked_keys: continue # novelty gate: previously reverted without a new rationale edits.append(edit) return edits def run_round( target: Path, manifest: BenchmarkManifest, *, ledger_path: Path, journal_path: Path, ruleset_dir: Path = DEFAULT_RULESET_DIR, policy: dict[str, CwePolicy] | None = None, recall_floor: float = 0.9, fp_ceiling: float = 0.1, pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, mechanism_candidates: Path | None = None, mechanism_store: Path | None = None, scripted_mechanism_edits: Sequence[MechanismEdit] | None = None, ) -> RoundOutcome: policy = policy if policy is not None else load_policy() rounds = load_journal(journal_path) round_index = next_round_index(rounds) blocked = reverted_edit_keys(rounds) current_ledger = read_rule_ledger(ledger_path) before_findings, before_rules = _scan(target, ruleset_dir, current_ledger, policy) before = _evaluate(target, manifest, before_findings, before_rules, policy) (ledger_path.parent / "rule_signals.json").parent.mkdir(parents=True, exist_ok=True) _write_signals(ledger_path.parent / "rule_signals.json", build_rule_signals(before.result)) ruleset_by_id = {rule.rule_id: rule for rule in before_rules} edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked) # corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit. mechanism_edits: list[MechanismEdit] = [] candidates_store = None if mechanism_candidates is not None and mechanism_store is not None and pair_cases: from ..semantic.mechanisms import MechanismStore candidates_store = MechanismStore(mechanism_candidates) mechanism_edits = ( list(scripted_mechanism_edits) if scripted_mechanism_edits is not None else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store) ) blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms] if not edits and not mechanism_edits: return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before) validator = EvolveValidator() try: for edit in edits: validator.validate(edit, ruleset_by_id, policy) if mechanism_edits and candidates_store is not None and mechanism_store is not None: from ..semantic.mechanisms import MechanismStore by_id = {record.id: record for record in candidates_store.load()} admitted_by_id = {record.id: record for record in MechanismStore(mechanism_store).load()} for mechanism_edit in mechanism_edits: validator.validate_mechanism(mechanism_edit, by_id, admitted=admitted_by_id) except StrictValidationError as exc: _record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits) return _outcome( round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before, mechanism_edits=mechanism_edits, ) candidate = edits_to_ledger(edits, current_ledger) try: _load_with(ruleset_dir, candidate) # replay smoke gate: candidate ruleset must boot except Exception as exc: # noqa: BLE001 — any boot failure rejects the round _record(journal_path, round_index, edits, "rejected", before, before, f"replay_failed: {exc}") return _outcome(round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before) after_findings, after_rules = _scan(target, ruleset_dir, candidate, policy) after = _evaluate(target, manifest, after_findings, after_rules, policy) accepted = ( after.recall >= recall_floor and after.fp_rate < fp_ceiling and after.score >= before.score and after.matched >= before.matched ) profiles_before: dict[str, dict[str, float]] = {} profiles_after: dict[str, dict[str, float]] = {} regressed: list[str] = [] small: list[str] = [] if accepted and pair_cases: # pair-corpus-honesty Req 7.2: a change may not help one provenance profile by hurting another. profiles_before = evaluate_profiles(pair_cases, before_rules) profiles_after = evaluate_profiles(pair_cases, after_rules) if len(profiles_before) > MAX_PROFILES: raise ValueError(f"at most {MAX_PROFILES} provenance profiles are supported; got {sorted(profiles_before)}") regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs) if regressed: accepted = False mechanisms_before: dict[str, dict[str, float]] = {} mechanisms_after: dict[str, dict[str, float]] = {} mechanism_reason = "" if accepted and mechanism_edits and candidates_store is not None and mechanism_store is not None and pair_cases: # The lever changes what variant search finds, so its gate is the same per-profile clause over the holdout pairs # scored with the store before and after; an admission that leaks or lowers any profile is reverted byte for byte. mechanisms_before = evaluate_mechanism_profiles(pair_cases, mechanism_store) snapshot = apply_mechanism_edits(mechanism_edits, candidates_store, mechanism_store) mechanisms_after = evaluate_mechanism_profiles(pair_cases, mechanism_store) mech_regressed, mech_small = profile_regressions( mechanisms_before, mechanisms_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs ) leaking = _leaking_admissions(mechanism_edits, pair_cases, mechanism_store) improved = any( mechanisms_after[p]["pair_correct"] > mechanisms_before.get(p, {}).get("pair_correct", 0.0) for p in mechanisms_after ) or any(edit.action == "retract" for edit in mechanism_edits) if mech_regressed or leaking or not improved: snapshot.restore() accepted = False regressed = regressed + mech_regressed if leaking: mechanism_reason = f"mechanism_leak:{leaking[0]}" # an admission that fires on a holdout fixed side (Req 5.3) elif mech_regressed: mechanism_reason = f"profile_regression:{mech_regressed[0]}" else: mechanism_reason = "mechanism_no_gain" small = sorted(set(small) | set(mech_small)) if accepted: if edits: write_rule_ledger(ledger_path, candidate) reason = "accepted" elif mechanism_reason: reason = mechanism_reason elif regressed: reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions else: reason = "reverted" # ledger_path untouched -> byte-for-byte revert _record( journal_path, round_index, edits, "accepted" if accepted else "reverted", before, after, reason, regressed, small, mechanism_edits=mechanism_edits, mechanisms_before=mechanisms_before, mechanisms_after=mechanisms_after, candidates=candidates_store, ) outcome = _outcome( round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, mechanism_edits=mechanism_edits ) return RoundOutcome( **{ **outcome.__dict__, "profile_regressions": regressed, "profiles_under_minimum": small, "per_profile_before": profiles_before, "per_profile_after": profiles_after, "mechanisms_before": mechanisms_before, "mechanisms_after": mechanisms_after, } ) def run_improvement( target: Path, manifest: BenchmarkManifest, *, ledger_path: Path, journal_path: Path, ruleset_dir: Path = DEFAULT_RULESET_DIR, policy: dict[str, CwePolicy] | None = None, max_rounds: int = 5, recall_floor: float = 0.9, fp_ceiling: float = 0.1, pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, mechanism_candidates: Path | None = None, mechanism_store: Path | None = None, ) -> list[RoundOutcome]: """Run improvement rounds until convergence (no new proposals) or ``max_rounds``.""" outcomes: list[RoundOutcome] = [] for _ in range(max_rounds): outcome = run_round( target, manifest, ledger_path=ledger_path, journal_path=journal_path, ruleset_dir=ruleset_dir, policy=policy, recall_floor=recall_floor, fp_ceiling=fp_ceiling, pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, mechanism_candidates=mechanism_candidates, mechanism_store=mechanism_store, ) outcomes.append(outcome) if outcome.reason == "no_proposals": break return outcomes @dataclass(frozen=True) class _Eval: result: BenchmarkResult recall: float fp_rate: float score: int matched: int def _scan( target: Path, ruleset_dir: Path, ledger: Mapping[str, dict[str, object]], policy: dict[str, CwePolicy] ) -> tuple[list[StaticFinding], tuple[PatternRule, ...]]: rules = _load_with(ruleset_dir, ledger) _, targets = preprocess_repository(target) targets = attach_reachability_hints(targets, analyze_entry_points(target, targets)) rankings = rank_targets(targets) findings = quick_scan_findings(target, targets, rankings, rules, policy) reported = [finding for finding in findings if finding.status != "shadow"] return reported, rules def _evaluate( target: Path, manifest: BenchmarkManifest, findings: list[StaticFinding], rules: tuple[PatternRule, ...], policy: dict[str, CwePolicy] ) -> _Eval: run = BenchmarkRun(benchmark_run_id="improve", root=target, manifest=manifest) result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None) metrics = result.metrics cwe_by_rule = {rule.rule_id: rule.cwe for rule in rules} rule_cwe_by_id = {f.finding_id: cwe_by_rule.get(f.finding_id.split(":", 1)[0], "") for f in findings} score = build_score_artifact(findings, rule_cwe_by_id, policy).project_score recall = metrics.recall if metrics.recall is not None else 1.0 fp_rate = metrics.false_positive_findings_total / metrics.actual_findings_total if metrics.actual_findings_total else 0.0 return _Eval(result=result, recall=recall, fp_rate=fp_rate, score=score, matched=metrics.matched_findings_total) def _load_with(ruleset_dir: Path, ledger: Mapping[str, dict[str, object]]) -> tuple[PatternRule, ...]: import json with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: handle.write(json.dumps(ledger)) temp_path = Path(handle.name) try: return load_ruleset(ruleset_dir, temp_path) finally: temp_path.unlink(missing_ok=True) def _write_signals(path: Path, signals: list[dict[str, object]]) -> None: import json path.write_text(json.dumps(signals, indent=2, sort_keys=True) + "\n") def _outcome( round_index: int, *, accepted: bool, reason: str, edits: list[RuleStatusEdit], before: _Eval, after: _Eval, mechanism_edits: list[MechanismEdit] | None = None, ) -> RoundOutcome: return RoundOutcome( round=round_index, accepted=accepted, reason=reason, edits=edits, mechanism_edits=list(mechanism_edits or []), recall_before=before.recall, recall_after=after.recall, fp_before=before.fp_rate, fp_after=after.fp_rate, score_before=before.score, score_after=after.score, matched_before=before.matched, matched_after=after.matched, ) def _record( journal_path: Path, round_index: int, edits: list[RuleStatusEdit], outcome: str, before: _Eval, after: _Eval, reason: str = "", profile_regressions: list[str] | None = None, profiles_under_minimum: list[str] | None = None, mechanism_edits: Sequence[MechanismEdit] = (), mechanisms_before: dict[str, dict[str, float]] | None = None, mechanisms_after: dict[str, dict[str, float]] | None = None, candidates: object | None = None, ) -> None: pairs_by_id: dict[str, list[str]] = {} if candidates is not None: pairs_by_id = {record.id: list(record.pairs) for record in candidates.load()} # type: ignore[attr-defined] append_round( journal_path, { "round": round_index, "outcome": outcome, "reason": reason or outcome, "profile_regressions": list(profile_regressions or []), "profiles_under_minimum": list(profiles_under_minimum or []), "edits": [ {"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale} for e in edits ] + [ { "key": m.key(), "lever": m.lever, "action": m.action, "mechanism_id": m.mechanism_id, "source": m.source, "pairs": pairs_by_id.get(m.mechanism_id, []), "rationale": m.rationale, } for m in mechanism_edits ], "mechanisms_before": mechanisms_before or {}, "mechanisms_after": mechanisms_after or {}, "recall_before": round(before.recall, 4), "recall_after": round(after.recall, 4), "fp_before": round(before.fp_rate, 4), "fp_after": round(after.fp_rate, 4), "score_before": before.score, "score_after": after.score, }, )