mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
feat(improve): proposals from plane memory through the unchanged gate (harnessx-removal task 3)
improve/memory.py: rules M1 (enabled->shadow on repeated false alerts across >=2 repositories, no agreed or site_match hit) and M2 (shadow->enabled on >=2 agreed declared sites no enabled rule catches); M3 disputed families are advisory (signals.json) and never reach the validator. The train-on-test guard drops rows of --qualify-population, gated benchmark cases and holdout pair repositories before any rule sees a row. run_round(proposals=...) adds memory edits after the benchmark's through the unchanged validator and gate; provenance (row ids, object keys and versions, counts, exclusions, index digest) goes to memory_proposals.jsonl and the journal's evidence field; proposal_outcome rows are written back after a non-dry run. ousast improve --memory is opt-in; default improve output is byte-identical to the task-1 baseline. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
52d28b5121
commit
d104166697
@@ -205,6 +205,22 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"--profile-tolerance", type=float, default=0.0, help="allowed per-profile drop in pair_correct/Youden before rejecting"
|
||||
)
|
||||
improve.add_argument("--min-holdout-pairs", type=int, default=5, help="profiles with fewer holdout pairs are reported, not gated")
|
||||
improve.add_argument(
|
||||
"--memory",
|
||||
nargs="?",
|
||||
const="",
|
||||
default=None,
|
||||
metavar="STORE",
|
||||
help="also propose rule-status edits from the plane memory store (a directory, file:// or minio:// URL; "
|
||||
"no value: OUSAST_MEMORY or the default store); off by default",
|
||||
)
|
||||
improve.add_argument(
|
||||
"--qualify-population",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME",
|
||||
help="population a memory proposal will be qualified on; its rows never propose (repeatable)",
|
||||
)
|
||||
|
||||
pairs = subparsers.add_parser(
|
||||
"pairs",
|
||||
@@ -367,6 +383,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
pair_catalog=None if args.no_pair_gate else args.pair_catalog,
|
||||
profile_tolerance=args.profile_tolerance,
|
||||
min_holdout_pairs=args.min_holdout_pairs,
|
||||
memory=args.memory,
|
||||
qualify_populations=tuple(args.qualify_population),
|
||||
guard_catalog=args.pair_catalog,
|
||||
)
|
||||
if args.command == "pairs":
|
||||
return _pairs(
|
||||
@@ -929,6 +948,9 @@ def _improve(
|
||||
pair_catalog: Path | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
memory: str | None = None,
|
||||
qualify_populations: tuple[str, ...] = (),
|
||||
guard_catalog: Path | None = None,
|
||||
) -> int:
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
||||
@@ -940,6 +962,7 @@ def _improve(
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
policy = load_policy()
|
||||
session = _memory_session(memory, manifest, target, ruleset_dir, qualify_populations, guard_catalog) if memory is not None else None
|
||||
|
||||
# The loop writes its accepted ledger where `scan`/`benchmark` read it, so a
|
||||
# subsequent scan of this target automatically picks up the improved ruleset.
|
||||
@@ -965,11 +988,22 @@ def _improve(
|
||||
pair_cases=pair_cases,
|
||||
profile_tolerance=profile_tolerance,
|
||||
min_holdout_pairs=min_holdout_pairs,
|
||||
proposals=session.proposals(ledger_path, journal_path) if session else (),
|
||||
)
|
||||
_print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run)
|
||||
if session is not None:
|
||||
session.finish(outcomes, journal_path, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def _memory_session(
|
||||
memory: str, manifest: object, target: Path, ruleset_dir: Path, populations: tuple[str, ...], catalog: Path | None
|
||||
) -> Any:
|
||||
from .improve.memory import MemorySession # lazy: the default `improve` path never touches the store
|
||||
|
||||
return MemorySession.open(memory, manifest, target, ruleset_dir, populations, catalog)
|
||||
|
||||
|
||||
def _pairs(
|
||||
catalog: Path,
|
||||
slice_name: str,
|
||||
|
||||
@@ -5,8 +5,8 @@ edits (auto-shadow precision-draggers) -> validate -> replay smoke -> re-benchma
|
||||
hard acceptance gate (recall >= floor AND FP < ceiling AND project score not regressed
|
||||
AND no matched-finding regression) -> accept (persist the ledger) or revert byte-for-byte.
|
||||
|
||||
The edit *proposer* here is the benchmark's per-rule signal; the LLM ``MetaAgent.evolve``
|
||||
proposer plugs into this same validated/gated machinery when the HarnessX extra is present.
|
||||
The edit *proposers* are the benchmark's per-rule signal and, opt-in, the plane's memory store
|
||||
(:mod:`.memory`, ``run_round(proposals=...)``); both go through this same validator and gate.
|
||||
The project score is the optimization reward; the recall/FP gate is a separate hard
|
||||
feasibility constraint that is never folded into the reward.
|
||||
"""
|
||||
@@ -27,6 +27,7 @@ from ..rank import rank_targets
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger
|
||||
from ..scoring import build_score_artifact
|
||||
from .journal import append_round, load_journal, next_round_index, reverted_edit_keys
|
||||
from .memory import MemoryProposal, applicable, proposal_id_of, write_sidecar
|
||||
from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger
|
||||
|
||||
|
||||
@@ -159,6 +160,7 @@ def run_round(
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
proposals: Sequence[MemoryProposal] = (),
|
||||
) -> RoundOutcome:
|
||||
policy = policy if policy is not None else load_policy()
|
||||
rounds = load_journal(journal_path)
|
||||
@@ -173,6 +175,14 @@ def run_round(
|
||||
|
||||
ruleset_by_id = {rule.rule_id: rule for rule in before_rules}
|
||||
edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
|
||||
if proposals:
|
||||
# harnessx-removal Req 6.2: memory proposals join after the benchmark's edits; the first edit per key wins.
|
||||
taken = {edit.key() for edit in edits}
|
||||
for proposal in applicable(proposals, ruleset_by_id, current_ledger, rounds):
|
||||
if proposal.edit.key() not in taken:
|
||||
taken.add(proposal.edit.key())
|
||||
edits.append(proposal.edit)
|
||||
write_sidecar(journal_path, round_index, [p for p in proposals if p.edit in edits])
|
||||
refusals: list[dict[str, object]] = []
|
||||
if not edits:
|
||||
return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals)
|
||||
@@ -260,6 +270,7 @@ def run_improvement(
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
proposals: Sequence[MemoryProposal] = (),
|
||||
) -> list[RoundOutcome]:
|
||||
"""Run improvement rounds until convergence (no new proposals) or ``max_rounds``."""
|
||||
outcomes: list[RoundOutcome] = []
|
||||
@@ -276,6 +287,7 @@ def run_improvement(
|
||||
pair_cases=pair_cases,
|
||||
profile_tolerance=profile_tolerance,
|
||||
min_holdout_pairs=min_holdout_pairs,
|
||||
proposals=proposals,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if outcome.reason == "no_proposals":
|
||||
@@ -382,10 +394,7 @@ def _record(
|
||||
"reason": reason or outcome,
|
||||
"profile_regressions": list(profile_regressions or []),
|
||||
"profiles_under_minimum": list(profiles_under_minimum or []),
|
||||
"edits": [
|
||||
{"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale}
|
||||
for e in edits
|
||||
],
|
||||
"edits": [_journal_edit(e) for e in edits],
|
||||
"recall_before": round(before.recall, 4),
|
||||
"recall_after": round(after.recall, 4),
|
||||
"fp_before": round(before.fp_rate, 4),
|
||||
@@ -394,3 +403,14 @@ def _record(
|
||||
"score_after": after.score,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _journal_edit(edit: RuleStatusEdit) -> dict[str, object]:
|
||||
entry: dict[str, object] = {
|
||||
"key": edit.key(), "lever": edit.lever, "rule_id": edit.rule_id, "from": edit.from_status, "to": edit.to_status,
|
||||
"rationale": edit.rationale,
|
||||
} # fmt: skip
|
||||
proposal_id = proposal_id_of(edit)
|
||||
if proposal_id is not None:
|
||||
entry["evidence"] = proposal_id # only a memory edit carries it; every other entry is unchanged byte for byte
|
||||
return entry
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
"""Rule-status proposals from the plane's memory store (harnessx-removal Req 6.2/6.4, design section 5).
|
||||
|
||||
The store (:mod:`..plane.memory`) holds what plane runs learned per repository and pin. Two deterministic rules turn
|
||||
its ``alert`` and ``verdict`` rows into :class:`RuleStatusEdit` proposals that :func:`.evolve.run_round` validates
|
||||
and gates exactly like the benchmark's own edits; a third reads dispute rates as advisory routing input only.
|
||||
|
||||
Join key: an ``alert`` and a ``verdict`` match when repo, pin and ``path::function`` are equal. A ``disputed``
|
||||
final counts as no evidence in either direction.
|
||||
|
||||
- **M1, repeated false alerts** (``enabled -> shadow``): at least :data:`M1_MIN_FALSE` alerts of the rule are false
|
||||
-- at a ``rejected`` candidate, or on a ``fixed`` pin inside the declared fix range (the alert row says
|
||||
``in_fix_range: true``, or its ``path::function`` is a ``site_match`` site of the same repository) -- across at
|
||||
least :data:`M1_MIN_REPOS` repositories, and no alert of the rule sits at an ``agreed`` or ``site_match``
|
||||
candidate.
|
||||
- **M2, missed declared sites** (``shadow -> enabled``): the shadow rule's alerts hit at least :data:`M2_MIN_HITS`
|
||||
``agreed`` candidates with ``site_match`` true across at least :data:`M2_MIN_REPOS` repositories, no enabled rule
|
||||
alerts at those candidates, the rule has no ``fixed``-pin alert at the same function, and at most
|
||||
:data:`M2_MAX_REJECTED` alert at a ``rejected`` candidate.
|
||||
- **M3, disputed families**: families with at least :data:`M3_MIN_CANDIDATES` candidates and a dispute rate of at
|
||||
least :data:`M3_MIN_DISPUTE_RATE`. Advisory (``signals.json``); never an edit, never shown to the validator.
|
||||
|
||||
**Train-on-test guard** (:class:`Guard`), applied before any rule sees a row: a row is dropped when its population
|
||||
is one the proposal will be qualified on, its (repo, pin) is a case of the gated benchmark manifest, or its
|
||||
repository is a holdout pair's repository. The pair catalog records a holdout pair's *fix* commit, not the
|
||||
vulnerable pin (its parent), so the holdout clause matches by repository, a superset of (repo, pin). This is the
|
||||
rule the closed-loop leak broke, where a holdout pair taught the shape that then "recovered" it.
|
||||
|
||||
**Provenance**: each proposal's rationale is ``memory:<proposal_id>``, ``proposal_id`` being the sha256 of (rule,
|
||||
edit key, sorted evidence row ids, thresholds). The sidecar ``memory_proposals.jsonl`` next to the journal carries
|
||||
the evidence rows (id, repo, pin, run, task, object key and version -- MinIO's version id, a file's content hash),
|
||||
the counts that met each threshold, what the guard excluded and the store's index digest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..plane.memory import MemoryStore, Record, repo_key, row_id, rows_digest
|
||||
from ..ruleset import PatternRule
|
||||
from .validator import RuleStatusEdit
|
||||
|
||||
M1_MIN_FALSE = 3
|
||||
M1_MIN_REPOS = 2
|
||||
M2_MIN_HITS = 2
|
||||
M2_MIN_REPOS = 2
|
||||
M2_MAX_REJECTED = 1
|
||||
M3_MIN_CANDIDATES = 5
|
||||
M3_MIN_DISPUTE_RATE = 0.3
|
||||
THRESHOLDS: dict[str, dict[str, float]] = {
|
||||
"M1": {"min_false_alerts": M1_MIN_FALSE, "min_repos": M1_MIN_REPOS, "max_agreed_hits": 0},
|
||||
"M2": {"min_site_hits": M2_MIN_HITS, "min_repos": M2_MIN_REPOS, "max_rejected": M2_MAX_REJECTED, "max_fixed_pin_same_function": 0},
|
||||
"M3": {"min_candidates": M3_MIN_CANDIDATES, "min_dispute_rate": M3_MIN_DISPUTE_RATE},
|
||||
}
|
||||
RATIONALE_PREFIX = "memory:"
|
||||
SIDECAR = "memory_proposals.jsonl"
|
||||
SIGNALS = "signals.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemoryProposal:
|
||||
edit: RuleStatusEdit
|
||||
provenance: dict[str, Any]
|
||||
|
||||
@property
|
||||
def proposal_id(self) -> str:
|
||||
return str(self.provenance["proposal_id"])
|
||||
|
||||
|
||||
def proposal_id_of(edit: object) -> str | None:
|
||||
"""The proposal id a memory edit carries in its rationale; None for any other edit."""
|
||||
rationale = getattr(edit, "rationale", "")
|
||||
return rationale[len(RATIONALE_PREFIX) :] if isinstance(rationale, str) and rationale.startswith(RATIONALE_PREFIX) else None
|
||||
|
||||
|
||||
# --- the train-on-test guard ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tail(repo: str) -> str:
|
||||
"""``owner/name`` of a repository, lower-cased: the pair catalog names repositories without a host."""
|
||||
return "/".join(repo_key(repo).lower().split("/")[-2:])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Guard:
|
||||
populations: frozenset[str] = frozenset()
|
||||
gated_cases: frozenset[tuple[str, str]] = frozenset() # (repo_key, pin) of the benchmark being gated
|
||||
holdout_repos: frozenset[str] = frozenset() # owner/name of every holdout pair
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls, populations: Iterable[str] = (), gated_cases: Iterable[tuple[str, str]] = (), holdout_pairs: Iterable[object] = ()
|
||||
) -> Guard:
|
||||
repos = {_tail(str(getattr(p, "repo", ""))) for p in holdout_pairs if getattr(p, "repo", "")}
|
||||
return cls(frozenset(populations), frozenset((repo_key(r), p) for r, p in gated_cases), frozenset(repos))
|
||||
|
||||
def reason(self, row: Mapping[str, Any]) -> str | None:
|
||||
if row.get("population") in self.populations:
|
||||
return "population"
|
||||
if (row.get("repo"), row.get("pin")) in self.gated_cases:
|
||||
return "gated_case"
|
||||
if _tail(str(row.get("repo"))) in self.holdout_repos:
|
||||
return "holdout_pair"
|
||||
return None
|
||||
|
||||
def apply(self, records: Iterable[Record]) -> tuple[list[Record], dict[str, Any]]:
|
||||
"""(kept records, what was excluded: row count, and counts per population / repo@pin per clause)."""
|
||||
kept: list[Record] = []
|
||||
excluded: dict[str, Any] = {"rows": 0, "population": {}, "gated_case": {}, "holdout_pair": {}}
|
||||
for record in records:
|
||||
why = self.reason(record.row)
|
||||
if why is None:
|
||||
kept.append(record)
|
||||
continue
|
||||
excluded["rows"] += 1
|
||||
label = str(record.row.get("population")) if why == "population" else f"{record.row.get('repo')}@{record.row.get('pin')}"
|
||||
excluded[why][label] = excluded[why].get(label, 0) + 1
|
||||
for clause in ("population", "gated_case", "holdout_pair"):
|
||||
excluded[clause] = dict(sorted(excluded[clause].items()))
|
||||
return kept, excluded
|
||||
|
||||
|
||||
# --- the rules -----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cand(row: Mapping[str, Any]) -> str:
|
||||
return f"{row.get('path')}::{row.get('function')}"
|
||||
|
||||
|
||||
def _status(rule_id: str, ruleset_by_id: Mapping[str, PatternRule], ledger: Mapping[str, dict[str, object]]) -> str | None:
|
||||
rule = ruleset_by_id.get(rule_id)
|
||||
return None if rule is None else str(ledger.get(rule_id, {}).get("status", rule.status))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _View:
|
||||
verdicts: dict[tuple[str, str, str], Record] = field(default_factory=dict) # (repo, pin, candidate)
|
||||
sites: set[tuple[str, str]] = field(default_factory=set) # (repo, candidate) with site_match true
|
||||
alerts: dict[str, list[Record]] = field(default_factory=dict) # rule_id -> alerts
|
||||
|
||||
@classmethod
|
||||
def of(cls, records: Sequence[Record]) -> _View:
|
||||
view = cls()
|
||||
for rec in records:
|
||||
row = rec.row
|
||||
if row["kind"] == "verdict":
|
||||
view.verdicts[(row["repo"], row["pin"], str(row.get("candidate")))] = rec
|
||||
if row.get("site_match") is True:
|
||||
view.sites.add((row["repo"], str(row.get("candidate"))))
|
||||
elif row["kind"] == "alert" and row.get("rule_id"):
|
||||
view.alerts.setdefault(str(row["rule_id"]), []).append(rec)
|
||||
return view
|
||||
|
||||
def verdict(self, alert: Mapping[str, Any]) -> Record | None:
|
||||
return self.verdicts.get((alert["repo"], alert["pin"], _cand(alert)))
|
||||
|
||||
def fixed_in_range(self, alert: Mapping[str, Any]) -> bool:
|
||||
if alert.get("pin_role") != "fixed":
|
||||
return False
|
||||
flag = alert.get("in_fix_range")
|
||||
return bool(flag) if flag is not None else (alert["repo"], _cand(alert)) in self.sites
|
||||
|
||||
|
||||
def _m1(rule_id: str, view: _View) -> tuple[list[Record], dict[str, int]] | None:
|
||||
evidence: list[Record] = []
|
||||
repos: set[str] = set()
|
||||
false_alerts = 0
|
||||
for alert in view.alerts.get(rule_id, []):
|
||||
verdict = view.verdict(alert.row)
|
||||
final = verdict.row.get("final") if verdict else None
|
||||
if final == "agreed" or (verdict is not None and verdict.row.get("site_match") is True):
|
||||
return None # the rule caught something the plane agreed on: never demote it on this evidence
|
||||
if final == "rejected" or view.fixed_in_range(alert.row):
|
||||
false_alerts += 1
|
||||
repos.add(alert.row["repo"])
|
||||
evidence.extend([alert, verdict] if verdict is not None and final == "rejected" else [alert])
|
||||
if false_alerts < M1_MIN_FALSE or len(repos) < M1_MIN_REPOS:
|
||||
return None
|
||||
return evidence, {"false_alerts": false_alerts, "repos": len(repos), "agreed_hits": 0}
|
||||
|
||||
|
||||
def _m2(rule_id: str, view: _View, enabled: set[str]) -> tuple[list[Record], dict[str, int]] | None:
|
||||
alerts = view.alerts.get(rule_id, [])
|
||||
hits: dict[tuple[str, str, str], tuple[Record, Record]] = {}
|
||||
rejected = 0
|
||||
for alert in alerts:
|
||||
verdict = view.verdict(alert.row)
|
||||
if verdict is None or alert.row.get("pin_role") == "fixed":
|
||||
continue
|
||||
if verdict.row.get("final") == "rejected":
|
||||
rejected += 1
|
||||
elif verdict.row.get("final") == "agreed" and verdict.row.get("site_match") is True:
|
||||
hits[(alert.row["repo"], alert.row["pin"], _cand(alert.row))] = (alert, verdict)
|
||||
if not hits:
|
||||
return None
|
||||
caught = {
|
||||
(a.row["repo"], a.row["pin"], _cand(a.row)) for rid in enabled if rid != rule_id for a in view.alerts.get(rid, [])
|
||||
} # fmt: skip
|
||||
missed = {k: v for k, v in hits.items() if k not in caught}
|
||||
fixed_same = sum(
|
||||
1 for a in alerts if a.row.get("pin_role") == "fixed" and any((a.row["repo"], _cand(a.row)) == (r, c) for r, _, c in missed)
|
||||
)
|
||||
repos = {repo for repo, _, _ in missed}
|
||||
if len(missed) < M2_MIN_HITS or len(repos) < M2_MIN_REPOS or fixed_same > 0 or rejected > M2_MAX_REJECTED:
|
||||
return None
|
||||
evidence = [rec for key in sorted(missed) for rec in missed[key]]
|
||||
return evidence, {"site_hits": len(missed), "repos": len(repos), "rejected": rejected, "fixed_pin_same_function": 0}
|
||||
|
||||
|
||||
def _evidence(records: Sequence[Record]) -> list[dict[str, Any]]:
|
||||
unique = {rec.row["id"]: rec for rec in records}
|
||||
return [
|
||||
{"id": i, "kind": r.row["kind"], "repo": r.row["repo"], "pin": r.row["pin"], "run": r.row["run"], "task": r.row["task"],
|
||||
"key": r.key, "version": r.version}
|
||||
for i, r in sorted(unique.items())
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def evidence_digest(evidence_ids: Iterable[str]) -> str:
|
||||
return hashlib.sha256("\n".join(sorted(evidence_ids)).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _proposal(
|
||||
rule: str, edit_key: str, rule_id: str, records: Sequence[Record], counts: dict[str, int], excluded: dict[str, Any], index: str
|
||||
) -> dict[str, Any]:
|
||||
evidence = _evidence(records)
|
||||
ids = [e["id"] for e in evidence]
|
||||
seed = json.dumps([rule, edit_key, ids, THRESHOLDS[rule]], sort_keys=True, separators=(",", ":"))
|
||||
return {
|
||||
"proposal_id": hashlib.sha256(seed.encode("utf-8")).hexdigest(),
|
||||
"rule": rule,
|
||||
"edit_key": edit_key,
|
||||
"rule_id": rule_id,
|
||||
"evidence": evidence,
|
||||
"evidence_digest": evidence_digest(ids),
|
||||
"counts": counts,
|
||||
"thresholds": THRESHOLDS[rule],
|
||||
"excluded": excluded,
|
||||
"index_digest": index,
|
||||
}
|
||||
|
||||
|
||||
def propose_from_memory(
|
||||
rows: Sequence[Record],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
journal_rounds: Sequence[Mapping[str, object]],
|
||||
*,
|
||||
excluded: Guard,
|
||||
index_digest: str = "",
|
||||
) -> list[MemoryProposal]:
|
||||
"""M1 and M2 over the rows the guard keeps, minus proposals already reverted on the same evidence."""
|
||||
kept, dropped = excluded.apply(rows)
|
||||
view = _View.of(kept)
|
||||
statuses = {rid: _status(rid, ruleset_by_id, current_ledger) for rid in ruleset_by_id}
|
||||
enabled = {rid for rid, status in statuses.items() if status == "enabled"}
|
||||
proposals: list[MemoryProposal] = []
|
||||
for rule_id in sorted(view.alerts):
|
||||
status = statuses.get(rule_id)
|
||||
if status == "enabled" and (m1 := _m1(rule_id, view)):
|
||||
name, found, to_status = "M1", m1, "shadow"
|
||||
elif status == "shadow" and (m2 := _m2(rule_id, view, enabled)):
|
||||
name, found, to_status = "M2", m2, "enabled"
|
||||
else:
|
||||
continue # unknown to the ruleset (the loop cannot add rules), disabled, or no rule fired
|
||||
key = RuleStatusEdit(rule_id, status, to_status).key()
|
||||
provenance = _proposal(name, key, rule_id, found[0], found[1], dropped, index_digest)
|
||||
edit = RuleStatusEdit(rule_id, status, to_status, rationale=RATIONALE_PREFIX + provenance["proposal_id"])
|
||||
proposals.append(MemoryProposal(edit, provenance))
|
||||
return applicable(proposals, ruleset_by_id, current_ledger, journal_rounds)
|
||||
|
||||
|
||||
def applicable(
|
||||
proposals: Sequence[MemoryProposal],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
journal_rounds: Sequence[Mapping[str, object]],
|
||||
) -> list[MemoryProposal]:
|
||||
"""Proposals whose rule still has the ``from`` status and whose evidence no reverted round already tried."""
|
||||
tried: set[tuple[str, str]] = set()
|
||||
for entry in journal_rounds:
|
||||
edits = entry.get("edits")
|
||||
if entry.get("outcome") == "reverted" and isinstance(edits, list):
|
||||
tried |= {(str(e.get("key")), str(e.get("evidence"))) for e in edits if isinstance(e, dict) and e.get("evidence")}
|
||||
return [
|
||||
p for p in proposals
|
||||
if _status(p.edit.rule_id, ruleset_by_id, current_ledger) == p.edit.from_status and (p.edit.key(), p.proposal_id) not in tried
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def disputed_families(rows: Sequence[Record], guard: Guard) -> list[dict[str, Any]]:
|
||||
"""M3: families disputed often enough to route differently. Advisory only: never an edit."""
|
||||
kept, _ = guard.apply(rows)
|
||||
per: dict[str, list[int]] = {}
|
||||
for rec in kept:
|
||||
if rec.row["kind"] == "verdict" and rec.row.get("family"):
|
||||
tally = per.setdefault(str(rec.row["family"]), [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += rec.row.get("final") == "disputed"
|
||||
return [
|
||||
{"family": fam, "candidates": n, "disputed": d, "dispute_rate": round(d / n, 4), "advice": "schedule pass c up front"}
|
||||
for fam, (n, d) in sorted(per.items())
|
||||
if n >= M3_MIN_CANDIDATES and d / n >= M3_MIN_DISPUTE_RATE
|
||||
]
|
||||
|
||||
|
||||
# --- the store, the sidecar and the outcomes -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def index_digest(store: MemoryStore) -> str:
|
||||
return rows_digest(store.index())
|
||||
|
||||
|
||||
def write_sidecar(journal_path: Path, round_index: int, proposals: Sequence[MemoryProposal]) -> None:
|
||||
"""Append each proposal once (by id) to ``memory_proposals.jsonl`` next to the journal."""
|
||||
if not proposals:
|
||||
return
|
||||
path = journal_path.with_name(SIDECAR)
|
||||
seen = {json.loads(line)["proposal_id"] for line in path.read_text().splitlines() if line.strip()} if path.is_file() else set()
|
||||
new = [p for p in proposals if p.proposal_id not in seen]
|
||||
if not new:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a") as sink:
|
||||
for p in new:
|
||||
sink.write(json.dumps({**p.provenance, "round": round_index}, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
|
||||
|
||||
def read_sidecar(journal_path: Path) -> dict[str, dict[str, Any]]:
|
||||
path = journal_path.with_name(SIDECAR)
|
||||
if not path.is_file():
|
||||
return {}
|
||||
return {row["proposal_id"]: row for row in (json.loads(line) for line in path.read_text().splitlines() if line.strip())}
|
||||
|
||||
|
||||
def write_signals(directory: Path, signals: Sequence[Mapping[str, Any]]) -> Path:
|
||||
path = directory / SIGNALS
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"advisory": True, "disputed_families": list(signals), "thresholds": THRESHOLDS["M3"]}, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def outcome_of(reason: str, accepted: bool) -> str | None:
|
||||
if accepted:
|
||||
return "accepted"
|
||||
if reason == "no_proposals":
|
||||
return None
|
||||
if reason.startswith(("validation_failed", "replay_failed")):
|
||||
return "rejected"
|
||||
return "reverted"
|
||||
|
||||
|
||||
def outcome_rows(
|
||||
edits: Sequence[RuleStatusEdit],
|
||||
outcome: str,
|
||||
round_index: int,
|
||||
reason: str,
|
||||
proposals: Mapping[str, MemoryProposal],
|
||||
gate_run: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""One ``proposal_outcome`` row per (repository, pin) of each memory edit's evidence."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
for edit in edits:
|
||||
pid = proposal_id_of(edit)
|
||||
proposal = proposals.get(pid or "")
|
||||
if proposal is None:
|
||||
continue
|
||||
by_place: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for ev in proposal.provenance["evidence"]:
|
||||
by_place.setdefault((ev["repo"], ev["pin"]), ev)
|
||||
for (repo, pin), _ in sorted(by_place.items()):
|
||||
task = f"round-{round_index}"
|
||||
rows.append(
|
||||
{
|
||||
"id": row_id("proposal_outcome", gate_run, task, f"{pid}:{repo}:{pin}"), "kind": "proposal_outcome",
|
||||
"repo": repo, "pin": pin, "run": gate_run, "task": task, "population": "improve-loop", "split": "loop",
|
||||
"image": "host", "edit_key": edit.key(), "proposal_id": pid,
|
||||
"evidence_digest": proposal.provenance["evidence_digest"], "outcome": outcome, "reason": reason,
|
||||
"gate_run": gate_run,
|
||||
}
|
||||
) # fmt: skip
|
||||
return rows
|
||||
|
||||
|
||||
def gate_run_name(manifest_name: str) -> str:
|
||||
return f"improve:{manifest_name}:{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}"
|
||||
|
||||
|
||||
# --- the `ousast improve --memory` session -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def benchmark_cases(manifest: object) -> frozenset[tuple[str, str]]:
|
||||
"""(repo, pin) of the gated benchmark's cases. Local sources (the only kind `resolve_benchmark_source` accepts)
|
||||
name no repository pin, so they exclude nothing here; a pinned source would list its case."""
|
||||
source = getattr(manifest, "source", None)
|
||||
repo, pin = getattr(source, "repo", ""), getattr(source, "pin", "")
|
||||
return frozenset({(repo_key(repo), pin)}) if repo and pin else frozenset()
|
||||
|
||||
|
||||
def store_spec(value: str) -> str | None:
|
||||
"""``--memory`` value -> ``open_store`` spec: empty uses OUSAST_MEMORY or the default, a bare path is a FileStore."""
|
||||
if not value:
|
||||
return None
|
||||
return value if "://" in value else f"file://{Path(value).resolve()}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemorySession:
|
||||
"""One `improve --memory` invocation: the store read once, proposals per round, outcomes written back."""
|
||||
|
||||
store: MemoryStore
|
||||
guard: Guard
|
||||
records: list[Record]
|
||||
ruleset_dir: Path
|
||||
gate_run: str
|
||||
made: dict[str, MemoryProposal] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def open(
|
||||
cls, spec: str, manifest: object, target: Path, ruleset_dir: Path, populations: Sequence[str], catalog: Path | None
|
||||
) -> MemorySession:
|
||||
from ..pairs import load_pair_catalog, select_split
|
||||
from ..plane.memory import FileStore, open_store
|
||||
|
||||
store = open_store(store_spec(spec))
|
||||
if isinstance(store, FileStore) and not (store.root / "index.jsonl").is_file():
|
||||
raise SystemExit(f"memory store {store.describe()} has no index.jsonl: an unread store would propose nothing")
|
||||
if catalog is None or not catalog.is_file():
|
||||
raise SystemExit(f"--memory needs the pair catalog for the holdout guard: {catalog} is not a file")
|
||||
holdout = select_split(load_pair_catalog(catalog), "holdout")
|
||||
guard = Guard.build(populations, benchmark_cases(manifest), holdout)
|
||||
records = store.rows()
|
||||
print(
|
||||
f"memory={store.describe()} index_entries={len(store.index())} rows={len(records)} "
|
||||
f"guard: populations={sorted(guard.populations) or '-'} gated_cases={len(guard.gated_cases)} "
|
||||
f"holdout_repos={len(guard.holdout_repos)}"
|
||||
)
|
||||
return cls(store, guard, records, ruleset_dir, gate_run_name(str(getattr(manifest, "name", "manifest"))))
|
||||
|
||||
def proposals(self, ledger_path: Path, journal_path: Path) -> list[MemoryProposal]:
|
||||
from ..ruleset import load_ruleset, read_rule_ledger
|
||||
from .journal import load_journal
|
||||
|
||||
rules = load_ruleset(self.ruleset_dir, ledger_path if ledger_path.is_file() else None)
|
||||
found = propose_from_memory(
|
||||
self.records,
|
||||
{r.rule_id: r for r in rules},
|
||||
read_rule_ledger(ledger_path),
|
||||
load_journal(journal_path),
|
||||
excluded=self.guard,
|
||||
index_digest=index_digest(self.store),
|
||||
)
|
||||
kept, excluded = self.guard.apply(self.records)
|
||||
kinds: dict[str, int] = {}
|
||||
for rec in kept:
|
||||
kinds[rec.row["kind"]] = kinds.get(rec.row["kind"], 0) + 1
|
||||
print(f"memory_rows kept={len(kept)} by_kind={json.dumps(dict(sorted(kinds.items())))}")
|
||||
print(f"memory_excluded={json.dumps(excluded, sort_keys=True)}")
|
||||
for p in found:
|
||||
print(f"memory_proposal {p.edit.key()} rule={p.provenance['rule']} id={p.proposal_id} evidence={len(p.provenance['evidence'])}")
|
||||
print(f"memory_proposals={len(found)}")
|
||||
self.made.update({p.proposal_id: p for p in found})
|
||||
return found
|
||||
|
||||
def finish(self, outcomes: Sequence[Any], journal_path: Path, *, dry_run: bool) -> None:
|
||||
signals = disputed_families(self.records, self.guard)
|
||||
path = write_signals(journal_path.parent, signals)
|
||||
print(f"memory_signals disputed_families={[s['family'] for s in signals] or '-'} ({path.name}, advisory)")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for o in outcomes:
|
||||
outcome = outcome_of(o.reason, o.accepted)
|
||||
if outcome is not None:
|
||||
rows += outcome_rows(o.edits, outcome, o.round, o.reason, self.made, self.gate_run)
|
||||
if dry_run:
|
||||
print(f"memory_outcomes={len(rows)} (dry run: not written to the store)")
|
||||
return
|
||||
if rows:
|
||||
self.store.put_rows(rows)
|
||||
print(f"memory_outcomes={len(rows)} written to {self.store.describe()}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Guard",
|
||||
"MemorySession",
|
||||
"benchmark_cases",
|
||||
"store_spec",
|
||||
"M1_MIN_FALSE",
|
||||
"M1_MIN_REPOS",
|
||||
"M2_MAX_REJECTED",
|
||||
"M2_MIN_HITS",
|
||||
"M2_MIN_REPOS",
|
||||
"M3_MIN_CANDIDATES",
|
||||
"M3_MIN_DISPUTE_RATE",
|
||||
"MemoryProposal",
|
||||
"THRESHOLDS",
|
||||
"applicable",
|
||||
"disputed_families",
|
||||
"evidence_digest",
|
||||
"gate_run_name",
|
||||
"index_digest",
|
||||
"outcome_of",
|
||||
"outcome_rows",
|
||||
"propose_from_memory",
|
||||
"proposal_id_of",
|
||||
"read_sidecar",
|
||||
"write_sidecar",
|
||||
"write_signals",
|
||||
]
|
||||
Reference in New Issue
Block a user