mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
chore(p0): remove the improve lever and the plane improvement loop
Off every active line (PATHFINDER-2026-10-13/03-unified-proposal.md, cuts). Deleted: src/openultrasast/improve/ (1,175 lines), the ousast improve subcommand, plane/tasks/loop.py and loop.yaml with the --loop generation, the memory-snapshot step in memory.seed, pairs.build_pair_signals, and the four improve tests plus the loop cases in other tests. Docs no longer describe the lever. Net: 3,073 lines deleted, 83 added. Targeted tests: 154 passed; 3,436 tests collect. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
d2010d0693
commit
94e96c6885
+4
-182
@@ -6,7 +6,6 @@ import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
@@ -39,10 +38,8 @@ from .complexity.ledger import persist_verdicts
|
||||
from .config import ModelLayerConfig, ObligationsConfig, ResolvedConfig, RetiredConfigError, load_config, load_dotenv
|
||||
from .findings import StaticFinding, quick_scan_findings, uncovered_languages, write_findings
|
||||
from .fusion import FusionDecision, fuse_findings
|
||||
from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
from .hunter import run_hunter_pool, write_hunter_trajectories
|
||||
from .improve import RoundOutcome, run_improvement
|
||||
from .learn.cli import add_commands as add_learn_commands
|
||||
from .learn.cli import run as run_learn
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
@@ -234,55 +231,9 @@ def _main(argv: list[str] | None) -> int:
|
||||
benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick")
|
||||
benchmark.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
||||
|
||||
improve = subparsers.add_parser(
|
||||
"improve",
|
||||
help="run the bounded self-improvement loop against a benchmark manifest and update the loop-owned ruleset ledger",
|
||||
)
|
||||
improve.add_argument("manifest", type=Path)
|
||||
improve.add_argument("--max-rounds", type=int, default=5)
|
||||
improve.add_argument("--recall-floor", type=float, default=RECALL_FLOOR)
|
||||
improve.add_argument("--fp-ceiling", type=float, default=FALSE_POSITIVE_CEILING)
|
||||
improve.add_argument(
|
||||
"--ledger", type=Path, default=None, help="ruleset ledger path (default: <target>/.openultrasast/calibration/rule_policy.json)"
|
||||
)
|
||||
improve.add_argument("--journal", type=Path, default=None, help="improvement journal path (default: alongside the ledger)")
|
||||
improve.add_argument(
|
||||
"--ruleset-dir", type=Path, default=DEFAULT_RULESET_DIR, help="ruleset directory to improve (default: the bundled ruleset)"
|
||||
)
|
||||
improve.add_argument("--dry-run", action="store_true", help="run rounds against a throwaway ledger; never touch the target's ledger")
|
||||
improve.add_argument(
|
||||
"--pair-catalog",
|
||||
type=Path,
|
||||
default=DEFAULT_CATALOG,
|
||||
help="pair catalog whose holdout split gates every round per provenance profile",
|
||||
)
|
||||
improve.add_argument(
|
||||
"--no-pair-gate", action="store_true", help="skip the per-profile holdout clause (faster; not for accepted ledgers)"
|
||||
)
|
||||
improve.add_argument(
|
||||
"--profile-tolerance", type=float, default=0.0, help="allowed per-profile drop in pair_correct/Youden before rejecting"
|
||||
)
|
||||
improve.add_argument("--min-holdout-pairs", type=int, default=5, help="profiles with fewer holdout pairs are reported, not gated")
|
||||
improve.add_argument(
|
||||
"--memory",
|
||||
nargs="?",
|
||||
const="",
|
||||
default=None,
|
||||
metavar="STORE",
|
||||
help="also propose rule-status edits from the plane memory store (a directory, file:// or s3:// URL; "
|
||||
"no value: OUSAST_MEMORY or the default store); off by default",
|
||||
)
|
||||
improve.add_argument(
|
||||
"--qualify-population",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="NAME",
|
||||
help="population a memory proposal will be qualified on; its rows never propose (repeatable)",
|
||||
)
|
||||
|
||||
pairs = subparsers.add_parser(
|
||||
"pairs",
|
||||
help="scan isolated vuln-vs-fixed pairs (TP on vuln, silent on fix) and emit improve-loop signals",
|
||||
help="scan isolated vuln-vs-fixed pairs (TP on vuln, silent on fix) and report misses and fixed-side false positives",
|
||||
)
|
||||
pairs.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG)
|
||||
pairs.add_argument("--slice", choices=SLICE_NAMES, default="all")
|
||||
@@ -392,17 +343,6 @@ def _main(argv: list[str] | None) -> int:
|
||||
plane_ws.add_argument(
|
||||
"--runner-image", type=Path, help="with --validation-set: re-pin the task templates to the image in this file (ops/ax/up.sh)"
|
||||
)
|
||||
plane_ws.add_argument(
|
||||
"--loop", action="store_true",
|
||||
help="with --validation-set: append the improvement loop (alerts per case, then snapshot, measure, propose, improve)",
|
||||
) # fmt: skip
|
||||
plane_ws.add_argument("--loop-commit", help="with --loop: the commit under test the gate runs at (default: git rev-parse HEAD)")
|
||||
plane_ws.add_argument("--loop-repo", help="with --loop: this repository's URL (default: git remote get-url origin)")
|
||||
plane_ws.add_argument("--loop-manifest", help="with --loop: the benchmark manifest the proposals are gated on")
|
||||
plane_ws.add_argument(
|
||||
"--qualify-population", action="append", default=[], metavar="NAME", help="with --loop: a population whose rows never propose"
|
||||
)
|
||||
|
||||
raw = list(argv if argv is not None else sys.argv[1:])
|
||||
if _advisory_hook(raw):
|
||||
# Git hook mode (stdin) without an explicit blocking opt-in: a check that cannot run must
|
||||
@@ -430,23 +370,6 @@ def _main(argv: list[str] | None) -> int:
|
||||
return _scan(args.path, args.config, args.mode, args.fail_on)
|
||||
if args.command == "benchmark":
|
||||
return _benchmark(args.manifest, args.config, args.mode)
|
||||
if args.command == "improve":
|
||||
return _improve(
|
||||
args.manifest,
|
||||
max_rounds=args.max_rounds,
|
||||
recall_floor=args.recall_floor,
|
||||
fp_ceiling=args.fp_ceiling,
|
||||
ledger=args.ledger,
|
||||
journal=args.journal,
|
||||
ruleset_dir=args.ruleset_dir,
|
||||
dry_run=args.dry_run,
|
||||
pair_catalog=None if args.no_pair_gate else args.pair_catalog,
|
||||
profile_tolerance=args.profile_tolerance,
|
||||
min_holdout_pairs=args.min_holdout_pairs,
|
||||
memory=args.memory,
|
||||
qualify_populations=tuple(args.qualify_population),
|
||||
guard_catalog=args.pair_catalog,
|
||||
)
|
||||
if args.command == "pairs":
|
||||
return _pairs(
|
||||
args.catalog,
|
||||
@@ -1103,75 +1026,6 @@ def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _improve(
|
||||
manifest_path: Path,
|
||||
*,
|
||||
max_rounds: int,
|
||||
recall_floor: float,
|
||||
fp_ceiling: float,
|
||||
ledger: Path | None,
|
||||
journal: Path | None,
|
||||
ruleset_dir: Path,
|
||||
dry_run: bool,
|
||||
pair_catalog: Path | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
memory: str | None = None,
|
||||
qualify_populations: tuple[str, ...] = (),
|
||||
guard_catalog: Path | None = None,
|
||||
) -> int:
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
||||
pair_cases: tuple[object, ...] = ()
|
||||
if pair_catalog is not None:
|
||||
if not pair_catalog.is_file():
|
||||
raise SystemExit(f"pair catalog is not a file: {pair_catalog} (pass --no-pair-gate to skip the per-profile holdout clause)")
|
||||
pair_cases = select_split(load_pair_catalog(pair_catalog), "holdout")
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
policy = load_policy()
|
||||
session = _memory_session(memory, manifest, target, ruleset_dir, qualify_populations, guard_catalog) if memory is not None else None
|
||||
|
||||
# The loop writes its accepted ledger where `scan`/`benchmark` read it, so a
|
||||
# subsequent scan of this target automatically picks up the improved ruleset.
|
||||
default_ledger = target / CALIBRATION_DIR / "rule_policy.json"
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-improve-") as scratch:
|
||||
if dry_run:
|
||||
ledger_path = Path(scratch) / "rule_policy.json"
|
||||
journal_path = Path(scratch) / "improve_journal.json"
|
||||
else:
|
||||
ledger_path = ledger or default_ledger
|
||||
journal_path = journal or ledger_path.with_name("improve_journal.json")
|
||||
|
||||
outcomes = run_improvement(
|
||||
target,
|
||||
manifest,
|
||||
ledger_path=ledger_path,
|
||||
journal_path=journal_path,
|
||||
ruleset_dir=ruleset_dir,
|
||||
policy=policy,
|
||||
max_rounds=max_rounds,
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
pair_cases=pair_cases,
|
||||
profile_tolerance=profile_tolerance,
|
||||
min_holdout_pairs=min_holdout_pairs,
|
||||
proposals=session.proposals(ledger_path, journal_path) if session else (),
|
||||
)
|
||||
_print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run)
|
||||
if session is not None:
|
||||
session.finish(outcomes, journal_path, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def _memory_session(
|
||||
memory: str, manifest: object, target: Path, ruleset_dir: Path, populations: tuple[str, ...], catalog: Path | None
|
||||
) -> Any:
|
||||
from .improve.memory import MemorySession # lazy: the default `improve` path never touches the store
|
||||
|
||||
return MemorySession.open(memory, manifest, target, ruleset_dir, populations, catalog)
|
||||
|
||||
|
||||
def _pairs(
|
||||
catalog: Path,
|
||||
slice_name: str,
|
||||
@@ -1473,29 +1327,6 @@ def _check_obligations(
|
||||
return statics, payload, cited
|
||||
|
||||
|
||||
def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None:
|
||||
print(f"manifest={manifest_path}")
|
||||
print(f"target={target}")
|
||||
accepted = [o for o in outcomes if o.accepted]
|
||||
for outcome in outcomes:
|
||||
edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-"
|
||||
print(
|
||||
f"round {outcome.round}: {outcome.reason} | "
|
||||
f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} "
|
||||
f"fp {outcome.fp_before:.2%}->{outcome.fp_after:.2%} "
|
||||
f"score {outcome.score_before}->{outcome.score_after} | edits: {edits}"
|
||||
)
|
||||
if outcome.profile_regressions or outcome.profiles_under_minimum:
|
||||
print(f" profiles: regressed={outcome.profile_regressions or '-'} under_minimum={outcome.profiles_under_minimum or '-'}")
|
||||
print(f"rounds={len(outcomes)} accepted={len(accepted)}")
|
||||
if dry_run:
|
||||
print("dry_run=true (no ledger or mechanism store written)")
|
||||
elif accepted:
|
||||
print(f"ledger={ledger_path}")
|
||||
else:
|
||||
print("ledger=unchanged (no round accepted)")
|
||||
|
||||
|
||||
def _load_scan_findings(run_dir: Path) -> list[StaticFinding]:
|
||||
return load_findings(run_dir / "findings.json")
|
||||
|
||||
@@ -1644,7 +1475,7 @@ def _plane_memory(
|
||||
assert run_manifest is not None
|
||||
seeded = memory.seed(run_manifest, store)
|
||||
if seeded:
|
||||
print(f"memory {store.describe()}: seeded {len(seeded)} tasks (reused facts, loop snapshot): {', '.join(seeded)}")
|
||||
print(f"memory {store.describe()}: seeded {len(seeded)} tasks (reused facts): {', '.join(seeded)}")
|
||||
return 0
|
||||
annotations: dict[str, str] = {}
|
||||
if run_manifest is not None:
|
||||
@@ -1665,7 +1496,7 @@ def _plane_memory(
|
||||
|
||||
|
||||
def _plane_increment(args: argparse.Namespace, atespace: str) -> int:
|
||||
from .plane.generate import LOOP_MANIFEST, Loop, increment
|
||||
from .plane.generate import increment
|
||||
|
||||
if args.candidates is None or args.triage is None:
|
||||
print("--validation-set needs --candidates and --triage", file=sys.stderr)
|
||||
@@ -1679,18 +1510,9 @@ def _plane_increment(args: argparse.Namespace, atespace: str) -> int:
|
||||
)
|
||||
if args.runner_image is not None:
|
||||
command += f" --runner-image {str(args.runner_image).replace(home, '~', 1)}"
|
||||
loop = None
|
||||
if args.loop:
|
||||
git = lambda *a: subprocess.run(["git", *a], capture_output=True, text=True, check=True).stdout.strip() # noqa: E731
|
||||
loop = Loop(
|
||||
repo=args.loop_repo or git("remote", "get-url", "origin"), commit=args.loop_commit or git("rev-parse", "HEAD"),
|
||||
manifest=args.loop_manifest or LOOP_MANIFEST, populations=tuple(args.qualify_population),
|
||||
) # fmt: skip
|
||||
command += f" --loop --loop-commit {loop.commit} --loop-repo {loop.repo} --loop-manifest {loop.manifest}"
|
||||
command += "".join(f" --qualify-population {p}" for p in loop.populations)
|
||||
written = increment(
|
||||
args.population, args.validation_set, args.candidates, args.triage, plane=args.plane, run_name=args.run, command=command,
|
||||
repos=args.repos, runner_image=args.runner_image, loop=loop, atespace=atespace,
|
||||
repos=args.repos, runner_image=args.runner_image, atespace=atespace,
|
||||
) # fmt: skip
|
||||
print(f"{len(written)} manifests written under {args.plane} (Run {args.run})")
|
||||
return 0
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
"""Bounded, gated self-improvement loop over the ruleset/policy governance data."""
|
||||
|
||||
from ..pairs import build_pair_signals
|
||||
from .evolve import RoundOutcome, build_rule_signals, propose_status_edits, run_improvement, run_round
|
||||
from .journal import load_journal, reverted_edit_keys
|
||||
from .validator import (
|
||||
VALID_LEVERS,
|
||||
EvolveBounds,
|
||||
EvolveValidator,
|
||||
PolicyConstantEdit,
|
||||
RuleStatusEdit,
|
||||
StrictValidationError,
|
||||
edits_to_ledger,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"EvolveBounds",
|
||||
"EvolveValidator",
|
||||
"PolicyConstantEdit",
|
||||
"RoundOutcome",
|
||||
"RuleStatusEdit",
|
||||
"StrictValidationError",
|
||||
"VALID_LEVERS",
|
||||
"build_pair_signals",
|
||||
"build_rule_signals",
|
||||
"edits_to_ledger",
|
||||
"load_journal",
|
||||
"propose_status_edits",
|
||||
"reverted_edit_keys",
|
||||
"run_improvement",
|
||||
"run_round",
|
||||
]
|
||||
@@ -1,416 +0,0 @@
|
||||
"""Deterministic self-improvement loop (tasks 7.4, 7.5, 7.6).
|
||||
|
||||
One round: benchmark with the current ledger -> per-rule signals -> propose bounded
|
||||
edits (auto-shadow precision-draggers) -> validate -> replay smoke -> re-benchmark ->
|
||||
hard acceptance gate (recall >= floor AND FP < ceiling AND project score not regressed
|
||||
AND no matched-finding regression) -> accept (persist the ledger) or revert byte-for-byte.
|
||||
|
||||
The edit *proposers* are the benchmark's per-rule signal and, opt-in, the plane's memory store
|
||||
(:mod:`.memory`, ``run_round(proposals=...)``); both go through this same validator and gate.
|
||||
The project score is the optimization reward; the recall/FP gate is a separate hard
|
||||
feasibility constraint that is never folded into the reward.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ..benchmark import BenchmarkManifest, BenchmarkResult, BenchmarkRun, evaluate_benchmark
|
||||
from ..findings import StaticFinding, quick_scan_findings
|
||||
from ..mapping import analyze_entry_points, attach_reachability_hints
|
||||
from ..policy import CwePolicy, load_policy
|
||||
from ..preprocess import preprocess_repository
|
||||
from ..rank import rank_targets
|
||||
from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger
|
||||
from ..scoring import build_score_artifact
|
||||
from .journal import append_round, load_journal, next_round_index, reverted_edit_keys
|
||||
from .memory import MemoryProposal, applicable, proposal_id_of, write_sidecar
|
||||
from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoundOutcome:
|
||||
round: int
|
||||
accepted: bool
|
||||
reason: str
|
||||
edits: list[RuleStatusEdit] = field(default_factory=list)
|
||||
recall_before: float = 0.0
|
||||
recall_after: float = 0.0
|
||||
fp_before: float = 0.0
|
||||
fp_after: float = 0.0
|
||||
score_before: int = 0
|
||||
score_after: int = 0
|
||||
matched_before: int = 0
|
||||
matched_after: int = 0
|
||||
profile_regressions: list[str] = field(default_factory=list)
|
||||
profiles_under_minimum: list[str] = field(default_factory=list)
|
||||
per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict)
|
||||
# learning-harness Req 5.2: the candidates this round declined to learn from, by pair name.
|
||||
degradations: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
|
||||
MAX_PROFILES = 4
|
||||
|
||||
|
||||
def evaluate_profiles(
|
||||
pair_cases: Sequence[object],
|
||||
rules: tuple[PatternRule, ...],
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Pair-corpus metrics per provenance profile under ``rules`` (holdout split, no inventory sidecar).
|
||||
|
||||
Only ``seeded`` and ``reviewed`` pairs gate (Req 9.3); ``advisory`` and ``title`` pairs are scored by ``pairs``, never here.
|
||||
|
||||
Deliberately split-blind: this *measures* a ruleset, it never learns from a pair, so every pair is scored and the
|
||||
teacher rule has nothing to say here. What the rule governs is the store these rules were built from
|
||||
(`semantic.seed`, `semantic.loo`), and a filter here would only shrink the evidence (learning-harness Req 5.1).
|
||||
"""
|
||||
from ..pairs import GATING_TIERS, PairCase, evaluate_catalog, select_tier, select_vendored
|
||||
|
||||
# Gates score vendored pairs only (Req 10.4); pointer pairs are for `pairs --pointers`, never for the improve gate.
|
||||
cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS))
|
||||
if not cases:
|
||||
return {}
|
||||
result = evaluate_catalog(cases, ruleset=rules, inventory_sidecar=False)
|
||||
return {
|
||||
profile: {
|
||||
"pairs": float(metrics.pairs),
|
||||
"pair_correct": float(metrics.pair_correct),
|
||||
"pair_pass_rate": metrics.pair_pass_rate,
|
||||
"youden": metrics.youden,
|
||||
}
|
||||
for profile, metrics in result.per_profile.items()
|
||||
}
|
||||
|
||||
|
||||
def profile_regressions(
|
||||
before: dict[str, dict[str, float]],
|
||||
after: dict[str, dict[str, float]],
|
||||
*,
|
||||
tolerance: float = 0.0,
|
||||
min_pairs: int = 5,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""(regressed profiles, profiles under the minimum).
|
||||
|
||||
A profile regresses when its pair pass rate or Youden drops by more than ``tolerance``
|
||||
(both are rates in [-1, 1], so one tolerance applies to both).
|
||||
"""
|
||||
regressed: list[str] = []
|
||||
small: list[str] = []
|
||||
for profile in sorted(set(before) | set(after)):
|
||||
old = before.get(profile, {"pairs": 0.0, "pair_correct": 0.0, "pair_pass_rate": 0.0, "youden": 0.0})
|
||||
new = after.get(profile, old)
|
||||
if old["pairs"] < min_pairs:
|
||||
small.append(profile)
|
||||
continue
|
||||
old_rate = old.get("pair_pass_rate", old["pair_correct"] / old["pairs"] if old["pairs"] else 0.0)
|
||||
new_rate = new.get("pair_pass_rate", new["pair_correct"] / new["pairs"] if new["pairs"] else 0.0)
|
||||
if new_rate < old_rate - tolerance or new["youden"] < old["youden"] - tolerance:
|
||||
regressed.append(profile)
|
||||
return regressed, small
|
||||
|
||||
|
||||
def build_rule_signals(result: BenchmarkResult) -> list[dict[str, object]]:
|
||||
"""Convert a benchmark result into per-rule loop signals (recall: miss; precision: fp)."""
|
||||
signals: list[dict[str, object]] = []
|
||||
for miss in result.misses:
|
||||
signals.append({"rule_id": miss.rule_id or "", "signal": "miss", "cwe": miss.cwe, "path": miss.path})
|
||||
for fp in result.false_positives:
|
||||
signals.append({"rule_id": fp.rule_id, "signal": "fp", "path": fp.path, "line": fp.line})
|
||||
return sorted(signals, key=lambda item: (str(item.get("signal")), str(item.get("rule_id")), str(item.get("path"))))
|
||||
|
||||
|
||||
def propose_status_edits(
|
||||
per_rule: Mapping[str, dict[str, int]],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
blocked_keys: set[str],
|
||||
) -> list[RuleStatusEdit]:
|
||||
"""Auto-shadow precision-draggers: enabled rules with false positives -> shadow (never delete)."""
|
||||
edits: list[RuleStatusEdit] = []
|
||||
for rule_id, counts in sorted(per_rule.items()):
|
||||
if counts.get("false_positives", 0) <= 0:
|
||||
continue
|
||||
rule = ruleset_by_id.get(rule_id)
|
||||
if rule is None:
|
||||
continue
|
||||
current_status = str(current_ledger.get(rule_id, {}).get("status", rule.status))
|
||||
if current_status != "enabled":
|
||||
continue # already staged or disabled
|
||||
edit = RuleStatusEdit(rule_id=rule_id, from_status="enabled", to_status="shadow", rationale="")
|
||||
if edit.key() in blocked_keys:
|
||||
continue # novelty gate: previously reverted without a new rationale
|
||||
edits.append(edit)
|
||||
return edits
|
||||
|
||||
|
||||
def run_round(
|
||||
target: Path,
|
||||
manifest: BenchmarkManifest,
|
||||
*,
|
||||
ledger_path: Path,
|
||||
journal_path: Path,
|
||||
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
recall_floor: float = 0.9,
|
||||
fp_ceiling: float = 0.1,
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
proposals: Sequence[MemoryProposal] = (),
|
||||
) -> RoundOutcome:
|
||||
policy = policy if policy is not None else load_policy()
|
||||
rounds = load_journal(journal_path)
|
||||
round_index = next_round_index(rounds)
|
||||
blocked = reverted_edit_keys(rounds)
|
||||
current_ledger = read_rule_ledger(ledger_path)
|
||||
|
||||
before_findings, before_rules = _scan(target, ruleset_dir, current_ledger, policy)
|
||||
before = _evaluate(target, manifest, before_findings, before_rules, policy)
|
||||
(ledger_path.parent / "rule_signals.json").parent.mkdir(parents=True, exist_ok=True)
|
||||
_write_signals(ledger_path.parent / "rule_signals.json", build_rule_signals(before.result))
|
||||
|
||||
ruleset_by_id = {rule.rule_id: rule for rule in before_rules}
|
||||
edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked)
|
||||
if proposals:
|
||||
# harnessx-removal Req 6.2: memory proposals join after the benchmark's edits; the first edit per key wins.
|
||||
taken = {edit.key() for edit in edits}
|
||||
for proposal in applicable(proposals, ruleset_by_id, current_ledger, rounds):
|
||||
if proposal.edit.key() not in taken:
|
||||
taken.add(proposal.edit.key())
|
||||
edits.append(proposal.edit)
|
||||
write_sidecar(journal_path, round_index, [p for p in proposals if p.edit in edits])
|
||||
refusals: list[dict[str, object]] = []
|
||||
if not edits:
|
||||
return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals)
|
||||
|
||||
validator = EvolveValidator()
|
||||
try:
|
||||
for edit in edits:
|
||||
validator.validate(edit, ruleset_by_id, policy)
|
||||
except StrictValidationError as exc:
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, str(exc))
|
||||
return _outcome(
|
||||
round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before, degradations=refusals
|
||||
)
|
||||
|
||||
candidate = edits_to_ledger(edits, current_ledger)
|
||||
try:
|
||||
_load_with(ruleset_dir, candidate) # replay smoke gate: candidate ruleset must boot
|
||||
except Exception as exc: # noqa: BLE001 — any boot failure rejects the round
|
||||
_record(journal_path, round_index, edits, "rejected", before, before, f"replay_failed: {exc}")
|
||||
return _outcome(
|
||||
round_index, accepted=False, reason="replay_failed", edits=edits, before=before, after=before, degradations=refusals
|
||||
)
|
||||
|
||||
after_findings, after_rules = _scan(target, ruleset_dir, candidate, policy)
|
||||
after = _evaluate(target, manifest, after_findings, after_rules, policy)
|
||||
|
||||
accepted = (
|
||||
after.recall >= recall_floor and after.fp_rate < fp_ceiling and after.score >= before.score and after.matched >= before.matched
|
||||
)
|
||||
profiles_before: dict[str, dict[str, float]] = {}
|
||||
profiles_after: dict[str, dict[str, float]] = {}
|
||||
regressed: list[str] = []
|
||||
small: list[str] = []
|
||||
if accepted and pair_cases:
|
||||
# pair-corpus-honesty Req 7.2: a change may not help one provenance profile by hurting another.
|
||||
profiles_before = evaluate_profiles(pair_cases, before_rules)
|
||||
profiles_after = evaluate_profiles(pair_cases, after_rules)
|
||||
if len(profiles_before) > MAX_PROFILES:
|
||||
raise ValueError(f"at most {MAX_PROFILES} provenance profiles are supported; got {sorted(profiles_before)}")
|
||||
regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs)
|
||||
if regressed:
|
||||
accepted = False
|
||||
if accepted:
|
||||
if edits:
|
||||
write_rule_ledger(ledger_path, candidate)
|
||||
reason = "accepted"
|
||||
elif regressed:
|
||||
reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions
|
||||
else:
|
||||
reason = "reverted" # ledger_path untouched -> byte-for-byte revert
|
||||
_record(
|
||||
journal_path,
|
||||
round_index,
|
||||
edits,
|
||||
"accepted" if accepted else "reverted",
|
||||
before,
|
||||
after,
|
||||
reason,
|
||||
regressed,
|
||||
small,
|
||||
)
|
||||
outcome = _outcome(round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, degradations=refusals)
|
||||
return RoundOutcome(
|
||||
**{
|
||||
**outcome.__dict__,
|
||||
"profile_regressions": regressed,
|
||||
"profiles_under_minimum": small,
|
||||
"per_profile_before": profiles_before,
|
||||
"per_profile_after": profiles_after,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def run_improvement(
|
||||
target: Path,
|
||||
manifest: BenchmarkManifest,
|
||||
*,
|
||||
ledger_path: Path,
|
||||
journal_path: Path,
|
||||
ruleset_dir: Path = DEFAULT_RULESET_DIR,
|
||||
policy: dict[str, CwePolicy] | None = None,
|
||||
max_rounds: int = 5,
|
||||
recall_floor: float = 0.9,
|
||||
fp_ceiling: float = 0.1,
|
||||
pair_cases: Sequence[object] | None = None,
|
||||
profile_tolerance: float = 0.0,
|
||||
min_holdout_pairs: int = 5,
|
||||
proposals: Sequence[MemoryProposal] = (),
|
||||
) -> list[RoundOutcome]:
|
||||
"""Run improvement rounds until convergence (no new proposals) or ``max_rounds``."""
|
||||
outcomes: list[RoundOutcome] = []
|
||||
for _ in range(max_rounds):
|
||||
outcome = run_round(
|
||||
target,
|
||||
manifest,
|
||||
ledger_path=ledger_path,
|
||||
journal_path=journal_path,
|
||||
ruleset_dir=ruleset_dir,
|
||||
policy=policy,
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
pair_cases=pair_cases,
|
||||
profile_tolerance=profile_tolerance,
|
||||
min_holdout_pairs=min_holdout_pairs,
|
||||
proposals=proposals,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
if outcome.reason == "no_proposals":
|
||||
break
|
||||
return outcomes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Eval:
|
||||
result: BenchmarkResult
|
||||
recall: float
|
||||
fp_rate: float
|
||||
score: int
|
||||
matched: int
|
||||
|
||||
|
||||
def _scan(
|
||||
target: Path, ruleset_dir: Path, ledger: Mapping[str, dict[str, object]], policy: dict[str, CwePolicy]
|
||||
) -> tuple[list[StaticFinding], tuple[PatternRule, ...]]:
|
||||
rules = _load_with(ruleset_dir, ledger)
|
||||
_, targets = preprocess_repository(target)
|
||||
targets = attach_reachability_hints(targets, analyze_entry_points(target, targets))
|
||||
rankings = rank_targets(targets)
|
||||
findings = quick_scan_findings(target, targets, rankings, rules, policy)
|
||||
reported = [finding for finding in findings if finding.status != "shadow"]
|
||||
return reported, rules
|
||||
|
||||
|
||||
def _evaluate(
|
||||
target: Path, manifest: BenchmarkManifest, findings: list[StaticFinding], rules: tuple[PatternRule, ...], policy: dict[str, CwePolicy]
|
||||
) -> _Eval:
|
||||
run = BenchmarkRun(benchmark_run_id="improve", root=target, manifest=manifest)
|
||||
result = evaluate_benchmark(run=run, mode="quick", findings=findings, scan_id=None, scan_run_dir=None)
|
||||
metrics = result.metrics
|
||||
cwe_by_rule = {rule.rule_id: rule.cwe for rule in rules}
|
||||
rule_cwe_by_id = {f.finding_id: cwe_by_rule.get(f.finding_id.split(":", 1)[0], "") for f in findings}
|
||||
score = build_score_artifact(findings, rule_cwe_by_id, policy).project_score
|
||||
recall = metrics.recall if metrics.recall is not None else 1.0
|
||||
fp_rate = metrics.false_positive_findings_total / metrics.actual_findings_total if metrics.actual_findings_total else 0.0
|
||||
return _Eval(result=result, recall=recall, fp_rate=fp_rate, score=score, matched=metrics.matched_findings_total)
|
||||
|
||||
|
||||
def _load_with(ruleset_dir: Path, ledger: Mapping[str, dict[str, object]]) -> tuple[PatternRule, ...]:
|
||||
import json
|
||||
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle:
|
||||
handle.write(json.dumps(ledger))
|
||||
temp_path = Path(handle.name)
|
||||
try:
|
||||
return load_ruleset(ruleset_dir, temp_path)
|
||||
finally:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _write_signals(path: Path, signals: list[dict[str, object]]) -> None:
|
||||
import json
|
||||
|
||||
path.write_text(json.dumps(signals, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _outcome(
|
||||
round_index: int,
|
||||
*,
|
||||
accepted: bool,
|
||||
reason: str,
|
||||
edits: list[RuleStatusEdit],
|
||||
before: _Eval,
|
||||
after: _Eval,
|
||||
degradations: list[dict[str, object]] | None = None,
|
||||
) -> RoundOutcome:
|
||||
return RoundOutcome(
|
||||
round=round_index,
|
||||
accepted=accepted,
|
||||
reason=reason,
|
||||
edits=edits,
|
||||
degradations=list(degradations or []),
|
||||
recall_before=before.recall,
|
||||
recall_after=after.recall,
|
||||
fp_before=before.fp_rate,
|
||||
fp_after=after.fp_rate,
|
||||
score_before=before.score,
|
||||
score_after=after.score,
|
||||
matched_before=before.matched,
|
||||
matched_after=after.matched,
|
||||
)
|
||||
|
||||
|
||||
def _record(
|
||||
journal_path: Path,
|
||||
round_index: int,
|
||||
edits: list[RuleStatusEdit],
|
||||
outcome: str,
|
||||
before: _Eval,
|
||||
after: _Eval,
|
||||
reason: str = "",
|
||||
profile_regressions: list[str] | None = None,
|
||||
profiles_under_minimum: list[str] | None = None,
|
||||
) -> None:
|
||||
append_round(
|
||||
journal_path,
|
||||
{
|
||||
"round": round_index,
|
||||
"outcome": outcome,
|
||||
"reason": reason or outcome,
|
||||
"profile_regressions": list(profile_regressions or []),
|
||||
"profiles_under_minimum": list(profiles_under_minimum or []),
|
||||
"edits": [_journal_edit(e) for e in edits],
|
||||
"recall_before": round(before.recall, 4),
|
||||
"recall_after": round(after.recall, 4),
|
||||
"fp_before": round(before.fp_rate, 4),
|
||||
"fp_after": round(after.fp_rate, 4),
|
||||
"score_before": before.score,
|
||||
"score_after": after.score,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _journal_edit(edit: RuleStatusEdit) -> dict[str, object]:
|
||||
entry: dict[str, object] = {
|
||||
"key": edit.key(), "lever": edit.lever, "rule_id": edit.rule_id, "from": edit.from_status, "to": edit.to_status,
|
||||
"rationale": edit.rationale,
|
||||
} # fmt: skip
|
||||
proposal_id = proposal_id_of(edit)
|
||||
if proposal_id is not None:
|
||||
entry["evidence"] = proposal_id # only a memory edit carries it; every other entry is unchanged byte for byte
|
||||
return entry
|
||||
@@ -1,53 +0,0 @@
|
||||
"""Improvement journal: per-round audit + novelty memory (task 7.3, 7.9).
|
||||
|
||||
Persistent JSON record of every self-improvement round so each rule/policy edit is
|
||||
attributable and reversible, and so a previously-reverted edit is not re-proposed
|
||||
without a new rationale (the novelty gate).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_journal(path: Path) -> list[dict[str, object]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
payload = json.loads(path.read_text())
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
|
||||
def write_journal(path: Path, rounds: list[dict[str, object]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(rounds, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def reverted_edit_keys(rounds: list[dict[str, object]]) -> set[str]:
|
||||
"""Edit keys from reverted rounds — blocked from re-proposal unless a new rationale appears."""
|
||||
keys: set[str] = set()
|
||||
for entry in rounds:
|
||||
if entry.get("outcome") != "reverted":
|
||||
continue
|
||||
for edit in _edits(entry):
|
||||
key = edit.get("key")
|
||||
# Rule edits are blocked only without a new rationale; mechanism edits are blocked outright (the loop
|
||||
# always writes a rationale for them, and a reverted store edit must not be re-proposed every round).
|
||||
if isinstance(key, str) and (not edit.get("rationale") or edit.get("lever") == "mechanisms"):
|
||||
keys.add(key)
|
||||
return keys
|
||||
|
||||
|
||||
def next_round_index(rounds: list[dict[str, object]]) -> int:
|
||||
return len(rounds) + 1
|
||||
|
||||
|
||||
def append_round(path: Path, entry: dict[str, object]) -> None:
|
||||
rounds = load_journal(path)
|
||||
rounds.append(entry)
|
||||
write_journal(path, rounds)
|
||||
|
||||
|
||||
def _edits(entry: dict[str, object]) -> list[dict[str, object]]:
|
||||
edits = entry.get("edits")
|
||||
return [edit for edit in edits if isinstance(edit, dict)] if isinstance(edits, list) else []
|
||||
@@ -1,549 +0,0 @@
|
||||
"""Rule-status proposals from the plane's memory store (harnessx-removal Req 6.2/6.4, design section 5).
|
||||
|
||||
The store (:mod:`..plane.memory`) holds what plane runs learned per repository and pin. Two deterministic rules turn
|
||||
its ``alert`` and ``verdict`` rows into :class:`RuleStatusEdit` proposals that :func:`.evolve.run_round` validates
|
||||
and gates exactly like the benchmark's own edits; a third reads dispute rates as advisory routing input only.
|
||||
|
||||
Join key: an ``alert`` and a ``verdict`` match when repo, pin and ``path::function`` are equal. A ``disputed``
|
||||
final counts as no evidence in either direction.
|
||||
|
||||
- **M1, repeated false alerts** (``enabled -> shadow``): at least :data:`M1_MIN_FALSE` alerts of the rule are false
|
||||
-- at a ``rejected`` candidate, or on a ``fixed`` pin inside the declared fix range (the alert row says
|
||||
``in_fix_range: true``, or its ``path::function`` is a ``site_match`` site of the same repository) -- across at
|
||||
least :data:`M1_MIN_REPOS` repositories, and no alert of the rule sits at an ``agreed`` or ``site_match``
|
||||
candidate.
|
||||
- **M2, missed declared sites** (``shadow -> enabled``): the shadow rule's alerts hit at least :data:`M2_MIN_HITS`
|
||||
``agreed`` candidates with ``site_match`` true across at least :data:`M2_MIN_REPOS` repositories, no enabled rule
|
||||
alerts at those candidates, the rule has no ``fixed``-pin alert at the same function, and at most
|
||||
:data:`M2_MAX_REJECTED` alert at a ``rejected`` candidate.
|
||||
- **M3, disputed families**: families with at least :data:`M3_MIN_CANDIDATES` candidates and a dispute rate of at
|
||||
least :data:`M3_MIN_DISPUTE_RATE`. Advisory (``signals.json``); never an edit, never shown to the validator.
|
||||
|
||||
**Coverage**: M1 and M2 skip every repository and pin whose ``coverage`` rows record the rule's language only as
|
||||
``none`` (quick mode had no rule for it and no engine alerts were produced): a rule cannot be demoted or promoted
|
||||
on evidence that could not exist. The skipped alerts are counted per ``repo@pin`` under ``coverage_none`` in the
|
||||
proposal's ``excluded``.
|
||||
|
||||
**Train-on-test guard** (:class:`Guard`), applied before any rule sees a row: a row is dropped when its population
|
||||
is one the proposal will be qualified on, its (repo, pin) is a case of the gated benchmark manifest, or its
|
||||
repository is a holdout pair's repository. The pair catalog records a holdout pair's *fix* commit, not the
|
||||
vulnerable pin (its parent), so the holdout clause matches by repository, a superset of (repo, pin). This is the
|
||||
rule the closed-loop leak broke, where a holdout pair taught the shape that then "recovered" it.
|
||||
|
||||
**Provenance**: each proposal's rationale is ``memory:<proposal_id>``, ``proposal_id`` being the sha256 of (rule,
|
||||
edit key, sorted evidence row ids, thresholds). The sidecar ``memory_proposals.jsonl`` next to the journal carries
|
||||
the evidence rows (id, repo, pin, run, task, object key and version -- the bucket's object version id, a file's content hash),
|
||||
the counts that met each threshold, what the guard excluded and the store's index digest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..plane.memory import MemoryStore, Record, repo_key, row_id, rows_digest
|
||||
from ..ruleset import PatternRule
|
||||
from .validator import RuleStatusEdit
|
||||
|
||||
M1_MIN_FALSE = 3
|
||||
M1_MIN_REPOS = 2
|
||||
M2_MIN_HITS = 2
|
||||
M2_MIN_REPOS = 2
|
||||
M2_MAX_REJECTED = 1
|
||||
M3_MIN_CANDIDATES = 5
|
||||
M3_MIN_DISPUTE_RATE = 0.3
|
||||
THRESHOLDS: dict[str, dict[str, float]] = {
|
||||
"M1": {"min_false_alerts": M1_MIN_FALSE, "min_repos": M1_MIN_REPOS, "max_agreed_hits": 0},
|
||||
"M2": {"min_site_hits": M2_MIN_HITS, "min_repos": M2_MIN_REPOS, "max_rejected": M2_MAX_REJECTED, "max_fixed_pin_same_function": 0},
|
||||
"M3": {"min_candidates": M3_MIN_CANDIDATES, "min_dispute_rate": M3_MIN_DISPUTE_RATE},
|
||||
}
|
||||
RATIONALE_PREFIX = "memory:"
|
||||
SIDECAR = "memory_proposals.jsonl"
|
||||
SIGNALS = "signals.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemoryProposal:
|
||||
edit: RuleStatusEdit
|
||||
provenance: dict[str, Any]
|
||||
|
||||
@property
|
||||
def proposal_id(self) -> str:
|
||||
return str(self.provenance["proposal_id"])
|
||||
|
||||
|
||||
def proposal_id_of(edit: object) -> str | None:
|
||||
"""The proposal id a memory edit carries in its rationale; None for any other edit."""
|
||||
rationale = getattr(edit, "rationale", "")
|
||||
return rationale[len(RATIONALE_PREFIX) :] if isinstance(rationale, str) and rationale.startswith(RATIONALE_PREFIX) else None
|
||||
|
||||
|
||||
# --- the train-on-test guard ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tail(repo: str) -> str:
|
||||
"""``owner/name`` of a repository, lower-cased: the pair catalog names repositories without a host."""
|
||||
return "/".join(repo_key(repo).lower().split("/")[-2:])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Guard:
|
||||
populations: frozenset[str] = frozenset()
|
||||
gated_cases: frozenset[tuple[str, str]] = frozenset() # (repo_key, pin) of the benchmark being gated
|
||||
holdout_repos: frozenset[str] = frozenset() # owner/name of every holdout pair
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls, populations: Iterable[str] = (), gated_cases: Iterable[tuple[str, str]] = (), holdout_pairs: Iterable[object] = ()
|
||||
) -> Guard:
|
||||
repos = {_tail(str(getattr(p, "repo", ""))) for p in holdout_pairs if getattr(p, "repo", "")}
|
||||
return cls(frozenset(populations), frozenset((repo_key(r), p) for r, p in gated_cases), frozenset(repos))
|
||||
|
||||
def reason(self, row: Mapping[str, Any]) -> str | None:
|
||||
if row.get("population") in self.populations:
|
||||
return "population"
|
||||
if (row.get("repo"), row.get("pin")) in self.gated_cases:
|
||||
return "gated_case"
|
||||
if _tail(str(row.get("repo"))) in self.holdout_repos:
|
||||
return "holdout_pair"
|
||||
return None
|
||||
|
||||
def apply(self, records: Iterable[Record]) -> tuple[list[Record], dict[str, Any]]:
|
||||
"""(kept records, what was excluded: row count, and counts per population / repo@pin per clause)."""
|
||||
kept: list[Record] = []
|
||||
excluded: dict[str, Any] = {"rows": 0, "population": {}, "gated_case": {}, "holdout_pair": {}}
|
||||
for record in records:
|
||||
why = self.reason(record.row)
|
||||
if why is None:
|
||||
kept.append(record)
|
||||
continue
|
||||
excluded["rows"] += 1
|
||||
label = str(record.row.get("population")) if why == "population" else f"{record.row.get('repo')}@{record.row.get('pin')}"
|
||||
excluded[why][label] = excluded[why].get(label, 0) + 1
|
||||
for clause in ("population", "gated_case", "holdout_pair"):
|
||||
excluded[clause] = dict(sorted(excluded[clause].items()))
|
||||
return kept, excluded
|
||||
|
||||
|
||||
# --- the rules -----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cand(row: Mapping[str, Any]) -> str:
|
||||
return f"{row.get('path')}::{row.get('function')}"
|
||||
|
||||
|
||||
def _status(rule_id: str, ruleset_by_id: Mapping[str, PatternRule], ledger: Mapping[str, dict[str, object]]) -> str | None:
|
||||
rule = ruleset_by_id.get(rule_id)
|
||||
return None if rule is None else str(ledger.get(rule_id, {}).get("status", rule.status))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _View:
|
||||
verdicts: dict[tuple[str, str, str], Record] = field(default_factory=dict) # (repo, pin, candidate)
|
||||
sites: set[tuple[str, str]] = field(default_factory=set) # (repo, candidate) with site_match true
|
||||
alerts: dict[str, list[Record]] = field(default_factory=dict) # rule_id -> alerts
|
||||
coverage: dict[tuple[str, str, str], set[str]] = field(default_factory=dict) # (repo, pin, language) -> recorded kinds
|
||||
|
||||
@classmethod
|
||||
def of(cls, records: Sequence[Record]) -> _View:
|
||||
"""One alert per (repo, pin, rule, path, line, function, pin role): every loop run re-alerts an unchanged pin under a
|
||||
new run name, and a repeat is the same evidence, not more of it."""
|
||||
view = cls()
|
||||
seen: set[tuple[object, ...]] = set()
|
||||
for rec in records:
|
||||
row = rec.row
|
||||
if row["kind"] == "coverage" and row.get("language"):
|
||||
view.coverage.setdefault((row["repo"], row["pin"], str(row["language"])), set()).add(str(row.get("coverage")))
|
||||
elif row["kind"] == "verdict":
|
||||
view.verdicts[(row["repo"], row["pin"], str(row.get("candidate")))] = rec
|
||||
if row.get("site_match") is True:
|
||||
view.sites.add((row["repo"], str(row.get("candidate"))))
|
||||
elif row["kind"] == "alert" and row.get("rule_id"):
|
||||
place = (row["repo"], row["pin"], row["rule_id"], *(row.get(k) for k in ("path", "line", "function", "pin_role")))
|
||||
if place not in seen:
|
||||
seen.add(place)
|
||||
view.alerts.setdefault(str(row["rule_id"]), []).append(rec)
|
||||
return view
|
||||
|
||||
def verdict(self, alert: Mapping[str, Any]) -> Record | None:
|
||||
return self.verdicts.get((alert["repo"], alert["pin"], _cand(alert)))
|
||||
|
||||
def fixed_in_range(self, alert: Mapping[str, Any]) -> bool:
|
||||
if alert.get("pin_role") != "fixed":
|
||||
return False
|
||||
flag = alert.get("in_fix_range")
|
||||
return bool(flag) if flag is not None else (alert["repo"], _cand(alert)) in self.sites
|
||||
|
||||
def blind(self, repo: str, pin: str, languages: Iterable[str]) -> bool:
|
||||
"""True when a language of the rule was recorded at this repository and pin with coverage ``none`` only: no
|
||||
rule could fire there, so what the rule did or did not say there is no evidence either way."""
|
||||
return any(self.coverage.get((repo, pin, language)) == {"none"} for language in languages)
|
||||
|
||||
def seeing(self, languages_by_rule: Mapping[str, Sequence[str]]) -> tuple[_View, dict[str, int]]:
|
||||
"""This view without the alerts at a repository and pin blind to their rule's language, and how many were
|
||||
dropped per ``repo@pin``."""
|
||||
kept: dict[str, list[Record]] = {}
|
||||
dropped: dict[str, int] = {}
|
||||
for rule_id, alerts in self.alerts.items():
|
||||
languages = languages_by_rule.get(rule_id, ())
|
||||
for alert in alerts:
|
||||
if self.blind(alert.row["repo"], alert.row["pin"], languages):
|
||||
label = f"{alert.row['repo']}@{alert.row['pin']}"
|
||||
dropped[label] = dropped.get(label, 0) + 1
|
||||
else:
|
||||
kept.setdefault(rule_id, []).append(alert)
|
||||
return _View(self.verdicts, self.sites, kept, self.coverage), dict(sorted(dropped.items()))
|
||||
|
||||
|
||||
def _m1(rule_id: str, view: _View) -> tuple[list[Record], dict[str, int]] | None:
|
||||
evidence: list[Record] = []
|
||||
repos: set[str] = set()
|
||||
false_alerts = 0
|
||||
for alert in view.alerts.get(rule_id, []):
|
||||
verdict = view.verdict(alert.row)
|
||||
final = verdict.row.get("final") if verdict else None
|
||||
if final == "agreed" or (verdict is not None and verdict.row.get("site_match") is True):
|
||||
return None # the rule caught something the plane agreed on: never demote it on this evidence
|
||||
if final == "rejected" or view.fixed_in_range(alert.row):
|
||||
false_alerts += 1
|
||||
repos.add(alert.row["repo"])
|
||||
evidence.extend([alert, verdict] if verdict is not None and final == "rejected" else [alert])
|
||||
if false_alerts < M1_MIN_FALSE or len(repos) < M1_MIN_REPOS:
|
||||
return None
|
||||
return evidence, {"false_alerts": false_alerts, "repos": len(repos), "agreed_hits": 0}
|
||||
|
||||
|
||||
def _m2(rule_id: str, view: _View, enabled: set[str]) -> tuple[list[Record], dict[str, int]] | None:
|
||||
alerts = view.alerts.get(rule_id, [])
|
||||
hits: dict[tuple[str, str, str], tuple[Record, Record]] = {}
|
||||
rejected = 0
|
||||
for alert in alerts:
|
||||
verdict = view.verdict(alert.row)
|
||||
if verdict is None or alert.row.get("pin_role") == "fixed":
|
||||
continue
|
||||
if verdict.row.get("final") == "rejected":
|
||||
rejected += 1
|
||||
elif verdict.row.get("final") == "agreed" and verdict.row.get("site_match") is True:
|
||||
hits[(alert.row["repo"], alert.row["pin"], _cand(alert.row))] = (alert, verdict)
|
||||
if not hits:
|
||||
return None
|
||||
caught = {
|
||||
(a.row["repo"], a.row["pin"], _cand(a.row)) for rid in enabled if rid != rule_id for a in view.alerts.get(rid, [])
|
||||
} # fmt: skip
|
||||
missed = {k: v for k, v in hits.items() if k not in caught}
|
||||
fixed_same = sum(
|
||||
1 for a in alerts if a.row.get("pin_role") == "fixed" and any((a.row["repo"], _cand(a.row)) == (r, c) for r, _, c in missed)
|
||||
)
|
||||
repos = {repo for repo, _, _ in missed}
|
||||
if len(missed) < M2_MIN_HITS or len(repos) < M2_MIN_REPOS or fixed_same > 0 or rejected > M2_MAX_REJECTED:
|
||||
return None
|
||||
evidence = [rec for key in sorted(missed) for rec in missed[key]]
|
||||
return evidence, {"site_hits": len(missed), "repos": len(repos), "rejected": rejected, "fixed_pin_same_function": 0}
|
||||
|
||||
|
||||
def _evidence(records: Sequence[Record]) -> list[dict[str, Any]]:
|
||||
unique = {rec.row["id"]: rec for rec in records}
|
||||
return [
|
||||
{"id": i, "kind": r.row["kind"], "repo": r.row["repo"], "pin": r.row["pin"], "run": r.row["run"], "task": r.row["task"],
|
||||
"key": r.key, "version": r.version}
|
||||
for i, r in sorted(unique.items())
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def evidence_digest(evidence_ids: Iterable[str]) -> str:
|
||||
return hashlib.sha256("\n".join(sorted(evidence_ids)).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _proposal(
|
||||
rule: str, edit_key: str, rule_id: str, records: Sequence[Record], counts: dict[str, int], excluded: dict[str, Any], index: str
|
||||
) -> dict[str, Any]:
|
||||
evidence = _evidence(records)
|
||||
ids = [e["id"] for e in evidence]
|
||||
seed = json.dumps([rule, edit_key, ids, THRESHOLDS[rule]], sort_keys=True, separators=(",", ":"))
|
||||
return {
|
||||
"proposal_id": hashlib.sha256(seed.encode("utf-8")).hexdigest(),
|
||||
"rule": rule,
|
||||
"edit_key": edit_key,
|
||||
"rule_id": rule_id,
|
||||
"evidence": evidence,
|
||||
"evidence_digest": evidence_digest(ids),
|
||||
"counts": counts,
|
||||
"thresholds": THRESHOLDS[rule],
|
||||
"excluded": excluded,
|
||||
"index_digest": index,
|
||||
}
|
||||
|
||||
|
||||
def propose_from_memory(
|
||||
rows: Sequence[Record],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
journal_rounds: Sequence[Mapping[str, object]],
|
||||
*,
|
||||
excluded: Guard,
|
||||
index_digest: str = "",
|
||||
) -> list[MemoryProposal]:
|
||||
"""M1 and M2 over the rows the guard keeps, minus proposals already reverted on the same evidence."""
|
||||
kept, dropped = excluded.apply(rows)
|
||||
view, blind = _View.of(kept).seeing({rid: rule.languages for rid, rule in ruleset_by_id.items()})
|
||||
if blind:
|
||||
dropped = {**dropped, "coverage_none": blind}
|
||||
statuses = {rid: _status(rid, ruleset_by_id, current_ledger) for rid in ruleset_by_id}
|
||||
enabled = {rid for rid, status in statuses.items() if status == "enabled"}
|
||||
proposals: list[MemoryProposal] = []
|
||||
for rule_id in sorted(view.alerts):
|
||||
status = statuses.get(rule_id)
|
||||
if status == "enabled" and (m1 := _m1(rule_id, view)):
|
||||
name, found, to_status = "M1", m1, "shadow"
|
||||
elif status == "shadow" and (m2 := _m2(rule_id, view, enabled)):
|
||||
name, found, to_status = "M2", m2, "enabled"
|
||||
else:
|
||||
continue # unknown to the ruleset (the loop cannot add rules), disabled, or no rule fired
|
||||
key = RuleStatusEdit(rule_id, status, to_status).key()
|
||||
provenance = _proposal(name, key, rule_id, found[0], found[1], dropped, index_digest)
|
||||
edit = RuleStatusEdit(rule_id, status, to_status, rationale=RATIONALE_PREFIX + provenance["proposal_id"])
|
||||
proposals.append(MemoryProposal(edit, provenance))
|
||||
return applicable(proposals, ruleset_by_id, current_ledger, journal_rounds)
|
||||
|
||||
|
||||
def applicable(
|
||||
proposals: Sequence[MemoryProposal],
|
||||
ruleset_by_id: Mapping[str, PatternRule],
|
||||
current_ledger: Mapping[str, dict[str, object]],
|
||||
journal_rounds: Sequence[Mapping[str, object]],
|
||||
) -> list[MemoryProposal]:
|
||||
"""Proposals whose rule still has the ``from`` status and whose evidence no reverted round already tried."""
|
||||
tried: set[tuple[str, str]] = set()
|
||||
for entry in journal_rounds:
|
||||
edits = entry.get("edits")
|
||||
if entry.get("outcome") == "reverted" and isinstance(edits, list):
|
||||
tried |= {(str(e.get("key")), str(e.get("evidence"))) for e in edits if isinstance(e, dict) and e.get("evidence")}
|
||||
return [
|
||||
p for p in proposals
|
||||
if _status(p.edit.rule_id, ruleset_by_id, current_ledger) == p.edit.from_status and (p.edit.key(), p.proposal_id) not in tried
|
||||
] # fmt: skip
|
||||
|
||||
|
||||
def disputed_families(rows: Sequence[Record], guard: Guard) -> list[dict[str, Any]]:
|
||||
"""M3: families disputed often enough to route differently. Advisory only: never an edit."""
|
||||
kept, _ = guard.apply(rows)
|
||||
per: dict[str, list[int]] = {}
|
||||
for rec in kept:
|
||||
if rec.row["kind"] == "verdict" and rec.row.get("family"):
|
||||
tally = per.setdefault(str(rec.row["family"]), [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += rec.row.get("final") == "disputed"
|
||||
return [
|
||||
{"family": fam, "candidates": n, "disputed": d, "dispute_rate": round(d / n, 4), "advice": "schedule pass c up front"}
|
||||
for fam, (n, d) in sorted(per.items())
|
||||
if n >= M3_MIN_CANDIDATES and d / n >= M3_MIN_DISPUTE_RATE
|
||||
]
|
||||
|
||||
|
||||
# --- the store, the sidecar and the outcomes -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def index_digest(store: MemoryStore) -> str:
|
||||
return rows_digest(store.index())
|
||||
|
||||
|
||||
def write_sidecar(journal_path: Path, round_index: int, proposals: Sequence[MemoryProposal]) -> None:
|
||||
"""Append each proposal once (by id) to ``memory_proposals.jsonl`` next to the journal."""
|
||||
if not proposals:
|
||||
return
|
||||
path = journal_path.with_name(SIDECAR)
|
||||
seen = {json.loads(line)["proposal_id"] for line in path.read_text().splitlines() if line.strip()} if path.is_file() else set()
|
||||
new = [p for p in proposals if p.proposal_id not in seen]
|
||||
if not new:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a") as sink:
|
||||
for p in new:
|
||||
sink.write(json.dumps({**p.provenance, "round": round_index}, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
|
||||
|
||||
def read_sidecar(journal_path: Path) -> dict[str, dict[str, Any]]:
|
||||
path = journal_path.with_name(SIDECAR)
|
||||
if not path.is_file():
|
||||
return {}
|
||||
return {row["proposal_id"]: row for row in (json.loads(line) for line in path.read_text().splitlines() if line.strip())}
|
||||
|
||||
|
||||
def write_signals(directory: Path, signals: Sequence[Mapping[str, Any]]) -> Path:
|
||||
path = directory / SIGNALS
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"advisory": True, "disputed_families": list(signals), "thresholds": THRESHOLDS["M3"]}, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def outcome_of(reason: str, accepted: bool) -> str | None:
|
||||
if accepted:
|
||||
return "accepted"
|
||||
if reason == "no_proposals":
|
||||
return None
|
||||
if reason.startswith(("validation_failed", "replay_failed")):
|
||||
return "rejected"
|
||||
return "reverted"
|
||||
|
||||
|
||||
def outcome_rows(
|
||||
edits: Sequence[RuleStatusEdit],
|
||||
outcome: str,
|
||||
round_index: int,
|
||||
reason: str,
|
||||
proposals: Mapping[str, MemoryProposal],
|
||||
gate_run: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""One ``proposal_outcome`` row per (repository, pin) of each memory edit's evidence."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
for edit in edits:
|
||||
pid = proposal_id_of(edit)
|
||||
proposal = proposals.get(pid or "")
|
||||
if proposal is None:
|
||||
continue
|
||||
by_place: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for ev in proposal.provenance["evidence"]:
|
||||
by_place.setdefault((ev["repo"], ev["pin"]), ev)
|
||||
for (repo, pin), _ in sorted(by_place.items()):
|
||||
task = f"round-{round_index}"
|
||||
rows.append(
|
||||
{
|
||||
"id": row_id("proposal_outcome", gate_run, task, f"{pid}:{repo}:{pin}"), "kind": "proposal_outcome",
|
||||
"repo": repo, "pin": pin, "run": gate_run, "task": task, "population": "improve-loop", "split": "loop",
|
||||
"image": "host", "edit_key": edit.key(), "proposal_id": pid,
|
||||
"evidence_digest": proposal.provenance["evidence_digest"], "outcome": outcome, "reason": reason,
|
||||
"gate_run": gate_run,
|
||||
}
|
||||
) # fmt: skip
|
||||
return rows
|
||||
|
||||
|
||||
def gate_run_name(manifest_name: str) -> str:
|
||||
return f"improve:{manifest_name}:{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}"
|
||||
|
||||
|
||||
# --- the `ousast improve --memory` session -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def benchmark_cases(manifest: object) -> frozenset[tuple[str, str]]:
|
||||
"""(repo, pin) of the gated benchmark's cases. Local sources (the only kind `resolve_benchmark_source` accepts)
|
||||
name no repository pin, so they exclude nothing here; a pinned source would list its case."""
|
||||
source = getattr(manifest, "source", None)
|
||||
repo, pin = getattr(source, "repo", ""), getattr(source, "pin", "")
|
||||
return frozenset({(repo_key(repo), pin)}) if repo and pin else frozenset()
|
||||
|
||||
|
||||
def store_spec(value: str) -> str | None:
|
||||
"""``--memory`` value -> ``open_store`` spec: empty uses OUSAST_MEMORY or the default, a bare path is a FileStore."""
|
||||
if not value:
|
||||
return None
|
||||
return value if "://" in value else f"file://{Path(value).resolve()}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemorySession:
|
||||
"""One `improve --memory` invocation: the store read once, proposals per round, outcomes written back."""
|
||||
|
||||
store: MemoryStore
|
||||
guard: Guard
|
||||
records: list[Record]
|
||||
ruleset_dir: Path
|
||||
gate_run: str
|
||||
made: dict[str, MemoryProposal] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def open(
|
||||
cls, spec: str, manifest: object, target: Path, ruleset_dir: Path, populations: Sequence[str], catalog: Path | None
|
||||
) -> MemorySession:
|
||||
from ..pairs import load_pair_catalog, select_split
|
||||
from ..plane.memory import FileStore, open_store
|
||||
|
||||
store = open_store(store_spec(spec))
|
||||
if isinstance(store, FileStore) and not (store.root / "index.jsonl").is_file():
|
||||
raise SystemExit(f"memory store {store.describe()} has no index.jsonl: an unread store would propose nothing")
|
||||
if catalog is None or not catalog.is_file():
|
||||
raise SystemExit(f"--memory needs the pair catalog for the holdout guard: {catalog} is not a file")
|
||||
holdout = select_split(load_pair_catalog(catalog), "holdout")
|
||||
guard = Guard.build(populations, benchmark_cases(manifest), holdout)
|
||||
records = store.rows()
|
||||
print(
|
||||
f"memory={store.describe()} index_entries={len(store.index())} rows={len(records)} "
|
||||
f"guard: populations={sorted(guard.populations) or '-'} gated_cases={len(guard.gated_cases)} "
|
||||
f"holdout_repos={len(guard.holdout_repos)}"
|
||||
)
|
||||
return cls(store, guard, records, ruleset_dir, gate_run_name(str(getattr(manifest, "name", "manifest"))))
|
||||
|
||||
def proposals(self, ledger_path: Path, journal_path: Path) -> list[MemoryProposal]:
|
||||
from ..ruleset import load_ruleset, read_rule_ledger
|
||||
from .journal import load_journal
|
||||
|
||||
rules = load_ruleset(self.ruleset_dir, ledger_path if ledger_path.is_file() else None)
|
||||
found = propose_from_memory(
|
||||
self.records,
|
||||
{r.rule_id: r for r in rules},
|
||||
read_rule_ledger(ledger_path),
|
||||
load_journal(journal_path),
|
||||
excluded=self.guard,
|
||||
index_digest=index_digest(self.store),
|
||||
)
|
||||
kept, excluded = self.guard.apply(self.records)
|
||||
kinds: dict[str, int] = {}
|
||||
for rec in kept:
|
||||
kinds[rec.row["kind"]] = kinds.get(rec.row["kind"], 0) + 1
|
||||
print(f"memory_rows kept={len(kept)} by_kind={json.dumps(dict(sorted(kinds.items())))}")
|
||||
print(f"memory_excluded={json.dumps(excluded, sort_keys=True)}")
|
||||
for p in found:
|
||||
print(f"memory_proposal {p.edit.key()} rule={p.provenance['rule']} id={p.proposal_id} evidence={len(p.provenance['evidence'])}")
|
||||
print(f"memory_proposals={len(found)}")
|
||||
self.made.update({p.proposal_id: p for p in found})
|
||||
return found
|
||||
|
||||
def finish(self, outcomes: Sequence[Any], journal_path: Path, *, dry_run: bool) -> None:
|
||||
signals = disputed_families(self.records, self.guard)
|
||||
path = write_signals(journal_path.parent, signals)
|
||||
print(f"memory_signals disputed_families={[s['family'] for s in signals] or '-'} ({path.name}, advisory)")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for o in outcomes:
|
||||
outcome = outcome_of(o.reason, o.accepted)
|
||||
if outcome is not None:
|
||||
rows += outcome_rows(o.edits, outcome, o.round, o.reason, self.made, self.gate_run)
|
||||
if dry_run:
|
||||
print(f"memory_outcomes={len(rows)} (dry run: not written to the store)")
|
||||
return
|
||||
if rows:
|
||||
self.store.put_rows(rows)
|
||||
print(f"memory_outcomes={len(rows)} written to {self.store.describe()}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Guard",
|
||||
"MemorySession",
|
||||
"benchmark_cases",
|
||||
"store_spec",
|
||||
"M1_MIN_FALSE",
|
||||
"M1_MIN_REPOS",
|
||||
"M2_MAX_REJECTED",
|
||||
"M2_MIN_HITS",
|
||||
"M2_MIN_REPOS",
|
||||
"M3_MIN_CANDIDATES",
|
||||
"M3_MIN_DISPUTE_RATE",
|
||||
"MemoryProposal",
|
||||
"THRESHOLDS",
|
||||
"applicable",
|
||||
"disputed_families",
|
||||
"evidence_digest",
|
||||
"gate_run_name",
|
||||
"index_digest",
|
||||
"outcome_of",
|
||||
"outcome_rows",
|
||||
"propose_from_memory",
|
||||
"proposal_id_of",
|
||||
"read_sidecar",
|
||||
"write_sidecar",
|
||||
"write_signals",
|
||||
]
|
||||
@@ -1,125 +0,0 @@
|
||||
"""Bounded-lever edit model + validator for the self-improvement loop (tasks 7.1, 7.2).
|
||||
|
||||
The loop may only adjust governance *data* through two bounded levers and never the
|
||||
two authorities a human/upstream owns: detection **pattern text** and the 0-5 CWE
|
||||
**severity**. Edits are typed so pattern/severity changes are structurally
|
||||
impossible; the validator additionally enforces status/bounds/staging/resolution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..policy import CwePolicy
|
||||
from ..ruleset import PatternRule
|
||||
from ..ruleset.store import VALID_STATUS
|
||||
|
||||
# The only levers the loop may pull (mirrors the design's journal._VALID_LEVERS).
|
||||
VALID_LEVERS = frozenset({"rule", "policy", "mechanisms"})
|
||||
_IDENTIFIER = re.compile(r"^[A-Za-z_$][\w$]*$")
|
||||
MECHANISM_ACTIONS = frozenset({"admit", "retract"})
|
||||
# Policy constants the loop may tune; the 0-5 severity is intentionally absent.
|
||||
TUNABLE_POLICY_CONSTANTS = frozenset({"K", "MIN_SCORE"})
|
||||
|
||||
|
||||
class StrictValidationError(ValueError):
|
||||
"""Raised when a proposed loop edit violates the bounded-lever contract."""
|
||||
|
||||
def __init__(self, message: str, *, kind: str = "rule_change") -> None:
|
||||
super().__init__(message)
|
||||
self.kind = kind
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuleStatusEdit:
|
||||
"""A `rule`-lever edit: change a rule's status / evidence floor / precision estimate."""
|
||||
|
||||
rule_id: str
|
||||
from_status: str
|
||||
to_status: str
|
||||
rationale: str = ""
|
||||
lever: str = "rule"
|
||||
|
||||
def key(self) -> str:
|
||||
return f"rule:{self.rule_id}:{self.from_status}->{self.to_status}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyConstantEdit:
|
||||
"""A `policy`-lever edit: tune a score constant (K / MIN_SCORE). Never severity."""
|
||||
|
||||
name: str
|
||||
from_value: float
|
||||
to_value: float
|
||||
rationale: str = ""
|
||||
lever: str = "policy"
|
||||
|
||||
def key(self) -> str:
|
||||
return f"policy:{self.name}:{self.from_value}->{self.to_value}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvolveBounds:
|
||||
k_range: tuple[float, float] = (20.0, 200.0)
|
||||
min_score_range: tuple[int, int] = (0, 100)
|
||||
|
||||
|
||||
class EvolveValidator:
|
||||
"""Enforces the bounded-lever safety contract; raises StrictValidationError on any breach."""
|
||||
|
||||
def __init__(self, bounds: EvolveBounds | None = None) -> None:
|
||||
self._bounds = bounds or EvolveBounds()
|
||||
|
||||
def validate(self, edit: object, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None:
|
||||
if isinstance(edit, RuleStatusEdit):
|
||||
self._validate_rule(edit, ruleset_by_id, policy)
|
||||
elif isinstance(edit, PolicyConstantEdit):
|
||||
self._validate_policy(edit)
|
||||
else:
|
||||
raise StrictValidationError(f"unknown edit type {type(edit).__name__}", kind="rule_change")
|
||||
|
||||
def _validate_rule(self, edit: RuleStatusEdit, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None:
|
||||
if edit.to_status not in VALID_STATUS:
|
||||
raise StrictValidationError(f"invalid status {edit.to_status!r} for {edit.rule_id}")
|
||||
rule = ruleset_by_id.get(edit.rule_id)
|
||||
if rule is None:
|
||||
raise StrictValidationError(f"edit targets unknown rule_id {edit.rule_id!r} (loop cannot add rules)")
|
||||
if edit.from_status == "enabled" and edit.to_status == "disabled":
|
||||
raise StrictValidationError(f"{edit.rule_id}: enabled->disabled jump is forbidden; stage via shadow first")
|
||||
if rule.cwe not in policy:
|
||||
raise StrictValidationError(f"{edit.rule_id}: CWE {rule.cwe} does not resolve in the policy")
|
||||
|
||||
def _validate_policy(self, edit: PolicyConstantEdit) -> None:
|
||||
if edit.name not in TUNABLE_POLICY_CONSTANTS:
|
||||
raise StrictValidationError(
|
||||
f"policy constant {edit.name!r} is not loop-tunable (severity is upstream-owned)", kind="policy_change"
|
||||
)
|
||||
if edit.name == "K" and not (self._bounds.k_range[0] <= edit.to_value <= self._bounds.k_range[1]):
|
||||
raise StrictValidationError(f"K={edit.to_value} out of bounds {self._bounds.k_range}", kind="policy_change")
|
||||
if edit.name == "MIN_SCORE" and not (self._bounds.min_score_range[0] <= edit.to_value <= self._bounds.min_score_range[1]):
|
||||
raise StrictValidationError(f"MIN_SCORE={edit.to_value} out of bounds {self._bounds.min_score_range}", kind="policy_change")
|
||||
|
||||
|
||||
def edits_to_ledger(edits: list[RuleStatusEdit], base: Mapping[str, dict[str, object]]) -> dict[str, dict[str, object]]:
|
||||
"""Overlay rule-status edits onto a copy of the base ledger."""
|
||||
ledger: dict[str, dict[str, object]] = {key: dict(value) for key, value in base.items()}
|
||||
for edit in edits:
|
||||
entry = dict(ledger.get(edit.rule_id, {}))
|
||||
entry["status"] = edit.to_status
|
||||
ledger[edit.rule_id] = entry
|
||||
return ledger
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EvolveBounds",
|
||||
"EvolveValidator",
|
||||
"MECHANISM_ACTIONS",
|
||||
"PolicyConstantEdit",
|
||||
"RuleStatusEdit",
|
||||
"StrictValidationError",
|
||||
"TUNABLE_POLICY_CONSTANTS",
|
||||
"VALID_LEVERS",
|
||||
"edits_to_ledger",
|
||||
]
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Vuln-vs-fixed pair corpus: TP on the vuln tree, silent on the fix tree.
|
||||
|
||||
Isolated trees keep the two sides from contaminating each other (a `safe.py`
|
||||
sitting next to `app.py` cannot measure false positives). Pair outcomes feed
|
||||
the improvement loop as miss (vuln) and fp (fix) signals.
|
||||
sitting next to `app.py` cannot measure false positives). Pair outcomes report
|
||||
misses on vulnerable code and false positives on fixed code.
|
||||
|
||||
pair-corpus-honesty: the overlay scorer counts ``coverage`` records with a
|
||||
source as detections and as leaks, matches by function when a label names one,
|
||||
@@ -61,7 +61,7 @@ FunctionRanges = dict[str, tuple[tuple[str, int, int], ...]]
|
||||
SLICE_NAMES = ("all", "local", "github", "sast", "vfc", "vibe-py", "vfc-js", "agent-vfc", "owasp")
|
||||
PROVENANCES = frozenset({"human", "agent", "mixed", "synthetic"})
|
||||
SPLITS = frozenset({"train", "holdout"})
|
||||
# Req 9: how a label was established. Only seeded and reviewed pairs steer the improve loop.
|
||||
# Req 9: how a label was established.
|
||||
REVIEW_TIERS = frozenset({"seeded", "advisory", "title", "reviewed"})
|
||||
GATING_TIERS = frozenset({"seeded", "reviewed"})
|
||||
# Req 10: pointer pairs are harvested into a cache outside the repository, never vendored.
|
||||
@@ -442,7 +442,7 @@ def evaluate_catalog(
|
||||
inventory_sidecar: bool = True,
|
||||
pointers: bool | None = None,
|
||||
) -> PairEvalResult:
|
||||
"""Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan.
|
||||
"""Score every case. ``ruleset`` selects the rules to score; ``inventory_sidecar=False`` skips the second scan.
|
||||
|
||||
Pointer pairs (``vendored = false``) are harvested into the cache when ``pointers`` (or OPENULTRASAST_PAIRS_NETWORK=1) allows the
|
||||
network, otherwise skipped with a ``pointer_pair_skipped`` degradation (Req 10.2, 10.3).
|
||||
@@ -497,14 +497,14 @@ def evaluate_catalog(
|
||||
|
||||
|
||||
def build_pair_signals(outcomes: Sequence[PairOutcome], *, split: str | None = None) -> list[dict[str, object]]:
|
||||
"""Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms.
|
||||
"""Miss and false-positive signals, tagged with the pair name, profile, and mechanisms.
|
||||
|
||||
``split`` restricts the signals to one split, so a learning path teaches from the train rows only (Req 5.1).
|
||||
"""
|
||||
signals: list[dict[str, object]] = []
|
||||
for outcome in outcomes:
|
||||
if outcome.known_limit is not None:
|
||||
continue # unachievable by construction; the loop must not chase it
|
||||
continue # excluded from the achievable metrics
|
||||
if split is not None and outcome.split != split:
|
||||
continue
|
||||
common: dict[str, object] = {
|
||||
@@ -653,7 +653,7 @@ def select_slice(cases: Sequence[PairCase], slice_name: str | None) -> tuple[Pai
|
||||
|
||||
|
||||
def select_tier(cases: Sequence[PairCase], tiers: Iterable[str]) -> tuple[PairCase, ...]:
|
||||
"""Pairs whose review tier is in ``tiers`` (the improve loop passes GATING_TIERS)."""
|
||||
"""Pairs whose review tier is in ``tiers``."""
|
||||
wanted = frozenset(tiers)
|
||||
return tuple(case for case in cases if case.review_tier in wanted)
|
||||
|
||||
|
||||
@@ -20,13 +20,6 @@ under ``--plane`` (default ``plane/``), everything a Run over the set needs besi
|
||||
with budgets, annotated with the population and split (``openultrasast.io/population``/``split``: the files'
|
||||
stems) the memory rows carry.
|
||||
|
||||
``--loop`` (harnessx-removal design section 6) appends the improvement loop: per case ``alerts`` (quick-mode rules
|
||||
on the vulnerable and the fixed pin, which gets its Workspace ``<case>-fixed``; ``case.json`` then also carries the
|
||||
fix's new-side ranges) feeding ``remember``, and once ``memory-snapshot`` (the store's rows after the train-on-test
|
||||
guard, written on the host by ``memory.seed``) -> ``loop-measure`` -> ``loop-propose`` -> ``loop-improve``, the last
|
||||
two bound to this repository at the commit under test. All of them model-free with budget ``{usd: 0, calls: 0}``;
|
||||
the gate's verdict is ``<run>/loop-improve/gate.json``, and adopting an accepted ledger stays a maintainer commit.
|
||||
|
||||
Each ``repo-facts`` Task carries ``openultrasast.io/memory-key`` (repository, pin, candidates digest, runner image
|
||||
digest): the key under which stored facts are reused instead of recomputed (``plane/memory.py``, ``seed``).
|
||||
|
||||
@@ -67,12 +60,12 @@ import yaml
|
||||
from ..model.endpoint import price_of
|
||||
from .budget import cost_of
|
||||
from .manifests import GIT_COMMITS_ANNOTATION, RUN_API_VERSION, Task, load_manifests
|
||||
from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SNAPSHOT_ANNOTATION, SPLIT_ANNOTATION, memory_key
|
||||
from .memory import MEMORY_KEY_ANNOTATION, POPULATION_ANNOTATION, SPLIT_ANNOTATION, memory_key
|
||||
from .reconciler import _ax_name, _doc
|
||||
from .tasks.repo_facts import _digest
|
||||
from .tasks.verify import MAX_STEPS, units_of
|
||||
|
||||
__all__ = ["CaseInputs", "Loop", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
|
||||
__all__ = ["CaseInputs", "fix_ranges", "increment", "read_cases", "render", "repin_templates"]
|
||||
|
||||
REFERENCE_USD_PER_CANDIDATE = 0.022 # two passes: verifier-batched-check-2026-09-29.json cost.usd_per_candidate_two_passes
|
||||
REFERENCE_MODEL = "deepseek-flash" # verify_batched.py's default (DEFAULT_DETECTOR_MODEL) in the batched check
|
||||
@@ -86,12 +79,7 @@ CALLS_PER_HUNT = MAX_STEPS + 2 # every tool step, the answer, one empty-content
|
||||
ATESPACE = "default" # the default when no profile is given; ``increment`` takes ``profile.atespace``
|
||||
CASE_PATH = "/workspace/case"
|
||||
INPUTS_PATH = "/workspace/inputs"
|
||||
FIXED_PATH = "/workspace/fixed"
|
||||
PROJECT_PATH = "/workspace/project"
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "loop", "features", "roles")
|
||||
FIXED_RANGES_BASIS = "git diff -U0 <vulnerable> <fixed>, new side: the fix's lines at the fixed pin"
|
||||
LOOP_MANIFEST = "benchmarks/manifests/java-spring-boot-vulnerable.toml" # the equality baseline's `improve` manifest
|
||||
LOOP_CATALOG = "benchmarks/pairs/catalog.toml"
|
||||
TEMPLATES = ("repo-facts", "verify", "agree", "remember", "alerts", "features", "roles")
|
||||
|
||||
Candidate = tuple[str, str, int]
|
||||
|
||||
@@ -105,20 +93,6 @@ class CaseInputs:
|
||||
recorded_usd: float
|
||||
triage_usd: float = 0.0
|
||||
ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
|
||||
fixed_ranges: Mapping[str, Sequence[tuple[int, int]]] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Loop:
|
||||
"""The improvement loop appended to a Run (``--loop``, design section 6): ``repo`` at ``commit`` is this
|
||||
repository at the commit under test, where ``manifest`` and ``catalog`` (paths inside it) gate the proposals;
|
||||
rows of ``populations`` never propose (the train-on-test guard)."""
|
||||
|
||||
repo: str
|
||||
commit: str
|
||||
manifest: str = LOOP_MANIFEST
|
||||
catalog: str = LOOP_CATALOG
|
||||
populations: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def fix_ranges(repo: Path, vulnerable: str, fixed: str, side: str = "old") -> dict[str, list[tuple[int, int]]]:
|
||||
@@ -165,11 +139,10 @@ def _scan_rows(triage_dir: Path, case_id: str) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def read_cases(
|
||||
validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None, *, fixed_side: bool = False
|
||||
validation_set: Path, candidates_dir: Path, triage_dir: Path, population: Path, repos: Path | None = None
|
||||
) -> list[CaseInputs]:
|
||||
"""The set's cases in id order, each with its candidates, the triage-kept subset, the recorded cost and its
|
||||
triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges (and, with
|
||||
``fixed_side``, its new-side ranges, which the loop's `alerts` task marks fixed-pin alerts by)."""
|
||||
triage share, and -- with ``repos`` (one clone per case id) -- the fix's old-side ranges."""
|
||||
subset: dict[str, list[list[str]]] = json.loads(validation_set.read_text(encoding="utf-8"))
|
||||
records = {c["id"]: c for c in tomllib.loads(population.read_text(encoding="utf-8"))["case"]}
|
||||
cases: list[CaseInputs] = []
|
||||
@@ -199,11 +172,6 @@ def read_cases(
|
||||
recorded_usd=round(sum(float(r.get("usd") or 0.0) for r in rows), 6),
|
||||
triage_usd=_triage_usd(rows),
|
||||
ranges=None if repos is None else fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"]),
|
||||
fixed_ranges=(
|
||||
fix_ranges(repos / case_id, records[case_id]["vulnerable"], records[case_id]["fixed"], side="new")
|
||||
if repos is not None and fixed_side
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
return cases
|
||||
@@ -226,9 +194,6 @@ def _case_record(item: CaseInputs) -> dict[str, Any]:
|
||||
if item.ranges is not None:
|
||||
record["ranges"] = {path: [list(span) for span in spans] for path, spans in item.ranges.items()}
|
||||
record["ranges_basis"] = RANGES_BASIS
|
||||
if item.fixed_ranges is not None:
|
||||
record["fixed_ranges"] = {path: [list(span) for span in spans] for path, spans in item.fixed_ranges.items()}
|
||||
record["fixed_ranges_basis"] = FIXED_RANGES_BASIS
|
||||
record["sites_in_set"] = [s for s in sites if s in in_set]
|
||||
record["cost"] = {
|
||||
"candidates_before_triage": len(item.candidates), "recorded_usd": item.recorded_usd,
|
||||
@@ -304,95 +269,38 @@ def _free() -> dict[str, int]:
|
||||
return {"usd": 0, "calls": 0} # model-free: budget.py refuses any call, so a stray one fails loudly
|
||||
|
||||
|
||||
def _chain_inputs(case_id: str, alerts: bool) -> dict[str, str]:
|
||||
"""Every artifact of a case's chain -- facts, the three passes with their summaries, the final agree -- and, in a
|
||||
loop, its alerts with their coverage."""
|
||||
def _chain_inputs(case_id: str) -> dict[str, str]:
|
||||
"""Every artifact of a case's chain: facts, the three passes with their summaries, and the final agreement."""
|
||||
inputs = {"facts": f"{case_id}-facts/facts.json", "facts_summary": f"{case_id}-facts/summary.json"}
|
||||
for label in ("a", "b", "c"):
|
||||
inputs.update({f"pass_{label}": f"{case_id}-v{label}/units.jsonl", f"summary_{label}": f"{case_id}-v{label}/summary.json"})
|
||||
inputs["agreed"] = f"{case_id}-final/agreed.json"
|
||||
if alerts:
|
||||
inputs["alerts"] = f"{case_id}-alerts/alerts.jsonl"
|
||||
inputs["alerts_summary"] = f"{case_id}-alerts/summary.json" # the coverage: whether a zero could mean clean
|
||||
return inputs
|
||||
|
||||
|
||||
def _features(item: CaseInputs, templates: Mapping[str, Task], alerts: bool) -> tuple[dict, dict]:
|
||||
def _features(item: CaseInputs, templates: Mapping[str, Task]) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `features` (learned-decision-engine design section 1): model-free, bound to the
|
||||
vulnerable Workspace for the functions' spans and entry-point distance; the chain's artifacts as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo"}
|
||||
task = _task(templates["features"], f"features-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = _chain_inputs(case_id, alerts)
|
||||
inputs = _chain_inputs(case_id)
|
||||
entry = {"name": f"{case_id}-features", "task": f"features-{case_id}", "inputs": inputs, "outputs": ["features.jsonl", "summary.json"]}
|
||||
return task, {**entry, "budget": _free()}
|
||||
|
||||
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str, alerts: bool) -> tuple[dict, dict]:
|
||||
def _remember(item: CaseInputs, templates: Mapping[str, Task], population: str, split: str) -> tuple[dict, dict]:
|
||||
"""(Task, Run entry) of the case's `remember` (design section 4): bound to the vulnerable Workspace for the
|
||||
repository and pin its rows carry; every artifact of the case's chain (and, in a loop, its alerts) and the case's
|
||||
repository and pin its rows carry; every artifact of the case's chain and the case's
|
||||
feature records as input."""
|
||||
case_id = item.case["id"]
|
||||
env = {"OUSAST_POPULATION": population or "unknown", "OUSAST_SPLIT": split or "unknown"}
|
||||
task = _task(templates["remember"], f"remember-{case_id}", env, [(_ax_name(case_id, "vulnerable"), CASE_PATH)])
|
||||
inputs = {**_chain_inputs(case_id, alerts), "features": f"{case_id}-features/features.jsonl"}
|
||||
inputs = {**_chain_inputs(case_id), "features": f"{case_id}-features/features.jsonl"}
|
||||
entry = {"name": f"{case_id}-remember", "task": f"remember-{case_id}", "inputs": inputs, "outputs": ["memory.jsonl"], "budget": _free()}
|
||||
return task, entry
|
||||
|
||||
|
||||
def _alerts(item: CaseInputs, templates: Mapping[str, Task], atespace: str = ATESPACE) -> tuple[dict, dict, dict]:
|
||||
"""(fixed-pin Workspace, Task, Run entry) of the case's `alerts`: quick-mode rules on both pins."""
|
||||
case = item.case
|
||||
git = [{"name": "repo", "repo": case["repo"], "dir": "repo", "depth": 1}]
|
||||
fixed = _doc("Workspace", _ax_name(case["id"], "fixed"), {"git": git}, atespace)
|
||||
fixed["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={case['fixed']}"}
|
||||
bindings = [(_ax_name(case["id"], "vulnerable"), CASE_PATH), (fixed["metadata"]["name"], FIXED_PATH)]
|
||||
bindings.append((_ax_name(case["id"], "inputs"), INPUTS_PATH))
|
||||
env = {
|
||||
"OUSAST_WORKSPACE_DIR": f"{CASE_PATH}/repo", "OUSAST_FIXED_DIR": f"{FIXED_PATH}/repo",
|
||||
"OUSAST_INPUT_CASE": f"{INPUTS_PATH}/case.json", "OUSAST_VULNERABLE_PIN": str(case["vulnerable"]),
|
||||
"OUSAST_FIXED_PIN": str(case["fixed"]),
|
||||
} # fmt: skip
|
||||
task = _task(templates["alerts"], f"alerts-{case['id']}", env, bindings)
|
||||
entry = {"name": f"{case['id']}-alerts", "task": f"alerts-{case['id']}", "outputs": ["alerts.jsonl", "summary.json"], "budget": _free()}
|
||||
entry["serialize"] = "engine"
|
||||
return fixed, task, entry
|
||||
|
||||
|
||||
def _loop(
|
||||
cases: Sequence[CaseInputs], templates: Mapping[str, Task], loop: Loop, atespace: str = ATESPACE
|
||||
) -> tuple[dict, list[dict], list[dict]]:
|
||||
"""(project Workspace, Tasks, Run entries) of the loop's singletons: snapshot -> measure -> propose -> improve."""
|
||||
git = [{"name": "repo", "repo": loop.repo, "dir": "repo", "depth": 1}]
|
||||
project = _doc("Workspace", _ax_name("openultrasast", loop.commit[:12]), {"git": git}, atespace)
|
||||
project["metadata"]["annotations"] = {GIT_COMMITS_ANNOTATION: f"repo={loop.commit}"}
|
||||
bound = [(project["metadata"]["name"], PROJECT_PATH)]
|
||||
env = {
|
||||
"OUSAST_PROJECT_DIR": f"{PROJECT_PATH}/repo", "OUSAST_LOOP_MANIFEST": loop.manifest, "OUSAST_LOOP_CATALOG": loop.catalog,
|
||||
"OUSAST_QUALIFY_POPULATIONS": json.dumps(sorted(loop.populations)),
|
||||
} # fmt: skip
|
||||
snapshot = _task(templates["loop"], "memory-snapshot", {}, [], ["snapshot"])
|
||||
guard = {"catalog": loop.catalog, "manifest": loop.manifest, "populations": sorted(loop.populations)}
|
||||
snapshot["metadata"].setdefault("annotations", {})[SNAPSHOT_ANNOTATION] = json.dumps(guard, sort_keys=True, separators=(",", ":"))
|
||||
tasks = [
|
||||
snapshot, _task(templates["loop"], "loop-measure", {}, [], ["measure"]),
|
||||
_task(templates["loop"], "loop-propose", env, bound, ["propose"]),
|
||||
_task(templates["loop"], "loop-improve", env, bound, ["improve"]),
|
||||
] # fmt: skip
|
||||
remembered = {f"remember_{c.case['id']}": f"{c.case['id']}-remember/memory.jsonl" for c in cases}
|
||||
entries = [
|
||||
{"name": "memory-snapshot", "task": "memory-snapshot", "outputs": ["rows.jsonl", "index.json"], "budget": _free()},
|
||||
{"name": "loop-measure", "task": "loop-measure", "inputs": {**remembered, "snapshot": "memory-snapshot/rows.jsonl"},
|
||||
"outputs": ["measure.json", "rows.jsonl"], "budget": _free()},
|
||||
{"name": "loop-propose", "task": "loop-propose",
|
||||
"inputs": {"measure": "loop-measure/measure.json", "rows": "loop-measure/rows.jsonl", "index": "memory-snapshot/index.json"},
|
||||
"outputs": ["proposals.jsonl", "memory_proposals.jsonl", "signals.json"], "budget": _free()},
|
||||
{"name": "loop-improve", "task": "loop-improve", "inputs": {"proposals": "loop-propose/proposals.jsonl"},
|
||||
"outputs": ["gate.json", "journal.json"], "budget": _free(), "serialize": "engine"},
|
||||
] # fmt: skip
|
||||
return project, tasks, entries
|
||||
|
||||
|
||||
def render(
|
||||
cases: Sequence[CaseInputs],
|
||||
templates: Mapping[str, Task],
|
||||
@@ -401,13 +309,11 @@ def render(
|
||||
*,
|
||||
population: str = "",
|
||||
split: str = "",
|
||||
loop: Loop | None = None,
|
||||
atespace: str = ATESPACE,
|
||||
) -> dict[str, str]:
|
||||
"""Relative path under the plane root -> file text, for every generated manifest. ``population`` and ``split``
|
||||
(the population file's and the set's stems) become the Run's annotations, which the memory rows carry. Each
|
||||
case ends in `features` -> `remember`; ``loop`` adds per case `alerts` (and the fixed-pin Workspace), and once the
|
||||
snapshot, `loop-measure`, `loop-propose` and `loop-improve` (design section 6), all model-free."""
|
||||
case ends in `features` -> `remember`."""
|
||||
header = f"# generated by `{command}`; do not edit, regenerate.\n"
|
||||
out: dict[str, str] = {}
|
||||
tasks: list[dict[str, Any]] = []
|
||||
@@ -454,22 +360,12 @@ def render(
|
||||
tasks.sort(key=lambda doc: 1 if doc["metadata"]["name"].startswith("verify-c-") else 0) # stable: the old Tasks first
|
||||
entries.extend(tiebreak) # appended: the first Run's entries stay byte-identical, so a rerun keeps their state
|
||||
for item in cases: # appended after the tie-break for the same reason
|
||||
if loop is not None:
|
||||
fixed, alerts_task, alerts_entry = _alerts(item, templates, atespace)
|
||||
out[f"workspaces/{fixed['metadata']['name']}.yaml"] = _dump(header, [fixed])
|
||||
tasks.append(alerts_task)
|
||||
entries.append(alerts_entry)
|
||||
features_task, features_entry = _features(item, templates, loop is not None)
|
||||
features_task, features_entry = _features(item, templates)
|
||||
tasks.append(features_task)
|
||||
entries.append(features_entry)
|
||||
remember_task, remember_entry = _remember(item, templates, population, split, loop is not None)
|
||||
remember_task, remember_entry = _remember(item, templates, population, split)
|
||||
tasks.append(remember_task)
|
||||
entries.append(remember_entry)
|
||||
if loop is not None:
|
||||
project, loop_tasks, loop_entries = _loop(cases, templates, loop, atespace)
|
||||
out[f"workspaces/{project['metadata']['name']}.yaml"] = _dump(header, [project])
|
||||
tasks.extend(loop_tasks)
|
||||
entries.extend(loop_entries)
|
||||
out[f"tasks/{run_name}.yaml"] = _dump(header, tasks)
|
||||
annotations = {k: v for k, v in ((POPULATION_ANNOTATION, population), (SPLIT_ANNOTATION, split)) if v}
|
||||
metadata: dict[str, Any] = {"name": run_name, **({"annotations": annotations} if annotations else {})}
|
||||
@@ -519,24 +415,21 @@ def increment(
|
||||
command: str,
|
||||
repos: Path | None = None,
|
||||
runner_image: Path | None = None,
|
||||
loop: Loop | None = None,
|
||||
atespace: str | None = None,
|
||||
) -> list[Path]:
|
||||
"""Write the generated manifests under ``plane`` from the templates in ``plane/tasks`` (re-pinned first when
|
||||
``runner_image`` is given), with the improvement loop when ``loop`` is given; returns the paths. Workspaces go
|
||||
``runner_image`` is given); returns the paths. Workspaces go
|
||||
to ``atespace`` (default: the profile's)."""
|
||||
if atespace is None:
|
||||
from .profile import load_profile
|
||||
|
||||
atespace = load_profile().atespace
|
||||
if loop is not None and repos is None:
|
||||
raise ValueError("--loop needs --repos: the alerts task marks fixed-pin alerts by the fix's new-side ranges")
|
||||
if runner_image is not None:
|
||||
repin_templates(plane, runner_image)
|
||||
loaded = load_manifests([plane / "tasks" / f"{name}.yaml" for name in TEMPLATES])
|
||||
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos, fixed_side=loop is not None)
|
||||
cases = read_cases(validation_set, candidates_dir, triage_dir, population, repos)
|
||||
files = render(
|
||||
cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, loop=loop, atespace=atespace
|
||||
cases, loaded.tasks, run_name, command, population=population.stem, split=validation_set.stem, atespace=atespace
|
||||
)
|
||||
written: list[Path] = []
|
||||
for relative, text in files.items():
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
One store, keyed by repository and pin, holds what plane runs learned: repository facts by content hash, and rows
|
||||
of the kinds in ``KINDS`` (``facts``, ``verdict``, ``unit_cost``, ``alert``, ``coverage``, ``features``, ...) that
|
||||
``remember`` (:mod:`.tasks.remember`) derives from a run's delivered artifacts, plus the loop's and the decision
|
||||
``remember`` (:mod:`.tasks.remember`) derives from a run's delivered artifacts, plus the decision
|
||||
engine's own. Layout, identical for both backends::
|
||||
|
||||
index.jsonl one row per ingested (run, task): sha256 of its rows, row count
|
||||
@@ -34,10 +34,6 @@ Fact reuse: a facts entry is valid for (repo, pin, candidates digest, runner ima
|
||||
writes as the Task annotation ``openultrasast.io/memory-key``. :func:`seed` runs before a Run: for every entry whose
|
||||
Task carries a key with stored facts and no state yet, it writes ``<run>/<task>/facts.json`` and a ``done``
|
||||
summary and marks the task done under the run lock (``reconciler.mark_done``), so the reconciler skips it.
|
||||
|
||||
The loop's ``memory-snapshot`` (a Task annotated ``openultrasast.io/memory-snapshot`` with the train-on-test guard's
|
||||
parameters) is seeded the same way: a sandboxed task cannot read this store, so :func:`seed` writes the store's rows
|
||||
after the guard into ``<run>/memory-snapshot/`` (:func:`.tasks.loop.write_snapshot`) and marks it done.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -62,7 +58,6 @@ log = logging.getLogger(__name__)
|
||||
MEMORY_KEY_ANNOTATION = "openultrasast.io/memory-key"
|
||||
POPULATION_ANNOTATION = "openultrasast.io/population"
|
||||
SPLIT_ANNOTATION = "openultrasast.io/split"
|
||||
SNAPSHOT_ANNOTATION = "openultrasast.io/memory-snapshot" # the loop's guard: {"catalog", "manifest", "populations"}
|
||||
# The decision engine's kinds (learned-decision-engine, Data Models): `features` from the `features` task through
|
||||
# `remember`; `label`, `decision`, `experiment`, `arm_outcome` and `experiment_result` for the later tasks; `example`
|
||||
# (a labelled case of the local memory: label + feature record + excerpt sha) from `ousast learn memory build`.
|
||||
@@ -984,8 +979,7 @@ def ingest(run_dir: Path, store: MemoryStore | None = None) -> list[IngestResult
|
||||
def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root: Path | None = None) -> list[str]:
|
||||
"""Reuse stored facts for a Run's tasks whose ``openultrasast.io/memory-key`` matches (Req 6.1): each such task
|
||||
without a directory or a state gets ``facts.json`` and a ``done`` summary and is marked done under the run
|
||||
lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes.
|
||||
A ``memory-snapshot`` Task gets the store's rows after the guard its annotation names, marked done likewise."""
|
||||
lock. Returns the seeded task names. A changed pin, candidate set or runner image finds nothing and recomputes."""
|
||||
from . import reconciler
|
||||
|
||||
run_spec, manifests = reconciler.load_run(Path(run_manifest))
|
||||
@@ -995,16 +989,8 @@ def seed(run_manifest: Path, store: MemoryStore | None = None, *, results_root:
|
||||
seeded: list[str] = []
|
||||
for entry in run_spec.tasks:
|
||||
annotations = manifests.tasks[entry.task].metadata.annotations
|
||||
text, snapshot = annotations.get(MEMORY_KEY_ANNOTATION), annotations.get(SNAPSHOT_ANNOTATION)
|
||||
if not (text or snapshot) or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending":
|
||||
continue
|
||||
if snapshot:
|
||||
from .tasks.loop import write_snapshot # lazy: the guard reads the pair catalog and the benchmark manifest
|
||||
|
||||
info = write_snapshot(base / entry.name, store, json.loads(snapshot))
|
||||
reused = {"snapshot": store.describe(), **{k: info[k] for k in ("rows", "kept", "index_digest")}}
|
||||
reconciler.mark_done(run_spec.metadata.name, entry.name, reused=reused, results_root=results_root)
|
||||
seeded.append(entry.name)
|
||||
text = annotations.get(MEMORY_KEY_ANNOTATION)
|
||||
if not text or (base / entry.name).exists() or state.get(entry.name, {}).get("status", "pending") != "pending":
|
||||
continue
|
||||
key = parse_memory_key(str(text))
|
||||
where = {"candidates_digest": key["candidates"], "image": key["image"]}
|
||||
@@ -1037,7 +1023,6 @@ __all__ = [
|
||||
"POPULATION_ANNOTATION",
|
||||
"QUERYABLE",
|
||||
"QUERY_FIELDS",
|
||||
"SNAPSHOT_ANNOTATION",
|
||||
"SPLIT_ANNOTATION",
|
||||
"FileStore",
|
||||
"IngestResult",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""`alerts` (harnessx-removal Requirement 6.3, design section 6): what the deterministic rules say about a case.
|
||||
|
||||
No model, budget ``{usd: 0, calls: 0}``, one per case in a ``--loop`` Run, serialized like ``repo-facts`` (a full-tree
|
||||
No model, budget ``{usd: 0, calls: 0}``, serialized like ``repo-facts`` (a full-tree
|
||||
scan per pin). It runs the quick-mode scan -- ``preprocess_repository``, the entry-point reachability hints,
|
||||
``rank_targets`` and ``quick_scan_findings`` over the image's bundled ruleset, enabled and shadow rules alike, exactly
|
||||
what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on its fixed pin, and writes one row per
|
||||
@@ -12,8 +12,7 @@ what ``ousast scan --mode quick`` runs -- on the case's vulnerable pin and on it
|
||||
candidates' ``path::function``; ``<global>`` outside any function); a language those patterns do not cover keeps the
|
||||
scan's own reachability attribution. ``in_fix_range`` says whether the line lies inside the fix: the old-side ranges
|
||||
of ``case.json`` on the vulnerable pin, the new-side ``fixed_ranges`` on the fixed pin (null when ``case.json`` has
|
||||
none). A fixed-pin alert inside the fix is a false alert on code the fix wrote: rule M1 of ``improve/memory.py``
|
||||
counts it.
|
||||
none). A fixed-pin alert inside the fix is a false alert on code the fix wrote.
|
||||
|
||||
**Coverage** (an honest zero): quick mode has rules only for the languages its ruleset declares
|
||||
(:func:`quick_languages`, derived from the bundled ruleset directory, never listed by hand). ``summary.json`` records
|
||||
|
||||
@@ -1,302 +0,0 @@
|
||||
"""The improvement loop as plane tasks (harnessx-removal Requirement 6.3, design section 6).
|
||||
|
||||
``command: ["loop", <step>]``; every step is model-free with budget ``{usd: 0, calls: 0}`` (``budget.py`` refuses any
|
||||
call, so a stray one fails loudly), and each writes ``summary.json`` under the task contract. Records travel as
|
||||
``{"key", "version", "row"}`` lines: the store object a row came from and its version (the bucket's object version id, a file's
|
||||
content hash), which a proposal's provenance cites (Req 6.4).
|
||||
|
||||
- ``snapshot``: never runs in the sandbox, which cannot read the host's store. ``ousast plane run`` seeds it before
|
||||
the Run (``memory.seed`` -> :func:`write_snapshot`): the store's rows after the train-on-test guard to
|
||||
``rows.jsonl``, and ``index.json`` (store, index digest, row counts, what the guard excluded). Run in a sandbox it
|
||||
fails, naming the seed that did not happen.
|
||||
- ``measure``: every case's ``remember`` rows (inputs ``remember_<case>``) plus the snapshot -> ``measure.json``,
|
||||
the increment's metrics for this run, the store and both (declared sites agreed, usd per candidate, dispute rates
|
||||
per family, alerts per rule and pin) -- and ``rows.jsonl``, the union by row id that ``propose`` reads.
|
||||
- ``propose``: ``improve/memory.py``'s rules M1/M2 over those rows after the guard (``OUSAST_QUALIFY_POPULATIONS``, the
|
||||
gated manifest's cases, the pair catalog's holdout pairs), against the benchmark target's ledger and journal in
|
||||
this repository at the commit under test (``OUSAST_PROJECT_DIR``) -> ``proposals.jsonl`` (edit + provenance),
|
||||
``memory_proposals.jsonl`` (the sidecar) and ``signals.json`` (M3, advisory).
|
||||
- ``improve``: one ``evolve.run_round`` with those proposals -- the unchanged validator and gate, the pair catalog's
|
||||
holdout clause included -- on copies of the target's ledger and journal -> ``gate.json`` (the ``RoundOutcome``
|
||||
and which edits came from memory), ``journal.json``, the sidecar, and ``rule_policy.json`` only when the round
|
||||
was accepted; ``memory.jsonl`` holds the ``proposal_outcome`` rows the host ingests after the Run. Adopting an
|
||||
accepted ledger stays a maintainer commit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..memory import REMEMBER_OUTPUT, MemoryStore, Record, validate_row
|
||||
|
||||
INPUT_PREFIX = "OUSAST_INPUT_REMEMBER_"
|
||||
CALIBRATION = Path(".openultrasast") / "calibration" # cli.CALIBRATION_DIR: where `improve` keeps a target's ledger
|
||||
ADOPTION = "adoption is a maintainer commit: copy rule_policy.json to {ledger} (README, Rule-level loop)"
|
||||
|
||||
|
||||
# --- records -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _record_line(record: Record) -> str:
|
||||
return json.dumps({"key": record.key, "version": record.version, "row": record.row}, sort_keys=True, separators=(",", ":")) + "\n"
|
||||
|
||||
|
||||
def read_records(path: Path) -> list[Record]:
|
||||
records: list[Record] = []
|
||||
for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if raw.strip():
|
||||
item = json.loads(raw)
|
||||
records.append(Record(validate_row(item["row"], f"{path}:{number}"), str(item["key"]), item.get("version")))
|
||||
return records
|
||||
|
||||
|
||||
def write_records(path: Path, records: Iterable[Record]) -> None:
|
||||
path.write_text("".join(_record_line(r) for r in sorted(records, key=lambda r: str(r.row["id"]))), encoding="utf-8")
|
||||
|
||||
|
||||
def _guard(params: Mapping[str, Any], root: Path) -> Any:
|
||||
from ...benchmark import load_benchmark_manifest
|
||||
from ...improve.memory import Guard, benchmark_cases
|
||||
from ...pairs import load_pair_catalog, select_split
|
||||
|
||||
manifest = load_benchmark_manifest(root / str(params["manifest"]))
|
||||
holdout = select_split(load_pair_catalog(Path(str(params["catalog"]))), "holdout")
|
||||
return Guard.build([str(p) for p in params.get("populations") or []], benchmark_cases(manifest), holdout)
|
||||
|
||||
|
||||
def write_snapshot(directory: Path, store: MemoryStore, params: Mapping[str, Any], root: Path | None = None) -> dict[str, Any]:
|
||||
"""The store's rows after the guard ``params`` names (``catalog``, ``manifest``, ``populations``; paths relative to
|
||||
``root``, the working directory) into ``directory``: ``rows.jsonl``, ``index.json`` and a ``done`` summary."""
|
||||
from ...improve.memory import index_digest
|
||||
|
||||
here = Path.cwd()
|
||||
try:
|
||||
os.chdir(root or here) # the pair catalog names its files relative to the repository root
|
||||
guard = _guard(params, Path.cwd())
|
||||
finally:
|
||||
os.chdir(here)
|
||||
directory.mkdir(parents=True, exist_ok=False) # after the guard loaded: a failed load leaves no half-seeded task
|
||||
records = store.rows()
|
||||
kept, excluded = guard.apply(records)
|
||||
write_records(directory / "rows.jsonl", kept)
|
||||
info = {
|
||||
"store": store.describe(), "index_entries": len(store.index()), "index_digest": index_digest(store), "rows": len(records),
|
||||
"kept": len(kept), "excluded": excluded, "guard": dict(params),
|
||||
} # fmt: skip
|
||||
(directory / "index.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
summary = {"status": "done", "units_done": 1, "units_total": 1, "usd": None, "calls": 0, "usage": {}, "model": None, "reused": info}
|
||||
(directory / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return info
|
||||
|
||||
|
||||
# --- measure -------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def metrics(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
"""The increment's metrics over verdict, unit-cost and alert rows (a disputed final is neither agreed nor not)."""
|
||||
verdicts = [r for r in rows if r["kind"] == "verdict"]
|
||||
finals = {f: sum(1 for v in verdicts if v.get("final") == f) for f in ("agreed", "rejected", "disputed")}
|
||||
costs = [r.get("usd") for r in rows if r["kind"] == "unit_cost"]
|
||||
usd = None if any(c is None for c in costs) else round(sum(float(c or 0) for c in costs), 6)
|
||||
families: dict[str, list[int]] = {}
|
||||
for v in verdicts:
|
||||
tally = families.setdefault(str(v.get("family")), [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += v.get("final") == "disputed"
|
||||
alerts: dict[str, dict[str, int]] = {}
|
||||
for a in (r for r in rows if r["kind"] == "alert"):
|
||||
role = "fixed" if a.get("pin_role") == "fixed" else "vulnerable"
|
||||
tally_a = alerts.setdefault(str(a.get("rule_id")), {"vulnerable": 0, "fixed": 0, "fixed_in_fix_range": 0})
|
||||
tally_a[role] += 1
|
||||
tally_a["fixed_in_fix_range"] += role == "fixed" and a.get("in_fix_range") is True
|
||||
coverage: dict[str, dict[str, int]] = {} # language -> pins per coverage kind: a `none` pin's zero alerts mean nothing
|
||||
for c in (r for r in rows if r["kind"] == "coverage"):
|
||||
tally_c = coverage.setdefault(str(c.get("language")), {"quick": 0, "engine": 0, "none": 0})
|
||||
tally_c[str(c.get("coverage"))] = tally_c.get(str(c.get("coverage")), 0) + 1
|
||||
return {
|
||||
"cases": len({(v["repo"], v["pin"]) for v in verdicts}),
|
||||
"candidates": len(verdicts),
|
||||
**finals,
|
||||
"tiebreaks": sum(1 for v in verdicts if v.get("tiebreak")),
|
||||
"declared_sites_matched": sum(1 for v in verdicts if v.get("site_match") is True),
|
||||
"declared_sites_agreed": sum(1 for v in verdicts if v.get("site_match") is True and v.get("final") == "agreed"),
|
||||
"usd": usd,
|
||||
"usd_per_candidate": None if usd is None or not verdicts else round(usd / len(verdicts), 6),
|
||||
"dispute_rate": {f: {"candidates": n, "disputed": d, "rate": round(d / n, 4)} for f, (n, d) in sorted(families.items())},
|
||||
"alerts": dict(sorted(alerts.items())),
|
||||
"coverage": dict(sorted(coverage.items())),
|
||||
}
|
||||
|
||||
|
||||
def measure(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
run = env.get("OUSAST_RUN") or "unknown"
|
||||
current: list[Record] = []
|
||||
read: dict[str, int] = {}
|
||||
for name in sorted(k for k in env if k.startswith(INPUT_PREFIX)):
|
||||
path = Path(env[name])
|
||||
data = path.read_bytes()
|
||||
read[name.removeprefix(INPUT_PREFIX).lower()] = len(data)
|
||||
version = hashlib.sha256(data).hexdigest()
|
||||
for number, raw in enumerate(data.decode("utf-8").splitlines(), 1):
|
||||
if raw.strip():
|
||||
row = validate_row(json.loads(raw), f"{path}:{number}")
|
||||
current.append(Record(row, f"runs/{run}/{row['task']}/{REMEMBER_OUTPUT}", version))
|
||||
if not current:
|
||||
raise ValueError(f"read no remember rows from {len(read)} inputs ({read}): an unread run is not an empty one")
|
||||
snapshot = read_records(Path(env["OUSAST_INPUT_SNAPSHOT"])) if env.get("OUSAST_INPUT_SNAPSHOT") else []
|
||||
merged = {str(r.row["id"]): r for r in snapshot}
|
||||
merged.update({str(r.row["id"]): r for r in current}) # this run's row wins over its earlier ingest
|
||||
write_records(output / "rows.jsonl", merged.values())
|
||||
report = {
|
||||
"run": run, "inputs_bytes": read, "rows": {"run": len(current), "snapshot": len(snapshot), "union": len(merged)},
|
||||
"run_metrics": metrics([r.row for r in current]), "store_metrics": metrics([r.row for r in snapshot]),
|
||||
"union_metrics": metrics([r.row for r in merged.values()]),
|
||||
} # fmt: skip
|
||||
(output / "measure.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return {"rows": report["rows"]}
|
||||
|
||||
|
||||
# --- propose and improve -------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _project(env: Mapping[str, str]) -> tuple[Any, Path, Path, Path]:
|
||||
"""(manifest, target, ledger, journal) in the repository at the commit under test; the working directory
|
||||
becomes the repository root, where the pair catalog's relative paths resolve."""
|
||||
from ...benchmark import load_benchmark_manifest, resolve_benchmark_source
|
||||
|
||||
project = Path(env.get("OUSAST_PROJECT_DIR") or "")
|
||||
if not env.get("OUSAST_PROJECT_DIR") or not (project / str(env.get("OUSAST_LOOP_MANIFEST"))).is_file():
|
||||
raise ValueError(f"OUSAST_PROJECT_DIR {project} holds no benchmark manifest {env.get('OUSAST_LOOP_MANIFEST')}")
|
||||
os.chdir(project)
|
||||
manifest_path = Path(str(env["OUSAST_LOOP_MANIFEST"]))
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
ledger = target / CALIBRATION / "rule_policy.json"
|
||||
return manifest, target, ledger, ledger.with_name("improve_journal.json")
|
||||
|
||||
|
||||
def _params(env: Mapping[str, str]) -> dict[str, Any]:
|
||||
populations = json.loads(env.get("OUSAST_QUALIFY_POPULATIONS") or "[]")
|
||||
return {"catalog": env.get("OUSAST_LOOP_CATALOG"), "manifest": env.get("OUSAST_LOOP_MANIFEST"), "populations": populations}
|
||||
|
||||
|
||||
def propose(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
from ...improve.journal import load_journal, next_round_index
|
||||
from ...improve.memory import disputed_families, propose_from_memory, write_sidecar, write_signals
|
||||
from ...ruleset import DEFAULT_RULESET_DIR, load_ruleset, read_rule_ledger
|
||||
|
||||
records = read_records(Path(env["OUSAST_INPUT_ROWS"]))
|
||||
index = json.loads(Path(env["OUSAST_INPUT_INDEX"]).read_text(encoding="utf-8"))
|
||||
_, _, ledger, journal = _project(env)
|
||||
guard = _guard(_params(env), Path.cwd())
|
||||
rules = load_ruleset(DEFAULT_RULESET_DIR, ledger if ledger.is_file() else None)
|
||||
rounds = load_journal(journal)
|
||||
found = propose_from_memory(
|
||||
records, {r.rule_id: r for r in rules}, read_rule_ledger(ledger), rounds, excluded=guard, index_digest=str(index["index_digest"])
|
||||
)
|
||||
lines = [json.dumps({"edit": asdict(p.edit), "provenance": p.provenance}, sort_keys=True) + "\n" for p in found]
|
||||
(output / "proposals.jsonl").write_text("".join(lines), encoding="utf-8")
|
||||
sidecar = output / "journal.json"
|
||||
write_sidecar(sidecar, next_round_index(rounds), found)
|
||||
sidecar.with_name("memory_proposals.jsonl").touch() # present when nothing was proposed, as a declared output
|
||||
write_signals(output, disputed_families(records, guard))
|
||||
kept, excluded = guard.apply(records)
|
||||
return {"proposals": len(found), "rows": len(records), "kept": len(kept), "excluded": excluded["rows"]}
|
||||
|
||||
|
||||
def improve(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
from ...gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from ...improve.evolve import run_round
|
||||
from ...improve.memory import MemoryProposal, outcome_of, outcome_rows, proposal_id_of
|
||||
from ...improve.validator import RuleStatusEdit
|
||||
from ...pairs import load_pair_catalog, select_split
|
||||
from ...policy import load_policy
|
||||
from ...ruleset import DEFAULT_RULESET_DIR
|
||||
|
||||
path = Path(env["OUSAST_INPUT_PROPOSALS"])
|
||||
items = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
proposals = [MemoryProposal(RuleStatusEdit(**item["edit"]), item["provenance"]) for item in items]
|
||||
manifest, target, ledger, journal = _project(env)
|
||||
pairs = select_split(load_pair_catalog(Path(str(env["OUSAST_LOOP_CATALOG"]))), "holdout")
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-loop-") as scratch:
|
||||
work_ledger = Path(scratch) / "rule_policy.json"
|
||||
if ledger.is_file():
|
||||
shutil.copyfile(ledger, work_ledger)
|
||||
if journal.is_file():
|
||||
shutil.copyfile(journal, output / "journal.json")
|
||||
outcome = run_round(
|
||||
target, manifest, ledger_path=work_ledger, journal_path=output / "journal.json", ruleset_dir=DEFAULT_RULESET_DIR,
|
||||
policy=load_policy(), recall_floor=RECALL_FLOOR, fp_ceiling=FALSE_POSITIVE_CEILING, pair_cases=pairs, proposals=proposals,
|
||||
) # fmt: skip
|
||||
if outcome.accepted and work_ledger.is_file():
|
||||
shutil.copyfile(work_ledger, output / "rule_policy.json")
|
||||
(output / "journal.json").touch()
|
||||
(output / "memory_proposals.jsonl").touch()
|
||||
memory_edits = [e.key() for e in outcome.edits if proposal_id_of(e)]
|
||||
verdict = {
|
||||
"accepted": outcome.accepted, "reason": outcome.reason, "round": outcome.round, "proposals": len(proposals),
|
||||
"memory_edits": memory_edits, "benchmark_edits": [e.key() for e in outcome.edits if not proposal_id_of(e)],
|
||||
"manifest": env.get("OUSAST_LOOP_MANIFEST"), "outcome": asdict(outcome),
|
||||
"adoption": ADOPTION.format(ledger=ledger.as_posix()) if outcome.accepted else None,
|
||||
} # fmt: skip
|
||||
(output / "gate.json").write_text(json.dumps(verdict, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
|
||||
status = outcome_of(outcome.reason, outcome.accepted)
|
||||
by_id = {p.proposal_id: p for p in proposals}
|
||||
gate_run = f"plane:{env.get('OUSAST_RUN') or 'unknown'}"
|
||||
rows = outcome_rows(outcome.edits, status, outcome.round, outcome.reason, by_id, gate_run) if status else []
|
||||
(output / REMEMBER_OUTPUT).write_text("".join(json.dumps(r, sort_keys=True) + "\n" for r in rows), encoding="utf-8")
|
||||
return {"accepted": outcome.accepted, "reason": outcome.reason, "edits": len(outcome.edits), "memory_edits": len(memory_edits)}
|
||||
|
||||
|
||||
def snapshot(env: Mapping[str, str], output: Path) -> dict[str, Any]:
|
||||
raise RuntimeError(
|
||||
"memory-snapshot ran in the sandbox: `ousast plane run` writes it on the host from the memory store before the Run "
|
||||
"starts (memory.seed), so the seed did not happen; rerun through `ousast plane run`"
|
||||
)
|
||||
|
||||
|
||||
STEPS: dict[str, Callable[[Mapping[str, str], Path], dict[str, Any]]] = {
|
||||
"snapshot": snapshot, "measure": measure, "propose": propose, "improve": improve,
|
||||
} # fmt: skip
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None) -> int:
|
||||
args = list(sys.argv[1:] if argv is None else argv)
|
||||
env = environ if environ is not None else os.environ
|
||||
output = Path(env["OUSAST_OUTPUT_DIR"]) if env.get("OUSAST_OUTPUT_DIR") else None
|
||||
summary: dict[str, Any] = {"status": "failed", "units_done": 0, "units_total": 1, "usd": 0, "calls": 0, "usage": {}, "model": None}
|
||||
here = Path.cwd()
|
||||
try:
|
||||
if output is None:
|
||||
raise ValueError("OUSAST_OUTPUT_DIR is not set")
|
||||
if len(args) != 1 or args[0] not in STEPS:
|
||||
raise ValueError(f"expected one step of {sorted(STEPS)}, got {args}")
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
summary.update(step=args[0], **STEPS[args[0]](env, output))
|
||||
summary.update(status="done", units_done=1)
|
||||
except Exception: # noqa: BLE001 -- a crash is `failed` with its traceback
|
||||
summary["reason"] = traceback.format_exc()[-2000:]
|
||||
finally:
|
||||
os.chdir(here)
|
||||
if output is not None:
|
||||
(output / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True, default=str) + "\n", encoding="utf-8")
|
||||
shown = {k: summary.get(k) for k in ("status", "step", "proposals", "accepted", "reason", "rows")}
|
||||
print(json.dumps(shown, default=str), file=sys.stderr)
|
||||
return 0 if summary["status"] == "done" else 2
|
||||
|
||||
|
||||
__all__ = ["main", "measure", "metrics", "propose", "improve", "read_records", "write_records", "write_snapshot"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -20,8 +20,8 @@ class RulesetError(ValueError):
|
||||
class PatternRule:
|
||||
"""A detection rule expressed as governed data (no rule-local severity).
|
||||
|
||||
``pattern`` is human-PR-only; the self-improvement loop may only change
|
||||
``status``/``min_evidence_level``/``precision_estimate`` via the ledger.
|
||||
Patterns change through reviewed commits. The ledger overlays
|
||||
``status``, ``min_evidence_level`` and ``precision_estimate``.
|
||||
"""
|
||||
|
||||
rule_id: str
|
||||
@@ -42,10 +42,10 @@ class PatternRule:
|
||||
|
||||
|
||||
def load_ruleset(directory: Path = DEFAULT_RULESET_DIR, ledger: Path | None = None, *, priors: Priors = "all") -> tuple[PatternRule, ...]:
|
||||
"""Load every ``*.toml`` rule file under ``directory`` and apply the loop ledger.
|
||||
"""Load every ``*.toml`` rule file under ``directory`` and apply the rule policy ledger.
|
||||
|
||||
Rules are returned sorted by ``rule_id`` for determinism. The optional ledger
|
||||
(``rule_policy.json``) overlays loop-owned ``status``/``min_evidence_level``/
|
||||
(``rule_policy.json``) overlays ``status``/``min_evidence_level``/
|
||||
``precision_estimate`` per ``rule_id``.
|
||||
|
||||
``priors`` keeps or drops the rules tagged ``framework``/``library``: ``"all"`` (the default, today's rules),
|
||||
@@ -103,12 +103,12 @@ def write_ruleset(path: Path, rules: Iterable[PatternRule]) -> None:
|
||||
|
||||
|
||||
def read_rule_ledger(path: Path) -> dict[str, dict[str, object]]:
|
||||
"""Read the loop-owned ``rule_policy.json`` overlay (status/floors per rule_id)."""
|
||||
"""Read the ``rule_policy.json`` overlay (status/floors per rule_id)."""
|
||||
return _load_ledger(path)
|
||||
|
||||
|
||||
def write_rule_ledger(path: Path, entries: dict[str, dict[str, object]]) -> None:
|
||||
"""Write the loop-owned ``rule_policy.json`` overlay (used by the evolve loop)."""
|
||||
"""Write the ``rule_policy.json`` overlay."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(entries, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user