mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 12:09:27 +00:00
feat(cli): add ousast improve — run the self-improvement loop from the CLI
Expose the bounded, gated self-improvement loop as a first-class subcommand so it can be steered from the command line / opencode. - `ousast improve <manifest>` resolves the target from the benchmark manifest, runs the loop to convergence, and writes accepted edits to the loop-owned ledger at <target>/.openultrasast/calibration/rule_policy.json — exactly where `scan` and `benchmark` read it, so a subsequent scan of that target automatically picks up the improved ruleset (the loop closes end-to-end). - Flags: --max-rounds, --recall-floor, --fp-ceiling, --ledger, --journal, --ruleset-dir, and --dry-run (runs against a throwaway ledger, touches nothing). - Per-round output: reason, recall/fp/score deltas, and the edits applied. - tests/test_cli_improve.py: accept path writes the loop-owned ledger (noisy rule auto-shadowed), --dry-run never touches it, missing manifest fails loudly. - README: quickstart entry + a "Rule-level loop: ousast improve" subsection documenting the bounded levers, hard acceptance gate, and the HarnessX MetaAgent as the optional proposer over the same gated machinery. 131 passed, 3 skipped; ruff + mypy clean; detection gate PASS. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
649bd42fc3
commit
760df7c809
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
@@ -27,10 +28,12 @@ from .calibration import (
|
||||
)
|
||||
from .config import load_config
|
||||
from .findings import StaticFinding, quick_scan_findings, write_findings
|
||||
from .gate import FALSE_POSITIVE_CEILING, RECALL_FLOOR
|
||||
from .harness import HarnessRuntime, HarnessTraceWriter, write_harness_config
|
||||
from .harness_ext import has_harnessx
|
||||
from .hunter import run_hunter_pool, write_hunter_trajectories
|
||||
from .hunter_harness import HxScanOrchestrator
|
||||
from .improve import RoundOutcome, run_improvement
|
||||
from .index import build_code_chunks
|
||||
from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints
|
||||
from .policy import assert_rules_resolve, load_policy
|
||||
@@ -77,6 +80,23 @@ def main(argv: list[str] | None = None) -> int:
|
||||
benchmark.add_argument("--mode", choices=("quick", "standard", "deep"), default="quick")
|
||||
benchmark.add_argument("--config", type=Path, default=Path("openultrasast.toml"))
|
||||
|
||||
improve = subparsers.add_parser(
|
||||
"improve",
|
||||
help="run the bounded self-improvement loop against a benchmark manifest and update the loop-owned ruleset ledger",
|
||||
)
|
||||
improve.add_argument("manifest", type=Path)
|
||||
improve.add_argument("--max-rounds", type=int, default=5)
|
||||
improve.add_argument("--recall-floor", type=float, default=RECALL_FLOOR)
|
||||
improve.add_argument("--fp-ceiling", type=float, default=FALSE_POSITIVE_CEILING)
|
||||
improve.add_argument(
|
||||
"--ledger", type=Path, default=None, help="ruleset ledger path (default: <target>/.openultrasast/calibration/rule_policy.json)"
|
||||
)
|
||||
improve.add_argument("--journal", type=Path, default=None, help="improvement journal path (default: alongside the ledger)")
|
||||
improve.add_argument(
|
||||
"--ruleset-dir", type=Path, default=DEFAULT_RULESET_DIR, help="ruleset directory to improve (default: the bundled ruleset)"
|
||||
)
|
||||
improve.add_argument("--dry-run", action="store_true", help="run rounds against a throwaway ledger; never touch the target's ledger")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "scan":
|
||||
return _scan(args.path, args.config, args.mode, args.fail_on)
|
||||
@@ -84,6 +104,17 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return _index(args.path, args.config, args.chunk_lines)
|
||||
if args.command == "benchmark":
|
||||
return _benchmark(args.manifest, args.config, args.mode)
|
||||
if args.command == "improve":
|
||||
return _improve(
|
||||
args.manifest,
|
||||
max_rounds=args.max_rounds,
|
||||
recall_floor=args.recall_floor,
|
||||
fp_ceiling=args.fp_ceiling,
|
||||
ledger=args.ledger,
|
||||
journal=args.journal,
|
||||
ruleset_dir=args.ruleset_dir,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
return 2
|
||||
|
||||
|
||||
@@ -318,6 +349,70 @@ def _benchmark(manifest_path: Path, config_path: Path, mode: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _improve(
|
||||
manifest_path: Path,
|
||||
*,
|
||||
max_rounds: int,
|
||||
recall_floor: float,
|
||||
fp_ceiling: float,
|
||||
ledger: Path | None,
|
||||
journal: Path | None,
|
||||
ruleset_dir: Path,
|
||||
dry_run: bool,
|
||||
) -> int:
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
raise SystemExit(f"benchmark manifest is not a file: {manifest_path}")
|
||||
manifest = load_benchmark_manifest(manifest_path)
|
||||
target = resolve_benchmark_source(manifest_path, manifest)
|
||||
policy = load_policy()
|
||||
|
||||
# The loop writes its accepted ledger where `scan`/`benchmark` read it, so a
|
||||
# subsequent scan of this target automatically picks up the improved ruleset.
|
||||
default_ledger = target / CALIBRATION_DIR / "rule_policy.json"
|
||||
with tempfile.TemporaryDirectory(prefix="ousast-improve-") as scratch:
|
||||
if dry_run:
|
||||
ledger_path = Path(scratch) / "rule_policy.json"
|
||||
journal_path = Path(scratch) / "improve_journal.json"
|
||||
else:
|
||||
ledger_path = ledger or default_ledger
|
||||
journal_path = journal or ledger_path.with_name("improve_journal.json")
|
||||
|
||||
outcomes = run_improvement(
|
||||
target,
|
||||
manifest,
|
||||
ledger_path=ledger_path,
|
||||
journal_path=journal_path,
|
||||
ruleset_dir=ruleset_dir,
|
||||
policy=policy,
|
||||
max_rounds=max_rounds,
|
||||
recall_floor=recall_floor,
|
||||
fp_ceiling=fp_ceiling,
|
||||
)
|
||||
_print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None:
|
||||
print(f"manifest={manifest_path}")
|
||||
print(f"target={target}")
|
||||
accepted = [o for o in outcomes if o.accepted]
|
||||
for outcome in outcomes:
|
||||
edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-"
|
||||
print(
|
||||
f"round {outcome.round}: {outcome.reason} | "
|
||||
f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} "
|
||||
f"fp {outcome.fp_before:.2%}->{outcome.fp_after:.2%} "
|
||||
f"score {outcome.score_before}->{outcome.score_after} | edits: {edits}"
|
||||
)
|
||||
print(f"rounds={len(outcomes)} accepted={len(accepted)}")
|
||||
if dry_run:
|
||||
print("dry_run=true (no ledger written)")
|
||||
elif accepted:
|
||||
print(f"ledger={ledger_path}")
|
||||
else:
|
||||
print("ledger=unchanged (no round accepted)")
|
||||
|
||||
|
||||
def _load_scan_findings(run_dir: Path) -> list[StaticFinding]:
|
||||
return load_findings(run_dir / "findings.json")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user