mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(cli): add ousast improve — run the self-improvement loop from the CLI
Expose the bounded, gated self-improvement loop as a first-class subcommand so it can be steered from the command line / opencode. - `ousast improve <manifest>` resolves the target from the benchmark manifest, runs the loop to convergence, and writes accepted edits to the loop-owned ledger at <target>/.openultrasast/calibration/rule_policy.json — exactly where `scan` and `benchmark` read it, so a subsequent scan of that target automatically picks up the improved ruleset (the loop closes end-to-end). - Flags: --max-rounds, --recall-floor, --fp-ceiling, --ledger, --journal, --ruleset-dir, and --dry-run (runs against a throwaway ledger, touches nothing). - Per-round output: reason, recall/fp/score deltas, and the edits applied. - tests/test_cli_improve.py: accept path writes the loop-owned ledger (noisy rule auto-shadowed), --dry-run never touches it, missing manifest fails loudly. - README: quickstart entry + a "Rule-level loop: ousast improve" subsection documenting the bounded levers, hard acceptance gate, and the HarnessX MetaAgent as the optional proposer over the same gated machinery. 131 passed, 3 skipped; ruff + mypy clean; detection gate PASS. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
649bd42fc3
commit
760df7c809
@@ -0,0 +1,60 @@
|
||||
"""`ousast improve` CLI: drives the self-improvement loop and the loop-owned ledger."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from openultrasast.cli import main
|
||||
from openultrasast.ruleset import PatternRule, read_rule_ledger, write_ruleset
|
||||
|
||||
|
||||
def _rule(rule_id: str, pattern: str) -> PatternRule:
|
||||
return PatternRule(rule_id=rule_id, title=rule_id, languages=("python",), cwe="CWE-95", tags=("injection",), pattern=pattern)
|
||||
|
||||
|
||||
def _expected_block(rule_id: str) -> str:
|
||||
return (
|
||||
'[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n'
|
||||
f'line = 2\nrule_id = "{rule_id}"\nsink = "eval"\nevidence = "eval executes input"\n'
|
||||
)
|
||||
|
||||
|
||||
def _scaffold(tmp_path: Path) -> tuple[Path, Path]:
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "app.py").write_text("def f(x):\n return eval(x)\ndef g():\n print('hello')\n")
|
||||
ruleset_dir = tmp_path / "ruleset"
|
||||
write_ruleset(ruleset_dir / "python" / "rules.toml", [_rule("good-eval", r"\beval\s*\("), _rule("noisy-print", r"\bprint\s*\(")])
|
||||
manifest = tmp_path / "bench.toml"
|
||||
manifest.write_text('name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' + _expected_block("good-eval"))
|
||||
return repo, ruleset_dir
|
||||
|
||||
|
||||
def test_improve_accepts_and_writes_loop_owned_ledger(tmp_path: Path) -> None:
|
||||
repo, ruleset_dir = _scaffold(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
|
||||
exit_code = main(["improve", str(manifest), "--ruleset-dir", str(ruleset_dir)])
|
||||
|
||||
assert exit_code == 0
|
||||
# The ledger lands exactly where `scan`/`benchmark` read it for this target.
|
||||
ledger = repo / ".openultrasast" / "calibration" / "rule_policy.json"
|
||||
assert ledger.exists()
|
||||
assert read_rule_ledger(ledger)["noisy-print"]["status"] == "shadow"
|
||||
journal = repo / ".openultrasast" / "calibration" / "improve_journal.json"
|
||||
assert journal.exists()
|
||||
|
||||
|
||||
def test_improve_dry_run_never_touches_the_targets_ledger(tmp_path: Path) -> None:
|
||||
repo, ruleset_dir = _scaffold(tmp_path)
|
||||
manifest = tmp_path / "bench.toml"
|
||||
|
||||
exit_code = main(["improve", str(manifest), "--ruleset-dir", str(ruleset_dir), "--dry-run"])
|
||||
|
||||
assert exit_code == 0
|
||||
assert not (repo / ".openultrasast" / "calibration" / "rule_policy.json").exists()
|
||||
|
||||
|
||||
def test_improve_rejects_missing_manifest(tmp_path: Path) -> None:
|
||||
import pytest
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
main(["improve", str(tmp_path / "nope.toml")])
|
||||
Reference in New Issue
Block a user