diff --git a/.kiro/specs/corpus-seeded-mechanisms/design.md b/.kiro/specs/corpus-seeded-mechanisms/design.md index 76cd1c0..85424ba 100644 --- a/.kiro/specs/corpus-seeded-mechanisms/design.md +++ b/.kiro/specs/corpus-seeded-mechanisms/design.md @@ -97,12 +97,12 @@ src/openultrasast/ │ ├── seed.py # export_mechanisms(cases, store) -> ExportReport (maintainer command `mechanisms export`) │ └── loo.py # evaluate_loo(cases, *, store_factory) -> LooResult; payload ├── findings.py # origin "variant" via tags/finding_id prefix "variant:" (no schema change) -├── cli.py # MAP: variant search after overlay; manifest counters; `pairs --loo`; `mechanisms export --slice` +├── cli.py # MAP: variant search after overlay; manifest counters; `pairs --loo`; `mechanisms export --slice`; `improve --mechanism-candidates/--mechanism-store/--no-mechanisms` (defaults: target calibration dir, then cwd) ├── improve/validator.py# MechanismEdit lever "mechanisms" -├── improve/evolve.py # proposer from loo misses/leaks; gate unchanged +├── improve/evolve.py # proposer from loo misses/leaks; mechanism edits decided by the per-profile clause over variant search (see §Lever) └── reports.py # mechanism id, guard, provenance on findings tests/ -├── test_variants.py, test_loo.py, test_mechanism_export.py, test_improve.py (lever) +├── test_variants.py, test_variant_search.py, test_variant_scan.py, test_mechanism_export.py, test_mechanism_seed.py, test_loo.py, test_mechanism_lever.py, test_mechanism_loop.py, test_mechanism_reports.py ``` ### Modified Files @@ -222,7 +222,9 @@ Only `seeded | reviewed` pairs seed; every trusted pair is held out once. `known ### Lever -`MechanismEdit(lever="mechanisms", action="admit"|"retract", mechanism_id, source="loo"|"export", rationale)`. Validator: id must exist in the exporter's candidate set; shape guard in `GUARD_KINDS`; no free text fields beyond rationale. Proposer: admit candidates whose leave-one-out hit recovers a currently missed holdout pair in some profile; retract admitted records that leak on holdout fixed sides. Gate unchanged; revert restores `mechanisms.jsonl` byte for byte. +`MechanismEdit(lever="mechanisms", action="admit"|"retract", mechanism_id, source="loo"|"export", rationale)`. Validator: id must exist in the exporter's candidate set; shape guard in `GUARD_KINDS`; no free text fields beyond rationale. Proposer: admit candidates whose leave-one-out hit recovers a currently missed holdout pair in some profile; retract admitted records that leak on holdout fixed sides. Application: admit appends the exporter record, retract appends a `retracted = true` tombstone (`MechanismStore.load` folds it away; the log stays append-only); `StoreSnapshot.restore` reverts byte for byte. A retraction may also name a record admitted in the scan store but no longer a candidate. + +Gate (implemented 2026-09-05, deviation from the first draft's "gate unchanged"): the rule gate and its per-profile clause are unchanged for rule edits; mechanism edits are decided by the *same* `profile_regressions` clause applied to `evaluate_mechanism_profiles` (holdout pairs scored per provenance profile by variant search with the store before and after, via `loo.score_pair_with_store`), because the overlay scorer cannot see what the store changes. Rejection reasons: `mechanism_leak:` (an admission fires on a holdout fixed side, §Error Handling), `profile_regression:`, `mechanism_no_gain` (no profile gains a correct pair and nothing is retracted); every rejection restores the store byte for byte and the reverted key is blocked in later rounds regardless of rationale. The exporter writes `mechanism-candidates.jsonl`; only the lever writes the scan-time `mechanisms.jsonl`. ## Data Models diff --git a/README.md b/README.md index f9ae7de..24d27d0 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ uv run ousast benchmark benchmarks/manifests/python-vulnerable.toml --mode quick uv run ousast pairs uv run ousast pairs --slice sast # OWASP Benchmark Java/Python + Juliet uv run ousast pairs --slice vibe-py --pointers # nightly: also harvest non-vendored pointer pairs into ~/.cache +uv run ousast mechanisms export --slice vibe-py # seed mechanism candidates from trusted pairs (offline); `ousast improve` admits them under the holdout gate +uv run ousast pairs --slice vibe-py --loo # leave-one-out recall of the corpus as a teacher uv run python -m openultrasast.pair_gate # Run the bounded self-improvement loop over the ruleset (benchmark-driven) diff --git a/docs/examples.md b/docs/examples.md index 5bb53c7..ef0d09a 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -37,6 +37,9 @@ uv run ousast pairs # local fixtures + vendored Git uv run ousast pairs --slice github --json # honesty dashboard as JSON uv run ousast pairs --slice sast # OWASP Benchmark + Juliet Youden (TPR-FPR) uv run ousast pairs --slice vibe-py --pointers # nightly: pointer pairs via the local cache (network) +uv run ousast mechanisms export --slice vibe-py # maintainer: seed mechanism candidates from trusted pairs (offline) +uv run ousast improve bench.toml --pair-catalog benchmarks/pairs/catalog.toml # the mechanisms lever admits candidates the holdout pairs prove; the scan then searches them +uv run ousast pairs --slice vibe-py --loo # leave-one-out: the corpus's own detection rate (reports/loo.json) uv run python -m openultrasast.pair_gate # CI: local pairs must all pass ``` diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index fdcbb7f..451a2f1 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -148,6 +148,19 @@ def main(argv: list[str] | None = None) -> int: "--profile-tolerance", type=float, default=0.0, help="allowed per-profile drop in pair_correct/Youden before rejecting" ) improve.add_argument("--min-holdout-pairs", type=int, default=5, help="profiles with fewer holdout pairs are reported, not gated") + improve.add_argument( + "--mechanism-candidates", + type=Path, + default=None, + help="exporter candidates the mechanisms lever may admit (default: /.openultrasast/calibration/mechanism-candidates.jsonl, then ./.openultrasast/calibration/mechanism-candidates.jsonl)", + ) + improve.add_argument( + "--mechanism-store", + type=Path, + default=None, + help="scan-time mechanisms.jsonl the lever writes (default: /.openultrasast/calibration/mechanisms.jsonl)", + ) + improve.add_argument("--no-mechanisms", action="store_true", help="disable the mechanisms lever for this run") pairs = subparsers.add_parser( "pairs", @@ -175,7 +188,10 @@ def main(argv: list[str] | None = None) -> int: export.add_argument("--slice", choices=SLICE_NAMES, default="all") export.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) export.add_argument( - "--store", type=Path, default=Path(CALIBRATION_DIR) / "mechanisms.jsonl", help="append-only mechanisms.jsonl the scan reads" + "--store", + type=Path, + default=Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl", + help="append-only candidates file the improve lever admits from (the scan reads mechanisms.jsonl next to it; never written here)", ) export.add_argument("--json", action="store_true") @@ -199,6 +215,9 @@ def main(argv: list[str] | None = None) -> int: pair_catalog=None if args.no_pair_gate else args.pair_catalog, profile_tolerance=args.profile_tolerance, min_holdout_pairs=args.min_holdout_pairs, + mechanism_candidates=args.mechanism_candidates, + mechanism_store=args.mechanism_store, + mechanisms_enabled=not args.no_mechanisms, ) if args.command == "pairs": return _pairs( @@ -784,6 +803,9 @@ def _improve( pair_catalog: Path | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, + mechanism_candidates: Path | None = None, + mechanism_store: Path | None = None, + mechanisms_enabled: bool = True, ) -> int: if not manifest_path.exists() or not manifest_path.is_file(): raise SystemExit(f"benchmark manifest is not a file: {manifest_path}") @@ -794,6 +816,19 @@ def _improve( pair_cases = select_split(load_pair_catalog(pair_catalog), "holdout") manifest = load_benchmark_manifest(manifest_path) target = resolve_benchmark_source(manifest_path, manifest) + # corpus-seeded-mechanisms Req 5: the lever runs when a candidates file exists (exporter output) and the pair gate is on. + # Candidates: an explicit path, else the target's calibration dir, else the working directory's (where `mechanisms export` + # writes by default). The store always lives where a scan of this target reads it unless overridden. + calibration = target / CALIBRATION_DIR + candidates_path: Path | None = mechanism_candidates + if candidates_path is None: + for candidate in (calibration / "mechanism-candidates.jsonl", Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl"): + if candidate.is_file(): + candidates_path = candidate + break + if not mechanisms_enabled or not pair_cases or candidates_path is None or not candidates_path.is_file(): + candidates_path = None + store_path = mechanism_store if mechanism_store is not None else calibration / "mechanisms.jsonl" policy = load_policy() # The loop writes its accepted ledger where `scan`/`benchmark` read it, so a @@ -803,6 +838,11 @@ def _improve( if dry_run: ledger_path = Path(scratch) / "rule_policy.json" journal_path = Path(scratch) / "improve_journal.json" + if candidates_path is not None: # a dry run must not mutate the scan-time store either + scratch_store = Path(scratch) / "mechanisms.jsonl" + if store_path.is_file(): + scratch_store.write_bytes(store_path.read_bytes()) + store_path = scratch_store else: ledger_path = ledger or default_ledger journal_path = journal or ledger_path.with_name("improve_journal.json") @@ -820,6 +860,8 @@ def _improve( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, + mechanism_candidates=candidates_path, + mechanism_store=store_path if candidates_path is not None else None, ) _print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run) return 0 @@ -951,7 +993,13 @@ def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, t print(f"target={target}") accepted = [o for o in outcomes if o.accepted] for outcome in outcomes: - edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-" + edits = ( + ", ".join( + [f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits] + + [f"{m.action} {m.mechanism_id}" for m in outcome.mechanism_edits] + ) + or "-" + ) print( f"round {outcome.round}: {outcome.reason} | " f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} " @@ -962,7 +1010,7 @@ def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, t print(f" profiles: regressed={outcome.profile_regressions or '-'} under_minimum={outcome.profiles_under_minimum or '-'}") print(f"rounds={len(outcomes)} accepted={len(accepted)}") if dry_run: - print("dry_run=true (no ledger written)") + print("dry_run=true (no ledger or mechanism store written)") elif accepted: print(f"ledger={ledger_path}") else: diff --git a/src/openultrasast/improve/evolve.py b/src/openultrasast/improve/evolve.py index 5fa8bea..2bc64f0 100644 --- a/src/openultrasast/improve/evolve.py +++ b/src/openultrasast/improve/evolve.py @@ -292,10 +292,13 @@ def run_round( try: for edit in edits: validator.validate(edit, ruleset_by_id, policy) - if mechanism_edits and candidates_store is not None: + if mechanism_edits and candidates_store is not None and mechanism_store is not None: + from ..semantic.mechanisms import MechanismStore + by_id = {record.id: record for record in candidates_store.load()} + admitted_by_id = {record.id: record for record in MechanismStore(mechanism_store).load()} for mechanism_edit in mechanism_edits: - validator.validate_mechanism(mechanism_edit, by_id) + validator.validate_mechanism(mechanism_edit, by_id, admitted=admitted_by_id) except StrictValidationError as exc: _record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits) return _outcome( @@ -416,6 +419,8 @@ def run_improvement( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, + mechanism_candidates: Path | None = None, + mechanism_store: Path | None = None, ) -> list[RoundOutcome]: """Run improvement rounds until convergence (no new proposals) or ``max_rounds``.""" outcomes: list[RoundOutcome] = [] @@ -432,6 +437,8 @@ def run_improvement( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, + mechanism_candidates=mechanism_candidates, + mechanism_store=mechanism_store, ) outcomes.append(outcome) if outcome.reason == "no_proposals": @@ -507,6 +514,7 @@ def _outcome( accepted=accepted, reason=reason, edits=edits, + mechanism_edits=list(mechanism_edits or []), recall_before=before.recall, recall_after=after.recall, fp_before=before.fp_rate, diff --git a/src/openultrasast/improve/journal.py b/src/openultrasast/improve/journal.py index 5910282..e866190 100644 --- a/src/openultrasast/improve/journal.py +++ b/src/openultrasast/improve/journal.py @@ -31,7 +31,9 @@ def reverted_edit_keys(rounds: list[dict[str, object]]) -> set[str]: continue for edit in _edits(entry): key = edit.get("key") - if isinstance(key, str) and not edit.get("rationale"): + # Rule edits are blocked only without a new rationale; mechanism edits are blocked outright (the loop + # always writes a rationale for them, and a reverted store edit must not be re-proposed every round). + if isinstance(key, str) and (not edit.get("rationale") or edit.get("lever") == "mechanisms"): keys.add(key) return keys diff --git a/src/openultrasast/improve/validator.py b/src/openultrasast/improve/validator.py index df09f09..807518e 100644 --- a/src/openultrasast/improve/validator.py +++ b/src/openultrasast/improve/validator.py @@ -118,13 +118,20 @@ class EvolveValidator: else: raise StrictValidationError(f"unknown edit type {type(edit).__name__}", kind="rule_change") - def validate_mechanism(self, edit: MechanismEdit, candidates: Mapping[str, object]) -> None: - """Req 5.2: only records the exporter derived from trusted pairs, with a closed guard and identifier-only shape text.""" + def validate_mechanism( + self, edit: MechanismEdit, candidates: Mapping[str, object], admitted: Mapping[str, object] | None = None + ) -> None: + """Req 5.2: only records the exporter derived from trusted pairs, with a closed guard and identifier-only shape text. + + A retraction may also name a record that is admitted in the scan store but no longer in the candidate set. + """ from ..semantic.variants import GUARD_KINDS, SOURCE_KINDS if edit.action not in MECHANISM_ACTIONS: raise StrictValidationError(f"mechanism edit action {edit.action!r} is not admit or retract", kind="mechanism_change") record = candidates.get(edit.mechanism_id) + if record is None and edit.action == "retract" and admitted is not None: + record = admitted.get(edit.mechanism_id) if record is None: raise StrictValidationError( f"unknown mechanism {edit.mechanism_id!r}: not in the exporter candidate set", kind="mechanism_change" diff --git a/src/openultrasast/semantic/loo.py b/src/openultrasast/semantic/loo.py index e0237d7..cf1ffcd 100644 --- a/src/openultrasast/semantic/loo.py +++ b/src/openultrasast/semantic/loo.py @@ -36,6 +36,7 @@ class LooOutcome: hits_fixed: int teaches: bool # the pair contributed at least one shape when others were held out leaked_by: tuple[str, ...] = () # mechanism record ids that hit the fixed side + degradations: tuple[dict[str, object], ...] = () # from variant search on either side def to_dict(self) -> dict[str, object]: return { @@ -95,9 +96,22 @@ def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = Non per_mechanism=_group(outcomes, lambda item: item.mechanisms), skipped=tuple(skipped), teaching_pairs=sum(1 for shapes in lessons.values() if shapes), + degradations=_unique_degradations(outcomes), ) +def _unique_degradations(outcomes: Sequence[LooOutcome]) -> tuple[dict[str, object], ...]: + seen: set[str] = set() + unique: list[dict[str, object]] = [] + for outcome in outcomes: + for item in outcome.degradations: + key = repr(sorted(item.items())) + if key not in seen: + seen.add(key) + unique.append(item) + return tuple(unique) + + def _hold_out(case: PairCase, targets: Sequence[PairCase], lessons: dict[str, list[Lesson]], facts: SemanticFacts) -> LooOutcome: with tempfile.TemporaryDirectory(prefix="ousast-loo-") as scratch: root = Path(scratch) @@ -127,8 +141,9 @@ def score_pair_with_store(case: PairCase, store: MechanismStore, facts: Semantic base = root if root is not None else Path(scratch) vuln_root = _materialize(base / "vuln", case.vuln_file, case.relpath) fix_root = _materialize(base / "fixed", case.fixed_file, case.relpath) - vuln_hits = search_tree(vuln_root, _targets(vuln_root), store, facts, max_mechanisms=len(records) or 1).hits - fix_hits = search_tree(fix_root, _targets(fix_root), store, facts, max_mechanisms=len(records) or 1).hits + vuln_search = search_tree(vuln_root, _targets(vuln_root), store, facts, max_mechanisms=len(records) or 1) + fix_search = search_tree(fix_root, _targets(fix_root), store, facts, max_mechanisms=len(records) or 1) + vuln_hits, fix_hits = vuln_search.hits, fix_search.hits text = case.vuln_file.read_text(errors="ignore") ranges = named_function_ranges(case.relpath, text, _parse_language(case)) or () found: list[tuple[str, tuple[str, ...]]] = [] @@ -150,6 +165,7 @@ def score_pair_with_store(case: PairCase, store: MechanismStore, facts: Semantic hits_fixed=len(fix_hits), teaches=False, leaked_by=tuple(sorted({hit.mechanism_id for hit in fix_hits})), + degradations=tuple(vuln_search.degradations) + tuple(fix_search.degradations), ) diff --git a/tests/test_loo.py b/tests/test_loo.py index a6edaf2..fe6fe76 100644 --- a/tests/test_loo.py +++ b/tests/test_loo.py @@ -105,3 +105,54 @@ def test_cli_pairs_loo_writes_the_artifact_offline(tmp_path: Path, monkeypatch: payload = json.loads(buf.getvalue()) assert payload["per_slice"]["vibe-py"]["pairs"] == 35 and "found_by" in payload["outcomes"][0] assert json.loads((tmp_path / "loo.json").read_text()) == payload + + +HELPER_SINK = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n\n\ndef ping(host):\n return run('ping ' + host)\n" +HELPER_SINK_FIX = ( + "import subprocess\n\n\ndef run(cmd):\n subprocess.run(cmd.split())\n\n\ndef ping(host):\n return run('ping ' + host)\n" +) + + +def test_hits_outside_the_labeled_function_or_off_label_never_count(tmp_path: Path) -> None: + """Round-3 mutants: a hit must sit inside the labeled function and name the labeled mechanism or sink.""" + from openultrasast.semantic.loo import evaluate_loo + + teacher = _case(tmp_path, "t", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78") + outside = _case( + tmp_path, "outside", HELPER_SINK, HELPER_SINK_FIX, function="ping", cwe="CWE-78" + ) # sink call lives in `run`, not `ping` + off_label = PairCase( + **{ + **_case(tmp_path, "offlabel", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78").__dict__, + "expected": ( + ExpectedFinding( + cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="ping", mechanism="missing_auth_guard" + ), + ), + } + ) + result = evaluate_loo([teacher, outside, off_label]) + by_name = {outcome.pair: outcome for outcome in result.outcomes} + assert by_name["outside"].hits_vuln >= 1 and not by_name["outside"].detected + assert by_name["offlabel"].hits_vuln >= 1 and not by_name["offlabel"].detected + + +def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from openultrasast.semantic.loo import evaluate_loo + + monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0") + js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n" + (tmp_path / "j-v.js").write_text(js) + (tmp_path / "j-f.js").write_text(js) + case = PairCase( + **{ + **_toy(tmp_path)[0].__dict__, + "name": "j", + "language": "javascript", + "vuln_file": tmp_path / "j-v.js", + "fixed_file": tmp_path / "j-f.js", + "relpath": "app.js", + } + ) + result = evaluate_loo([*_toy(tmp_path), case]) + assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations) diff --git a/tests/test_mechanism_lever.py b/tests/test_mechanism_lever.py index 9512f18..103ed7b 100644 --- a/tests/test_mechanism_lever.py +++ b/tests/test_mechanism_lever.py @@ -83,3 +83,30 @@ def test_apply_mechanism_edits_admits_into_the_scan_store_and_reverts_byte_for_b snapshot.restore() assert scan_store.read_bytes() == before # byte-for-byte revert of the whole round assert b.id != a.id + + +def test_retract_is_allowed_for_an_admitted_record_that_left_the_candidate_set(tmp_path: Path) -> None: + from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError + + scan = MechanismStore(tmp_path / "scan.jsonl") + stale = _record(scan, sink_name="popen") + validator = EvolveValidator() + with pytest.raises(StrictValidationError, match="unknown mechanism"): + validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=stale.id), {}, admitted={stale.id: stale}) + validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=stale.id), {}, admitted={stale.id: stale}) + + +def test_export_defaults_to_a_candidates_file_and_never_writes_the_scan_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import io + from contextlib import redirect_stdout + + from openultrasast.cli import main + + monkeypatch.chdir(tmp_path) + (tmp_path / "benchmarks").symlink_to(Path(__file__).resolve().parents[1] / "benchmarks") + buf = io.StringIO() + with redirect_stdout(buf): + assert main(["mechanisms", "export", "--slice", "vibe-py", "--json"]) == 0 + payload = __import__("json").loads(buf.getvalue()) + assert payload["store"].endswith("mechanism-candidates.jsonl") + assert not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() diff --git a/tests/test_mechanism_loop.py b/tests/test_mechanism_loop.py index 3bddaae..56ef60e 100644 --- a/tests/test_mechanism_loop.py +++ b/tests/test_mechanism_loop.py @@ -157,5 +157,162 @@ def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for mechanism_store=scan_store, scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")], ) - assert not forced.accepted and forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression") + assert not forced.accepted and (forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression")) assert scan_store.read_bytes() == before # byte-for-byte revert + + +def test_no_gain_admission_is_rejected_and_reverted_mechanism_edits_are_not_reproposed(tmp_path: Path) -> None: + from openultrasast.improve.evolve import run_round + from openultrasast.improve.journal import reverted_edit_keys + from openultrasast.improve.validator import MechanismEdit + + env = _round_env(tmp_path) + candidates = MechanismStore(tmp_path / "candidates.jsonl") + unrelated = append_from_pair( + candidates, _shape("popen"), summary="s", cwe="CWE-78", pair="teacher-c", provenance="human", tier="seeded" + ) + holdout = [_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89")] + scan_store = tmp_path / "scan" / "mechanisms.jsonl" + ledger = tmp_path / "rule_policy.json" + journal = tmp_path / "journal.json" + kwargs: dict[str, object] = { + "ledger_path": ledger, + "journal_path": journal, + "ruleset_dir": env["ruleset_dir"], + "pair_cases": holdout, + "min_holdout_pairs": 1, + "mechanism_candidates": candidates.log_path, + "mechanism_store": scan_store, + } + edit = MechanismEdit(action="admit", mechanism_id=unrelated.id, source="export", rationale="scripted: recovers nothing") + outcome = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] + assert not outcome.accepted and outcome.reason == "mechanism_no_gain" + assert not scan_store.exists() or scan_store.read_bytes() == b"" + assert edit.key() in reverted_edit_keys(__import__("json").loads(journal.read_text())) + # the same reverted edit, offered again with a rationale, is filtered before validation (novelty gate) + again = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] + assert again.reason == "no_proposals" and again.mechanism_edits == [] + + +def _holdout_catalog(tmp_path: Path) -> Path: + (tmp_path / "x-v.py").write_text(EXEC_VULN) + (tmp_path / "x-f.py").write_text(EXEC_FIX) + catalog = tmp_path / "pairs.toml" + catalog.write_text( + '[[pair]]\nname = "x"\nslice = "github"\nlanguage = "python"\nvuln = "x-v.py"\nfixed = "x-f.py"\nrelpath = "app.py"\n' + 'split = "holdout"\nreview_tier = "reviewed"\nreviewer = "t"\n\n' + '[[pair.expected]]\ncwe = "CWE-89"\nclass = "sql injection"\npath = "app.py"\nfunction = "lookup"\n' + 'sink = "execute"\nmechanism = "source_reaches_sink"\n' + ) + return catalog + + +def test_ousast_improve_admits_candidates_into_the_scan_store_by_default(tmp_path: Path) -> None: + """Req 5: the lever is reachable from the operator command; export -> candidates -> improve -> mechanisms.jsonl the scan reads.""" + from openultrasast.cli import main + + env = _round_env(tmp_path) + repo = Path(str(env["target"])) + calibration = repo / ".openultrasast" / "calibration" + candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") + exec_record = append_from_pair( + candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" + ) + catalog = _holdout_catalog(tmp_path) + manifest = tmp_path / "bench.toml" + assert ( + main( + ["improve", str(manifest), "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] + ) + == 0 + ) + admitted = MechanismStore(calibration / "mechanisms.jsonl").load() + assert [r.id for r in admitted] == [exec_record.id] + journal = json.loads((calibration / "improve_journal.json").read_text()) + assert any(e["lever"] == "mechanisms" and e["mechanism_id"] == exec_record.id for entry in journal for e in entry["edits"]) + + +def test_run_improvement_threads_mechanism_paths(tmp_path: Path) -> None: + from openultrasast.improve.evolve import run_improvement + from openultrasast.pairs import load_pair_catalog + + env = _round_env(tmp_path) + candidates = MechanismStore(tmp_path / "candidates.jsonl") + exec_record = append_from_pair( + candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" + ) + cases = load_pair_catalog(_holdout_catalog(tmp_path)) + scan_store = tmp_path / "scan" / "mechanisms.jsonl" + outcomes = run_improvement( + env["target"], + env["manifest"], + ledger_path=tmp_path / "ledger.json", + journal_path=tmp_path / "journal.json", # type: ignore[arg-type] + ruleset_dir=env["ruleset_dir"], + pair_cases=cases, + min_holdout_pairs=1, # type: ignore[arg-type] + mechanism_candidates=candidates.log_path, + mechanism_store=scan_store, + max_rounds=3, + ) + assert any(o.accepted and o.mechanism_edits for o in outcomes) + assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] + assert outcomes[-1].reason == "no_proposals" # converges once the candidate is admitted + + +def _cli_env(tmp_path: Path) -> tuple[dict[str, object], Path, Path]: + env = _round_env(tmp_path) + calibration = Path(str(env["target"])) / ".openultrasast" / "calibration" + candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") + append_from_pair(candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded") + return env, calibration, _holdout_catalog(tmp_path) + + +def test_dry_run_and_no_mechanisms_never_write_the_scan_store(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + from openultrasast.cli import main + + env, calibration, catalog = _cli_env(tmp_path) + base = [ + "improve", + str(tmp_path / "bench.toml"), + "--ruleset-dir", + str(env["ruleset_dir"]), + "--pair-catalog", + str(catalog), + "--min-holdout-pairs", + "1", + ] + assert main([*base, "--dry-run"]) == 0 + assert not (calibration / "mechanisms.jsonl").exists() # a dry run mutates nothing under the target + assert main([*base, "--no-mechanisms"]) == 0 + assert not (calibration / "mechanisms.jsonl").exists() + (calibration / "mechanism-candidates.jsonl").unlink() + capsys.readouterr() # the dry run legitimately reports its scratch admission; judge only the candidate-less round below + assert main(base) == 0 + assert not (calibration / "mechanisms.jsonl").exists() # no candidates file: the lever is inert, the round is a plain rule round + out = capsys.readouterr().out + assert "admit " not in out and "retract " not in out + + +def test_documented_default_flow_connects_export_to_improve(tmp_path: Path, monkeypatch, capsys) -> None: # type: ignore[no-untyped-def] + """README flow: `ousast mechanisms export` (cwd default) then `ousast improve --pair-catalog` admits into the target's store.""" + from openultrasast.cli import main + + env = _round_env(tmp_path) + catalog = _holdout_catalog(tmp_path) + monkeypatch.chdir(tmp_path) + # export a candidates file exactly as documented (cwd default), seeded from a tiny trusted catalog of our own + assert main(["mechanisms", "export", "--catalog", str(catalog), "--slice", "all"]) == 0 + exported = tmp_path / ".openultrasast" / "calibration" / "mechanism-candidates.jsonl" + assert exported.is_file() and not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() + # the target is a different directory (the benchmark fixture); improve must still find the cwd candidates by default + assert ( + main( + ["improve", "bench.toml", "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] + ) + == 0 + ) + target_store = Path(str(env["target"])) / ".openultrasast" / "calibration" / "mechanisms.jsonl" + assert MechanismStore(target_store).load() # admitted where the scan of that target reads + out = capsys.readouterr().out + assert "admit corpus:" in out # the accepted round names its mechanism edits