diff --git a/.kiro/specs/corpus-seeded-mechanisms/tasks.md b/.kiro/specs/corpus-seeded-mechanisms/tasks.md index 8a432d2..bde203c 100644 --- a/.kiro/specs/corpus-seeded-mechanisms/tasks.md +++ b/.kiro/specs/corpus-seeded-mechanisms/tasks.md @@ -36,29 +36,29 @@ - _Requirements: 3.4, 3.5, 6.2_ - _Depends: 2.2_ -- [ ] 3. Measurement: leave-one-out -- [ ] 3.1 Evaluate the corpus leave-one-out +- [x] 3. Measurement: leave-one-out +- [x] 3.1 Evaluate the corpus leave-one-out - For each trusted pair, seed a temporary store from the others, search both excerpts, and score detection and silence with the pair rules; report per slice, profile, mechanism, and per pair which mechanism found it; write `loo.json`. - Observable: `ousast pairs --slice vibe-py --loo --json` reports recall, silence, Youden per slice/profile/mechanism and `found_by` per pair; the run is offline. - _Requirements: 4.1, 4.2, 4.3, 4.4_ - _Boundary: LOO_ - _Depends: 2.2_ -- [ ] 3.2 Record the leave-one-out baseline +- [x] 3.2 Record the leave-one-out baseline - Roadmap gains a leave-one-out table per slice next to the pair baselines; not a gate. - Observable: roadmap table present; gates unchanged. - _Requirements: 4.4, 7.2_ - _Depends: 3.1_ -- [ ] 4. Integration: improve lever -- [ ] 4.1 Add the mechanisms lever with a closed validator +- [x] 4. Integration: improve lever +- [x] 4.1 Add the mechanisms lever with a closed validator - `MechanismEdit` admit/retract by record id from the exporter's candidate set; validator rejects free-form shapes and unknown guard kinds; journal records pair provenance and metrics. - Observable: a free-form edit is rejected; an admit edit of an exported record passes validation and appears in the journal. - _Requirements: 5.1, 5.2, 5.5_ - _Boundary: Lever_ - _Depends: 2.1_ -- [ ] 4.2 Propose admissions from leave-one-out misses and retractions from leaks under the existing gate +- [x] 4.2 Propose admissions from leave-one-out misses and retractions from leaks under the existing gate - Admit candidates that recover a currently missed holdout pair; retract admitted records that leak on holdout fixed sides; existing accept gate and per-profile clause decide; rejection reverts the store byte for byte. - Observable: a scripted candidate that recovers a holdout pair is accepted and persisted; one that leaks on a fixed side is rejected and the store is unchanged. - _Requirements: 5.3, 5.4_ @@ -89,5 +89,9 @@ - Task 2.3 (2026-09-05, RED first: 3 of 4 tests in `tests/test_variant_scan.py` failing — no variant finding, no overlay merge, variant not offered to the sandbox; the quick-mode test passed trivially and guards the invariant; then green): `[variants] enabled=true, max_mechanisms=500` config; MAP runs `_search_variants` after the overlay (standard/deep only; skipped when facts fail to load), appends variant findings, rewrites overlay.json with merged ids, and the manifest carries `variants = {mechanisms_searched, files_searched, findings, merged_into_overlay}`; in REGRESS variant findings join the candidate list after the promotions with a zero-score hotspot each, under the same safety check and cap. Verified with the fake sandbox: a variant of a project-specific sink with no inventory rule is offered. Suite 504 passed / 4 skipped, extra-free 491 / 17, statics clean, three gates PASS. - Task 3.1 (2026-09-05, RED first: 3 failing tests in `tests/test_loo.py` — `ModuleNotFoundError: semantic.loo` x2, `pairs --loo` unknown flag; the two API tests went green on the first implementation): `semantic/loo.py` `evaluate_loo(cases)` holds every scorable pair out once (known_limit and uncached pointer pairs skipped with a reason), seeds a temporary store from the *other* `seeded|reviewed` pairs (advisory/title pairs are scored as targets, never teach), materializes both sides, runs `search_tree`, and scores: detected when a hit lies inside the labeled function and names the labeled mechanism or sink; silent when the fixed side yields no hit; `found_by` lists the record id and the pairs that taught it; metrics (pairs, detected, silent, recall, silence, Youden, teaching) per slice, profile and mechanism; `to_dict()` is the `loo.json` artifact. Shape derivation is duplicated from `seed.py` (`_lessons`) pending a shared helper once group 2 is accepted. First measurement, vibe-py vendored (35 pairs, 1.0 s): 11 detected (recall 0.314), 21 silent (0.600), Youden -0.086, 16 teaching pairs; per mechanism source_reaches_sink 27 pairs 9 detected / 19 silent, missing_auth_guard 4 / 2 / 2, permissive_default 3 / 0 / 0, path_join_user_input 1 / 0 / 0. The 14 leaks are the coarse `execute/1 parameter` shape firing on Real-Vuln's false-positive trap twins: the honest over-match rate the `mechanisms` lever (group 4) retracts on. - Review round 2 (2026-09-05) APPROVED 2.1, 2.2, 2.3 (reviewer re-simulated all three REDs, killed 11 of 12 mutants, ran the export offline in a network namespace and reproduced the counts, confirmed gates byte-identical to HEAD; judged the zero-score-hotspot candidate path a closer fit to Req 3.4 than the forced path). Suggestions carried into round 3: a test that a hit on a `demote`/no-flow overlay record is not merged (the surviving mutant); `search_tree` skips a malformed store row with `variants_store_row_invalid` instead of aborting the scan; design §VariantSearch/§Modified Files synced (`SearchResult`, `regress/candidate.py` untouched); `_load_variants` uses `_int_value`; the exporter passes line and sink together; `no_labeled_function` skip reason; one shared `hotspot_from_finding` helper. +- Round-2 suggestions applied (2026-09-05): `hits_to_findings` covered by `test_hits_never_merge_into_demoted_or_flowless_records`; `search_tree` skips a malformed store row with `variants_store_row_invalid` (the store loader keeps the raw shape dict and validation happens where the shape is used); `_load_variants` uses `_int_value`; `seed.pair_lessons(case, facts)` is the one derivation shared by the exporter and leave-one-out and passes line and sink together; `no_labeled_function` skip reason; `regress.candidate.hotspot_from_finding` is the one zero-score hotspot helper (forced sev-5 inventory and variants); design §VariantSearch/§Modified Files synced. Task 3.1 CLI: `pairs --slice S --loo [--loo-out P]` writes `loo.json` (default `reports/loo.json`) and prints per-slice recall/silence/Youden/teaching; task 3.2: roadmap carries the leave-one-out table (vibe-py 11/35 detected, 21 silent, Youden -0.086, 16 teaching; local/sast/github/vfc-js 0 teaching because their tiers never seed or their labels name no sink call). +- Task 4.1 (2026-09-05, RED first: 3 failing tests in `tests/test_mechanism_lever.py`, `ImportError: MechanismEdit`; then green after the store tombstone landed): `MechanismEdit(action, mechanism_id, source, rationale)` with lever `mechanisms` (`VALID_LEVERS` extended); `EvolveValidator.validate_mechanism(edit, candidates)` accepts only admit/retract of a record present in the exporter's candidate set with `origin = corpus`, a shape whose guard and source kinds are closed and whose `sink_name`/`language`/`mechanism` are identifiers (a forged guard `if user not in ALLOWED` or a sink `os.system(cmd + ' ')` is rejected); `apply_mechanism_edits(edits, candidates, scan_store) -> StoreSnapshot` appends the exporter record for admit and a `retracted = true` tombstone for retract (the log stays append-only; `MechanismStore.load()` folds tombstones away), `StoreSnapshot.restore()` is the byte-for-byte revert. +- Task 4.2 (2026-09-05, RED first: 2 failing tests in `tests/test_mechanism_loop.py` — `evaluate_mechanism_profiles` missing, `run_round` rejected `mechanism_candidates`; then green): `evolve.propose_mechanism_edits(holdout, candidates, scan_store)` retracts admitted records that leak on a holdout fixed side and admits a candidate only when a trial store holding it alone recovers a currently missed holdout pair and leaks nowhere; `evaluate_mechanism_profiles` scores the holdout pairs with a given store per provenance profile (pairs, pair_correct = detected and silent, pass rate, Youden); `run_round(..., mechanism_candidates=, mechanism_store=, scripted_mechanism_edits=)` validates mechanism edits with the same validator, applies them after the rule gate passed, re-scores, and rejects with byte-for-byte revert on `mechanism_leak:`, `profile_regression:` (the existing per-profile clause over the before/after store metrics) or `mechanism_no_gain`; reverted keys are blocked in later rounds like rule edits; the journal records each mechanism edit with its pair provenance and `mechanisms_before`/`mechanisms_after`. This is the closed loop: leave-one-out misses propose, holdout leaks retract, the gate decides, the store is the only thing that changes. Design deviation recorded: the gate for mechanism edits is the per-profile clause computed by variant search (the change they make) rather than by the overlay scorer, which cannot see mechanisms; the rule gate and its clause are unchanged. +- Review round 3 (2026-09-05) APPROVED 3.1, 3.2, 4.1, 4.2 (reviewer re-simulated the three REDs, reproduced the roadmap table offline in a network namespace, confirmed no gate reads loo.json and gates byte-identical to HEAD, ran 16 mutants: 12 killed). Judged the variant-search-based per-profile clause for mechanism edits acceptable and required (design §Error Handling's leak row is only checkable through variant search). Carried into round 4: tests pinning the four surviving mutants (hit outside the labeled function, hit not naming the label, `mechanism_no_gain`, reverted mechanism keys re-proposed); forward `search_tree` degradations into `LooResult`; design §Lever sync (deviation, tombstone, reasons, `score_pair_with_store`); novelty gate keyed on the mechanisms lever regardless of rationale; exporter must not re-admit a standing tombstone (default `--store` to a candidates file); retraction allowed for admitted records absent from the candidate set; the `not a and b or c` precedence in the loop test. - Seeding reads `PairCase.review_tier` from `pair-corpus-honesty` task 8.1; until it lands, sast and vibe-py rows default to `advisory`/`seeded` per that design and only vibe-py seeds. - Guard classification is heuristic and closed; `none` is allowed and reported, never used to demote. diff --git a/src/openultrasast/improve/evolve.py b/src/openultrasast/improve/evolve.py index 989d7be..5fa8bea 100644 --- a/src/openultrasast/improve/evolve.py +++ b/src/openultrasast/improve/evolve.py @@ -26,8 +26,9 @@ from ..preprocess import preprocess_repository from ..rank import rank_targets from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger from ..scoring import build_score_artifact +from ..semantic.facts import SemanticFacts from .journal import append_round, load_journal, next_round_index, reverted_edit_keys -from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger +from .validator import EvolveValidator, MechanismEdit, RuleStatusEdit, StrictValidationError, apply_mechanism_edits, edits_to_ledger @dataclass(frozen=True) @@ -45,6 +46,9 @@ class RoundOutcome: matched_before: int = 0 matched_after: int = 0 profile_regressions: list[str] = field(default_factory=list) + mechanism_edits: list[MechanismEdit] = field(default_factory=list) + mechanisms_before: dict[str, dict[str, float]] = field(default_factory=dict) + mechanisms_after: dict[str, dict[str, float]] = field(default_factory=dict) profiles_under_minimum: list[str] = field(default_factory=list) per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict) per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict) @@ -79,6 +83,103 @@ def evaluate_profiles( } +def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]: + """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers).""" + from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored + from ..semantic.loo import score_pair_with_store + from ..semantic.mechanisms import MechanismStore + + cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) + store = MechanismStore(store_path) + by_profile: dict[str, list[tuple[bool, bool]]] = {} + for case in cases: + outcome = score_pair_with_store(case, store, _facts()) + by_profile.setdefault(case.provenance, []).append((outcome.detected, outcome.silent)) + result: dict[str, dict[str, float]] = {} + for profile, items in sorted(by_profile.items()): + pairs = len(items) + correct = sum(1 for detected, silent in items if detected and silent) + recall = sum(1 for detected, _ in items if detected) / pairs + silence = sum(1 for _, silent in items if silent) / pairs + result[profile] = { + "pairs": float(pairs), + "pair_correct": float(correct), + "pair_pass_rate": correct / pairs, + "youden": recall + silence - 1.0, + } + return result + + +def propose_mechanism_edits(pair_cases: Sequence[object], candidates: object, store_path: Path) -> list[MechanismEdit]: + """Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker.""" + from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored + from ..semantic.loo import score_pair_with_store + from ..semantic.mechanisms import MechanismStore, corpus_mechanisms + + cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) + if not cases: + return [] + facts = _facts() + scan = MechanismStore(store_path) + admitted = {record.id: record for record in corpus_mechanisms(scan.load())} + current = {case.name: score_pair_with_store(case, scan, facts) for case in cases} + edits: list[MechanismEdit] = [] + leakers = sorted({mechanism_id for outcome in current.values() for mechanism_id in outcome.leaked_by if mechanism_id in admitted}) + for mechanism_id in leakers: + leaked = sorted(name for name, outcome in current.items() if mechanism_id in outcome.leaked_by) + edits.append( + MechanismEdit( + action="retract", mechanism_id=mechanism_id, source="loo", rationale=f"leaks on the fixed side of {', '.join(leaked)}" + ) + ) + missed = [case for case in cases if not current[case.name].detected] + for record in corpus_mechanisms(candidates.load()): # type: ignore[attr-defined] + if record.id in admitted or not missed: + continue + with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch: + trial = MechanismStore(Path(scratch) / "trial.jsonl") + trial.append(record) + recovered = [case.name for case in missed if score_pair_with_store(case, trial, facts).detected] + leaks = [case.name for case in cases if not score_pair_with_store(case, trial, facts).silent] + if recovered and not leaks: + edits.append( + MechanismEdit( + action="admit", + mechanism_id=record.id, + source="loo", + rationale=f"recovers holdout {', '.join(recovered)}; taught by {', '.join(record.pairs)}", + ) + ) + return edits + + +def _facts() -> SemanticFacts: + from ..semantic.facts import FactLoadError, load_facts + + try: + return load_facts() + except FactLoadError: + return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) + + +def _leaking_admissions(edits: Sequence[MechanismEdit], pair_cases: Sequence[object], store_path: Path) -> list[str]: + """Admitted record ids that hit the fixed side of any holdout pair with the store as it stands after the edits.""" + from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored + from ..semantic.loo import score_pair_with_store + from ..semantic.mechanisms import MechanismStore + + admitted = {edit.mechanism_id for edit in edits if edit.action == "admit"} + if not admitted: + return [] + cases = select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS) + store = MechanismStore(store_path) + facts = _facts() + leaking: set[str] = set() + for case in cases: + leaking |= set(score_pair_with_store(case, store, facts).leaked_by) & admitted + return sorted(leaking) + + def profile_regressions( before: dict[str, dict[str, float]], after: dict[str, dict[str, float]], @@ -153,6 +254,9 @@ def run_round( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, + mechanism_candidates: Path | None = None, + mechanism_store: Path | None = None, + scripted_mechanism_edits: Sequence[MechanismEdit] | None = None, ) -> RoundOutcome: policy = policy if policy is not None else load_policy() rounds = load_journal(journal_path) @@ -167,16 +271,42 @@ def run_round( ruleset_by_id = {rule.rule_id: rule for rule in before_rules} edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked) - if not edits: + # corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit. + mechanism_edits: list[MechanismEdit] = [] + candidates_store = None + if mechanism_candidates is not None and mechanism_store is not None and pair_cases: + from ..semantic.mechanisms import MechanismStore + + candidates_store = MechanismStore(mechanism_candidates) + mechanism_edits = ( + list(scripted_mechanism_edits) + if scripted_mechanism_edits is not None + else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store) + ) + blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked + mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms] + if not edits and not mechanism_edits: return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before) validator = EvolveValidator() try: for edit in edits: validator.validate(edit, ruleset_by_id, policy) + if mechanism_edits and candidates_store is not None: + by_id = {record.id: record for record in candidates_store.load()} + for mechanism_edit in mechanism_edits: + validator.validate_mechanism(mechanism_edit, by_id) except StrictValidationError as exc: - _record(journal_path, round_index, edits, "rejected", before, before, str(exc)) - return _outcome(round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before) + _record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits) + return _outcome( + round_index, + accepted=False, + reason=f"validation_failed: {exc}", + edits=edits, + before=before, + after=before, + mechanism_edits=mechanism_edits, + ) candidate = edits_to_ledger(edits, current_ledger) try: @@ -204,15 +334,61 @@ def run_round( regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs) if regressed: accepted = False + mechanisms_before: dict[str, dict[str, float]] = {} + mechanisms_after: dict[str, dict[str, float]] = {} + mechanism_reason = "" + if accepted and mechanism_edits and candidates_store is not None and mechanism_store is not None and pair_cases: + # The lever changes what variant search finds, so its gate is the same per-profile clause over the holdout pairs + # scored with the store before and after; an admission that leaks or lowers any profile is reverted byte for byte. + mechanisms_before = evaluate_mechanism_profiles(pair_cases, mechanism_store) + snapshot = apply_mechanism_edits(mechanism_edits, candidates_store, mechanism_store) + mechanisms_after = evaluate_mechanism_profiles(pair_cases, mechanism_store) + mech_regressed, mech_small = profile_regressions( + mechanisms_before, mechanisms_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs + ) + leaking = _leaking_admissions(mechanism_edits, pair_cases, mechanism_store) + improved = any( + mechanisms_after[p]["pair_correct"] > mechanisms_before.get(p, {}).get("pair_correct", 0.0) for p in mechanisms_after + ) or any(edit.action == "retract" for edit in mechanism_edits) + if mech_regressed or leaking or not improved: + snapshot.restore() + accepted = False + regressed = regressed + mech_regressed + if leaking: + mechanism_reason = f"mechanism_leak:{leaking[0]}" # an admission that fires on a holdout fixed side (Req 5.3) + elif mech_regressed: + mechanism_reason = f"profile_regression:{mech_regressed[0]}" + else: + mechanism_reason = "mechanism_no_gain" + small = sorted(set(small) | set(mech_small)) if accepted: - write_rule_ledger(ledger_path, candidate) + if edits: + write_rule_ledger(ledger_path, candidate) reason = "accepted" + elif mechanism_reason: + reason = mechanism_reason elif regressed: reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions else: reason = "reverted" # ledger_path untouched -> byte-for-byte revert - _record(journal_path, round_index, edits, "accepted" if accepted else "reverted", before, after, reason, regressed, small) - outcome = _outcome(round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after) + _record( + journal_path, + round_index, + edits, + "accepted" if accepted else "reverted", + before, + after, + reason, + regressed, + small, + mechanism_edits=mechanism_edits, + mechanisms_before=mechanisms_before, + mechanisms_after=mechanisms_after, + candidates=candidates_store, + ) + outcome = _outcome( + round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, mechanism_edits=mechanism_edits + ) return RoundOutcome( **{ **outcome.__dict__, @@ -220,6 +396,8 @@ def run_round( "profiles_under_minimum": small, "per_profile_before": profiles_before, "per_profile_after": profiles_after, + "mechanisms_before": mechanisms_before, + "mechanisms_after": mechanisms_after, } ) @@ -314,7 +492,16 @@ def _write_signals(path: Path, signals: list[dict[str, object]]) -> None: path.write_text(json.dumps(signals, indent=2, sort_keys=True) + "\n") -def _outcome(round_index: int, *, accepted: bool, reason: str, edits: list[RuleStatusEdit], before: _Eval, after: _Eval) -> RoundOutcome: +def _outcome( + round_index: int, + *, + accepted: bool, + reason: str, + edits: list[RuleStatusEdit], + before: _Eval, + after: _Eval, + mechanism_edits: list[MechanismEdit] | None = None, +) -> RoundOutcome: return RoundOutcome( round=round_index, accepted=accepted, @@ -341,7 +528,14 @@ def _record( reason: str = "", profile_regressions: list[str] | None = None, profiles_under_minimum: list[str] | None = None, + mechanism_edits: Sequence[MechanismEdit] = (), + mechanisms_before: dict[str, dict[str, float]] | None = None, + mechanisms_after: dict[str, dict[str, float]] | None = None, + candidates: object | None = None, ) -> None: + pairs_by_id: dict[str, list[str]] = {} + if candidates is not None: + pairs_by_id = {record.id: list(record.pairs) for record in candidates.load()} # type: ignore[attr-defined] append_round( journal_path, { @@ -353,7 +547,21 @@ def _record( "edits": [ {"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale} for e in edits + ] + + [ + { + "key": m.key(), + "lever": m.lever, + "action": m.action, + "mechanism_id": m.mechanism_id, + "source": m.source, + "pairs": pairs_by_id.get(m.mechanism_id, []), + "rationale": m.rationale, + } + for m in mechanism_edits ], + "mechanisms_before": mechanisms_before or {}, + "mechanisms_after": mechanisms_after or {}, "recall_before": round(before.recall, 4), "recall_after": round(after.recall, 4), "fp_before": round(before.fp_rate, 4), diff --git a/tests/test_mechanism_loop.py b/tests/test_mechanism_loop.py new file mode 100644 index 0000000..3bddaae --- /dev/null +++ b/tests/test_mechanism_loop.py @@ -0,0 +1,161 @@ +"""corpus-seeded-mechanisms task 4.2: the improve loop admits mechanisms that recover holdout pairs and retracts leakers.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from openultrasast.benchmark import ExpectedFinding, load_benchmark_manifest +from openultrasast.pairs import PairCase +from openultrasast.ruleset import PatternRule, write_ruleset +from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair +from openultrasast.semantic.variants import Shape + +EXEC_VULN = ( + "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" + + " c.execute('SELECT * FROM t WHERE u = ' + user)\n" +) +EXEC_FIX = ( + "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" + + " c.execute('SELECT * FROM t WHERE u = ?', (user,))\n" +) +SYS_VULN = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n" +SYS_LEAKY_FIX = ( + "import os\n\n\ndef ping(host):\n log(host)\n os.system('ping ' + host)\n" # the 'fix' still calls the sink: a leaking shape +) + + +def _shape(sink: str, guard: str = "none") -> Shape: + return Shape( + language="python", + sink_name=sink, + arity=1, + source_positions=(0,), + source_kinds=("parameter",), + guard=guard, + mechanism="source_reaches_sink", + ) + + +def _pair(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase: + (tmp_path / f"{name}-v.py").write_text(vuln) + (tmp_path / f"{name}-f.py").write_text(fixed) + return PairCase( + name=name, + slice="github", + language="python", + origin="test", + vuln_file=tmp_path / f"{name}-v.py", + fixed_file=tmp_path / f"{name}-f.py", + relpath="app.py", + expected=( + ExpectedFinding( + cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink" + ), + ), + min_recall=1.0, + fix_policy="silent", + provenance=provenance, + split="holdout", + review_tier="reviewed", + reviewer="t", + ) + + +def _round_env(tmp_path: Path) -> dict[str, object]: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def f(x):\n return eval(x)\n") + rd = tmp_path / "ruleset" / "python" + write_ruleset( + rd / "rules.toml", + [PatternRule(rule_id="good-eval", title="e", languages=("python",), cwe="CWE-95", tags=("injection",), pattern=r"\beval\s*\(")], + ) + manifest_path = tmp_path / "bench.toml" + manifest_path.write_text( + 'name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' + '[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n' + 'line = 2\nrule_id = "good-eval"\nsink = "eval"\nevidence = "eval executes input"\n' + ) + return {"target": repo, "manifest": load_benchmark_manifest(manifest_path), "ruleset_dir": tmp_path / "ruleset"} + + +def test_proposer_admits_recovering_records_and_retracts_leakers(tmp_path: Path) -> None: + from openultrasast.improve.evolve import evaluate_mechanism_profiles, propose_mechanism_edits + + candidates = MechanismStore(tmp_path / "candidates.jsonl") + exec_record = append_from_pair( + candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" + ) + sys_record = append_from_pair( + candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" + ) + holdout = [ + _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), + _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78", provenance="agent"), + ] + scan_store = tmp_path / "scan" / "mechanisms.jsonl" + edits = propose_mechanism_edits(holdout, candidates, scan_store) + assert [(e.action, e.mechanism_id) for e in edits] == [("admit", exec_record.id)] # sys leaks on y's fixed side: never proposed + assert "x" in edits[0].rationale + # once a leaking record is admitted (by a human or an earlier round), the proposer asks to retract it + from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits + + apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=sys_record.id)], candidates, scan_store) + edits = propose_mechanism_edits(holdout, candidates, scan_store) + assert ("retract", sys_record.id) in [(e.action, e.mechanism_id) for e in edits] + profiles = evaluate_mechanism_profiles(holdout, scan_store) + assert profiles["agent"]["pairs"] == 1.0 and profiles["agent"]["pair_correct"] == 0.0 # y: detected but leaked + + +def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for_byte(tmp_path: Path) -> None: + from openultrasast.improve.evolve import run_round + from openultrasast.improve.validator import MechanismEdit + + env = _round_env(tmp_path) + candidates = MechanismStore(tmp_path / "candidates.jsonl") + exec_record = append_from_pair( + candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" + ) + sys_record = append_from_pair( + candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" + ) + holdout = [ + _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), + _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78"), + ] + scan_store = tmp_path / "scan" / "mechanisms.jsonl" + ledger = tmp_path / "rule_policy.json" + journal = tmp_path / "journal.json" + outcome = run_round( + env["target"], + env["manifest"], + ledger_path=ledger, + journal_path=journal, + ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] + pair_cases=holdout, + min_holdout_pairs=1, + mechanism_candidates=candidates.log_path, + mechanism_store=scan_store, + ) + assert outcome.accepted and outcome.reason == "accepted" + assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] + entry = json.loads(journal.read_text())[0] + mech_edits = [e for e in entry["edits"] if e["lever"] == "mechanisms"] + assert mech_edits and mech_edits[0]["mechanism_id"] == exec_record.id and mech_edits[0]["pairs"] == ["teacher-a"] + assert entry["mechanisms_before"]["human"]["pair_correct"] == 0.0 and entry["mechanisms_after"]["human"]["pair_correct"] == 1.0 + before = scan_store.read_bytes() + forced = run_round( + env["target"], + env["manifest"], + ledger_path=ledger, + journal_path=journal, + ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] + pair_cases=holdout, + min_holdout_pairs=1, + mechanism_candidates=candidates.log_path, + mechanism_store=scan_store, + scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")], + ) + assert not forced.accepted and forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression") + assert scan_store.read_bytes() == before # byte-for-byte revert