diff --git a/.kiro/specs/model-grounded-detection/spec.json b/.kiro/specs/model-grounded-detection/spec.json index b106720..83b4d93 100644 --- a/.kiro/specs/model-grounded-detection/spec.json +++ b/.kiro/specs/model-grounded-detection/spec.json @@ -1,9 +1,9 @@ { "feature_name": "model-grounded-detection", "created_at": "2026-09-08T00:00:00Z", - "updated_at": "2026-09-08T04:00:00Z", + "updated_at": "2026-09-08T10:00:00Z", "language": "en", - "phase": "tasks", + "phase": "implementation", "approvals": { "requirements": { "generated": true, @@ -15,14 +15,14 @@ }, "tasks": { "generated": true, - "approved": false + "approved": true } }, - "ready_for_implementation": false, + "ready_for_implementation": true, "supersedes": [ "learning-harness", "constrained-detector" ], "depends_on": [], - "approval_note": "2026-09-08 corrective brief drafted at the maintainer's direction ('can you produce a corrective spec that supersedes the project and sets a course'). Written after the learning-harness evolve rounds produced zero decisions distinguishable from noise, and after researching Clearwing (/home/mc/Source/clearwing, fork norandom/deepwing), Google Big Sleep, and the DARPA AIxCC 2025 finals. Not approved; requirements not generated. The central recommendation is to adopt Clearwing as the execution substrate rather than rebuild, and to contribute the web/logic detection path (taxonomy + IR candidate enumeration + obligation checker + a differential web oracle) that Clearwing's crash-oracle model does not reach. ; 2026-09-08 renamed execution-grounded-detection -> model-grounded-detection after the maintainer's correction: execution is not possible for all SAST targets and the bugs we own do not crash, so the arbiter is a program model built deterministically from code (IR candidate enumeration, binding-chain dataflow, obligation/guard-dominance) that checks LLM claims, producing a static evidence ladder (suspicion -> model_corroborated -> model_entailed); execution is a deferred tier for memory-safety-in-C only, adopted from Clearwing when a class needs it. ; 2026-09-08 requirements generated at the maintainer's direction ('write the requirements, and include to throw away broken parts. include the reduction of tests to what makes sense. I think there is an outgrowth problem here'): subtraction is Req 1-3 (remove the noise architecture, reduce 832 tests to surviving behavior, audit 99 modules for orphans) and precedes adoption of the Joern CPG arbiter (Req 4-8), the corpus as calibration set (Req 9), and the deferred Clearwing execution tier (Req 10). ; 2026-09-08 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); design generated: subtraction is phase 1 (gate-protected, ~5000 lines removed before any Joern), a Joern spike on the injection family is phase 2 and can end the design before the integration cost is paid. ; 2026-09-08 design approved and tasks generated ('approve the design and generate the tasks. add infos so that Opus can follow the tasks without running into complexity escalations'): tasks carry an agent-guidance preamble with the eighteen-defect lesson, the gate-baseline safety net, the zero-dep capability-detection pattern, and a full coupling map of every learning/mechanism touchpoint so the subtraction is not crash-driven discovery; group 1 is pure gate-protected subtraction, group 2 is a Joern go/no-go that gates groups 3-6." + "approval_note": "2026-09-08 corrective brief drafted at the maintainer's direction ('can you produce a corrective spec that supersedes the project and sets a course'). Written after the learning-harness evolve rounds produced zero decisions distinguishable from noise, and after researching Clearwing (/home/mc/Source/clearwing, fork norandom/deepwing), Google Big Sleep, and the DARPA AIxCC 2025 finals. Not approved; requirements not generated. The central recommendation is to adopt Clearwing as the execution substrate rather than rebuild, and to contribute the web/logic detection path (taxonomy + IR candidate enumeration + obligation checker + a differential web oracle) that Clearwing's crash-oracle model does not reach. ; 2026-09-08 renamed execution-grounded-detection -> model-grounded-detection after the maintainer's correction: execution is not possible for all SAST targets and the bugs we own do not crash, so the arbiter is a program model built deterministically from code (IR candidate enumeration, binding-chain dataflow, obligation/guard-dominance) that checks LLM claims, producing a static evidence ladder (suspicion -> model_corroborated -> model_entailed); execution is a deferred tier for memory-safety-in-C only, adopted from Clearwing when a class needs it. ; 2026-09-08 requirements generated at the maintainer's direction ('write the requirements, and include to throw away broken parts. include the reduction of tests to what makes sense. I think there is an outgrowth problem here'): subtraction is Req 1-3 (remove the noise architecture, reduce 832 tests to surviving behavior, audit 99 modules for orphans) and precedes adoption of the Joern CPG arbiter (Req 4-8), the corpus as calibration set (Req 9), and the deferred Clearwing execution tier (Req 10). ; 2026-09-08 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); design generated: subtraction is phase 1 (gate-protected, ~5000 lines removed before any Joern), a Joern spike on the injection family is phase 2 and can end the design before the integration cost is paid. ; 2026-09-08 design approved and tasks generated ('approve the design and generate the tasks. add infos so that Opus can follow the tasks without running into complexity escalations'): tasks carry an agent-guidance preamble with the eighteen-defect lesson, the gate-baseline safety net, the zero-dep capability-detection pattern, and a full coupling map of every learning/mechanism touchpoint so the subtraction is not crash-driven discovery; group 1 is pure gate-protected subtraction, group 2 is a Joern go/no-go that gates groups 3-6. ; 2026-09-08 tasks approved by the maintainer in session ('approve the tasks and move to 1.) as a whole'); implementing group 1 (subtraction) as a whole." } diff --git a/.kiro/specs/model-grounded-detection/tasks.md b/.kiro/specs/model-grounded-detection/tasks.md index 8719d3b..ea54e92 100644 --- a/.kiro/specs/model-grounded-detection/tasks.md +++ b/.kiro/specs/model-grounded-detection/tasks.md @@ -58,7 +58,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat ## Group 1 — Subtract and audit (gate-protected, no Joern yet) -- [ ] 1.1 Port the security vocabularies out before deleting their modules +- [x] 1.1 Port the security vocabularies out before deleting their modules - Create `src/openultrasast/model/specs.py` and move into it, as data, the knowledge the deleted modules carry: `_GUARD_PATTERNS` and the guard-kind vocabulary from `semantic/variants.py`; the sink token tables (from `benchmarks/pairs/catalog_gen.py::_SINK_TOKENS` and `semantic/facts.py` SinkFacts) as the seed @@ -70,7 +70,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat seed, and that the guard vocabulary is preserved verbatim. `model/specs.py` is read-only data — no LLM and no future optimiser writes it, and a test asserts the module exposes no writer (Req 7.4, the verifier boundary). - _Requirements: 1.2, 7.1, 7.2, 7.3, 7.4_ -- [ ] 1.2 Re-home the three retained modules +- [x] 1.2 Re-home the three retained modules - Move `learning/families.py` → `model/taxonomy.py`, `learning/candidates.py` → `model/candidates.py`, `learning/endpoint.py` → `model/endpoint.py`. Update their imports and every importer. `candidates.py` keeps its behaviour exactly (it is the suspicion-band feeder, Req 8.4) — its tests move with it and stay @@ -80,7 +80,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 4.4, 7.1_ - _Depends: 1.1_ -- [ ] 1.3 Detach the retained paths from the removed code +- [x] 1.3 Detach the retained paths from the removed code - `pairs.py`: remove the hunter-scoring path (`_evaluate_hunter_pair`, `_hunter_outcome`, `_family_metrics`, `PairEvalResult.per_family`, `build_pair_signals` split arg, the `k_runs`/`hunter` plumbing) and its `learning.*` imports; keep `_evaluate_overlay_pair`/`_evaluate_inventory_pair` and the `context_files` @@ -95,7 +95,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 1.3, 1.4, 11.2_ - _Depends: 1.2_ -- [ ] 1.4 Remove the noise architecture and the mechanism/evolve loop +- [x] 1.4 Remove the noise architecture and the mechanism/evolve loop - Delete `learning/{rounds,acceptance,proposer,journal,canaries,verifiers,difficulty,scoring,classify, detectors,split,publish,slices,predicates,judgments}.py` (everything under `learning/` except the three modules re-homed in 1.2, which are already gone from the directory), then remove the now-empty `learning/` @@ -110,7 +110,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 1.1, 1.2, 1.2b, 1.3, 1.4_ - _Depends: 1.3_ -- [ ] 1.5 Reduce the test suite to surviving behaviour +- [x] 1.5 Reduce the test suite to surviving behaviour - For every deleted module, delete the test files that existed only to pin it (`test_learning_*`, `test_mechanism_*`, `test_obligation_*` only where it tested the lever not the checker, `test_pair_*` hunter-scoring cases, etc.) in this same change. For every surviving test, confirm it maps to a surviving @@ -122,7 +122,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5_ - _Depends: 1.4_ -- [ ] 1.6 The module audit manifest +- [x] 1.6 The module audit manifest - Add `model/audit.py` that classifies every `src/**/*.py` as `load_bearing` (reachable from a gate, the scan CLI, or a surviving requirement), `standalone_capability` (its own CLI/users — e.g. `mcp.py`, `skills.py`), or `orphaned` (no importer, no entry, no requirement). Remove the orphans. Give @@ -250,3 +250,66 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat mistake this whole spec was written to stop. - `(P)` parallelism is deliberately absent: the subtraction is sequential by dependency, and the Joern spike gates everything after it. Do not fan out. + +## Implementation Notes — group 1 (2026-09-08) + +Landed as one change. Gate baseline captured before the first edit and re-checked after every step; the three +gates are **byte-identical to `2026-09-06-gate-baseline.txt`** at the end, as Req 1.4 and 11.2 require. + +**What the subtraction cost and bought.** 99 source modules -> 81; 832 tests -> 646 (642 passing, 4 skipped); +the whole `learning/` package (16 modules, ~4,100 lines) plus six `semantic/` modules removed. mypy clean over +82 files, ruff clean, core `dependencies = []` untouched. + +**Sequence that made it safe.** Vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion +third (1.4). Every import break during 1.4 was therefore a detachment miss in the importer, fixed there — no +module was ever restored to satisfy one. Two such misses appeared, both handled as a class rather than an +instance: `semantic/obligations/{operations,shapes}.py` both imported `trailing_name` from the deleted +`variants.py`, so the helper was re-homed to `semantic/ir.py` (its natural place — it is a pure helper over +`CallSite.name`) and both importers updated in the same edit. + +**Deviations from the design, each recorded with its reason.** + +1. *`improve/evolve.py` and `improve/validator.py` were stripped, not deleted.* The design's Removed list named + them whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop + that `ousast improve` still runs from benchmark evidence. That loop has no LLM in it, predates the noise + architecture, and is not what this feature supersedes; deleting the files would have taken a retained + capability with them. `test_subtraction_complete` now asserts the lever's absence **by symbol**, so it + cannot creep back. +2. *The `learning` CLI group became `model candidates`.* Deleting the group outright would have orphaned + `model/candidates.py` — the retained suspicion-band enumerator — by removing its only entry point, and + Req 3 would then have flagged it for deletion. The offline ceiling report survives as `ousast model + candidates`. +3. *`thinking` moved from `[learning]` to `[models]`.* It is a model setting, and `LearningConfig` was removed; + its coverage moved to `test_model_endpoint` rather than being dropped. +4. *Three helpers were re-homed rather than lost:* `Region` (promoted from `_Region`, since the detector module + that owned the type is gone), `labeled_family` and `labeled_spans` (from the deleted `rounds`/`scoring`). + `labeled_family` no longer falls back to the LLM classifier — an unlabeled pair is honestly `unknown`, the + same discipline the evidence ladder applies to a claim the model cannot arbitrate. + +**A measured gap, named rather than papered over (Req 9.2).** Task 1.1's observable asked that every family +have a `TaintSpec` or `DominanceSpec` seed. It cannot today: the retained sink facts have **no sink whose CWE +routes to `path`, `output_encoding`, `untrusted_destination` or `prototype`** — the same hole the entailment +ceiling diagnosed ("closed literal sink table: misses per-family sinks"). Rather than weaken the test, it now +asserts the seeded set *and* the gap set exactly, so neither can change silently, and task 4.2 must update it +when it authors those sinks. + +**Test reduction (Req 2), by cluster.** 34 files deleted whole: 21 `test_learning_*` (rounds, acceptance, +scoring, classify, detectors, proposer, journal, canaries, verifiers, publish, split, cli, scan, pairs_path, +cutoff, reliability, aggregation, classifier_report, pairs_evidence, round evidence x2, thinking), 4 +`test_mechanism_*`, `test_variants`, `test_variant_search`, `test_loo`, `test_obligation_export`, +`test_mechanism_reports`, `test_fixture_difficulty`. Partial trims where a file mixed retained and removed +behaviour: `test_semantic_pipeline` (3 mechanism tests of 12), `test_degradation_matrix` (3 rows for removed +capabilities), `test_pair_corpus` (the hunter scorer), `test_obligation_check` (the `known_fix` assertion, +which depended on the deleted store — the id contract, suspicion rung and tags it also covered are kept), +`test_index` (the bakeoff test). Two files re-homed with their subject: `test_learning_families` -> +`test_model_taxonomy`, `test_learning_endpoint` -> `test_model_endpoint`. + +**The audit (Req 3) found exactly one genuine orphan:** `vectorstore.py`, a store bakeoff harness with no +production caller and no CLI entry, alive only through its own test — removed with it. The five named +subsystems each carry a decision: `mcp`, `skills`, `fusion`, `harness_ext`, `hunter_harness` are all +`standalone_capability` (own entry point or protocol), joined by `stage_processors` and `slot_contract`, which +the zero-dependency guard in `test_gate` pins. Manifest committed at +`benchmarks/measurements/2026-09-08-module-audit.json`; a test asserts the tree still matches it. + +Group 2 (the Joern go/no-go) has not started. `semantic/engines.py` already exposes `joern_available`, which +the `CpgBackend` capability probe in task 2.1 should build on rather than duplicate. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json new file mode 100644 index 0000000..7cefec7 --- /dev/null +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -0,0 +1,888 @@ +{ + "counts": { + "load_bearing": 74, + "standalone_capability": 7 + }, + "generated": "2026-09-08", + "kept_with_a_lever_stripped": { + "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." + }, + "modules": 81, + "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", + "orphaned": [], + "removed_in_this_pass": { + "learning/ (16 modules)": "the noise architecture: rounds, acceptance, proposer, journal, canaries, verifiers, difficulty, scoring, classify, detectors, split, publish. Taxonomy, candidates and endpoint were re-homed to model/ first.", + "semantic/loo.py": "leave-one-out scoring of the mechanism store", + "semantic/mechanisms.py": "the mechanism store and the shape search that fed the optimisation loop", + "semantic/prove_budget.py": "embedding-ordered promotion budget, driven by the mechanism store", + "semantic/seed.py": "the mechanism exporter", + "semantic/variant_search.py": "variant search over the mechanism store", + "semantic/variants.py": "the shape-derivation optimisation surface; its guard vocabulary was ported verbatim to model/specs.py first, and trailing_name re-homed to semantic/ir.py", + "vectorstore.py": "a vector-store bakeoff harness with no production caller and no CLI entry, kept alive only by its own test -- the one genuine orphan the audit found" + }, + "rows": [ + { + "classification": "load_bearing", + "importers": [ + "cli", + "gate", + "improve.evolve", + "map_gate", + "mcp", + "pairs" + ], + "module": "benchmark", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "calibration", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "mcp" + ], + "module": "cli", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity.hints", + "complexity.ledger", + "complexity.map", + "map_gate", + "regress.candidate", + "semantic.prove_filter", + "tool_hunter" + ], + "module": "complexity", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.map" + ], + "module": "complexity.hints", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity", + "complexity.map", + "regress.candidate" + ], + "module": "complexity.ledger", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.hints", + "complexity.ledger", + "map_gate", + "regress.candidate", + "semantic.prove_filter", + "tool_hunter" + ], + "module": "complexity.map", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.hints", + "complexity.map" + ], + "module": "complexity.signals", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "harness", + "hunter_tools", + "model.endpoint", + "pairs", + "regress.candidate", + "regress.recipes", + "run" + ], + "module": "config", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "benchmark", + "calibration", + "cli", + "complexity.ledger", + "complexity.map", + "complexity.signals", + "fusion", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "rank", + "regress.candidate", + "reports", + "scoring.project_score", + "semantic.obligations.check", + "semantic.overlay", + "semantic.prove_filter", + "stage_processors", + "tool_hunter", + "verification", + "verify_judge" + ], + "module": "findings", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "fusion", + "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "slot_contract" + ], + "module": "harness", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli", + "fusion", + "hunter_harness", + "stage_processors", + "verify_judge" + ], + "module": "harness_ext", + "reason": "HarnessX extension surface; loaded by name when the optional harness is present" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_harness" + ], + "module": "hunter", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "hunter_harness", + "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp" + }, + { + "classification": "load_bearing", + "importers": [ + "tool_hunter" + ], + "module": "hunter_tools", + "reason": "imported by tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve.evolve" + ], + "module": "improve", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve" + ], + "module": "improve.evolve", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve", + "improve.evolve" + ], + "module": "improve.journal", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve", + "improve.evolve" + ], + "module": "improve.validator", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "skills" + ], + "module": "index", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "map_gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "gate", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "preprocess", + "stage_processors" + ], + "module": "mapping", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "mcp", + "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "model.specs" + ], + "module": "model", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.audit", + "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "model.candidates", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.endpoint", + "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.specs", + "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "model.specs" + ], + "module": "model.taxonomy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "pair_gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve", + "improve.evolve", + "model.candidates", + "pair_gate" + ], + "module": "pairs", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "findings", + "hunter", + "hunter_harness", + "improve.evolve", + "improve.validator", + "scoring.project_score", + "semantic.overlay", + "stage_processors" + ], + "module": "policy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "policy" + ], + "module": "policy.verycode", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity.hints", + "complexity.map", + "complexity.signals", + "findings", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "index", + "map_gate", + "mapping", + "model.candidates", + "pairs", + "rank", + "regress.candidate", + "semantic.overlay", + "skills", + "stage_processors" + ], + "module": "preprocess", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "provenance", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint", + "tool_hunter" + ], + "module": "provider", + "reason": "imported by model.endpoint, tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint", + "tool_hunter" + ], + "module": "provider.openrouter", + "reason": "imported by model.endpoint, tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "calibration", + "cli", + "findings", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "stage_processors" + ], + "module": "rank", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "harness", + "reports", + "semantic.obligations.intent" + ], + "module": "redaction", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "regress.candidate" + ], + "module": "regress", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress" + ], + "module": "regress.candidate", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.recipes", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.safety", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.verdict", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "reports", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "findings", + "hunter", + "hunter_harness", + "improve.evolve", + "improve.validator", + "pairs", + "stage_processors" + ], + "module": "ruleset", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve.validator", + "ruleset" + ], + "module": "ruleset.store", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "reports" + ], + "module": "run", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "regress.candidate", + "regress.recipes", + "regress.safety", + "regress.verdict" + ], + "module": "sandbox", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "sandbox" + ], + "module": "sandbox.probe", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress.candidate", + "regress.recipes", + "regress.safety", + "sandbox" + ], + "module": "sandbox.runner", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve.evolve", + "stage_processors" + ], + "module": "scoring", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "scoring" + ], + "module": "scoring.project_score", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic.cst", + "semantic.engines", + "semantic.functions", + "semantic.ir", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.prove_filter", + "semantic.taint" + ], + "module": "semantic", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.ir" + ], + "module": "semantic.cst", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.overlay" + ], + "module": "semantic.engines", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.cst", + "semantic.engines" + ], + "module": "semantic.extra", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.taint" + ], + "module": "semantic.facts", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "hunter_tools", + "pairs", + "semantic.obligations.shapes", + "semantic.overlay" + ], + "module": "semantic.functions", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "pairs", + "semantic.cst", + "semantic.functions", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.taint" + ], + "module": "semantic.ir", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic.obligations.check", + "semantic.obligations.dominance", + "semantic.obligations.intent", + "semantic.obligations.operations", + "semantic.obligations.policy", + "semantic.obligations.shapes", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "semantic.obligations", + "semantic.obligations.intent" + ], + "module": "semantic.obligations.check", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "pairs", + "semantic.obligations.check" + ], + "module": "semantic.obligations.dominance", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.specs", + "pairs", + "semantic.obligations", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.policy", + "semantic.obligations.shapes", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations.facts", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "semantic.obligations.intent", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.candidates", + "semantic.obligations.check", + "semantic.obligations.dominance", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations.operations", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "semantic.obligations.check" + ], + "module": "semantic.obligations.policy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.obligations", + "semantic.obligations.operations" + ], + "module": "semantic.obligations.shapes", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.obligations.check" + ], + "module": "semantic.obligations.siblings", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.prove_filter" + ], + "module": "semantic.overlay", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic" + ], + "module": "semantic.prove_filter", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.overlay" + ], + "module": "semantic.taint", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "hunter", + "hunter_harness" + ], + "module": "skills", + "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path" + }, + { + "classification": "standalone_capability", + "importers": [ + "stage_processors" + ], + "module": "slot_contract", + "reason": "the slot/contract protocol stage_processors implements" + }, + { + "classification": "standalone_capability", + "importers": [], + "module": "stage_processors", + "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "stages", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint" + ], + "module": "tool_hunter", + "reason": "imported by model.endpoint" + }, + { + "classification": "load_bearing", + "importers": [ + "calibration", + "cli", + "fusion", + "reports", + "verify_judge" + ], + "module": "verification", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "verify_judge", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + } + ], + "spec": "model-grounded-detection Req 3 (the outgrowth audit)", + "standalone": { + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "slot_contract": "the slot/contract protocol stage_processors implements", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + } +} diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 6d3d65b..e3113a4 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse import json import tempfile -from collections.abc import Callable, Sequence +from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path from time import perf_counter @@ -31,7 +31,6 @@ from .calibration import ( ) from .complexity import map as complexity_map from .complexity.ledger import persist_verdicts -from .complexity.map import Hotspot from .config import ObligationsConfig, load_config, load_dotenv from .findings import StaticFinding, quick_scan_findings, write_findings from .fusion import FusionDecision, fuse_findings_dispatch @@ -42,16 +41,13 @@ from .hunter import run_hunter_pool, write_hunter_trajectories from .hunter_harness import HxScanOrchestrator from .improve import RoundOutcome, run_improvement from .index import build_code_chunks -from .learning.families import FamilyTaxonomy from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints from .pair_gate import print_pair_metrics from .pairs import ( DEFAULT_CATALOG, SLICE_NAMES, - PairCase, evaluate_catalog, load_pair_catalog, - make_hunter_scan, result_payload, select_profile, select_slice, @@ -61,24 +57,19 @@ from .pairs import ( from .policy import assert_rules_resolve, load_policy from .preprocess import FileTarget, preprocess_repository, write_preprocess_artifact from .provenance import fingerprint -from .provider.openrouter import OpenRouterEmbeddingClient, OpenRouterError from .rank import rank_obligations, rank_targets, write_rankings from .regress import TRIGGERABLE, CandidateVerdict, run_regression, write_verdicts -from .regress.candidate import hotspot_from_finding from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from .ruleset import DEFAULT_RULESET_DIR, load_ruleset from .run import ScanRun, create_scan_run from .sandbox import resolve_sandbox_probe, resolve_sandbox_runner from .scoring import build_score_artifact from .semantic import ( - MechanismStore, OverlayRecord, adjudicate, - append_mechanism, filter_promoted_hotspots, finding_from_coverage, load_facts, - order_promotions, write_overlay, ) from .semantic.facts import FactLoadError, SemanticFacts @@ -101,7 +92,7 @@ class ScanOutcome: if TYPE_CHECKING: - from .learning.proposer import Proposer + pass def main(argv: list[str] | None = None) -> int: @@ -154,22 +145,6 @@ def main(argv: list[str] | None = None) -> int: "--profile-tolerance", type=float, default=0.0, help="allowed per-profile drop in pair_correct/Youden before rejecting" ) improve.add_argument("--min-holdout-pairs", type=int, default=5, help="profiles with fewer holdout pairs are reported, not gated") - improve.add_argument( - "--mechanism-candidates", - type=Path, - default=None, - help=( - "exporter candidates the mechanisms lever may admit (default: /.openultrasast/calibration/" - "mechanism-candidates.jsonl, then ./.openultrasast/calibration/mechanism-candidates.jsonl)" - ), - ) - improve.add_argument( - "--mechanism-store", - type=Path, - default=None, - help="scan-time mechanisms.jsonl the lever writes (default: /.openultrasast/calibration/mechanisms.jsonl)", - ) - improve.add_argument("--no-mechanisms", action="store_true", help="disable the mechanisms lever for this run") pairs = subparsers.add_parser( "pairs", @@ -181,72 +156,18 @@ def main(argv: list[str] | None = None) -> int: "--profile", choices=("all", "human", "agent", "mixed", "synthetic"), default="all", help="filter pairs by provenance profile" ) pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split") - pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)") - pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)") pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)") - pairs.add_argument( - "--loo", action="store_true", help="leave-one-out: score each pair against a store seeded from the other trusted pairs" - ) - pairs.add_argument("--loo-out", type=Path, default=None, help="where to write loo.json (default: reports/loo.json)") pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON") - pairs.add_argument( - "--k-runs", type=int, default=None, help="runs per pair on the hunter path; never fewer than three, and three by default" - ) subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration") - mechanisms = subparsers.add_parser("mechanisms", help="maintainer: mechanism memory seeded from trusted pairs") - mechanisms_sub = mechanisms.add_subparsers(dest="mechanisms_command", required=True) - export = mechanisms_sub.add_parser("export", help="derive mechanism records from seeded/reviewed pairs (offline)") - export.add_argument("--slice", choices=SLICE_NAMES, default="all") - export.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - export.add_argument( - "--store", - type=Path, - default=Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl", - help="append-only candidates file the improve lever admits from (the scan reads mechanisms.jsonl next to it; never written here)", - ) - export.add_argument("--json", action="store_true") - - learning = subparsers.add_parser("learning", help="maintainer: the classified detector set and its rounds") - learning_sub = learning.add_subparsers(dest="learning_command", required=True) - for name, help_text in ( - ("classify", "classify every labeled pair and report the classifier against itself"), - ("score", "score a catalog by family, with the denominator every number was computed over"), - ("baseline", "round zero: clone one detector into every family and measure each against itself"), - ("candidates", "offline: what a candidate-driven detector could find, per family and per slice"), - ("round", "one evolve round: propose one change for one family and let the evidence decide"), - ("publish", "regenerate every published number from the artifacts that produced it"), - ): - command = learning_sub.add_parser(name, help=help_text) - command.add_argument("--out", type=Path, default=Path(".openultrasast/learning"), help="where artifacts are read and written") - if name != "publish": - command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - command.add_argument("--slice", choices=SLICE_NAMES, default="all") - command.add_argument("--json", action="store_true") - if name == "baseline": - command.add_argument( - "--resume", - action="store_true", - help="keep the pairs a previous run of this baseline already measured and paid for", - ) - if name == "candidates": - command.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites") - if name in {"baseline", "round", "score"}: - command.add_argument("--k-runs", type=int, default=None, help="runs per pair; never fewer than three") - command.add_argument("--model", default=None, help="detector model; artifacts are keyed by it") - command.add_argument("--config", type=Path, default=Path("openultrasast.toml")) - if name == "round": - command.add_argument("--family", required=True) - command.add_argument("--cost-cap-usd", type=float, default=None) - command.add_argument( - "--proposals", - type=Path, - default=None, - help="a JSON list of proposals to try in order; without it the round asks the meta-agent", - ) - if name == "publish": - command.add_argument("--measurements", type=Path, default=Path("benchmarks/measurements")) - command.add_argument("--roadmap", type=Path, default=Path(".kiro/steering/roadmap.md")) + model_cmd = subparsers.add_parser("model", help="maintainer: the model layer that arbitrates detector claims") + model_sub = model_cmd.add_subparsers(dest="model_command", required=True) + candidates = model_sub.add_parser("candidates", help="offline: what the candidate enumerator can reach, per family and per slice") + candidates.add_argument("--out", type=Path, default=Path(".openultrasast/model"), help="where artifacts are written") + candidates.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + candidates.add_argument("--slice", choices=SLICE_NAMES, default="all") + candidates.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites") + candidates.add_argument("--json", action="store_true") args = parser.parse_args(argv) if args.command == "scan": @@ -268,9 +189,6 @@ def main(argv: list[str] | None = None) -> int: pair_catalog=None if args.no_pair_gate else args.pair_catalog, profile_tolerance=args.profile_tolerance, min_holdout_pairs=args.min_holdout_pairs, - mechanism_candidates=args.mechanism_candidates, - mechanism_store=args.mechanism_store, - mechanisms_enabled=not args.no_mechanisms, ) if args.command == "pairs": return _pairs( @@ -279,17 +197,10 @@ def main(argv: list[str] | None = None) -> int: json_out=args.json, profile=args.profile, split=args.split, - hunter=args.hunter, - hunter_model=args.hunter_model, pointers=args.pointers, - loo=args.loo, - loo_out=args.loo_out, - k_runs=args.k_runs, ) - if args.command == "mechanisms": - return _mechanisms_export(args.catalog, args.slice, args.store, json_out=args.json) - if args.command == "learning": - return _learning(args) + if args.command == "model": + return _model_candidates(args) if args.command == "mcp": from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional @@ -478,12 +389,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut built_map = None overlay_records: list[OverlayRecord] = [] wrote_overlay = False - variants_payload: dict[str, object] | None = None - variant_findings: list[StaticFinding] = [] - mechanisms_cited: dict[str, dict[str, object]] = {} obligations_payload: dict[str, object] | None = None - learning_payload: dict[str, object] | None = None - learning_cited: dict[str, dict[str, object]] = {} obligations_cited: dict[str, dict[str, object]] = {} if Stage.MAP in plan.requested: built_map = runtime.run_stage( @@ -517,19 +423,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if extra: findings = findings + extra write_findings(findings, findings_path) - if config.variants.enabled: - variant_findings, overlay_records, variants_payload, mechanisms_cited = _search_variants( - root=run.target, - targets=targets, - overlay_records=overlay_records, - facts=facts_or_error, - max_mechanisms=config.variants.max_mechanisms, - runtime=runtime, - ) - if variant_findings: - findings = findings + variant_findings - write_findings(findings, findings_path) - write_overlay(overlay_records, overlay_path) if config.obligations.enabled: obligation_findings, obligations_payload, obligations_cited = _check_obligations( root=run.target, @@ -543,18 +436,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if obligation_findings: findings = rank_obligations(findings + obligation_findings) write_findings(findings, findings_path) - if config.learning.enabled: - family_findings, learning_payload, learning_cited = _run_family_detectors( - root=run.target, - hotspots=built_map.hotspots if built_map is not None else (), - cases=None, - settings=config.learning, - config=config, - runtime=runtime, - ) - if family_findings: - findings = findings + family_findings - write_findings(findings, findings_path) plan = record_completed(plan, Stage.MAP) if not hunter_model: runtime.state["degradations"].append(skip_as_degradation(Stage.MAP, "hunter_model_unavailable")) @@ -588,33 +469,11 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut sandbox = resolve_sandbox_runner() hotspots = built_map.hotspots if built_map is not None else () hotspots = filter_promoted_hotspots(hotspots, overlay_records, findings) - store = MechanismStore(run.target / CALIBRATION_DIR / "mechanisms.jsonl") - embed_client = None - embed_model = config.embeddings.model - if embed_model: - try: - embed_client = OpenRouterEmbeddingClient.from_env() - except OpenRouterError: - embed_client = None - hotspots, budget_degradation = order_promotions( - hotspots, - overlay_records, - findings, - store=store, - client=embed_client, - model=embed_model, - ) - if budget_degradation: - runtime.state["degradations"].append({"stage": "prove_budget", "reason": budget_degradation}) promoted = [ finding for finding in findings if finding.finding_id in {record.proposal_id for record in overlay_records if record.disposition == "promote"} ] - # Variant findings are candidates after the promotions (Req 3.4): same safety check, same cap, lowest rank. - variant_ids = {finding.finding_id for finding in variant_findings} - promoted = promoted + [finding for finding in findings if finding.finding_id in variant_ids] - hotspots = hotspots + tuple(_hotspot_from_variant(finding) for finding in findings if finding.finding_id in variant_ids) cwe_by_rule_id = {rule.rule_id: rule.cwe for rule in ruleset} rule_cwe_by_finding = {finding.finding_id: cwe_by_rule_id.get(finding.finding_id.split(":", 1)[0], "") for finding in promoted} languages_by_path = {target.path: target.language for target in targets} @@ -640,7 +499,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if proven_ids and any(finding.finding_id.startswith("obligation:") for finding in findings): findings = rank_obligations(findings, proven_ids=proven_ids) write_findings(findings, findings_path) - _append_proven_mechanisms(store, records, overlay_records, findings, run.target) verdict_records = records wrote_verdicts = True plan = record_completed(plan, Stage.REGRESS) @@ -655,10 +513,8 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut complexity_map=built_map, verdicts=verdict_records if wrote_verdicts else None, overlay=overlay_records if wrote_overlay else None, - mechanisms=mechanisms_cited or None, obligations=obligations_cited or None, obligations_summary=obligations_payload or None, - learning=learning_cited or None, ), ) runtime.run_stage( @@ -668,9 +524,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut verifications, sarif_path, overlay=overlay_records if wrote_overlay else None, - mechanisms=mechanisms_cited or None, obligations=obligations_cited or None, - learning=learning_cited or None, ), ) runtime.run_stage( @@ -699,9 +553,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut complexity=complexity_payload, worth_fixing=worth_fixing_payload, provenance=provenance.to_dict(), - variants=variants_payload, obligations=obligations_payload, - learning=learning_payload, ), ) runtime.finish(status="succeeded") @@ -716,35 +568,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut ) -def _append_proven_mechanisms( - store: MechanismStore, - verdicts: tuple[CandidateVerdict, ...], - overlay_records: list[OverlayRecord], - findings: list[StaticFinding], - target: Path, -) -> None: - overlay_by_id = {record.proposal_id: record for record in overlay_records} - finding_by_id = {finding.finding_id: finding for finding in findings} - for verdict in verdicts: - if verdict.verdict != TRIGGERABLE: - continue - for finding_id in verdict.inventory_finding_ids: - record = overlay_by_id.get(finding_id) - if record is None or record.disposition != "promote": - continue - finding = finding_by_id.get(finding_id) - append_mechanism( - store, - summary=record.reason, - cwe=record.cwe, - language=record.language or verdict.language, - tags=finding.tags if finding is not None else (), - what_made_it_exploitable=record.reason, - source_finding_id=finding_id, - source_repo=str(target), - ) - - def _attach_tool_hunter( *, root: Path, @@ -900,9 +723,6 @@ def _improve( pair_catalog: Path | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, - mechanisms_enabled: bool = True, ) -> int: if not manifest_path.exists() or not manifest_path.is_file(): raise SystemExit(f"benchmark manifest is not a file: {manifest_path}") @@ -913,19 +733,6 @@ def _improve( pair_cases = select_split(load_pair_catalog(pair_catalog), "holdout") manifest = load_benchmark_manifest(manifest_path) target = resolve_benchmark_source(manifest_path, manifest) - # corpus-seeded-mechanisms Req 5: the lever runs when a candidates file exists (exporter output) and the pair gate is on. - # Candidates: an explicit path, else the target's calibration dir, else the working directory's (where `mechanisms export` - # writes by default). The store always lives where a scan of this target reads it unless overridden. - calibration = target / CALIBRATION_DIR - candidates_path: Path | None = mechanism_candidates - if candidates_path is None: - for candidate in (calibration / "mechanism-candidates.jsonl", Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl"): - if candidate.is_file(): - candidates_path = candidate - break - if not mechanisms_enabled or not pair_cases or candidates_path is None or not candidates_path.is_file(): - candidates_path = None - store_path = mechanism_store if mechanism_store is not None else calibration / "mechanisms.jsonl" policy = load_policy() # The loop writes its accepted ledger where `scan`/`benchmark` read it, so a @@ -935,11 +742,6 @@ def _improve( if dry_run: ledger_path = Path(scratch) / "rule_policy.json" journal_path = Path(scratch) / "improve_journal.json" - if candidates_path is not None: # a dry run must not mutate the scan-time store either - scratch_store = Path(scratch) / "mechanisms.jsonl" - if store_path.is_file(): - scratch_store.write_bytes(store_path.read_bytes()) - store_path = scratch_store else: ledger_path = ledger or default_ledger journal_path = journal or ledger_path.with_name("improve_journal.json") @@ -957,8 +759,6 @@ def _improve( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, - mechanism_candidates=candidates_path, - mechanism_store=store_path if candidates_path is not None else None, ) _print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run) return 0 @@ -971,30 +771,13 @@ def _pairs( json_out: bool, profile: str = "all", split: str = "all", - hunter: bool = False, - hunter_model: str | None = None, pointers: bool = False, - loo: bool = False, - loo_out: Path | None = None, - k_runs: int | None = None, ) -> int: if not catalog.exists() or not catalog.is_file(): raise SystemExit(f"pair catalog is not a file: {catalog}") cases = select_split(select_profile(select_slice(load_pair_catalog(catalog), slice_name), profile), split) - if loo: - return _pairs_loo(cases, loo_out or Path("reports") / "loo.json", json_out=json_out) - scan = None - if hunter: - config_path = Path("openultrasast.toml") - config = load_config(config_path if config_path.exists() else None) - model = hunter_model or config.models.hunter or "" - client = tool_hunter.resolve_hunter_client() - if model and client is not None: - from .learning.families import load_families - - scan = make_hunter_scan(client, model, taxonomy=load_families()) try: - result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None, k_runs=k_runs) + result = evaluate_catalog(cases, pointers=True if pointers else None) except ValueError as exc: raise SystemExit(f"--k-runs: {exc}") from exc if json_out: @@ -1005,50 +788,6 @@ def _pairs( return 0 -def _search_variants( - *, - root: Path, - targets: list[FileTarget], - overlay_records: list[OverlayRecord], - facts: SemanticFacts | FactLoadError, - max_mechanisms: int, - runtime: HarnessRuntime, -) -> tuple[list[StaticFinding], list[OverlayRecord], dict[str, object], dict[str, dict[str, object]]]: - """Third MAP proposer (corpus-seeded-mechanisms Req 3): shapes from the mechanism store, merged with overlay flows. - - Also returns the mechanisms the findings cite (summary, guard, pairs, cwe) for the reports (Req 6.1). - """ - from .semantic.mechanisms import corpus_mechanisms - from .semantic.variant_search import hits_to_findings, search_tree - - store = MechanismStore(root / CALIBRATION_DIR / "mechanisms.jsonl") - empty: dict[str, object] = {"mechanisms_searched": 0, "files_searched": 0, "findings": 0, "merged_into_overlay": 0} - if isinstance(facts, FactLoadError): - return [], overlay_records, empty, {} - result = runtime.run_stage("variants", lambda: search_tree(root, targets, store, facts, max_mechanisms=max_mechanisms)) - for degradation in result.degradations: - runtime.state["degradations"].append(degradation) - records_by_id = {record.id: record for record in corpus_mechanisms(store.load())} - summaries = {record.id: (record.summary, record.pairs, record.cwe) for record in records_by_id.values()} - findings, records = hits_to_findings(result.hits, overlay_records, summaries) - merged = sum(1 for before, after in zip(overlay_records, records, strict=True) if before.mechanism_id is None and after.mechanism_id) - payload: dict[str, object] = { - "mechanisms_searched": result.mechanisms_searched, - "files_searched": result.files_searched, - "findings": len(findings), - "merged_into_overlay": merged, - } - cited_ids = {tag.split(":", 1)[1] for finding in findings for tag in finding.tags if tag.startswith("mechanism:")} | { - str(record.mechanism_id) for record in records if record.mechanism_id - } - cited: dict[str, dict[str, object]] = { - mechanism_id: {"summary": record.summary, "guard": record.guard, "pairs": list(record.pairs), "cwe": record.cwe} - for mechanism_id, record in records_by_id.items() - if mechanism_id in cited_ids - } - return findings, records, payload, cited - - def _check_obligations( *, root: Path, @@ -1065,7 +804,7 @@ def _check_obligations( enter the sandbox candidate set. """ from .semantic.ir import parse_file - from .semantic.obligations import load_obligation_facts, obligation_mechanisms + from .semantic.obligations import load_obligation_facts from .semantic.obligations.check import check_obligations, findings_to_static from .semantic.obligations.dominance import OrderDominance from .semantic.obligations.policy import PolicyError, load_declared_policy @@ -1089,7 +828,6 @@ def _check_obligations( policy = load_declared_policy(policy_path) except PolicyError as exc: runtime.state["degradations"].append({"stage": "obligations", "reason": "policy_invalid", "detail": str(exc)[:200]}) - store = MechanismStore(root / CALIBRATION_DIR / "mechanisms.jsonl") result = runtime.run_stage( "obligations", lambda: check_obligations( @@ -1100,7 +838,7 @@ def _check_obligations( policy=policy, paths=(), dominance=OrderDominance(texts=texts), - store_shapes=obligation_mechanisms(store.load()), + store_shapes=(), # known-fix hints came from the deleted mechanism store min_siblings=settings.min_siblings, ), ) @@ -1138,292 +876,6 @@ def _check_obligations( return statics, payload, cited -def _run_family_detectors( - *, - root: Path, - hotspots: Sequence[object], - cases: object, - settings: object, - config: object, - runtime: HarnessRuntime, -) -> tuple[list[StaticFinding], dict[str, object], dict[str, dict[str, object]]]: - """Fifth MAP proposer (learning-harness Req 6.3, 7.1): one detector per admitted family, plus the generalist. - - Every claim passes through its family's verifier and carries what produced it. Nothing rises above - suspicion here: the verifier writes a tag, and the evidence ladder is another boundary's to move. - """ - del cases - from .learning.classify import classify_region - from .learning.detectors import DetectorConfigError, Region, load_family_configs, run_region_detectors - from .learning.endpoint import resolve_chat_endpoint, resolve_models - from .learning.families import FamiliesError, load_families - from .learning.verifiers import apply_verification, verifier_for - - empty: dict[str, dict[str, object]] = {} - configs_dir = root / str(getattr(settings, "configs_dir", ".openultrasast/learning/configs")) - try: - taxonomy = load_families(Path(p) if (p := getattr(settings, "families_path", None)) else None) - configs = load_family_configs(configs_dir, taxonomy) - except (FamiliesError, DetectorConfigError) as exc: - runtime.state["degradations"].append({"stage": "learning", "reason": "learning_configs_unavailable", "detail": str(exc)[:200]}) - return [], {}, empty - if not configs: - runtime.state["degradations"].append( - {"stage": "learning", "reason": "learning_configs_unavailable", "detail": f"no family configurations under {configs_dir}"} - ) - return [], {}, empty - resolved = resolve_chat_endpoint(config) # type: ignore[arg-type] - if resolved is None: - runtime.state["degradations"].append({"stage": "learning", "reason": "learning_endpoint_unavailable"}) - return [], {}, empty - client, endpoint = resolved - model = resolve_models(config)[0] # type: ignore[arg-type] - findings: list[StaticFinding] = [] - per_family: dict[str, int] = {} - per_tier: dict[str, int] = {} - abstained = 0 - seen: set[str] = set() - for hotspot in hotspots: - path = str(getattr(hotspot, "path", "")) - function = getattr(hotspot, "function_name", None) - key = f"{path}::{function}" - if not path or key in seen: - continue - seen.add(key) - answer = classify_region(root, path, function=function, taxonomy=taxonomy, client=None) - per_tier[answer.tier] = per_tier.get(answer.tier, 0) + 1 - abstained += 0 if answer.families else 1 - region = Region(path=path, function=str(function) if function else None, families=answer.families) - - def _run(region: Region = region) -> list[StaticFinding]: - return run_region_detectors(root, region, configs, client=client, model=model) - - produced = runtime.run_stage("learning", _run) - for finding in produced: - family_id = next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), "unknown") - verification = verifier_for(taxonomy.by_id(family_id)).verify(finding, root=root, language="python") - findings.append(apply_verification(finding, verification)) - per_family[family_id] = per_family.get(family_id, 0) + 1 - payload: dict[str, object] = { - "taxonomy_version": taxonomy.version, - "families": per_family, - # What the classifier did on *this* scan. Scoring it needs labels a maintainer stands behind, which a scan - # of someone else's repository does not have, so the block reports the routing rather than an accuracy. - "classifier": { - "regions": len(seen), - "per_tier": dict(sorted(per_tier.items())), - "abstained": abstained, - # Req 2.6/3.7: say out loud that no family declares a parent at this taxonomy version, so the - # partial credit for a parent-child relation never applied to any number here. - "hierarchical_credit": taxonomy.hierarchical, - }, - "round": None, - "endpoint": endpoint.provider, - "regions": len(seen), - } - cited: dict[str, dict[str, object]] = { - finding.finding_id: { - "family": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), ""), - "detector": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("detector:")), ""), - "verifier": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("verifier:")), ""), - } - for finding in findings - } - return findings, payload, cited - - -def _learning(args: argparse.Namespace) -> int: - """The maintainer's entry points into the classified detector set. Everything degrades with a reason.""" - from .learning.classify import measure_classifier, write_review_queue - from .learning.families import load_families - from .learning.publish import publish - - out: Path = args.out - taxonomy = load_families() - if args.learning_command == "publish": - report = publish(learning_dir=out, measurements_dir=args.measurements, roadmap=args.roadmap) - print(f"learning publish: {len(report.families)} families over {len(report.models)} model(s) -> {args.roadmap}") - for artifact in report.artifacts: - print(f" artifact {artifact}") - return 0 - cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice)) - if args.learning_command == "candidates": - # No model is called anywhere in this command. The ceiling is a property of the corpus and the enumerator, - # so it can end this feature before a cent is spent on it (constrained-detector task 1.3). - from .learning.candidates import ceiling - - coverage = ceiling(cases, taxonomy=taxonomy, generator=args.generator) - out.mkdir(parents=True, exist_ok=True) - payload = coverage.to_dict() - (out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - for family, block in sorted(coverage.per_family.items()): - print( - f"learning candidates {family}: {block['with_candidate']}/{block['pairs']} " - f"= {_as_float(block['ceiling']):.1%} ceiling, sites {block['sites']}" - ) - for slice_name, block in sorted(coverage.per_slice.items()): - print(f" slice {slice_name}: {_as_float(block['ceiling']):.1%} over {block['pairs']} pairs") - return 0 - if args.learning_command == "classify": - measured = measure_classifier(cases, taxonomy, client=None) - out.mkdir(parents=True, exist_ok=True) - payload = measured.to_dict() - (out / "classifier.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - written = write_review_queue(out / "review-queue.jsonl", measured.queue) - if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - print(f"learning classify: classified {measured.classified}, unknown {measured.unknown}, queued {written}") - return 0 - return _learning_run(args, cases, taxonomy) - - -def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: FamilyTaxonomy) -> int: - """score, baseline and round: the three that need a detector, and degrade with a reason without one.""" - from .learning.detectors import load_family_configs, write_default_configs - from .learning.endpoint import resolve_chat_endpoint, resolve_models - from .learning.journal import Archive, LearningJournal - from .learning.rounds import DEFAULT_SLICES, MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round - from .learning.scoring import aggregate - from .tool_hunter import _SYSTEM_PROMPT - - out: Path = args.out - config = load_config(args.config if args.config.exists() else None) - k_runs = args.k_runs or max(config.learning.k_runs, MIN_K_RUNS) - model = args.model or resolve_models(config)[0] - configs_dir = out / "configs" - if not configs_dir.exists(): - write_default_configs(configs_dir, taxonomy, prompt=_SYSTEM_PROMPT, version="0") # type: ignore[arg-type] - resolved = resolve_chat_endpoint(config) - if resolved is None: - print("learning: learning_endpoint_unavailable; scoring what can be scored without a detector") - client = resolved[0] if resolved else None - # The client meters itself; a run that never reads the meter publishes a cost of zero and can never trip its - # own cap (Req 8.3, 9.8). - meter = getattr(client, "cost_usd", None) - spent: Callable[[], float] | None = meter if callable(meter) else None - configs = load_family_configs(configs_dir, taxonomy) # type: ignore[arg-type] - - def factory(family_config: object) -> Callable[[Path], list[StaticFinding]]: - from .learning.detectors import Region, run_family_detector - - def scan(root: Path) -> list[StaticFinding]: - if client is None: - return [] - region = Region(path=_first_source(root), function=None) - return run_family_detector(root, region, family_config, client=client, model=model) # type: ignore[arg-type] - - return scan - - if args.learning_command == "score": - from .learning.rounds import score_case - - scores = [ - score_case(case, factory(configs[_label(case, taxonomy)]), taxonomy=taxonomy, runs=k_runs, family=_label(case, taxonomy)) - for case in cases - if _label(case, taxonomy) in configs - ] - cutoff = config.learning.cutoff_date or "" - families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, cutoff=cutoff).items()} - if args.json: - print(json.dumps({"families": families}, indent=2, sort_keys=True)) - else: - for name, block in sorted(families.items()): - raw = block.get("post_cutoff") - recent: dict[str, object] = raw if isinstance(raw, dict) else {} - tail = ( - f" post-cutoff({cutoff})={recent.get('scorable', 0)} undated={block['undated']}" - if cutoff - else " post-cutoff=none configured" - ) - print( - f"learning score {name}: scorable={block['scorable']} recall={block['recall']:.3f} youden={block['youden']:.3f}{tail}" - ) - return 0 - if args.learning_command == "baseline": - report = run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=configs_dir, - scan_factory=factory, - model=model, - out_dir=out, - k_runs=k_runs, - cutoff=config.learning.cutoff_date or "", - # `--slice` already chose the rows; without this the baseline's own default filters them out again and - # a run over the memory-safety slice reports nothing at all (Req 8.4). - slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES, - spent_usd=spent, - thinking=config.learning.thinking, - resume=args.resume, - ) - print(f"learning baseline cost: ${report.cost_usd:.4f}") - print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}") - return 0 - journal = LearningJournal(out / "journal.jsonl") - record = run_learning_round( - cases, - family=args.family, - taxonomy=taxonomy, - configs_dir=configs_dir, - proposer=_proposer(args, configs_dir, model, config.harnessx.provider), - scan_factory=factory, - model=model, - journal=journal, - archive=Archive(out / "archive.jsonl"), - floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES, config.learning.thinking), - out_dir=out, - cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, - minibatch=config.learning.minibatch, - k_runs=k_runs, - spent_usd=spent, - # Every other family the corpus actually carries, so a change that reaches beyond its own directory is - # measured rather than assumed impossible (Req 9.5). - sweep_families=tuple(sorted({_label(case, taxonomy) for case in cases} & set(configs) - {args.family})), - ) - print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason}) cost ${record.cost_usd:.4f}") - return 0 - - -def _as_float(value: object) -> float: - return float(value) if isinstance(value, int | float) else 0.0 - - -def _label(case: PairCase, taxonomy: object) -> str: - """The family that routes this pair: the maintainer's label when there is one, else the classifier's answer. - - Reading only the declared label would send every unlabeled row to the generalist, whose findings earn no credit - against any real family, so a scoreboard built that way reports structural zeros (Req 2.1, 8.2).""" - from .learning.rounds import _family_of - - return _family_of(case, taxonomy) # type: ignore[arg-type] - - -def _proposer(args: argparse.Namespace, configs_dir: Path, model: str, provider: str = "anthropic") -> Proposer: - """`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent.""" - from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer - - path = getattr(args, "proposals", None) - if path is None: - return HarnessXProposer(configs_dir=configs_dir, model=model, provider=provider) - rows = json.loads(Path(path).read_text(encoding="utf-8")) - return ScriptedProposer( - [ - Proposal( - hypothesis=str(row["hypothesis"]), - lever=row["lever"], - change=dict(row["change"]), - predicted_affected=tuple(row.get("predicted_affected", ())), - predicted_at_risk=tuple(row.get("predicted_at_risk", ())), - ) - for row in rows - ] - ) - - def _first_source(root: Path) -> str: from .preprocess import preprocess_repository @@ -1431,60 +883,12 @@ def _first_source(root: Path) -> str: return targets[0].path if targets else "." -def _hotspot_from_variant(finding: StaticFinding) -> Hotspot: - return hotspot_from_finding(finding, rationale="variant of a known mechanism (suspicion; sandbox may raise)") - - -def _pairs_loo(cases: Sequence[PairCase], out: Path, *, json_out: bool) -> int: - """Req 4: the corpus's own detection rate; written as an artifact, never a gate.""" - from .semantic.loo import evaluate_loo - - result = evaluate_loo(select_vendored(cases)) - payload = result.to_dict() - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - if json_out: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - for slice_name, metrics in result.per_slice.items(): - print( - f"loo {slice_name}: pairs={metrics['pairs']} detected={metrics['detected']} silent={metrics['silent']} " - f"youden={metrics['youden']:+.3f} teaching={metrics['teaching']}" - ) - print(f"artifact={out}") - return 0 - - -def _mechanisms_export(catalog: Path, slice_name: str, store_path: Path, *, json_out: bool) -> int: - from .semantic.mechanisms import MechanismStore - from .semantic.seed import export_mechanisms - - if not catalog.is_file(): - raise SystemExit(f"pair catalog is not a file: {catalog}") - cases = select_vendored(select_slice(load_pair_catalog(catalog), slice_name)) - report = export_mechanisms(cases, MechanismStore(store_path)) - payload = {"slice": slice_name, "store": str(store_path), "pairs": len(cases), **report.to_dict()} - if json_out: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - print(f"mechanisms export slice={slice_name} pairs={len(cases)} seeded={report.seeded} records={report.records} -> {store_path}") - for pair, reason in report.skipped: - print(f" skip {pair}: {reason}") - return 0 - - def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None: print(f"manifest={manifest_path}") print(f"target={target}") accepted = [o for o in outcomes if o.accepted] for outcome in outcomes: - edits = ( - ", ".join( - [f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits] - + [f"{m.action} {m.mechanism_id}" for m in outcome.mechanism_edits] - ) - or "-" - ) + edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-" print( f"round {outcome.round}: {outcome.reason} | " f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} " @@ -1506,6 +910,36 @@ def _load_scan_findings(run_dir: Path) -> list[StaticFinding]: return load_findings(run_dir / "findings.json") +def _model_candidates(args: argparse.Namespace) -> int: + """What the candidate enumerator can reach, per family and per slice. No model is called anywhere here. + + The ceiling is a property of the corpus and the enumerator, so it can end a design before a cent is spent + on it — which is what it did for the pattern-ruleset generator (12.1% against the IR's 96.6%). + """ + from .model.candidates import ceiling + from .model.taxonomy import load_families + + cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice)) + coverage = ceiling(cases, taxonomy=load_families(), generator=args.generator) + out: Path = args.out + out.mkdir(parents=True, exist_ok=True) + payload = coverage.to_dict() + (out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + for family, block in sorted(coverage.per_family.items()): + ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0 + print( + f"model candidates {family}: {block['with_candidate']}/{block['pairs']} " + f"= {ceiling_value:.1%} ceiling, sites {block['sites']}" + ) + for slice_name, block in sorted(coverage.per_slice.items()): + ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0 + print(f" slice {slice_name}: {ceiling_value:.1%} over {block['pairs']} pairs") + return 0 + + def _index(path: Path, config_path: Path, chunk_lines: int) -> int: if not path.exists() or not path.is_dir(): raise SystemExit(f"index path is not a directory: {path}") diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index b47e00d..69d2253 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -36,6 +36,11 @@ class ModelConfig: judge: str | None = None # second, independent judgement before anything is published as proven chat_base_url: str | None = None # chat endpoint; independent of the embedding endpoint chat_api_key_env: str | None = None # environment variable holding that endpoint's key + # Whether the chat model reasons before answering. Off by default because that is what the committed + # baseline was measured under: the provider silently ignores `temperature` while thinking, so thinking was + # disabled to make `temperature: 0` mean something. It did not — the measured run-to-run disagreement came + # from the agentic path, not the decoder — which is why the model layer arbitrates instead of averaging. + thinking: bool = False @dataclass(frozen=True) @@ -138,26 +143,6 @@ class VariantsConfig: max_mechanisms: int = 500 -@dataclass(frozen=True) -class LearningConfig: - # learning-harness: the classified detector set and its rounds. - enabled: bool = False # off until round zero has written the family configurations this reads - families_path: str | None = None # taxonomy file; the shipped ruleset when unset - configs_dir: str = ".openultrasast/learning/configs" - k_runs: int = 5 # runs per pair; a single run misses most real changes - round_cost_cap_usd: float = 10.0 - minibatch: int = 8 # train pairs in a round's first stage - detector_model: str | None = None - judge_model: str | None = None - cutoff_date: str | None = None # rows fixed after this date are reported as a separate slice - # Whether the detector model reasons before answering. Off by default because that is what round zero was - # measured under, not because it is known to be better: the provider silently ignores `temperature` while - # thinking, and thinking was disabled to make `temperature: 0` mean something. It did not — the measured - # disagreement between runs comes from the agentic path, not the decoder — so which mode wins is a question - # for a second baseline rather than for a default (Req 8.3). - thinking: bool = False - - @dataclass(frozen=True) class ObligationsConfig: # authorization-obligations: the obligation checker in MAP (standard/deep only). @@ -183,7 +168,6 @@ class ResolvedConfig: regress: RegressConfig = RegressConfig() variants: VariantsConfig = VariantsConfig() obligations: ObligationsConfig = ObligationsConfig() - learning: LearningConfig = LearningConfig() runs_dir: str = ".openultrasast/runs" @@ -210,7 +194,6 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig: regress=_load_regress(data.get("regress", {})), variants=_load_variants(data.get("variants", {})), obligations=_load_obligations(data.get("obligations", {})), - learning=_load_learning(data.get("learning", {})), runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"), ) @@ -238,6 +221,7 @@ def _load_models(value: object) -> ModelConfig: judge=_string(data.get("judge")), chat_base_url=_string(data.get("chat_base_url")), chat_api_key_env=_string(data.get("chat_api_key_env")), + thinking=bool(data.get("thinking", False)), ) @@ -385,22 +369,6 @@ def _load_variants(value: object) -> VariantsConfig: ) -def _load_learning(value: object) -> LearningConfig: - data = _section(value) - return LearningConfig( - enabled=bool(data.get("enabled", LearningConfig.enabled)), - families_path=_string(data.get("families_path")), - configs_dir=_string(data.get("configs_dir")) or LearningConfig.configs_dir, - k_runs=_int_value(data.get("k_runs"), LearningConfig.k_runs), - round_cost_cap_usd=_float_value(data.get("round_cost_cap_usd"), LearningConfig.round_cost_cap_usd), - minibatch=_int_value(data.get("minibatch"), LearningConfig.minibatch), - detector_model=_string(data.get("detector_model")), - judge_model=_string(data.get("judge_model")), - cutoff_date=_string(data.get("cutoff_date")), - thinking=bool(data.get("thinking", False)), - ) - - def _load_obligations(value: object) -> ObligationsConfig: data = _section(value) return ObligationsConfig( diff --git a/src/openultrasast/improve/evolve.py b/src/openultrasast/improve/evolve.py index bca4470..e64a11c 100644 --- a/src/openultrasast/improve/evolve.py +++ b/src/openultrasast/improve/evolve.py @@ -26,9 +26,8 @@ from ..preprocess import preprocess_repository from ..rank import rank_targets from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger from ..scoring import build_score_artifact -from ..semantic.facts import SemanticFacts from .journal import append_round, load_journal, next_round_index, reverted_edit_keys -from .validator import EvolveValidator, MechanismEdit, RuleStatusEdit, StrictValidationError, apply_mechanism_edits, edits_to_ledger +from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger @dataclass(frozen=True) @@ -46,9 +45,6 @@ class RoundOutcome: matched_before: int = 0 matched_after: int = 0 profile_regressions: list[str] = field(default_factory=list) - mechanism_edits: list[MechanismEdit] = field(default_factory=list) - mechanisms_before: dict[str, dict[str, float]] = field(default_factory=dict) - mechanisms_after: dict[str, dict[str, float]] = field(default_factory=dict) profiles_under_minimum: list[str] = field(default_factory=list) per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict) per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict) @@ -89,121 +85,6 @@ def evaluate_profiles( } -def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]: - """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers). - - Split-blind for the same reason as `evaluate_profiles`: the store is the thing the teacher rule governs, and this - function only reads it (learning-harness Req 5.1).""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore - - cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) - store = MechanismStore(store_path) - by_profile: dict[str, list[tuple[bool, bool]]] = {} - for case in cases: - outcome = score_pair_with_store(case, store, _facts()) - by_profile.setdefault(case.provenance, []).append((outcome.detected, outcome.silent)) - result: dict[str, dict[str, float]] = {} - for profile, items in sorted(by_profile.items()): - pairs = len(items) - correct = sum(1 for detected, silent in items if detected and silent) - recall = sum(1 for detected, _ in items if detected) / pairs - silence = sum(1 for _, silent in items if silent) / pairs - result[profile] = { - "pairs": float(pairs), - "pair_correct": float(correct), - "pair_pass_rate": correct / pairs, - "youden": recall + silence - 1.0, - } - return result - - -def propose_mechanism_edits( - pair_cases: Sequence[object], - candidates: object, - store_path: Path, - degradations: list[dict[str, object]] | None = None, -) -> list[MechanismEdit]: - """Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker.""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore, corpus_mechanisms - - cases_all = [case for case in pair_cases if isinstance(case, PairCase)] - cases = list(select_tier(select_vendored(cases_all), GATING_TIERS)) - if not cases: - return [] - facts = _facts() - scan = MechanismStore(store_path) - admitted = {record.id: record for record in corpus_mechanisms(scan.load())} - current = {case.name: score_pair_with_store(case, scan, facts) for case in cases} - edits: list[MechanismEdit] = [] - leakers = sorted({mechanism_id for outcome in current.values() for mechanism_id in outcome.leaked_by if mechanism_id in admitted}) - for mechanism_id in leakers: - leaked = sorted(name for name, outcome in current.items() if mechanism_id in outcome.leaked_by) - edits.append( - MechanismEdit( - action="retract", mechanism_id=mechanism_id, source="loo", rationale=f"leaks on the fixed side of {', '.join(leaked)}" - ) - ) - missed = [case for case in cases if not current[case.name].detected] - from ..learning.split import refuse_if_holdout - - for record in corpus_mechanisms(candidates.load()): # type: ignore[attr-defined] - if record.id in admitted or not missed: - continue - refusal = refuse_if_holdout(record.pairs, cases_all) - if refusal is not None: - # learning-harness Req 5.2: a shape a holdout pair taught can never recover that pair, and the run - # says which pairs it was rather than showing a shorter list with no reason. - if degradations is not None: - degradations.append(refusal.degradation()) - continue - with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch: - trial = MechanismStore(Path(scratch) / "trial.jsonl") - trial.append(record) - recovered = [case.name for case in missed if score_pair_with_store(case, trial, facts).detected] - leaks = [case.name for case in cases if not score_pair_with_store(case, trial, facts).silent] - if recovered and not leaks: - edits.append( - MechanismEdit( - action="admit", - mechanism_id=record.id, - source="loo", - rationale=f"recovers holdout {', '.join(recovered)}; taught by {', '.join(record.pairs)}", - ) - ) - return edits - - -def _facts() -> SemanticFacts: - from ..semantic.facts import FactLoadError, load_facts - - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) - - -def _leaking_admissions(edits: Sequence[MechanismEdit], pair_cases: Sequence[object], store_path: Path) -> list[str]: - """Admitted record ids that hit the fixed side of any holdout pair with the store as it stands after the edits.""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore - - admitted = {edit.mechanism_id for edit in edits if edit.action == "admit"} - if not admitted: - return [] - cases = select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS) - store = MechanismStore(store_path) - facts = _facts() - leaking: set[str] = set() - for case in cases: - leaking |= set(score_pair_with_store(case, store, facts).leaked_by) & admitted - return sorted(leaking) - - def profile_regressions( before: dict[str, dict[str, float]], after: dict[str, dict[str, float]], @@ -278,9 +159,6 @@ def run_round( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, - scripted_mechanism_edits: Sequence[MechanismEdit] | None = None, ) -> RoundOutcome: policy = policy if policy is not None else load_policy() rounds = load_journal(journal_path) @@ -295,46 +173,18 @@ def run_round( ruleset_by_id = {rule.rule_id: rule for rule in before_rules} edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked) - # corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit. - mechanism_edits: list[MechanismEdit] = [] refusals: list[dict[str, object]] = [] - candidates_store = None - if mechanism_candidates is not None and mechanism_store is not None and pair_cases: - from ..semantic.mechanisms import MechanismStore - - candidates_store = MechanismStore(mechanism_candidates) - mechanism_edits = ( - list(scripted_mechanism_edits) - if scripted_mechanism_edits is not None - else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store, degradations=refusals) - ) - blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked - mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms] - if not edits and not mechanism_edits: + if not edits: return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals) validator = EvolveValidator() try: for edit in edits: validator.validate(edit, ruleset_by_id, policy) - if mechanism_edits and candidates_store is not None and mechanism_store is not None: - from ..semantic.mechanisms import MechanismStore - - by_id = {record.id: record for record in candidates_store.load()} - admitted_by_id = {record.id: record for record in MechanismStore(mechanism_store).load()} - for mechanism_edit in mechanism_edits: - validator.validate_mechanism(mechanism_edit, by_id, admitted=admitted_by_id) except StrictValidationError as exc: - _record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits) + _record(journal_path, round_index, edits, "rejected", before, before, str(exc)) return _outcome( - round_index, - accepted=False, - reason=f"validation_failed: {exc}", - edits=edits, - before=before, - after=before, - mechanism_edits=mechanism_edits, - degradations=refusals, + round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before, degradations=refusals ) candidate = edits_to_ledger(edits, current_ledger) @@ -365,39 +215,10 @@ def run_round( regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs) if regressed: accepted = False - mechanisms_before: dict[str, dict[str, float]] = {} - mechanisms_after: dict[str, dict[str, float]] = {} - mechanism_reason = "" - if accepted and mechanism_edits and candidates_store is not None and mechanism_store is not None and pair_cases: - # The lever changes what variant search finds, so its gate is the same per-profile clause over the holdout pairs - # scored with the store before and after; an admission that leaks or lowers any profile is reverted byte for byte. - mechanisms_before = evaluate_mechanism_profiles(pair_cases, mechanism_store) - snapshot = apply_mechanism_edits(mechanism_edits, candidates_store, mechanism_store) - mechanisms_after = evaluate_mechanism_profiles(pair_cases, mechanism_store) - mech_regressed, mech_small = profile_regressions( - mechanisms_before, mechanisms_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs - ) - leaking = _leaking_admissions(mechanism_edits, pair_cases, mechanism_store) - improved = any( - mechanisms_after[p]["pair_correct"] > mechanisms_before.get(p, {}).get("pair_correct", 0.0) for p in mechanisms_after - ) or any(edit.action == "retract" for edit in mechanism_edits) - if mech_regressed or leaking or not improved: - snapshot.restore() - accepted = False - regressed = regressed + mech_regressed - if leaking: - mechanism_reason = f"mechanism_leak:{leaking[0]}" # an admission that fires on a holdout fixed side (Req 5.3) - elif mech_regressed: - mechanism_reason = f"profile_regression:{mech_regressed[0]}" - else: - mechanism_reason = "mechanism_no_gain" - small = sorted(set(small) | set(mech_small)) if accepted: if edits: write_rule_ledger(ledger_path, candidate) reason = "accepted" - elif mechanism_reason: - reason = mechanism_reason elif regressed: reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions else: @@ -412,20 +233,9 @@ def run_round( reason, regressed, small, - mechanism_edits=mechanism_edits, - mechanisms_before=mechanisms_before, - mechanisms_after=mechanisms_after, - candidates=candidates_store, ) outcome = _outcome( - round_index, - accepted=accepted, - reason=reason, - edits=edits, - before=before, - after=after, - mechanism_edits=mechanism_edits, - degradations=refusals, + round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, degradations=refusals ) return RoundOutcome( **{ @@ -434,8 +244,6 @@ def run_round( "profiles_under_minimum": small, "per_profile_before": profiles_before, "per_profile_after": profiles_after, - "mechanisms_before": mechanisms_before, - "mechanisms_after": mechanisms_after, } ) @@ -454,8 +262,6 @@ def run_improvement( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, ) -> list[RoundOutcome]: """Run improvement rounds until convergence (no new proposals) or ``max_rounds``.""" outcomes: list[RoundOutcome] = [] @@ -472,8 +278,6 @@ def run_improvement( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, - mechanism_candidates=mechanism_candidates, - mechanism_store=mechanism_store, ) outcomes.append(outcome) if outcome.reason == "no_proposals": @@ -542,7 +346,6 @@ def _outcome( edits: list[RuleStatusEdit], before: _Eval, after: _Eval, - mechanism_edits: list[MechanismEdit] | None = None, degradations: list[dict[str, object]] | None = None, ) -> RoundOutcome: return RoundOutcome( @@ -550,7 +353,6 @@ def _outcome( accepted=accepted, reason=reason, edits=edits, - mechanism_edits=list(mechanism_edits or []), degradations=list(degradations or []), recall_before=before.recall, recall_after=after.recall, @@ -573,14 +375,7 @@ def _record( reason: str = "", profile_regressions: list[str] | None = None, profiles_under_minimum: list[str] | None = None, - mechanism_edits: Sequence[MechanismEdit] = (), - mechanisms_before: dict[str, dict[str, float]] | None = None, - mechanisms_after: dict[str, dict[str, float]] | None = None, - candidates: object | None = None, ) -> None: - pairs_by_id: dict[str, list[str]] = {} - if candidates is not None: - pairs_by_id = {record.id: list(record.pairs) for record in candidates.load()} # type: ignore[attr-defined] append_round( journal_path, { @@ -592,21 +387,7 @@ def _record( "edits": [ {"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale} for e in edits - ] - + [ - { - "key": m.key(), - "lever": m.lever, - "action": m.action, - "mechanism_id": m.mechanism_id, - "source": m.source, - "pairs": pairs_by_id.get(m.mechanism_id, []), - "rationale": m.rationale, - } - for m in mechanism_edits ], - "mechanisms_before": mechanisms_before or {}, - "mechanisms_after": mechanisms_after or {}, "recall_before": round(before.recall, 4), "recall_after": round(after.recall, 4), "fp_before": round(before.fp_rate, 4), diff --git a/src/openultrasast/improve/validator.py b/src/openultrasast/improve/validator.py index 807518e..7440d27 100644 --- a/src/openultrasast/improve/validator.py +++ b/src/openultrasast/improve/validator.py @@ -8,11 +8,9 @@ impossible; the validator additionally enforces status/bounds/staging/resolution from __future__ import annotations -import json import re -from collections.abc import Mapping, Sequence -from dataclasses import asdict, dataclass -from pathlib import Path +from collections.abc import Mapping +from dataclasses import dataclass from ..policy import CwePolicy from ..ruleset import PatternRule @@ -62,40 +60,6 @@ class PolicyConstantEdit: return f"policy:{self.name}:{self.from_value}->{self.to_value}" -@dataclass(frozen=True) -class MechanismEdit: - """A `mechanisms`-lever edit (corpus-seeded-mechanisms Req 5): admit or retract one exporter record by id. - - The loop never writes a shape; it may only move a record the exporter derived from a trusted pair into or out of - the scan-time store. Free text is confined to ``rationale``. - """ - - action: str # admit | retract - mechanism_id: str - source: str = "loo" # loo | export - rationale: str = "" - lever: str = "mechanisms" - - def key(self) -> str: - return f"mechanisms:{self.action}:{self.mechanism_id}" - - -@dataclass(frozen=True) -class StoreSnapshot: - """Bytes of the scan-time mechanism store before a round; ``restore`` is the byte-for-byte revert.""" - - path: Path - content: bytes | None # None when the file did not exist - - def restore(self) -> None: - if self.content is None: - if self.path.exists(): - self.path.unlink() - return - self.path.parent.mkdir(parents=True, exist_ok=True) - self.path.write_bytes(self.content) - - @dataclass(frozen=True) class EvolveBounds: k_range: tuple[float, float] = (20.0, 200.0) @@ -113,49 +77,9 @@ class EvolveValidator: self._validate_rule(edit, ruleset_by_id, policy) elif isinstance(edit, PolicyConstantEdit): self._validate_policy(edit) - elif isinstance(edit, MechanismEdit): - raise StrictValidationError("mechanism edits need the exporter candidate set: use validate_mechanism", kind="mechanism_change") else: raise StrictValidationError(f"unknown edit type {type(edit).__name__}", kind="rule_change") - def validate_mechanism( - self, edit: MechanismEdit, candidates: Mapping[str, object], admitted: Mapping[str, object] | None = None - ) -> None: - """Req 5.2: only records the exporter derived from trusted pairs, with a closed guard and identifier-only shape text. - - A retraction may also name a record that is admitted in the scan store but no longer in the candidate set. - """ - from ..semantic.variants import GUARD_KINDS, SOURCE_KINDS - - if edit.action not in MECHANISM_ACTIONS: - raise StrictValidationError(f"mechanism edit action {edit.action!r} is not admit or retract", kind="mechanism_change") - record = candidates.get(edit.mechanism_id) - if record is None and edit.action == "retract" and admitted is not None: - record = admitted.get(edit.mechanism_id) - if record is None: - raise StrictValidationError( - f"unknown mechanism {edit.mechanism_id!r}: not in the exporter candidate set", kind="mechanism_change" - ) - if getattr(record, "origin", "") != "corpus": - raise StrictValidationError( - f"{edit.mechanism_id}: origin {getattr(record, 'origin', '')!r} is not corpus (only exporter records)", - kind="mechanism_change", - ) - shape = getattr(record, "shape", None) - if not isinstance(shape, dict): - raise StrictValidationError(f"{edit.mechanism_id}: record carries no shape", kind="mechanism_change") - if str(shape.get("guard")) not in GUARD_KINDS or str(getattr(record, "guard", "")) not in GUARD_KINDS: - raise StrictValidationError( - f"{edit.mechanism_id}: guard {shape.get('guard')!r} is not a closed guard kind", kind="mechanism_change" - ) - for field_name in ("sink_name", "language", "mechanism"): - value = str(shape.get(field_name, "")) - if not _IDENTIFIER.match(value.replace("-", "_")): - raise StrictValidationError(f"{edit.mechanism_id}: {field_name} {value!r} is not an identifier", kind="mechanism_change") - kinds = shape.get("source_kinds") or [] - if not isinstance(kinds, list) or any(str(kind) not in SOURCE_KINDS for kind in kinds): - raise StrictValidationError(f"{edit.mechanism_id}: source kinds {kinds!r} are not closed kinds", kind="mechanism_change") - def _validate_rule(self, edit: RuleStatusEdit, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None: if edit.to_status not in VALID_STATUS: raise StrictValidationError(f"invalid status {edit.to_status!r} for {edit.rule_id}") @@ -178,25 +102,6 @@ class EvolveValidator: raise StrictValidationError(f"MIN_SCORE={edit.to_value} out of bounds {self._bounds.min_score_range}", kind="policy_change") -def apply_mechanism_edits(edits: Sequence[MechanismEdit], candidates: object, scan_store: Path) -> StoreSnapshot: - """Apply admit/retract edits to the scan-time store; returns the snapshot whose ``restore`` reverts byte for byte. - - Admission appends the exporter record unchanged; retraction appends a tombstone row (``retracted = true``) so the - log stays append-only and ``MechanismStore.load`` folds the record away. - """ - snapshot = StoreSnapshot(path=scan_store, content=scan_store.read_bytes() if scan_store.exists() else None) - by_id = {record.id: record for record in candidates.load()} # type: ignore[attr-defined] - scan_store.parent.mkdir(parents=True, exist_ok=True) - with scan_store.open("a") as handle: - for edit in edits: - record = by_id[edit.mechanism_id] - payload = asdict(record) - if edit.action == "retract": - payload["retracted"] = True - handle.write(json.dumps(payload, sort_keys=True) + "\n") - return snapshot - - def edits_to_ledger(edits: list[RuleStatusEdit], base: Mapping[str, dict[str, object]]) -> dict[str, dict[str, object]]: """Overlay rule-status edits onto a copy of the base ledger.""" ledger: dict[str, dict[str, object]] = {key: dict(value) for key, value in base.items()} @@ -211,9 +116,6 @@ __all__ = [ "EvolveBounds", "EvolveValidator", "MECHANISM_ACTIONS", - "MechanismEdit", - "StoreSnapshot", - "apply_mechanism_edits", "PolicyConstantEdit", "RuleStatusEdit", "StrictValidationError", diff --git a/src/openultrasast/learning/__init__.py b/src/openultrasast/learning/__init__.py deleted file mode 100644 index 12f7d6f..0000000 --- a/src/openultrasast/learning/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""The learning harness: classified detectors improved one family at a time (learning-harness spec). - -Layers left to right, and nothing imports rightward: ``families`` -> ``classify`` -> ``scoring`` -> -``split`` -> ``detectors`` and ``verifiers`` -> ``rounds`` -> ``publish``. ``pairs`` and ``benchmark`` -never import this package; the corpus stores a family label as an opaque string and the classifier -validates it. -""" - -from __future__ import annotations - -from .families import FAMILY_IDS, VERIFIER_KINDS, FamiliesError, Family, FamilyTaxonomy, load_families - -__all__ = ["FAMILY_IDS", "VERIFIER_KINDS", "FamiliesError", "Family", "FamilyTaxonomy", "load_families"] diff --git a/src/openultrasast/learning/acceptance.py b/src/openultrasast/learning/acceptance.py deleted file mode 100644 index 46a1ebe..0000000 --- a/src/openultrasast/learning/acceptance.py +++ /dev/null @@ -1,146 +0,0 @@ -"""When a round is kept, and how it is undone (learning-harness, Req 6.2, 9.5). - -A round is accepted only when the family it targets does not go backwards on either the train or the -held-out pairs, at least one of them moves forward, and no other family flips more pairs than that -family showed against itself in round zero. A tie is a rejection: accepting one would let the loop -wander on noise and call it progress. - -A family with no measured floor gets no budget. Unmeasured is not the same as unlimited, and the -alternative is a family that never ran in round zero quietly absorbing every regression. - -Undoing is a directory restored byte for byte, files the round added removed, files it deleted put back. -A revert that only rewrites what it can see is not a revert. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path - -from .rounds import NoiseFloor - - -@dataclass(frozen=True) -class Verdict: - accepted: bool - reason: str - offenders: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return {"accepted": self.accepted, "reason": self.reason, "offenders": list(self.offenders)} - - -def decide( - *, - target: str, - train_delta: float, - holdout_delta: float, - sweep: Mapping[str, int], - floors: Mapping[str, NoiseFloor], -) -> Verdict: - """The acceptance rule, in one place, so no caller can soften it.""" - if train_delta < 0: - return Verdict(accepted=False, reason="train_regressed") - if holdout_delta < 0: - return Verdict(accepted=False, reason="holdout_regressed") - if train_delta == 0 and holdout_delta == 0: - return Verdict(accepted=False, reason="tie") - offenders = tuple( - family for family, flips in sorted(sweep.items()) if family != target and _gates(family, floors) and flips > _budget(family, floors) - ) - if offenders: - return Verdict(accepted=False, reason="collateral_regression", offenders=offenders) - return Verdict(accepted=True, reason="improved") - - -@dataclass(frozen=True) -class Reliability: - """Whether a stage's movement could be told from a coin flip.""" - - better: int - worse: int - p_value: float - reliable: bool - - def to_dict(self) -> dict[str, object]: - return {"better": self.better, "worse": self.worse, "p_value": self.p_value, "reliable": self.reliable} - - -def reliability(*, better: int, worse: int, alpha: float = 0.05) -> Reliability: - """The two-sided sign test over the pairs that changed, for the record rather than for the decision. - - `decide` compares raw deltas, as Req 9.5 specifies. On a detector whose measured floor says half its pairs - disagree with themselves between runs, a raw delta is mostly noise — which is what makes a deterministic - decoder look necessary. It is not: the alternative is an acceptance rule that models the score as noisy, the - way reflective-evolution and Bayesian prompt optimizers do, and keeps per-instance winners on a frontier - instead of demanding a single global improvement. Changing the rule is a maintainer's call, so every round - records this beside its decision and the journal can be read to answer the question with data. - """ - from .scoring import sign_test - - p_value = sign_test(better, worse) - return Reliability(better=better, worse=worse, p_value=p_value, reliable=bool(p_value < alpha and better != worse)) - - -def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool: - floor = floors.get(family) - return floor.gates if floor is not None else True - - -def _budget(family: str, floors: Mapping[str, NoiseFloor]) -> int: - floor = floors.get(family) - return floor.budget_flips if floor is not None else 0 - - -def within_budget(spent_usd: float, cap_usd: float) -> bool: - """A round may spend up to its cap; crossing it reverts the round rather than finishing it.""" - return spent_usd <= cap_usd - - -@dataclass(frozen=True) -class DirectorySnapshot: - """Every file of one directory as it was, so a rejected round leaves nothing behind.""" - - directory: Path - files: tuple[tuple[str, bytes], ...] - - @staticmethod - def of(directory: Path) -> DirectorySnapshot: - entries = [(path.relative_to(directory).as_posix(), path.read_bytes()) for path in sorted(directory.rglob("*")) if path.is_file()] - return DirectorySnapshot(directory=directory, files=tuple(entries)) - - def restore(self) -> None: - kept = {name for name, _ in self.files} - for path in sorted(self.directory.rglob("*"), reverse=True): - if path.is_file() and path.relative_to(self.directory).as_posix() not in kept: - path.unlink() - for name, content in self.files: - target = self.directory / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(content) - - -def digest_configs(configs_dir: Path) -> dict[str, str]: - """One digest per family directory, so an untouched family can be shown to be untouched.""" - digests: dict[str, str] = {} - for directory in sorted(path for path in configs_dir.iterdir() if path.is_dir()) if configs_dir.is_dir() else []: - digest = hashlib.sha256() - for path in sorted(directory.rglob("*")): - if path.is_file(): - digest.update(path.relative_to(directory).as_posix().encode()) - digest.update(path.read_bytes()) - digests[directory.name] = digest.hexdigest() - return digests - - -def bump_version(directory: Path, version: str) -> None: - """Write the accepted version into this family's settings, and only this family's.""" - settings = directory / "family.toml" - lines = settings.read_text(encoding="utf-8").splitlines() - rewritten = [f'version = "{version}"' if line.strip().startswith("version") else line for line in lines] - settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8") - - -__all__ = ["DirectorySnapshot", "Reliability", "Verdict", "bump_version", "decide", "digest_configs", "reliability", "within_budget"] diff --git a/src/openultrasast/learning/canaries.py b/src/openultrasast/learning/canaries.py deleted file mode 100644 index 82aea36..0000000 --- a/src/openultrasast/learning/canaries.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Canary snippets and the second judge (learning-harness, Req 7.2, 7.5). - -A canary is how a family says "the vulnerability fired": a snippet plants a marker only the -vulnerability can reveal, and the token appearing in the sandbox output is the oracle. A marker file an -injected command writes, a file under the scratch mount that only a traversal reaches, an object that -only an unsafe load reconstructs. The token is derived from the family, file and function, so one -family's oracle can never answer for another. - -The injection canary confirms untrusted input reaching an interpreter: command, code and template -execution. A query-language injection whose payload never reaches a shell is not confirmed by it and -stays a suspicion, which is the honest answer rather than a false negative dressed as a verdict. - -Nothing reaches `proven` on a machine's say-so alone: a second, independent judgement reads the snippet -and what the sandbox printed and either confirms or drops the claim back to corroboration. Templates -and oracle code live in the package, outside every directory a round may write. -""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass -from pathlib import Path - -from ..config import SandboxConfig -from ..findings import StaticFinding -from ..regress.candidate import DEFAULT_IMAGES, RegressionRunner -from ..regress.verdict import TRIGGERABLE -from ..sandbox.runner import SCRATCH_MOUNT, WORKSPACE_MOUNT, SandboxRunner -from ..tool_hunter import ChatClient -from .verifiers import Verification, VerifierKind, token_oracle - -_TOKEN_PREFIX = "ousast-canary-" -_JUDGE_SYSTEM = ( - "You check one security claim against what a sandbox actually printed. " - 'Answer with JSON only: {"confirms": true|false, "reason": ""}. ' - "Confirm only when the printed output can be explained by the claimed vulnerability and by nothing else." -) - - -def canary_token(family: str, path: str, function: str) -> str: - """The token the oracle looks for. Unique to the experiment, so one canary can never answer for another.""" - digest = hashlib.sha256(f"{family}|{path}|{function}".encode()).hexdigest()[:12] - return f"{_TOKEN_PREFIX}{digest}" - - -def payload_token(family: str, path: str, function: str) -> str: - """The value carried *inside* an injected payload, deliberately unrelated to the oracle token. - - A handler that merely echoes its input would otherwise print the oracle token and satisfy the oracle - without the vulnerability firing at all. The snippet prints the oracle token only after seeing this - value somewhere the payload alone could not have put it. - """ - digest = hashlib.sha256(f"payload|{family}|{path}|{function}".encode()).hexdigest()[:12] - return f"ousast-payload-{digest}" - - -def render_canary( - family: str, *, language: str, path: str, function: str, workspace: str = WORKSPACE_MOUNT, scratch: str = SCRATCH_MOUNT -) -> tuple[str, str] | None: - """The snippet that plants this family's canary and the token that proves it fired, or None. - - ``workspace`` and ``scratch`` default to the sandbox mounts; a test points them at real directories so - the snippet can be run for what it actually does rather than for what a fake runner says it does. - """ - template = _TEMPLATES.get((family, language)) - if template is None: - return None - token = canary_token(family, path, function) - module = path.replace("\\", "/") - snippet = template.format( - token=token, - payload=payload_token(family, path, function), - workspace=workspace, - scratch=scratch, - module=module, - function=function, - ) - return snippet, token - - -@dataclass(frozen=True) -class CanaryVerifier: - """Runs a family's canary in the sandbox and lets only its own token count as proof.""" - - runner: SandboxRunner - limits: SandboxConfig - images: dict[str, str] | None = None - kind: VerifierKind = "canary" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - family = next((tag.split(":", 1)[1] for tag in claim.tags if tag.startswith("family:")), "") - rendered = render_canary(family, language=language, path=claim.path, function=claim.function_name or "") - if rendered is None: - return Verification(rung="suspicion", reason="no_template") - snippet, token = rendered - images = self.images or dict(DEFAULT_IMAGES) - verdict = RegressionRunner(self.runner).run_recipe( - language, snippet, images.get(language, "ousast-missing-image"), self.limits, repo_root=root, oracle=token_oracle(token) - ) - if verdict.verdict == TRIGGERABLE: - return Verification(rung="proven", reason="oracle_fired", oracle_output=token) - return Verification(rung="suspicion", reason="oracle_quiet" if verdict.reason == "oracle_quiet" else verdict.reason) - - -def confirm_proven( - verification: Verification, claim: StaticFinding, *, client: ChatClient | None, model: str, snippet: str -) -> Verification: - """Req 7.5: a claim is published as proven only when a second, independent judgement agrees.""" - if verification.rung != "proven": - return verification - if client is None or not model: - return Verification(rung="static_corroboration", reason="judge_unavailable", oracle_output=verification.oracle_output) - question = ( - f"Claim: {claim.title}. {claim.rationale[:400]}\n" - f"Location: {claim.path} line {claim.line}, function {claim.function_name}.\n\n" - f"Snippet that was run:\n{snippet}\n\nWhat the sandbox printed:\n{verification.oracle_output}\n\n" - "Does that output demonstrate the claim? Answer as JSON." - ) - messages: list[dict[str, object]] = [{"role": "system", "content": _JUDGE_SYSTEM}, {"role": "user", "content": question}] - try: - response = client.complete(model=model, messages=messages, tools=[], json_object=True) # type: ignore[call-arg] - except TypeError: - response = client.complete(model=model, messages=messages, tools=[]) - except Exception: # noqa: BLE001 — a judge that cannot answer must not promote a claim - return Verification(rung="static_corroboration", reason="judge_unavailable", oracle_output=verification.oracle_output) - if _confirms(response.content): - return verification - return Verification(rung="static_corroboration", reason="judge_declined", oracle_output=verification.oracle_output) - - -def _confirms(content: str | None) -> bool: - if not isinstance(content, str) or "{" not in content: - return False - try: - payload = json.loads(content[content.index("{") : content.rindex("}") + 1]) - except (ValueError, json.JSONDecodeError): - return False - return isinstance(payload, dict) and payload.get("confirms") is True - - -# Templates are data, not prose: each plants a marker only its own vulnerability can reveal, imports the -# candidate from the read-only source mount, and prints the token on that path alone. A snippet that -# cannot reach the function prints nothing, which is a suspicion, not a proof. -_PYTHON_INJECTION = """# canary: untrusted input reaches an interpreter, which writes a marker only it could write. -# The payload carries a different value from the oracle token, so a handler that merely echoes its input -# cannot satisfy the oracle; only something that executed the payload can put PLANTED into the marker. -import sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} -MARKER = {scratch!r} + "/canary-injection.txt" -payload = "x; python3 -c \\"open(%r,'w').write(%r)\\"" % (MARKER, PLANTED) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - if callable(handler): - try: - handler(payload) - except Exception as inner: - print("handler raised:", type(inner).__name__) - import os - if os.path.exists(MARKER): - with open(MARKER, encoding="utf-8") as handle: - if PLANTED in handle.read(): - print("CANARY", CANARY) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_PYTHON_PATH = """# canary: a traversal reads a file the handler was never meant to open -import sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} -MARKER = {scratch!r} + "/canary-path.txt" -with open(MARKER, "w", encoding="utf-8") as handle: - handle.write(CANARY) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - result = handler(MARKER) if callable(handler) else None - if CANARY in str(result): - print("CANARY", CANARY) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_PYTHON_DESERIALIZATION = """# canary: an unsafe load reconstructs an object that announces itself -import pickle, sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} - - -class Marker: - def __reduce__(self): - return (print, ("CANARY", CANARY)) - - -blob = pickle.dumps(Marker()) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - if callable(handler): - handler(blob) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_TEMPLATES: dict[tuple[str, str], str] = { - ("injection", "python"): _PYTHON_INJECTION, - ("path", "python"): _PYTHON_PATH, - ("deserialization", "python"): _PYTHON_DESERIALIZATION, -} - - -__all__ = ["CanaryVerifier", "canary_token", "confirm_proven", "payload_token", "render_canary"] diff --git a/src/openultrasast/learning/classify.py b/src/openultrasast/learning/classify.py deleted file mode 100644 index 802b082..0000000 --- a/src/openultrasast/learning/classify.py +++ /dev/null @@ -1,399 +0,0 @@ -"""The auto-classifier: which families a pair, a finding or a region belongs to (learning-harness, Req 2). - -Three tiers, cheapest first. Tier one decides from what the repository already knows: a declared family -label, an obligation kind, a mechanism id, the rule a finding came from, or the labeled weakness. Tier -two asks the model one closed question in JSON mode, with thinking disabled so temperature applies, and -abstains when the answer is off-vocabulary or the reply's mean token log-probability is below the -threshold. Tier three is abstention. Every answer records which tier decided it, and answers are -multi-label because one function can owe more than one family. - -The classifier proposes; it never writes a label. It also owns validation of the labels the corpus -carries, because the corpus stores a family as an opaque string and this module owns the taxonomy. -""" - -from __future__ import annotations - -import json -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import Literal - -from ..findings import StaticFinding -from ..tool_hunter import ChatClient -from .families import FamilyTaxonomy - -Tier = Literal["static", "model", "none"] -MIN_CONFIDENCE = -1.0 # mean token log-probability; below this the model is guessing, so we abstain -_SYSTEM = ( - "You classify a piece of code or a labeled bug into vulnerability families for a security tool. " - 'Answer with JSON only, of the form {"families": ["", ...]}, using ids from the list you are given. ' - "Use an empty list when none of them fits. Do not explain and do not invent an id." -) - - -class FamilyLabelError(ValueError): - """Raised when a catalog row carries a family id outside the taxonomy.""" - - -@dataclass(frozen=True) -class Classification: - families: tuple[str, ...] # empty means unknown - tier: Tier - confidence: float | None = None # mean token log-probability when the model answered - - -def classify_pair( - case: object, taxonomy: FamilyTaxonomy, *, client: ChatClient | None, model: str = "", use_declared: bool = True -) -> Classification: - """The families a labeled pair belongs to: its declared label, else its mechanism, else its weakness. - - ``use_declared`` is False when the classifier is being measured against those same labels, so that - agreement is not the classifier reading back the answer it is scored on. - """ - families = _static_families_of_pair(case, taxonomy, use_declared=use_declared) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _pair_question(case)) - - -def classify_finding(finding: StaticFinding, taxonomy: FamilyTaxonomy, *, client: ChatClient | None, model: str = "") -> Classification: - """The families a finding belongs to: its own family tag, else what produced it, else the model.""" - families = _static_families_of_finding(finding, taxonomy) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _finding_question(finding)) - - -def classify_region( - root: Path, path: str, *, function: str | None, taxonomy: FamilyTaxonomy, client: ChatClient | None, model: str = "" -) -> Classification: - """The families a code region belongs to, from what the static tools already see in it.""" - families = _static_families_of_region(root, path, function, taxonomy) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _region_question(root, path, function)) - - -def validate_family_labels(cases: Sequence[object], taxonomy: FamilyTaxonomy) -> None: - """Every family a catalog row declares must be in the taxonomy. The corpus stores labels; this module owns them.""" - known = {family.id for family in taxonomy.families} - for case in cases: - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if label and label not in known: - name = getattr(case, "name", "?") - raise FamilyLabelError(f"pair {name}: unknown family {label!r}; expected one of {', '.join(sorted(known))}") - - -def _static_families_of_pair(case: object, taxonomy: FamilyTaxonomy, *, use_declared: bool = True) -> tuple[str, ...]: - declared: list[str] = [] - derived: list[str] = [] - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if use_declared and label and any(family.id == label for family in taxonomy.families): - declared.append(str(label)) - continue - obligation = getattr(row, "obligation", None) - if obligation: - derived.append("access_control") - mechanism = getattr(row, "mechanism", None) - by_mechanism = taxonomy.family_of_mechanism(str(mechanism)) if mechanism else None - if by_mechanism is not None and by_mechanism.id != "unknown": - derived.append(by_mechanism.id) - continue - by_cwe = taxonomy.family_of_cwe(str(getattr(row, "cwe", "") or "")) - if by_cwe is not None and by_cwe.id != "unknown": - derived.append(by_cwe.id) - return _ordered(declared or derived, taxonomy) - - -def _static_families_of_finding(finding: StaticFinding, taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - known = {family.id for family in taxonomy.families} - tagged = [tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:") and tag.split(":", 1)[1] in known] - if tagged: - return _ordered(tagged, taxonomy) - found: list[str] = [] - for tag in finding.tags: - if tag.startswith("obligation:") or tag.startswith("discharger:"): - found.append("access_control") - elif tag.startswith("mechanism:"): - family = taxonomy.family_of_mechanism(tag.split(":", 1)[1]) - if family is not None and family.id != "unknown": - found.append(family.id) - if finding.finding_id.startswith("obligation:"): - found.append("access_control") - rule_family = taxonomy.family_of_cwe(_rule_cwe(finding.finding_id.split(":", 1)[0])) - if rule_family is not None and rule_family.id != "unknown": - found.append(rule_family.id) - return _ordered(found, taxonomy) - - -def _static_families_of_region(root: Path, path: str, function: str | None, taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - from ..hunter_tools import PathEscapesRepo, flows, obligations - - found: list[str] = [] - if function: - try: - if obligations(root, path=path, function=function): - found.append("access_control") - for record in flows(root, path=path, function=function): - family = taxonomy.family_of_cwe(str(record.get("cwe") or "")) - if family is not None and family.id != "unknown": - found.append(family.id) - except (PathEscapesRepo, OSError): - return () - return _ordered(found, taxonomy) - - -def _ask(client: ChatClient | None, model: str, taxonomy: FamilyTaxonomy, question: str) -> Classification: - if client is None: - return Classification(families=(), tier="none") - catalog = "\n".join(f"- {family.id}: {family.description}" for family in taxonomy.families) - messages: list[dict[str, object]] = [ - {"role": "system", "content": _SYSTEM}, - {"role": "user", "content": f"Families:\n{catalog}\n\n{question}\n\nAnswer as JSON."}, - ] - try: - try: - response = client.complete(model=model, messages=messages, tools=[], json_object=True, logprobs=True) # type: ignore[call-arg] - except TypeError: # a client without the adapter's extras - response = client.complete(model=model, messages=messages, tools=[]) - except Exception: # noqa: BLE001 — the classifier is advisory; any failed call abstains, including in the fallback - return Classification(families=(), tier="none") - confidence = response.mean_logprob - if confidence is not None and confidence < MIN_CONFIDENCE: - return Classification(families=(), tier="none", confidence=confidence) - families = _ordered(_parse(response.content), taxonomy) - if not families: - return Classification(families=(), tier="none", confidence=confidence) - return Classification(families=families, tier="model", confidence=confidence) - - -def _parse(content: str | None) -> list[str]: - if not isinstance(content, str) or not content.strip(): - return [] - try: - payload = json.loads(content[content.index("{") : content.rindex("}") + 1]) - except (ValueError, json.JSONDecodeError): - return [] - families = payload.get("families") if isinstance(payload, dict) else None - if not isinstance(families, list): - return [] - return [str(item) for item in families if isinstance(item, str)] - - -def _ordered(names: Sequence[str], taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - """Taxonomy order, de-duplicated, unknown ids dropped, so two runs compare equal.""" - wanted = set(names) - return tuple(family.id for family in taxonomy.families if family.id in wanted and family.id != "unknown") - - -def _rule_cwe(rule_id: str) -> str: - from ..ruleset import DEFAULT_RULESET_DIR, load_ruleset - - for rule in load_ruleset(DEFAULT_RULESET_DIR): - if rule.rule_id == rule_id: - return rule.cwe - return "" - - -def _pair_question(case: object) -> str: - row = next(iter(getattr(case, "expected", ()) or ()), None) - parts = [f"Labeled bug in {getattr(case, 'relpath', '?')}"] - if row is not None: - parts.append( - f"function {getattr(row, 'function', '?')}, weakness {getattr(row, 'cwe', '?')}, mechanism {getattr(row, 'mechanism', '?')}" - ) - return "Which families does it belong to? " + "; ".join(parts) - - -def _finding_question(finding: StaticFinding) -> str: - return f"Which families does this finding belong to? {finding.title}. {finding.rationale[:400]}" - - -def _region_question(root: Path, path: str, function: str | None) -> str: - from ..hunter_tools import PathEscapesRepo, read_definition - - try: - found = read_definition(root, function or "", max_chars=2000) if function else None - except (PathEscapesRepo, OSError): - found = None - body = str(found["text"]) if found else "" - return f"Which families does this code belong to?\n{path}::{function}\n{body}" - - -@dataclass(frozen=True) -class QueueEntry: - """One disagreement between a maintainer label and the classifier, for a human to settle.""" - - pair: str - label: str - classified: tuple[str, ...] - tier: Tier - confidence: float | None = None - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "label": self.label, - "classified": list(self.classified), - "tier": self.tier, - "confidence": self.confidence, - } - - -@dataclass(frozen=True) -class ClassifierReport: - """How the classifier did on its own: coverage over the whole corpus, agreement where a reference exists. - - ``labeled`` is how many rows carry a maintainer family label. Agreement, the confusion matrix and both - baselines are computed over those rows only, so an unlabeled corpus reports coverage and says the - reference set is empty rather than inventing a score. - """ - - taxonomy_version: str - classified: int - labeled: int - agreement: float # credit for the classifier's first answer - oracle_router: float # credit for the best of its answers - random_router: float # a uniform pick over the real families - confusion: dict[str, dict[str, int]] = field(default_factory=dict) - per_tier: dict[str, int] = field(default_factory=dict) - per_family: dict[str, int] = field(default_factory=dict) - unknown: int = 0 - queue: tuple[QueueEntry, ...] = () - hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 2.6) - - def to_dict(self) -> dict[str, object]: - return { - "taxonomy_version": self.taxonomy_version, - "classified": self.classified, - "labeled": self.labeled, - "agreement": self.agreement, - "oracle_router": self.oracle_router, - "random_router": self.random_router, - "confusion": {label: dict(sorted(answers.items())) for label, answers in sorted(self.confusion.items())}, - "per_tier": dict(sorted(self.per_tier.items())), - "per_family": dict(sorted(self.per_family.items())), - "unknown": self.unknown, - "hierarchical_credit": self.hierarchical_credit, - "queue": [entry.to_dict() for entry in self.queue], - } - - -def measure_classifier( - cases: Sequence[object], - taxonomy: FamilyTaxonomy, - *, - client: ChatClient | None, - model: str = "", - reference_tiers: Sequence[str] = ("seeded", "reviewed"), -) -> ClassifierReport: - """Classify every case and score the classifier against the labels a maintainer stands behind.""" - rows = list(cases) - real = [family.id for family in taxonomy.families if family.id != "unknown"] - per_tier: dict[str, int] = {} - per_family: dict[str, int] = {} - confusion: dict[str, dict[str, int]] = {} - queue: list[QueueEntry] = [] - unknown = 0 - credits: list[float] = [] - oracles: list[float] = [] - for case in rows: - answer = classify_pair(case, taxonomy, client=client, model=model, use_declared=False) - per_tier[answer.tier] = per_tier.get(answer.tier, 0) + 1 - for family_id in answer.families: - per_family[family_id] = per_family.get(family_id, 0) + 1 - if not answer.families: - unknown += 1 - label = _reference_label(case, taxonomy, reference_tiers) - if label is None: - continue - first = answer.families[0] if answer.families else "unknown" - credits.append(_credit(taxonomy, label, (first,))) - oracles.append(_credit(taxonomy, label, answer.families or ("unknown",))) - confusion.setdefault(label, {})[first] = confusion.setdefault(label, {}).get(first, 0) + 1 - if credits[-1] < 1.0: - queue.append( - QueueEntry( - pair=str(getattr(case, "name", "?")), - label=label, - classified=answer.families, - tier=answer.tier, - confidence=answer.confidence, - ) - ) - labeled = len(credits) - return ClassifierReport( - taxonomy_version=taxonomy.version, - classified=len(rows), - labeled=labeled, - agreement=sum(credits) / labeled if labeled else 0.0, - oracle_router=sum(oracles) / labeled if labeled else 0.0, - random_router=(1.0 / len(real)) if real and labeled else 0.0, - confusion=confusion, - per_tier=per_tier, - per_family=per_family, - unknown=unknown, - queue=tuple(queue), - hierarchical_credit=taxonomy.hierarchical, - ) - - -def write_review_queue(path: Path, entries: Sequence[QueueEntry]) -> int: - """Append disagreements a human has not seen yet; a pair already queued is never duplicated.""" - path.parent.mkdir(parents=True, exist_ok=True) - seen: set[str] = set() - if path.exists(): - for line in path.read_text(encoding="utf-8").splitlines(): - if line.strip(): - try: - seen.add(str(json.loads(line).get("pair", ""))) - except json.JSONDecodeError: - continue - written = 0 - with path.open("a", encoding="utf-8") as handle: - for entry in entries: - if entry.pair in seen: - continue - handle.write(json.dumps(entry.to_dict(), sort_keys=True) + "\n") - seen.add(entry.pair) - written += 1 - return written - - -def _reference_label(case: object, taxonomy: FamilyTaxonomy, reference_tiers: Sequence[str]) -> str | None: - """The family a maintainer stands behind for this pair, or None. Only trusted tiers are a reference.""" - if str(getattr(case, "review_tier", "")) not in reference_tiers: - return None - known = {family.id for family in taxonomy.families} - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if label and str(label) in known: - return str(label) - return None - - -def _credit(taxonomy: FamilyTaxonomy, label: str, answers: Sequence[str]) -> float: - """1.0 for the same family, half for a parent or child, nothing for a lateral or fabricated answer.""" - best = 0.0 - for answer in answers: - relation = taxonomy.related(label, answer) - best = max(best, 1.0 if relation == "same" else 0.5 if relation == "parent_child" else 0.0) - return best - - -__all__ = [ - "MIN_CONFIDENCE", - "Classification", - "ClassifierReport", - "FamilyLabelError", - "QueueEntry", - "classify_finding", - "classify_pair", - "classify_region", - "measure_classifier", - "validate_family_labels", - "write_review_queue", -] diff --git a/src/openultrasast/learning/detectors.py b/src/openultrasast/learning/detectors.py deleted file mode 100644 index 5763cc5..0000000 --- a/src/openultrasast/learning/detectors.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Per-family detector configurations and the runner that uses them (learning-harness, Req 6.1-6.5). - -A detector is a configuration, not code: a system prompt, an evidence checklist, the curated tools it -may call, its step and cost budget, and the hard negatives and counterexamples earlier rounds left it. -Round zero writes one directory per family from the same prompt, so the baseline is controlled, and a -round rewrites exactly one directory and bumps that family's version. Every other family is frozen by -construction, because a round can only touch the directory it targets. - -The runner gives a detector the whole file rather than an excerpt around a hotspot, since context loss -is a documented cause of misjudging patched code, and tags each finding with exactly one family and the -configuration version that produced it, so a finding can always be traced back to what made it. -""" - -from __future__ import annotations - -import json -import tomllib -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path - -from ..findings import StaticFinding -from ..tool_hunter import HUNTER_TOOLS, ChatClient, run_tool_hunter -from .families import FamilyTaxonomy - -MAX_PROMPT_CHARS = 8000 # prompt plus checklist; unbounded prompts are the documented signature of overfitting -GENERALIST = "unknown" # the family every region also runs, so a bug outside the taxonomy is still looked for -CURATED_TOOLS = tuple(str(schema["function"]["name"]) for schema in HUNTER_TOOLS) # type: ignore[index] -_REQUIRED = ("prompt.md", "family.toml") - - -class DetectorConfigError(ValueError): - """Raised when a detector configuration is missing, over its cap, or names something outside the closed sets.""" - - -@dataclass(frozen=True) -class FamilyConfig: - family: str - version: str - prompt: str - checklist: str - tools: tuple[str, ...] - max_steps: int - max_cost_usd: float - max_chars: int - hard_negatives: tuple[Mapping[str, object], ...] = () - counterexamples: tuple[Mapping[str, object], ...] = () - - @property - def detector_id(self) -> str: - return f"{self.family}@{self.version}" - - -@dataclass(frozen=True) -class Region: - """A place a detector is pointed at: one file, optionally one function, and the families admitted for it.""" - - path: str - function: str | None = None - families: tuple[str, ...] = () - - -def load_family_configs(configs_dir: Path, taxonomy: FamilyTaxonomy) -> dict[str, FamilyConfig]: - """Every configuration under ``configs_dir``. A directory that is not a family, or a broken one, fails loud.""" - known = {family.id for family in taxonomy.families} - configs: dict[str, FamilyConfig] = {} - for directory in sorted(path for path in configs_dir.iterdir() if path.is_dir()) if configs_dir.is_dir() else []: - if directory.name not in known: - raise DetectorConfigError(f"{directory}: {directory.name!r} is not a family; expected one of {', '.join(sorted(known))}") - configs[directory.name] = _load_one(directory) - return configs - - -def _load_one(directory: Path) -> FamilyConfig: - for name in _REQUIRED: - if not (directory / name).is_file(): - raise DetectorConfigError(f"{directory}: {name} is required") - try: - settings = tomllib.loads((directory / "family.toml").read_text(encoding="utf-8")) - except tomllib.TOMLDecodeError as exc: - raise DetectorConfigError(f"{directory / 'family.toml'}: not valid TOML: {exc}") from exc - version = str(settings.get("version", "")).strip() - if not version: - raise DetectorConfigError(f"{directory}: version is required") - prompt = (directory / "prompt.md").read_text(encoding="utf-8").strip() - checklist = _read_optional(directory / "checklist.md") - if len(prompt) + len(checklist) > MAX_PROMPT_CHARS: - raise DetectorConfigError( - f"{directory}: prompt and checklist length {len(prompt) + len(checklist)} exceeds the cap of {MAX_PROMPT_CHARS}" - ) - tools = tuple(str(name) for name in settings.get("tools", []) or ()) - if not tools: - raise DetectorConfigError(f"{directory}: tools is required and must not be empty; a detector with no tools reports nothing") - unknown_tools = [name for name in tools if name not in CURATED_TOOLS] - if unknown_tools: - raise DetectorConfigError(f"{directory}: unknown tool(s) {', '.join(unknown_tools)}; expected from {', '.join(CURATED_TOOLS)}") - return FamilyConfig( - family=directory.name, - version=version, - prompt=prompt, - checklist=checklist, - tools=tools, - max_steps=int(settings.get("max_steps", 4)), - max_cost_usd=float(settings.get("max_cost_usd", 0.5)), - max_chars=int(settings.get("max_chars", 4000)), - hard_negatives=_read_jsonl(directory / "hard_negatives.jsonl"), - counterexamples=_read_jsonl(directory / "counterexamples.jsonl"), - ) - - -def run_family_detector(root: Path, region: Region, config: FamilyConfig, *, client: ChatClient, model: str) -> list[StaticFinding]: - """One family's detector over one region. Findings carry that family and the configuration that found them.""" - return run_tool_hunter( - root, - [], - client=client, - model=model, - max_steps=config.max_steps, - system_prompt=config.prompt, - user_prompt=_question(region, config), - context_files=(region.path,), - tools=config.tools, - max_chars=config.max_chars, - tags=(f"family:{config.family}", f"detector:{config.detector_id}"), - ) - - -def run_region_detectors( - root: Path, region: Region, configs: Mapping[str, FamilyConfig], *, client: ChatClient, model: str -) -> list[StaticFinding]: - """Every family the classifier admitted for this region, plus the generalist (Req 6.3).""" - wanted = [family for family in region.families if family in configs and family != GENERALIST] - if GENERALIST in configs: - wanted.append(GENERALIST) - findings: list[StaticFinding] = [] - for family in wanted: - findings.extend(run_family_detector(root, region, configs[family], client=client, model=model)) - return findings - - -def write_default_configs( - configs_dir: Path, taxonomy: FamilyTaxonomy, *, prompt: str, version: str, tools: Sequence[str] | None = None -) -> dict[str, Path]: - """Round zero: one directory per family from the same prompt, so the baseline is controlled (Req 8.1).""" - written: dict[str, Path] = {} - for family in taxonomy.families: - directory = configs_dir / family.id - directory.mkdir(parents=True, exist_ok=True) - (directory / "prompt.md").write_text(f"{prompt}\n\nFamily: {family.id}. {family.description}\n", encoding="utf-8") - (directory / "checklist.md").write_text("- name the file, the line and why the code is reachable\n", encoding="utf-8") - selected = list(tools) if tools is not None else list(CURATED_TOOLS) - (directory / "family.toml").write_text( - f'version = "{version}"\ntools = {json.dumps(selected)}\nmax_steps = 4\nmax_cost_usd = 0.5\nmax_chars = 4000\n', - encoding="utf-8", - ) - for name in ("hard_negatives.jsonl", "counterexamples.jsonl"): - if not (directory / name).exists(): - (directory / name).write_text("", encoding="utf-8") - written[family.id] = directory - return written - - -def _question(region: Region, config: FamilyConfig) -> str: - where = f"{region.path}::{region.function}" if region.function else region.path - parts = [f"Look for {config.family} problems in {where}. The whole file follows."] - if config.checklist: - parts.append(f"Evidence checklist:\n{config.checklist}") - if config.hard_negatives: - parts.append("Known non-issues, do not report these:\n" + "\n".join(_line(item) for item in config.hard_negatives)) - if config.counterexamples: - parts.append("Worked examples:\n" + "\n".join(_line(item) for item in config.counterexamples)) - parts.append("Reply with a JSON array of objects with path, line, title and rationale, after using a tool.") - return "\n\n".join(parts) - - -def _line(item: Mapping[str, object]) -> str: - return "- " + "; ".join(f"{key}: {value}" for key, value in sorted(item.items())) - - -def _read_optional(path: Path) -> str: - return path.read_text(encoding="utf-8").strip() if path.is_file() else "" - - -def _read_jsonl(path: Path) -> tuple[Mapping[str, object], ...]: - if not path.is_file(): - return () - rows: list[Mapping[str, object]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - payload = json.loads(line) - except json.JSONDecodeError as exc: - raise DetectorConfigError(f"{path}: not valid JSON lines: {exc}") from exc - if isinstance(payload, dict): - rows.append(payload) - return tuple(rows) - - -__all__ = [ - "CURATED_TOOLS", - "GENERALIST", - "MAX_PROMPT_CHARS", - "DetectorConfigError", - "FamilyConfig", - "Region", - "load_family_configs", - "run_family_detector", - "run_region_detectors", - "write_default_configs", -] diff --git a/src/openultrasast/learning/difficulty.py b/src/openultrasast/learning/difficulty.py deleted file mode 100644 index 8b4b273..0000000 --- a/src/openultrasast/learning/difficulty.py +++ /dev/null @@ -1,139 +0,0 @@ -"""How hard a piece of code makes an absence bug to see (learning-harness, Req 10.4). - -A unit fixture that is easier than the corpus is worse than no fixture: the checker passes on it, the number -looks like capability, and the corpus rows it claims to cover keep failing for reasons nobody is looking at. The -three dimensions below are the ones the corpus actually varies on, measured on the vulnerable side: - -- ``binding_hops`` — how many assignments sit between the identity source and the constrained use. `filter_by( - user_id=request.user.id)` is nothing to trace; `resp = token_validator(...)` then `User.query.filter_by( - username=resp['sub'])` then `filter_by(user=user, ...)` is two. -- ``registration_distance`` — 0 when the route decorator sits on the handler, 1 when a statement elsewhere in the - same file registers it, 2 when the registration lives in another document entirely (connexion's `operationId`, - an express router). Two is where the entry-point mapper stops being able to name the handler at all. -- ``guard_distance`` — 0 when a decorator carries the guard, 1 when the guard is a call in the body, 2 when the - handler carries no visible guard. A decorator is a pattern; a call in the body has to be understood. - -Higher is harder on every axis, so "strictly easier" is: no dimension harder, at least one easier. -""" - -from __future__ import annotations - -import ast -import re -from collections.abc import Sequence -from dataclasses import dataclass - -IDENTITY_SOURCES = ("current_user", "request.user", "session", "g.user", "auth", "token", "jwt", "claims", "sub", "principal") -GUARD_TOKENS = ("login_required", "token_required", "requires_auth", "authenticate", "authorize", "token_validator", "current_user") -_ROUTE_DECORATOR = re.compile(r"\.(route|get|post|put|patch|delete)\b") - - -@dataclass(frozen=True) -class DifficultyVector: - """One handler's difficulty. Higher is harder on every axis.""" - - function: str - binding_hops: int - registration_distance: int - guard_distance: int - - def axes(self) -> tuple[int, int, int]: - return (self.binding_hops, self.registration_distance, self.guard_distance) - - def to_dict(self) -> dict[str, object]: - return { - "function": self.function, - "binding_hops": self.binding_hops, - "registration_distance": self.registration_distance, - "guard_distance": self.guard_distance, - } - - -def difficulty_of(source: str, function: str, *, context: Sequence[str] = ()) -> DifficultyVector: - """The vector for one Python handler. ``context`` is any document that registers it from outside the file.""" - try: - tree = ast.parse(source) - except SyntaxError: - return DifficultyVector(function=function, binding_hops=0, registration_distance=2, guard_distance=2) - node = _function(tree, function) - if node is None: - return DifficultyVector(function=function, binding_hops=0, registration_distance=2, guard_distance=2) - decorators = [ast.unparse(item) for item in node.decorator_list] + _class_decorators(tree, node) - return DifficultyVector( - function=function, - binding_hops=_binding_hops(node), - registration_distance=_registration_distance(source, tree, function, decorators, context), - guard_distance=_guard_distance(node, decorators), - ) - - -def hardest(vectors: Sequence[DifficultyVector]) -> tuple[int, int, int]: - """The per-axis maximum over a set of handlers: what the set can be asked to cope with.""" - if not vectors: - return (0, 0, 0) - return tuple(max(row.axes()[index] for row in vectors) for index in range(3)) # type: ignore[return-value] - - -def strictly_easier(fixture: Sequence[DifficultyVector], corpus: Sequence[DifficultyVector]) -> bool: - """True when the fixture set reaches no further than the corpus on any axis and falls short on at least one. - - Per axis and over the whole set, because a suite covers a bug shape with several handlers between them: one - fixture handler may carry the long binding chain and another the missing guard. What must not happen is a suite - whose hardest case on some axis is easier than the corpus rows its tests claim to cover.""" - if not corpus: - return False - limit, reach = hardest(corpus), hardest(fixture) - return all(reach[index] <= limit[index] for index in range(3)) and any(reach[index] < limit[index] for index in range(3)) - - -def _function(tree: ast.AST, name: str) -> ast.FunctionDef | ast.AsyncFunctionDef | None: - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name: - return node - return None - - -def _class_decorators(tree: ast.AST, node: ast.AST) -> list[str]: - for klass in ast.walk(tree): - if isinstance(klass, ast.ClassDef) and any(child is node for child in klass.body): - return [ast.unparse(item) for item in klass.decorator_list] - return [] - - -def _binding_hops(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: - """Assignments that carry the identity forward before it constrains anything.""" - tainted: set[str] = set() - hops = 0 - for statement in ast.walk(node): - if not isinstance(statement, ast.Assign): - continue - text = ast.unparse(statement.value) - if any(token in text for token in IDENTITY_SOURCES) or any(name in text for name in tainted): - hops += 1 - for target in statement.targets: - tainted.update(part.id for part in ast.walk(target) if isinstance(part, ast.Name)) - return hops - - -def _registration_distance(source: str, tree: ast.AST, function: str, decorators: Sequence[str], context: Sequence[str]) -> int: - if any(_ROUTE_DECORATOR.search(item) for item in decorators): - return 0 - mention = re.compile(rf"(? int: - if any(token in item.lower() for item in decorators for token in GUARD_TOKENS): - return 0 - body = ast.unparse(node).lower() - return 1 if any(token in body for token in GUARD_TOKENS) else 2 - - -__all__ = ["GUARD_TOKENS", "IDENTITY_SOURCES", "DifficultyVector", "difficulty_of", "hardest", "strictly_easier"] diff --git a/src/openultrasast/learning/journal.py b/src/openultrasast/learning/journal.py deleted file mode 100644 index a14f808..0000000 --- a/src/openultrasast/learning/journal.py +++ /dev/null @@ -1,237 +0,0 @@ -"""The round journal, the per-family archive and the rejected buffer (learning-harness, Req 9.3, 9.6, 9.7, 11.3). - -The journal is append-only, one JSON line per round, and a round number is never reused, so the record -of what was tried cannot be quietly rewritten by the thing that is trying. Every round writes a line -whether it was accepted or not: a rejection is evidence too, and the rejected hypotheses are what stop -the proposer from offering the same idea twice. - -The archive keeps, per family, which configuration version wins which pair, rather than one champion. -Keeping only the best-overall candidate throws away a version that is the only one solving some pair, -which is how a search collapses onto one lineage. A later version takes a pair only by winning it. - -Nothing is written without passing redaction, because a trajectory or a hypothesis may quote the code -it was reasoning about. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..redaction import redact_secrets -from .scoring import PairFamilyScore - -Outcome = str # accepted | rejected | reverted_cost | refused_holdout - - -def _as_int(value: object) -> int: - return int(value) if isinstance(value, int | float | str) and not isinstance(value, bool) else 0 - - -def _as_float(value: object) -> float: - return float(value) if isinstance(value, int | float | str) and not isinstance(value, bool) else 0.0 - - -def _as_strings(value: object) -> tuple[str, ...]: - return tuple(str(item) for item in value) if isinstance(value, list | tuple) else () - - -def _as_sweep(value: object) -> dict[str, int]: - return {str(key): _as_int(item) for key, item in value.items()} if isinstance(value, Mapping) else {} - - -class JournalError(ValueError): - """Raised when a round would overwrite history.""" - - -@dataclass(frozen=True) -class Attribution: - """Whether the round moved what it said it would move, and what it moved by accident.""" - - flipped_predicted: int = 0 - flipped_unpredicted: int = 0 - precision: float = 0.0 - - def to_dict(self) -> dict[str, object]: - return { - "flipped_predicted": self.flipped_predicted, - "flipped_unpredicted": self.flipped_unpredicted, - "precision": self.precision, - } - - @staticmethod - def from_dict(payload: Mapping[str, object]) -> Attribution: - return Attribution( - flipped_predicted=_as_int(payload.get("flipped_predicted")), - flipped_unpredicted=_as_int(payload.get("flipped_unpredicted")), - precision=_as_float(payload.get("precision")), - ) - - -@dataclass(frozen=True) -class RoundRecord: - round: int - family: str - hypothesis: str - levers: tuple[str, ...] - predicted_affected: tuple[str, ...] - predicted_at_risk: tuple[str, ...] - outcome: Outcome - reason: str = "" - target_train_delta: float = 0.0 - target_holdout_delta: float = 0.0 - sweep: dict[str, int] = field(default_factory=dict) # family -> negative flips in the cross-family sweep - attribution: Attribution = Attribution() - cost_usd: float = 0.0 - taxonomy_version: str = "" - config_version: str = "" - # Pairs that moved each way in each stage, so the record can say whether the decision was distinguishable - # from noise. The decision itself is Req 9.5's raw-delta rule; this is the evidence beside it. - train_better: int = 0 - train_worse: int = 0 - holdout_better: int = 0 - holdout_worse: int = 0 - - def to_dict(self) -> dict[str, object]: - return { - "round": self.round, - "family": self.family, - "hypothesis": self.hypothesis, - "levers": list(self.levers), - "predicted_affected": list(self.predicted_affected), - "predicted_at_risk": list(self.predicted_at_risk), - "outcome": self.outcome, - "reason": self.reason, - "target_train_delta": self.target_train_delta, - "target_holdout_delta": self.target_holdout_delta, - "sweep": dict(sorted(self.sweep.items())), - "attribution": self.attribution.to_dict(), - "cost_usd": self.cost_usd, - "taxonomy_version": self.taxonomy_version, - "config_version": self.config_version, - **{f"train_{key}": value for key, value in self._reliability(self.train_better, self.train_worse).items()}, - **{f"holdout_{key}": value for key, value in self._reliability(self.holdout_better, self.holdout_worse).items()}, - } - - @staticmethod - def _reliability(better: int, worse: int) -> dict[str, object]: - from .acceptance import reliability - - result = reliability(better=better, worse=worse) - return {"better": result.better, "worse": result.worse, "p_value": result.p_value, "reliable": result.reliable} - - @staticmethod - def from_dict(payload: Mapping[str, object]) -> RoundRecord: - attribution = payload.get("attribution") - return RoundRecord( - round=_as_int(payload.get("round")), - family=str(payload.get("family", "")), - hypothesis=str(payload.get("hypothesis", "")), - levers=_as_strings(payload.get("levers")), - predicted_affected=_as_strings(payload.get("predicted_affected")), - predicted_at_risk=_as_strings(payload.get("predicted_at_risk")), - outcome=str(payload.get("outcome", "")), - reason=str(payload.get("reason", "")), - target_train_delta=_as_float(payload.get("target_train_delta")), - target_holdout_delta=_as_float(payload.get("target_holdout_delta")), - sweep=_as_sweep(payload.get("sweep")), - attribution=Attribution.from_dict(attribution if isinstance(attribution, Mapping) else {}), - cost_usd=_as_float(payload.get("cost_usd")), - taxonomy_version=str(payload.get("taxonomy_version", "")), - config_version=str(payload.get("config_version", "")), - ) - - -class LearningJournal: - """Append-only history of rounds. What was tried, what it predicted, and what actually happened.""" - - def __init__(self, path: Path) -> None: - self.path = path - - def append(self, record: RoundRecord) -> None: - existing = {item.round for item in self.rounds()} - if record.round in existing: - raise JournalError(f"round {record.round} is already journaled; history is append-only") - self.path.parent.mkdir(parents=True, exist_ok=True) - line = redact_secrets(json.dumps(record.to_dict(), sort_keys=True)) - with self.path.open("a", encoding="utf-8") as handle: - handle.write(line + "\n") - - def rounds(self) -> tuple[RoundRecord, ...]: - return tuple(RoundRecord.from_dict(payload) for payload in _read_jsonl(self.path)) - - def next_round(self) -> int: - rounds = self.rounds() - return max((item.round for item in rounds), default=0) + 1 - - def rejected_buffer(self, family: str) -> tuple[str, ...]: - """Hypotheses this family already tried and lost, so the proposer is not handed the same idea twice.""" - return tuple(item.hypothesis for item in self.rounds() if item.family == family and item.outcome != "accepted" and item.hypothesis) - - -class Archive: - """Per family, which configuration version wins which pair. Never one champion.""" - - def __init__(self, path: Path) -> None: - self.path = path - - def record(self, family: str, config_version: str, score: PairFamilyScore) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "family": family, - "config_version": config_version, - "pair": score.pair, - "outcome": score.outcome, - "flips": score.flips, - } - with self.path.open("a", encoding="utf-8") as handle: - handle.write(redact_secrets(json.dumps(payload, sort_keys=True)) + "\n") - - def winners(self, family: str) -> dict[str, str]: - """pair -> the earliest configuration version that scored it correct. - - Earliest, not latest: the point of the archive is that a version which solves a pair keeps its - claim to it, so a later version that loses the pair cannot quietly take it off the board. - """ - winners: dict[str, str] = {} - for payload in _read_jsonl(self.path): - if str(payload.get("family")) != family or str(payload.get("outcome")) != "pair_correct": - continue - winners.setdefault(str(payload.get("pair")), str(payload.get("config_version"))) - return winners - - -def _read_jsonl(path: Path) -> list[Mapping[str, object]]: - if not path.is_file(): - return [] - rows: list[Mapping[str, object]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(payload, dict): - rows.append(payload) - return rows - - -def summarize(records: Sequence[RoundRecord]) -> dict[str, object]: - """What the rounds add up to: how often a prediction landed, and what it cost.""" - accepted = [item for item in records if item.outcome == "accepted"] - predicted = sum(item.attribution.flipped_predicted for item in records) - unpredicted = sum(item.attribution.flipped_unpredicted for item in records) - return { - "rounds": len(records), - "accepted": len(accepted), - "flipped_predicted": predicted, - "flipped_unpredicted": unpredicted, - "prediction_precision": predicted / (predicted + unpredicted) if predicted + unpredicted else 0.0, - "cost_usd": round(sum(item.cost_usd for item in records), 4), - } - - -__all__ = ["Archive", "Attribution", "JournalError", "LearningJournal", "Outcome", "RoundRecord", "summarize"] diff --git a/src/openultrasast/learning/proposer.py b/src/openultrasast/learning/proposer.py deleted file mode 100644 index 7881657..0000000 --- a/src/openultrasast/learning/proposer.py +++ /dev/null @@ -1,318 +0,0 @@ -"""What a round proposes, and who may propose it (learning-harness, Req 9.1, 9.2, 5.4, 7.6). - -A proposer is handed facts, never scores it is about to be judged on: the misses and leaks of one family -on the train split, what each run actually did, and the hypotheses this family already tried and lost. -Self-diagnosis from free narrative confabulates; structured facts are what make a proposal answerable. - -A proposal is one hypothesis, one lever, and one file inside one family's directory. The narrowness is -the safety property: a round can only touch what it targets, so every other family is frozen by -construction rather than by promise, and the verifier code and the corpus stay outside every write root. -""" - -from __future__ import annotations - -import json -import shutil -import tempfile -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import Literal, Protocol - -from ..harness_ext import has_harnessx -from .detectors import MAX_PROMPT_CHARS, FamilyConfig -from .journal import LearningJournal -from .scoring import PairFamilyScore - -Lever = Literal["tool", "checklist", "counterexample", "memory", "prompt"] -LEVERS: tuple[Lever, ...] = ("tool", "checklist", "counterexample", "memory", "prompt") -# The only files a round may write, one per lever. Anything else is a proposal reaching beyond its family. -EDITABLE_FILES: dict[Lever, str] = { - "tool": "family.toml", - "checklist": "checklist.md", - "counterexample": "counterexamples.jsonl", - "memory": "hard_negatives.jsonl", - "prompt": "prompt.md", -} - - -@dataclass(frozen=True) -class PairFact: - """What one pair did, in terms a proposer can act on.""" - - pair: str - outcome: str - runs: tuple[str, ...] = () - other_findings: int = 0 - rung: str = "suspicion" - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "outcome": self.outcome, - "runs": list(self.runs), - "other_findings": self.other_findings, - "rung": self.rung, - } - - -@dataclass(frozen=True) -class FailureFacts: - family: str - config: FamilyConfig - misses: tuple[PairFact, ...] = () - leaks: tuple[PairFact, ...] = () - rejected: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "config_version": self.config.version, - "tools": list(self.config.tools), - "misses": [item.to_dict() for item in self.misses], - "leaks": [item.to_dict() for item in self.leaks], - "rejected": list(self.rejected), - } - - -@dataclass(frozen=True) -class Proposal: - hypothesis: str - lever: Lever - change: Mapping[str, str] # file name inside the family directory -> its new contents - predicted_affected: tuple[str, ...] = () - predicted_at_risk: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return { - "hypothesis": self.hypothesis, - "lever": self.lever, - "change": dict(self.change), - "predicted_affected": list(self.predicted_affected), - "predicted_at_risk": list(self.predicted_at_risk), - } - - -class Proposer(Protocol): - def propose(self, facts: FailureFacts) -> Proposal | None: ... - - -def build_failure_facts( - scores: Sequence[PairFamilyScore], - *, - family: str, - config: FamilyConfig, - journal: LearningJournal, - train_pairs: set[str], -) -> FailureFacts: - """The facts of one family's failures on the train split. A holdout pair never appears here (Req 5.4).""" - mine = [item for item in scores if item.family == family and item.pair in train_pairs and item.outcome != "unscorable"] - return FailureFacts( - family=family, - config=config, - misses=tuple(_fact(item, item.other_findings_vuln) for item in mine if item.outcome in {"both_silent", "reversed"}), - leaks=tuple(_fact(item, item.other_findings_fixed) for item in mine if item.outcome in {"both_flagged", "reversed"}), - rejected=journal.rejected_buffer(family), - ) - - -def refuse_proposal(proposal: Proposal, config: FamilyConfig) -> str | None: - """None when the proposal is one legal change to its own family; otherwise why it is refused.""" - if proposal.lever not in LEVERS: - return f"unknown lever {proposal.lever!r}; expected one of {', '.join(LEVERS)}" - if len(proposal.change) != 1: - return f"a round changes one file, not {len(proposal.change)}" - name, content = next(iter(proposal.change.items())) - if "/" in name or "\\" in name or name.startswith("."): - return f"{name!r} reaches outside the family directory" - if name != EDITABLE_FILES[proposal.lever]: - return f"{name!r} is not the file the {proposal.lever!r} lever writes ({EDITABLE_FILES[proposal.lever]})" - if name == "prompt.md" and len(content) + len(config.checklist) > MAX_PROMPT_CHARS: - return f"the proposed prompt exceeds the cap of {MAX_PROMPT_CHARS} characters" - if name == "checklist.md" and len(content) + len(config.prompt) > MAX_PROMPT_CHARS: - return f"the proposed checklist exceeds the cap of {MAX_PROMPT_CHARS} characters" - return None - - -def apply_proposal(proposal: Proposal, directory: Path) -> None: - """Write the one file, inside the one directory. Callers snapshot first so a rejection can revert.""" - directory.mkdir(parents=True, exist_ok=True) - for name, content in proposal.change.items(): - (directory / name).write_text(content, encoding="utf-8") - - -def write_roots(configs_dir: Path, family: str) -> tuple[Path, ...]: - """Everything a round may write: exactly one family's directory, and never the verifiers or the corpus.""" - return (configs_dir / family,) - - -@dataclass -class ScriptedProposer: - """A fixed sequence of proposals, for tests and for the first offline rounds.""" - - proposals: list[Proposal] = field(default_factory=list) - used: int = 0 - - def propose(self, facts: FailureFacts) -> Proposal | None: - del facts - if self.used >= len(self.proposals): - return None - self.used += 1 - return self.proposals[self.used - 1] - - -MetaAgentCall = Callable[..., str] - - -@dataclass -class HarnessXProposer: - """The meta-agent, with write access limited to a copy of the family directory it is targeting. - - It never edits the live tree. The round snapshots the family directory *after* a proposal exists, so a proposer - that wrote in place would have changed the configuration before there was anything to restore, and a rejection - could not put it back. The agent works in a scratch copy and the proposal is the diff of that copy, which also - means the one-file rule and the lever whitelist are enforced on what the agent actually did rather than on what - it claimed (Req 9.1, 9.2, 5.4). - """ - - configs_dir: Path - model: str - # The vendor the meta-agent talks to. Hard-coding anthropic here sent it to a provider with no key configured - # while the detector was talking to DeepSeek, and the round journalled that as "no proposal". - provider: str = "anthropic" - reason: str = "" - agent: MetaAgentCall | None = None # injected in tests; None composes MetaAgent behind harness_ext - - def write_roots(self, family: str) -> tuple[Path, ...]: - return write_roots(self.configs_dir, family) - - def propose(self, facts: FailureFacts) -> Proposal | None: - call = self.agent - if call is None: - if not has_harnessx(): - self.reason = "harnessx_unavailable" - return None - try: - call = _meta_agent_call(self.provider) - except Exception: # noqa: BLE001 — a proposer that cannot start proposes nothing and says so - self.reason = "harnessx_unavailable" - return None - family_dir = self.configs_dir / facts.family - with tempfile.TemporaryDirectory(prefix="ousast-propose-") as scratch: - workspace = Path(scratch) / facts.family - shutil.copytree(family_dir, workspace) - before = _directory_text(workspace) - try: - hypothesis = call(workspace=workspace, facts=facts, model=self.model) - except (TypeError, AttributeError, ImportError): - # "we called the agent wrong" is not "the agent declined", and folding one into the other is how a - # broken composition reads as a quiet round for as long as nobody looks. - raise - except Exception as exc: # noqa: BLE001 — an agent that failed proposed nothing, and the round says why - self.reason = f"meta_agent_failed: {type(exc).__name__}" - return None - after = _directory_text(workspace) - changed = sorted(name for name in set(before) | set(after) if before.get(name) != after.get(name)) - if not changed: - self.reason = "no_change_proposed" - return None - if len(changed) > 1: - self.reason = f"a round changes one file, not {len(changed)}: {', '.join(changed)}" - return None - name = changed[0] - lever: Lever | None = next((key for key, filename in EDITABLE_FILES.items() if filename == name), None) - if lever is None or name not in after: - self.reason = f"{name!r} is not a file any lever writes" - return None - self.reason = "" - return Proposal( - hypothesis=str(hypothesis or "").strip() or f"meta-agent edit to {name}", - lever=lever, - change={name: after[name]}, - predicted_affected=tuple(item.pair for item in facts.misses), - predicted_at_risk=tuple(item.pair for item in facts.leaks), - ) - - -def _directory_text(directory: Path) -> dict[str, str]: - """Every readable file directly inside ``directory``, by name. Nested paths are not a lever's business.""" - out: dict[str, str] = {} - for path in sorted(directory.iterdir()) if directory.is_dir() else []: - if path.is_file(): - try: - out[path.name] = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - return out - - -def _meta_agent_class() -> type: - """`MetaAgent` behind `harness_ext`. A seam, so the argument contract can be pinned without the extra.""" - from ..harness_ext import require_harnessx - - require_harnessx() - from harnessx.meta_harness.agent import MetaAgent # type: ignore[import-not-found] - - return MetaAgent # type: ignore[no-any-return] - - -def _model_config(model: str, provider: str = "anthropic") -> object: - """`MetaAgent(inner_model=...)` takes a `ModelConfig`, never a model name; the name goes to the provider.""" - from ..harness_ext import build_provider, require_harnessx - - require_harnessx() - from harnessx.core.model_config import ModelConfig # type: ignore[import-not-found] - - return ModelConfig(main=build_provider(model, provider)) - - -def _meta_agent_call(provider: str = "anthropic") -> MetaAgentCall: - """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy. - - `evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the - workspace afterwards, so the return value is only used for the hypothesis line. - """ - from ..harness_ext import require_harnessx - - require_harnessx() - - def call(*, workspace: Path, facts: FailureFacts, model: str) -> str: - import asyncio - - trajectories = workspace.parent / "trajectories" - trajectories.mkdir(parents=True, exist_ok=True) - (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8") - agent = _meta_agent_class()(inner_model=_model_config(model, provider), allowed_write_roots=(workspace,)) - output = workspace.parent / "meta-out" - output.mkdir(parents=True, exist_ok=True) - asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output)) - return f"meta-agent round for {facts.family}" - - return call - - -def _fact(score: PairFamilyScore, other: int) -> PairFact: - return PairFact(pair=score.pair, outcome=score.outcome, runs=score.runs, other_findings=other, rung=score.rung) - - -def facts_prompt(facts: FailureFacts) -> str: - """The facts as the proposer sees them: structured, and free of anything it will be judged on.""" - return json.dumps(facts.to_dict(), indent=2, sort_keys=True) - - -__all__ = [ - "EDITABLE_FILES", - "LEVERS", - "FailureFacts", - "HarnessXProposer", - "Lever", - "PairFact", - "Proposal", - "Proposer", - "ScriptedProposer", - "apply_proposal", - "build_failure_facts", - "facts_prompt", - "refuse_proposal", - "write_roots", -] diff --git a/src/openultrasast/learning/publish.py b/src/openultrasast/learning/publish.py deleted file mode 100644 index afa9550..0000000 --- a/src/openultrasast/learning/publish.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Every published number, regenerated from the artifacts that produced it (learning-harness, Req 11.1, 11.2, 4.5, 5.3). - -A number in a roadmap that nobody can regenerate is a claim, not a measurement. This writes them all -from one entry point, out of the artifacts committed beside them, and puts the denominator next to each: -how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise -the family shows against itself, and what the run cost per pair it got right. - -Running it twice without new artifacts changes nothing, and the roadmap section is replaced in place -rather than appended, so the file cannot accumulate stale numbers beside fresh ones. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import UTC, datetime -from pathlib import Path - -BEGIN = "" -END = "" -COMMAND = "uv run ousast learning publish" - - -@dataclass(frozen=True) -class Baseline: - """One model's round-zero artifact, read back.""" - - model: str - slices: tuple[str, ...] = () - metrics: dict[str, Mapping[str, object]] = field(default_factory=dict) - floors: dict[str, Mapping[str, object]] = field(default_factory=dict) - cost_usd: float = 0.0 - k_runs: int = 0 - - -@dataclass(frozen=True) -class PublishReport: - command: str - table: str - families: tuple[str, ...] = () - models: tuple[str, ...] = () - artifacts: tuple[Path, ...] = field(default_factory=tuple) - - -def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: str | None = None) -> PublishReport: - """Regenerate the family numbers, write the artifact, and replace the roadmap section in place.""" - stamp = date or datetime.now(UTC).strftime("%Y-%m-%d") - models = _baselines(learning_dir) - families = tuple(sorted({family for baseline in models.values() for family in baseline.metrics})) - lever = _lever(measurements_dir) - table = _table(models, families, lever) - artifacts: list[Path] = [] - if models: - measurements_dir.mkdir(parents=True, exist_ok=True) - artifact = measurements_dir / f"{stamp}-learning-families.json" - artifact.write_text( - json.dumps( - { - "generated": stamp, - "command": COMMAND, - "models": {name: dict(baseline.metrics) for name, baseline in sorted(models.items())}, - "floors": {name: dict(baseline.floors) for name, baseline in sorted(models.items())}, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - artifacts.append(artifact) - _replace_section(roadmap, table) - _supersede(roadmap) - return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts)) - - -def _label(model: str, slices: tuple[str, ...]) -> str: - """What identifies a run in the published table: the model, and the slice when it is not the default pair.""" - return f"{model} ({', '.join(slices)})" if slices else model - - -def _baselines(learning_dir: Path) -> dict[str, Baseline]: - root = learning_dir / "baseline" - found: dict[str, Baseline] = {} - for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []: - report = directory / "report.json" - if not report.is_file(): - continue - payload = json.loads(report.read_text(encoding="utf-8")) - # Keyed by model *and* slice. Keyed by model alone, a second slice silently replaced the first and the - # published table showed one of them as if it were the corpus (Req 3.1, 3.2). - model = str(payload.get("model", directory.name)) - slices = tuple(str(item) for item in (payload.get("slices") or ())) - found[_label(model, slices)] = Baseline( - model=model, - slices=slices, - metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)}, - floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)}, - cost_usd=float(payload.get("cost_usd", 0.0) or 0.0), - k_runs=int(payload.get("k_runs", 0) or 0), - ) - return found - - -def _lever(measurements_dir: Path) -> Mapping[str, object] | None: - path = measurements_dir / "2026-09-06-mechanism-lever-split.json" - if not path.is_file(): - return None - payload = json.loads(path.read_text(encoding="utf-8")) - return payload if isinstance(payload, dict) else None - - -def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Mapping[str, object] | None) -> str: - if not models: - return ( - f"Generated by `{COMMAND}`.\n\nNo baseline has been recorded yet, so there is nothing to publish. " - "Run round zero first; a number without the run that produced it is a claim, not a measurement.\n" - ) - lines = [ - f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.", - "", - ( - "| model | slice | family | scorable | unscorable | recall | silence | Youden " - "| at fixed FPR | noise floor | cost per correct pair |" - ), - "|---|---|---|---|---|---|---|---|---|---|---|", - ] - for _label_key, baseline in sorted(models.items()): - model = baseline.model - slice_name = ", ".join(baseline.slices) or "all" - for family in families: - row = baseline.metrics.get(family) - if row is None: - continue - outcomes = row.get("outcomes") - raw_correct = outcomes.get("pair_correct") if isinstance(outcomes, Mapping) else None - correct = int(raw_correct) if isinstance(raw_correct, int | float) else 0 - floor = baseline.floors.get(family) - raw_budget = floor.get("budget_flips") if floor is not None else None - budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0 - cost = baseline.cost_usd - lines.append( - f"| {model} | {slice_name} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) " - f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} " - f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |" - ) - lines += ["", _credit_note(models), ""] - cutoff = _cutoff_note(models, families) - if cutoff: - lines += [cutoff, ""] - if lever is not None: - lines += [_lever_note(lever), ""] - return "\n".join(lines) - - -def _cutoff_note(models: Mapping[str, Baseline], families: tuple[str, ...]) -> str: - """The post-cutoff slice, its cutoff and the undated count (Req 10.5). - - A model that may have read the fix cannot be measured on it the same way as one that cannot have, so the rows - fixed after the detector's training cutoff are reported apart. Rows the corpus records no date for are in - neither slice, and their number is here rather than folded silently into the whole.""" - rows = [ - (model, family, baseline.metrics[family]) - for model, baseline in sorted(models.items()) - for family in families - if family in baseline.metrics and str(baseline.metrics[family].get("cutoff") or "") - ] - if not rows: - return "No detector cutoff is configured, so no post-cutoff slice is reported; every figure above is over the whole corpus." - cutoff = str(rows[0][2].get("cutoff")) - lines = [ - f"Post-cutoff slice, fixed after {cutoff} and therefore beyond the detector's training data. " - "A row counts only when its earliest possible date is after the cutoff, so a year-precision date can " - "under-claim this slice but never over-claim it.", - "", - "| model | family | post-cutoff scorable | post-cutoff recall | post-cutoff Youden | undated |", - "|---|---|---|---|---|---|", - ] - for model, family, row in rows: - block = row.get("post_cutoff") - block = block if isinstance(block, Mapping) else {} - lines.append( - f"| {model} | {family} | {block.get('scorable', 0)} | {_num(block.get('recall'))} " - f"| {_num(block.get('youden'))} | {row.get('undated', 0)} |" - ) - return "\n".join(lines) - - -def _credit_note(models: Mapping[str, Baseline]) -> str: - hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values()) - if hierarchical: - return "A parent-child answer earned partial credit in these figures." - return "No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above." - - -def _lever_note(lever: Mapping[str, object]) -> str: - before = lever.get("before") - after = lever.get("after") - before_map = before if isinstance(before, Mapping) else {} - after_map = after if isinstance(after, Mapping) else {} - pairs = lever.get("holdout_pairs", 0) - return ( - f"Mechanism lever on the {lever.get('slice', '')} holdout: detected {after_map.get('detected')} of {pairs} at Youden " - f"{after_map.get('youden')} once only train-split pairs teach, against detected {before_map.get('detected')} at Youden " - f"{before_map.get('youden')} when every pair taught. The difference is what holdout pairs stopped teaching." - ) - - -def _reasons(row: Mapping[str, object]) -> str: - reasons = row.get("unscorable") or {} - if not isinstance(reasons, Mapping) or not reasons: - return "none" - return ", ".join(f"{name}: {count}" for name, count in sorted(reasons.items())) - - -def _num(value: object) -> str: - return f"{float(value):.3f}" if isinstance(value, int | float) else "-" - - -def _cost(cost: float, correct: int) -> str: - return f"{cost / correct:.2f}" if cost and correct else "-" - - -# Claims the split correction supersedes, as they were written before holdout pairs stopped teaching. Req 5.3 says -# the corrected number is published *in place of* the earlier one; a document that carries both asserts two numbers -# for one measurement and the reader has no way to tell which run is current. -_SUPERSEDED = ( - re.compile(r"human holdout profile pair_correct \d+ -> \d+ of \d+, Youden [-+]?\d+ -> [-+]?[\d.]+(?:, round \d+ `[^`]+`)?\."), - re.compile(r"The lever recovered \d+ of \d+ holdout pairs at Youden [-+][\d.]+\."), -) -_SUPERSEDED_NOTE = ( - "superseded by the learning-harness section below: that number was measured with holdout pairs teaching " - "(see `benchmarks/measurements/2026-09-06-mechanism-lever-split.json`)." -) - - -def _supersede(roadmap: Path) -> None: - """Rewrite every superseded lever claim to point at the regenerated section. Idempotent: the pattern is gone - afterwards, so a second run changes nothing.""" - if not roadmap.is_file(): - return - text = roadmap.read_text(encoding="utf-8") - updated = text - for pattern in _SUPERSEDED: - updated = pattern.sub(_SUPERSEDED_NOTE, updated) - if updated != text: - roadmap.write_text(updated, encoding="utf-8") - - -def _replace_section(roadmap: Path, table: str) -> None: - body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n" - text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else "" - if BEGIN in text and END in text: - head, _, rest = text.partition(BEGIN) - _, _, tail = rest.partition(END) - roadmap.write_text(head + body + tail.lstrip("\n"), encoding="utf-8") - return - roadmap.parent.mkdir(parents=True, exist_ok=True) - roadmap.write_text((text.rstrip("\n") + "\n\n" if text else "") + body, encoding="utf-8") - - -__all__ = ["BEGIN", "COMMAND", "END", "Baseline", "PublishReport", "publish"] diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py deleted file mode 100644 index 89b8bd0..0000000 --- a/src/openultrasast/learning/rounds.py +++ /dev/null @@ -1,761 +0,0 @@ -"""Round zero and the per-family noise floor (learning-harness, Req 8). - -Before anything learns, every family gets the same detector and is measured against itself. Running a -pair several times at the same settings does not give the same answer: the chat provider offers no seed, -so the spread between runs is real and unavoidable. That spread is the family's noise floor, and it -becomes the budget a later round must stay inside before a cross-family change counts as a regression. -Choosing a budget instead of measuring one would be a guess dressed as a threshold. - -Artifacts are keyed by detector model, so a second model is baselined by changing the model and running -again. Configurations are never touched by a model swap, which is what makes the two numbers comparable -and what makes reconsidering the model cheap. -""" - -from __future__ import annotations - -import json -import math -import re -import tempfile -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..findings import StaticFinding -from ..pairs import PairCase -from .classify import classify_pair -from .detectors import FamilyConfig, write_default_configs -from .families import FamilyTaxonomy -from .journal import Archive, Attribution, LearningJournal, RoundRecord -from .proposer import Proposal, Proposer -from .scoring import FamilyMetrics, PairFamilyScore, aggregate, score_pair_family - -DEFAULT_SLICES = ("vibe-py", "agent-vfc") # the web families, where real-world code and dangerous fruit coincide -NON_GATING_FAMILIES = frozenset({"memory"}) # measured on the vendored slice, never allowed to fail a round -DEFAULT_K_RUNS = 5 -Scan = Callable[[Path], list[StaticFinding]] - - -@dataclass(frozen=True) -class NoiseFloor: - """How much a family disagrees with itself between runs, and the budget that follows from it.""" - - family: str - k_runs: int - pairs: int - flaky_pairs: int - negative_flip_rate: float - budget_flips: int - gates: bool = True - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "k_runs": self.k_runs, - "pairs": self.pairs, - "flaky_pairs": self.flaky_pairs, - "negative_flip_rate": self.negative_flip_rate, - "budget_flips": self.budget_flips, - "gates": self.gates, - } - - -@dataclass(frozen=True) -class BaselineReport: - model: str - k_runs: int - taxonomy_version: str - slices: tuple[str, ...] - thinking: bool = False # the condition that produced these numbers, so two modes cannot be read as one - floors: dict[str, NoiseFloor] = field(default_factory=dict) - metrics: dict[str, dict[str, object]] = field(default_factory=dict) - # `/`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one - # pooled number hides which of them moved (Req 8.4). - per_slice: dict[str, dict[str, object]] = field(default_factory=dict) - scores: tuple[PairFamilyScore, ...] = () - cost_usd: float = 0.0 - - def to_dict(self) -> dict[str, object]: - return { - "model": self.model, - "k_runs": self.k_runs, - "taxonomy_version": self.taxonomy_version, - "slices": list(self.slices), - "thinking": self.thinking, - "cost_usd": self.cost_usd, - "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())}, - "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())}, - "per_slice": {name: dict(block) for name, block in sorted(self.per_slice.items())}, - } - - -def run_baseline( - cases: Sequence[PairCase], - *, - taxonomy: FamilyTaxonomy, - configs_dir: Path, - scan_factory: Callable[[FamilyConfig], Scan], - model: str, - out_dir: Path, - slices: Sequence[str] = DEFAULT_SLICES, - k_runs: int = DEFAULT_K_RUNS, - prompt: str | None = None, - spent_usd: Callable[[], float] | None = None, - cutoff: str = "", - thinking: bool = False, - resume: bool = False, -) -> BaselineReport: - """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts. - - The factory is the point. One shared `scan` would score every family with one detector, and since the generalist - tags its findings `family:unknown` — which earns no credit against any real family, by the taxonomy's own - decision that abstention is not a partial answer — every family but `unknown` would baseline at a structural - zero and the noise floors would all be nil (Req 8.2). - """ - from ..tool_hunter import _SYSTEM_PROMPT - from .detectors import load_family_configs - - if k_runs < MIN_K_RUNS: - raise ValueError(f"round zero scores at least three runs per pair; {k_runs} was asked for") - if not configs_dir.exists(): - write_default_configs(configs_dir, taxonomy, prompt=prompt or _SYSTEM_PROMPT, version="0") - configs = load_family_configs(configs_dir, taxonomy) - selected = [case for case in cases if case.slice in set(slices)] - # Every pair is written down as it is measured. Round zero over the vfc slice is 1760 detector runs and hours of - # paid calls; accumulating in memory and persisting once at the end means a timeout, a kill or a machine going - # away costs all of it. An exception already cost only one pair; nothing else did. - checkpoint = out_dir / "baseline" / _artifact_key(model, slices, thinking) / "scores.jsonl" - done = _checkpointed(checkpoint) if resume else {} - if not resume and checkpoint.exists(): - checkpoint.unlink() - scores = [] - for case in selected: - name = _family_of(case, taxonomy) - config = configs.get(name) - if config is None: - continue # a family with no configuration is not measured rather than measured with someone else's - if case.name in done: - scores.append(done[case.name]) - continue - try: - score = score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name) - scores.append(score) - _append_score(checkpoint, score) - except Exception: # noqa: BLE001 — a provider hiccup costs one pair, not the whole run - # Round zero is the most expensive thing this harness does. Raising here threw away everything already - # measured and paid for, so the only honest response to a flaky endpoint was to run it all again. - unreachable = PairFamilyScore( - pair=case.name, - family=name, - slice=case.slice, - fix_date=case.fix_date, - runs=(), - outcome="unscorable", - unscorable_reason="detector_unreachable", - ) - scores.append(unreachable) - _append_score(checkpoint, unreachable) - metrics = aggregate(scores, taxonomy=taxonomy, cutoff=cutoff) - floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()} - report = BaselineReport( - model=model, - k_runs=k_runs, - taxonomy_version=taxonomy.version, - slices=tuple(slices), - thinking=thinking, - floors=floors, - metrics={name: block.to_dict() for name, block in metrics.items()}, - per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()}, - scores=tuple(scores), - cost_usd=spent_usd() if spent_usd is not None else 0.0, - ) - _write(out_dir, report) - return report - - -def _append_score(path: Path, score: PairFamilyScore) -> None: - """One line per pair, flushed, so an interrupted run keeps what it measured.""" - path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "pair": score.pair, - "family": score.family, - "slice": score.slice, - "fix_date": score.fix_date, - "outcome": score.outcome, - "runs": list(score.runs), - "flips": score.flips, - "other_findings_vuln": score.other_findings_vuln, - "other_findings_fixed": score.other_findings_fixed, - "fabricated_vuln": score.fabricated_vuln, - "fabricated_fixed": score.fabricated_fixed, - "rung": score.rung, - "unscorable_reason": score.unscorable_reason, - } - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, sort_keys=True) + "\n") - handle.flush() - - -def _checkpointed(path: Path) -> dict[str, PairFamilyScore]: - """What a previous run of this exact baseline already measured and paid for.""" - if not path.is_file(): - return {} - out: dict[str, PairFamilyScore] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue # a line half-written when the run died is not a measurement - out[str(row["pair"])] = PairFamilyScore( - pair=str(row["pair"]), - family=str(row["family"]), - slice=str(row.get("slice", "")), - fix_date=str(row.get("fix_date", "")), - runs=tuple(row.get("runs", ())), - outcome=row["outcome"], - flips=int(row.get("flips", 0)), - other_findings_vuln=int(row.get("other_findings_vuln", 0)), - other_findings_fixed=int(row.get("other_findings_fixed", 0)), - fabricated_vuln=int(row.get("fabricated_vuln", 0)), - fabricated_fixed=int(row.get("fabricated_fixed", 0)), - rung=row.get("rung", "suspicion"), - unscorable_reason=row.get("unscorable_reason"), - ) - return out - - -def score_case(case: PairCase, scan: Scan, *, taxonomy: FamilyTaxonomy, runs: int, family: str | None = None) -> PairFamilyScore: - """Run one detector over both sides of one pair ``runs`` times and score it for its family. - - A pair the corpus already marked unscorable is never run: spending model calls on a row that cannot - be scored buys nothing and would put its noise into a floor it does not belong in. - """ - from ..pairs import _materialize_side, _overlay_scan - - name = family or _family_of(case, taxonomy) - if case.unscorable: - return PairFamilyScore( - pair=case.name, - family=name, - runs=(), - outcome="unscorable", - slice=case.slice, - fix_date=case.fix_date, - unscorable_reason=case.unscorable, - ) - with tempfile.TemporaryDirectory(prefix="ousast-round-") as scratch: - vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") - fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - ranges = _overlay_scan(vuln_root).ranges - fixed_ranges = _overlay_scan(fix_root).ranges - vuln_runs = [scan(vuln_root) for _ in range(max(runs, 1))] - fix_runs = [scan(fix_root) for _ in range(max(runs, 1))] - return score_pair_family( - case, name, vuln_runs, fix_runs, ranges=ranges, taxonomy=taxonomy, parse_ok=bool(ranges), fixed_ranges=fixed_ranges - ) - - -def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]: - """Every baseline written under ``out_dir``, model by model, so two detectors can be read side by side.""" - root = out_dir / "baseline" - table: dict[str, dict[str, dict[str, object]]] = {} - for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []: - report = directory / "report.json" - if not report.is_file(): - continue - payload = json.loads(report.read_text(encoding="utf-8")) - model = str(payload.get("model", directory.name)) - slices = tuple(str(item) for item in (payload.get("slices") or ())) - conditions = [*(() if not payload.get("thinking") else ("thinking",)), *(() if slices == DEFAULT_SLICES else slices)] - label = f"{model} ({', '.join(conditions)})" if conditions else model - table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} - return table - - -def _floor(family: str, metrics: FamilyMetrics, scores: Sequence[PairFamilyScore], k_runs: int) -> NoiseFloor: - """A family's own disagreement between runs, as a rate and as a whole number of pairs.""" - scorable = [item for item in scores if item.outcome != "unscorable"] - flaky = sum(1 for item in scorable if item.flips) - rate = flaky / len(scorable) if scorable else 0.0 - return NoiseFloor( - family=family, - k_runs=k_runs, - pairs=metrics.scorable, - flaky_pairs=flaky, - negative_flip_rate=rate, - budget_flips=math.ceil(rate * metrics.scorable) if metrics.scorable else 0, - gates=family not in NON_GATING_FAMILIES, - ) - - -def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str: - for row in case.expected: - if row.family: - return str(row.family) - answer = classify_pair(case, taxonomy, client=None) - return answer.families[0] if answer.families else "unknown" - - -def _artifact_key(model: str, slices: Sequence[str], thinking: bool = False) -> str: - """What identifies a baseline run: the detector model, and the slice selection when it is not the default. - - Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and - the published table shows one of them as if it were everything (Req 8.4).""" - parts = [_slug(model)] - if thinking: - parts.append("thinking") - if tuple(slices) != DEFAULT_SLICES: - parts.append(_slug("-".join(sorted(slices)))) - return "-".join(parts) - - -def _write(out_dir: Path, report: BaselineReport) -> None: - directory = out_dir / "baseline" / _artifact_key(report.model, report.slices, report.thinking) - directory.mkdir(parents=True, exist_ok=True) - payload = report.to_dict() - (directory / "noise-floors.json").write_text( - json.dumps({"model": report.model, "k_runs": report.k_runs, "floors": payload["floors"]}, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - (directory / "report.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _slug(model: str) -> str: - return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed" - - -def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = (), thinking: bool = False) -> dict[str, NoiseFloor]: - """The budgets a later round must stay inside, as round zero measured them for this model.""" - path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES, thinking) / "noise-floors.json" - if not path.is_file(): - return {} - payload = json.loads(path.read_text(encoding="utf-8")) - floors: dict[str, NoiseFloor] = {} - for name, block in (payload.get("floors") or {}).items(): - if isinstance(block, Mapping): - floors[str(name)] = NoiseFloor( - family=str(block.get("family", name)), - k_runs=int(block.get("k_runs", DEFAULT_K_RUNS) or DEFAULT_K_RUNS), - pairs=int(block.get("pairs", 0) or 0), - flaky_pairs=int(block.get("flaky_pairs", 0) or 0), - negative_flip_rate=float(block.get("negative_flip_rate", 0.0) or 0.0), - budget_flips=int(block.get("budget_flips", 0) or 0), - gates=bool(block.get("gates", True)), - ) - return floors - - -MIN_K_RUNS = 3 # Req 3.5: a single run misses most real changes, so no stage may score with fewer - - -def run_learning_round( - cases: Sequence[PairCase], - *, - family: str, - taxonomy: FamilyTaxonomy, - configs_dir: Path, - proposer: Proposer, - scan_factory: Callable[[FamilyConfig], Scan], - model: str, - journal: LearningJournal, - archive: Archive, - floors: Mapping[str, NoiseFloor], - out_dir: Path, - cost_cap_usd: float = 10.0, - minibatch: int = 8, - k_runs: int = MIN_K_RUNS, - sweep_families: Sequence[str] = (), - spent_usd: Callable[[], float] | None = None, - trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None, -) -> RoundRecord: - """One round: propose one change for one family, then let the evidence decide whether it stays. - - Staged so that the cheap answer comes first: a train minibatch, then the family's held-out pairs, - then every other family. Each stage scores several runs per pair, because a single run misses most - real changes. The proposer never sees a held-out pair, and the family directory is snapshotted before - the change so any outcome other than acceptance leaves the tree exactly as it was found. - """ - from .acceptance import DirectorySnapshot, bump_version, decide, within_budget - from .detectors import load_family_configs - from .proposer import apply_proposal, build_failure_facts, refuse_proposal - from .split import is_teacher, refuse_if_holdout - - if k_runs < MIN_K_RUNS: - raise ValueError(f"a stage scores at least three runs per pair; {k_runs} was asked for") - round_number = journal.next_round() # type: ignore[attr-defined] - directory = out_dir / "rounds" / str(round_number) - if directory.exists(): - from .journal import JournalError - - raise JournalError(f"round {round_number} already has a directory; history is append-only") - configs = load_family_configs(configs_dir, taxonomy) - config = configs[family] - train = [case for case in cases if is_teacher(case) and _family_of(case, taxonomy) == family] - holdout = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == family] - batch = train[:minibatch] - spent = 0.0 - - def _score(rows: Sequence[PairCase], scan: Scan, name: str) -> list[PairFamilyScore]: - return [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows] - - def _over_cap() -> bool: - """Req 9.8: cost is read after every stage, so a round stops rather than finishing and then complaining.""" - nonlocal spent - spent = spent_usd() if spent_usd is not None else 0.0 - return not within_budget(spent, cost_cap_usd) - - record = _round_record(round_number, family, None, taxonomy, config) - try: - before_scan = scan_factory(config) - before_train = _score(batch, before_scan, family) - before_hold = _score(holdout, before_scan, family) - except Exception as exc: - # These stages run before anything is applied, so there is nothing to revert — but they have already spent - # money, and a round that leaves no record is a round nobody can account for. Measured: round 5 died on a - # network read during its first stage and vanished. - _finish( - journal, - directory, - record, - outcome="reverted", - reason=f"stage_failed: {type(exc).__name__}", - cost=spent_usd() if spent_usd is not None else 0.0, - trajectories=trajectories, - ) - raise - if _over_cap(): - return _finish( - journal, directory, record, outcome="reverted_cost", reason="cost_cap_exceeded", cost=spent, trajectories=trajectories - ) - facts = build_failure_facts(before_train, family=family, config=config, journal=journal, train_pairs={case.name for case in train}) - proposal = proposer.propose(facts) # type: ignore[attr-defined] - record = _round_record(round_number, family, proposal, taxonomy, config) - if proposal is None: - # `no_proposal` covers two different worlds: a proposer that considered the facts and declined, and one - # that could not start at all. Losing that distinction hides a fixable configuration error as a quiet round. - why = str(getattr(proposer, "reason", "") or "") - return _finish( - journal, - directory, - record, - outcome="rejected", - reason=f"no_proposal: {why}" if why else "no_proposal", - cost=spent, - trajectories=trajectories, - ) - refusal = refuse_proposal(proposal, config) - if refusal is not None: - return _finish( - journal, directory, record, outcome="rejected", reason=refusal, proposal=proposal, cost=spent, trajectories=trajectories - ) - leak = refuse_if_holdout(_pairs_named_by(proposal, cases), cases) - if leak is not None: - # Req 5.2: a change a holdout pair taught, or that names one, is refused by name rather than measured. - return _finish( - journal, - directory, - record, - outcome="refused_holdout", - reason=leak.degradation()["detail"], # type: ignore[arg-type] - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - # The sweep's "before" is measured with the unchanged configurations, so what it later reports is what this - # round flipped and not what was already failing (Req 9.5). - before_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs) - if _over_cap(): - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - snapshot = DirectorySnapshot.of(configs_dir / family) - apply_proposal(proposal, configs_dir / family) - version = str(int(config.version) + 1) if config.version.isdigit() else f"{config.version}+1" - try: - after_config = load_family_configs(configs_dir, taxonomy)[family] - after_scan = scan_factory(after_config) - after_train = _score(batch, after_scan, family) - after_hold = _score(holdout, after_scan, family) - for score in after_train + after_hold: - archive.record(family, version, score) # type: ignore[attr-defined] - if _over_cap(): - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - scores=after_train + after_hold, - trajectories=trajectories, - ) - after_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs) - sweep, flipped_unpredicted = _negative_flips(before_sweep, after_sweep) - train_delta = _delta(before_train, after_train) - holdout_delta = _delta(before_hold, after_hold) - flipped = _flipped(before_train + before_hold, after_train + after_hold) - train_moves = _moves(before_train, after_train) - holdout_moves = _moves(before_hold, after_hold) - attribution = Attribution( - flipped_predicted=len(flipped), - flipped_unpredicted=len(flipped_unpredicted), - precision=len(flipped) / (len(flipped) + len(flipped_unpredicted)) if flipped or flipped_unpredicted else 0.0, - ) - if _over_cap(): - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - scores=after_train + after_hold, - trajectories=trajectories, - ) - verdict = decide(target=family, train_delta=train_delta, holdout_delta=holdout_delta, sweep=sweep, floors=floors) # type: ignore[arg-type] - if not verdict.accepted: - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="rejected", - reason=verdict.reason, - proposal=proposal, - cost=spent, - train_delta=train_delta, - holdout_delta=holdout_delta, - sweep=sweep, - attribution=attribution, - moves=(train_moves, holdout_moves), - flipped=flipped, - unpredicted=flipped_unpredicted, - scores=after_train + after_hold, - trajectories=trajectories, - ) - bump_version(configs_dir / family, version) - return _finish( - journal, - directory, - record, - outcome="accepted", - reason=verdict.reason, - proposal=proposal, - cost=spent, - train_delta=train_delta, - holdout_delta=holdout_delta, - sweep=sweep, - attribution=attribution, - moves=(train_moves, holdout_moves), - flipped=flipped, - unpredicted=flipped_unpredicted, - scores=after_train + after_hold, - version=version, - trajectories=trajectories, - ) - except Exception as exc: - # A stage that raises has already spent money and already changed the tree. Both facts belong in the - # journal before the exception leaves, or the next round starts from a state nothing describes. - snapshot.restore() - _finish( - journal, - directory, - record, - outcome="reverted", - reason=f"stage_failed: {type(exc).__name__}", - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - raise - - -def _pairs_named_by(proposal: Proposal, cases: Sequence[PairCase]) -> tuple[str, ...]: - """Every corpus pair this proposal names, in its predictions or in the file it writes. - - A memory entry that quotes a held-out pair is the loop learning the answer instead of the shape, and it looks - exactly like a legitimate hard negative until the holdout number is read as if it meant something.""" - haystack = "\n".join([*proposal.predicted_affected, *proposal.predicted_at_risk, *proposal.change.values()]) - # Substring first, regex only for the handful that survive it: the corpus is a few hundred rows and a round - # runs this once, but a per-round sweep of one regex per row is the kind of thing that stops being free. - named = [ - name - for case in cases - if (name := str(getattr(case, "name", ""))) and name in haystack and re.search(rf"(? dict[str, dict[str, str]]: - """Every other family's held-out pairs, family by family, as `{family: {pair: outcome}}`. - - Called once before the change and once after it. Scoring both sides is the only way to say which pairs *this* - round broke: on a real corpus most pairs are failing already, and counting failures would reject every round - for damage it did not do (Req 9.5). Unscorable rows are dropped here rather than counted as failures (Req 4.1). - """ - outcomes: dict[str, dict[str, str]] = {} - for name in families: - if name == target or name not in configs: - continue - rows = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == name] - scan = scan_factory(configs[name]) - scored = [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows] - outcomes[name] = {item.pair: item.outcome for item in scored if item.outcome != "unscorable"} - return outcomes - - -def _negative_flips(before: Mapping[str, Mapping[str, str]], after: Mapping[str, Mapping[str, str]]) -> tuple[dict[str, int], list[str]]: - """Pairs that were right before this round and are not right after it, per family and by name.""" - flips: dict[str, int] = {} - broken: list[str] = [] - for name, was in before.items(): - now = after.get(name, {}) - lost = sorted(pair for pair, outcome in was.items() if outcome == "pair_correct" and now.get(pair) != "pair_correct") - flips[name] = len(lost) - broken.extend(lost) - return flips, broken - - -def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> float: - """The change in how many pairs came out right, as a fraction of the pairs that could be scored.""" - scorable = [item for item in before if item.outcome != "unscorable"] - if not scorable: - return 0.0 - was = sum(1 for item in before if item.outcome == "pair_correct") - now = sum(1 for item in after if item.outcome == "pair_correct") - return (now - was) / len(scorable) - - -def _moves(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> tuple[int, int]: - """(pairs that became correct, pairs that stopped being correct) — what the sign test is run over.""" - was = {item.pair: item.outcome for item in before if item.outcome != "unscorable"} - better = worse = 0 - for item in after: - previous = was.get(item.pair) - if previous is None or item.outcome == "unscorable": - continue - if previous != "pair_correct" and item.outcome == "pair_correct": - better += 1 - elif previous == "pair_correct" and item.outcome != "pair_correct": - worse += 1 - return better, worse - - -def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]: - """Pairs this round turned right, by name and in a stable order.""" - was = {item.pair: item.outcome for item in before} - return sorted(item.pair for item in after if item.outcome == "pair_correct" and was.get(item.pair) not in {"pair_correct", None}) - - -def _round_record( - round_number: int, family: str, proposal: Proposal | None, taxonomy: FamilyTaxonomy, config: FamilyConfig -) -> dict[str, object]: - return { - "round": round_number, - "family": family, - "hypothesis": proposal.hypothesis if proposal is not None else "", - "levers": (proposal.lever,) if proposal is not None else (), - "predicted_affected": tuple(proposal.predicted_affected) if proposal is not None else (), - "predicted_at_risk": tuple(proposal.predicted_at_risk) if proposal is not None else (), - "taxonomy_version": taxonomy.version, - "config_version": config.version, - } - - -def _finish( - journal: LearningJournal, - directory: Path, - base: dict[str, object], - *, - outcome: str, - reason: str, - proposal: Proposal | None = None, - cost: float = 0.0, - train_delta: float = 0.0, - holdout_delta: float = 0.0, - sweep: Mapping[str, int] | None = None, - attribution: Attribution | None = None, - moves: tuple[tuple[int, int], tuple[int, int]] = ((0, 0), (0, 0)), - flipped: Sequence[str] = (), - unpredicted: Sequence[str] = (), - scores: Sequence[PairFamilyScore] = (), - version: str | None = None, - trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None, -) -> RoundRecord: - """Write the round directory and journal the outcome. Every outcome is journalled, not just the good ones.""" - from ..redaction import redact_secrets - - record = RoundRecord( - **base, # type: ignore[arg-type] - outcome=outcome, - reason=reason, - target_train_delta=train_delta, - target_holdout_delta=holdout_delta, - sweep=dict(sweep or {}), - attribution=attribution or Attribution(), - cost_usd=cost, - train_better=moves[0][0], - train_worse=moves[0][1], - holdout_better=moves[1][0], - holdout_worse=moves[1][1], - ) - if version is not None: - record = RoundRecord(**{**record.__dict__, "config_version": version}) - directory.mkdir(parents=True, exist_ok=True) - if proposal is not None: - (directory / "proposal.json").write_text( - redact_secrets(json.dumps(proposal.to_dict(), indent=2, sort_keys=True)) + "\n", - encoding="utf-8", - ) - (directory / "scores.json").write_text( - json.dumps([{"pair": item.pair, "family": item.family, "outcome": item.outcome} for item in scores], indent=2, sort_keys=True) - + "\n", - encoding="utf-8", - ) - (directory / "attribution.json").write_text( - json.dumps({"flipped_predicted": list(flipped), "flipped_unpredicted": list(unpredicted)}, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - rows = list(trajectories() if trajectories is not None else ()) - (directory / "trajectories.jsonl").write_text( - "".join(redact_secrets(json.dumps(row, sort_keys=True)) + "\n" for row in rows), encoding="utf-8" - ) - journal.append(record) # type: ignore[attr-defined] - return record - - -__all__ = [ - "DEFAULT_K_RUNS", - "MIN_K_RUNS", - "run_learning_round", - "DEFAULT_SLICES", - "NON_GATING_FAMILIES", - "BaselineReport", - "NoiseFloor", - "Scan", - "compare_baselines", - "load_noise_floors", - "run_baseline", - "score_case", -] diff --git a/src/openultrasast/learning/scoring.py b/src/openultrasast/learning/scoring.py deleted file mode 100644 index b324e36..0000000 --- a/src/openultrasast/learning/scoring.py +++ /dev/null @@ -1,405 +0,0 @@ -"""Class-aware, pair-wise scoring (learning-harness, Req 3, 4.1, 4.5). - -A pair is scored once per family. Detection means a finding whose family tag relates to the labeled -family, sitting inside the labeled function; the words in a finding are never read, because a text -token was exactly the proxy that made the old hunter number meaningless. Silence means no such finding -on the fixed side: findings of other families and findings outside the labeled function are counted -separately and are never called leaks. A pair that cannot be scored keeps its reason and is never run. - -Every evaluation runs a pair K times, because a single run misses most real changes. The recorded -outcome is the majority of the runs, and the runs that disagree with it are the pair's own flakiness. -""" - -from __future__ import annotations - -import math -from collections import Counter -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from typing import Literal - -from ..findings import StaticFinding -from .families import FamilyTaxonomy - -FamilyOutcome = Literal["pair_correct", "both_flagged", "both_silent", "reversed", "unscorable"] -# Families whose bug shape is the absence of a check on a reached handler: without the registration that -# makes the handler reachable, the pair says nothing either way, which is a corpus defect with its own name. -CONTEXT_DEPENDENT_FAMILIES = frozenset({"access_control"}) - - -def is_post_cutoff(fix_date: str, cutoff: str) -> bool: - """True when the *earliest possible* date of ``fix_date`` is after ``cutoff`` (Req 10.5). - - The corpus records dates at the precision it has: a full ISO day for a few rows, a year for almost all of them. - A year-precision row counts only when the whole year is after the cutoff, so a coarse date can under-claim the - clean slice but never over-claim it — and over-claiming is the one that would turn contamination into a result. - """ - if not fix_date or len(cutoff) != 10: - return False # a cutoff coarser than a day is the one over-claim path there is, so it is refused outright - earliest = fix_date if len(fix_date) == 10 else f"{fix_date}-01-01"[:10] - return earliest > cutoff - - -Rung = Literal["suspicion", "static_corroboration", "proven"] -_RUNGS: tuple[Rung, ...] = ("suspicion", "static_corroboration", "proven") -FunctionRanges = Mapping[str, Sequence[tuple[str, int, int]]] - - -@dataclass(frozen=True) -class PairFamilyScore: - """What one pair says about one family.""" - - pair: str - family: str - runs: tuple[FamilyOutcome, ...] - outcome: FamilyOutcome - slice: str = "" - other_findings_vuln: int = 0 # right place, another family: not a detection, never a leak - other_findings_fixed: int = 0 - fabricated_vuln: int = 0 # a family id outside the taxonomy: noise, and worth naming - fabricated_fixed: int = 0 - flips: int = 0 # runs that disagreed with the majority - rung: Rung = "suspicion" - unscorable_reason: str | None = None - fix_date: str = "" # the date the corpus recorded, at whatever precision it has (Req 10.5) - - -def score_pair_family( - case: object, - family: str, - vuln_runs: Sequence[Sequence[StaticFinding]], - fixed_runs: Sequence[Sequence[StaticFinding]], - *, - ranges: FunctionRanges, - taxonomy: FamilyTaxonomy, - parse_ok: bool = True, - entry_points: Sequence[str] | None = None, - fixed_ranges: FunctionRanges | None = None, -) -> PairFamilyScore: - """Score one pair for one family over K runs of the detector on each side. - - ``fixed_ranges`` is the *fixed* side's parse. A fix that adds a guard above the handler moves its first line, - so judging the fixed side against the vulnerable side's span reads a finding inside the fixed handler as - outside it: a leak recorded as silence, and always in the direction that flatters the number. - """ - name = str(getattr(case, "name", "?")) - slice_name = str(getattr(case, "slice", "") or "") - reason = unscorable_reason(case, parse_ok=parse_ok, ranges=ranges, entry_points=entry_points, family=family) - if reason is not None: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason=reason, - ) - spans = _spans(case, ranges) - if fixed_ranges is None: - fixed_spans = spans - else: - # A trap pair's fixed side holds a different, correctly guarded handler; `fix_function` names it. Without a - # span there, every fixed-side finding reads as outside the labeled function and the pair scores as silence. - fixed_spans = _spans(case, fixed_ranges, function=str(getattr(case, "fix_function", "") or "")) - if not fixed_spans: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason="unresolved_label:fixed", - ) - outcomes: list[FamilyOutcome] = [] - other_vuln = other_fixed = fabricated_vuln = fabricated_fixed = 0 - rung: Rung = "suspicion" - for index in range(max(len(vuln_runs), len(fixed_runs))): - vuln = list(vuln_runs[index]) if index < len(vuln_runs) else [] - fixed = list(fixed_runs[index]) if index < len(fixed_runs) else [] - hits_vuln = _hits(vuln, family, spans, taxonomy) - hits_fixed = _hits(fixed, family, fixed_spans, taxonomy) - other_vuln += _others(vuln, family, spans, taxonomy) - other_fixed += _others(fixed, family, fixed_spans, taxonomy) - fabricated_vuln += _fabricated(vuln, taxonomy) - fabricated_fixed += _fabricated(fixed, taxonomy) - rung = _best_rung(rung, hits_vuln) - outcomes.append(_outcome(bool(hits_vuln), bool(hits_fixed))) - if not outcomes: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason="no_runs", - ) - majority = Counter(outcomes).most_common(1)[0][0] - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=tuple(outcomes), - outcome=majority, - other_findings_vuln=other_vuln, - other_findings_fixed=other_fixed, - fabricated_vuln=fabricated_vuln, - fabricated_fixed=fabricated_fixed, - flips=sum(1 for item in outcomes if item != majority), - rung=rung, - ) - - -def _outcome(detected: bool, leaked: bool) -> FamilyOutcome: - if detected and not leaked: - return "pair_correct" - if detected and leaked: - return "both_flagged" - return "reversed" if leaked else "both_silent" - - -def unscorable_reason( - case: object, - *, - parse_ok: bool = True, - ranges: FunctionRanges, - entry_points: Sequence[str] | None = None, - family: str = "", -) -> str | None: - """Why this pair cannot be scored, or None. The reasons have different owners, so they must not be conflated. - - A maintainer-declared limit wins over anything computed. An unparsable side is a tooling gap - (``unsupported_language``), a labeled function no parsed range names is a corpus labelling defect - (``unresolved_label``), and a handler that arrived without the registration that makes it reachable is a - harvest defect (``missing_context``), which only matters for families whose bug shape is a missing check. - """ - declared = getattr(case, "unscorable", None) - if declared: - return str(declared) - if not parse_ok: - return "unsupported_language" - if not _spans(case, ranges): - return "unresolved_label" - labeled_family = family or next( - (str(getattr(row, "family", "") or "") for row in getattr(case, "expected", ()) or () if getattr(row, "family", None)), "" - ) - if entry_points is not None and labeled_family in CONTEXT_DEPENDENT_FAMILIES: - labeled = {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} - if not (labeled & set(entry_points)): - return "missing_context" - return None - - -def _spans(case: object, ranges: FunctionRanges, *, function: str = "") -> tuple[tuple[str, int, int], ...]: - """The (path, start, end) spans of the labeled functions. ``function`` overrides the label for the fixed side.""" - wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} - wanted.discard("") - found: list[tuple[str, int, int]] = [] - for path, entries in ranges.items(): - for name, start, end in entries: - if name in wanted: - found.append((path, start, end)) - return tuple(found) - - -def _inside(finding: StaticFinding, spans: Sequence[tuple[str, int, int]]) -> bool: - return finding.line is not None and any(path == finding.path and start <= finding.line <= end for path, start, end in spans) - - -def _families_of(finding: StaticFinding) -> tuple[str, ...]: - return tuple(tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")) - - -def _relation(finding: StaticFinding, family: str, taxonomy: FamilyTaxonomy) -> str: - relations = [taxonomy.related(family, candidate) for candidate in _families_of(finding)] - for wanted in ("same", "parent_child", "lateral"): - if wanted in relations: - return wanted - return "fabricated" if relations else "none" - - -def _hits( - findings: Sequence[StaticFinding], family: str, spans: Sequence[tuple[str, int, int]], taxonomy: FamilyTaxonomy -) -> list[StaticFinding]: - return [finding for finding in findings if _inside(finding, spans) and _relation(finding, family, taxonomy) in {"same", "parent_child"}] - - -def _others(findings: Sequence[StaticFinding], family: str, spans: Sequence[tuple[str, int, int]], taxonomy: FamilyTaxonomy) -> int: - return sum(1 for finding in findings if _relation(finding, family, taxonomy) == "lateral" or not _inside(finding, spans)) - - -def _fabricated(findings: Sequence[StaticFinding], taxonomy: FamilyTaxonomy) -> int: - known = {item.id for item in taxonomy.families} - return sum(1 for finding in findings if any(candidate not in known for candidate in _families_of(finding))) - - -def _best_rung(current: Rung, hits: Sequence[StaticFinding]) -> Rung: - best = current - for finding in hits: - for tag in finding.tags: - if tag.startswith("verifier:"): - candidate = tag.split(":", 1)[1] - if candidate in _RUNGS and _RUNGS.index(candidate) > _RUNGS.index(best): # type: ignore[arg-type] - best = candidate # type: ignore[assignment] - return best - - -@dataclass(frozen=True) -class FamilyMetrics: - """One family's numbers, with the denominator they were computed over and why rows fell out of it.""" - - family: str - taxonomy_version: str - scorable: int - unscorable: dict[str, int] = field(default_factory=dict) - outcomes: dict[str, int] = field(default_factory=dict) - recall: float = 0.0 # flagged the vulnerable side - silence: float = 0.0 # said nothing on the fixed side - youden: float = 0.0 - directional_bias: float = 0.0 # positive: flags both sides; negative: silent on both. Youden cannot tell them apart. - leak_rate: float = 0.0 - fixed_fpr_recall: float = 0.0 # recall, forfeited entirely when the leak rate exceeds the ceiling - negative_flips: int = 0 # pairs that were correct in the baseline and are not now - positive_flips: int = 0 - p_value: float | None = None # two-sided sign test over the discordant pairs - reliable_change: bool = False - hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 3.7) - cutoff: str = "" # the detector cutoff these numbers were split on, or "" when none is configured (Req 10.5) - undated: int = 0 # rows the corpus records no date for; they are in neither slice and are counted here - post_cutoff: dict[str, object] = field(default_factory=dict) # the same numbers over rows fixed after the cutoff - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "taxonomy_version": self.taxonomy_version, - "scorable": self.scorable, - "unscorable": dict(sorted(self.unscorable.items())), - "outcomes": dict(sorted(self.outcomes.items())), - "recall": self.recall, - "silence": self.silence, - "youden": self.youden, - "directional_bias": self.directional_bias, - "leak_rate": self.leak_rate, - "fixed_fpr_recall": self.fixed_fpr_recall, - "negative_flips": self.negative_flips, - "positive_flips": self.positive_flips, - "p_value": self.p_value, - "reliable_change": self.reliable_change, - "hierarchical_credit": self.hierarchical_credit, - "cutoff": self.cutoff, - "undated": self.undated, - "post_cutoff": dict(self.post_cutoff), - } - - -def aggregate( - scores: Sequence[PairFamilyScore], - *, - taxonomy: FamilyTaxonomy, - baseline: Sequence[PairFamilyScore] | None = None, - fpr_ceiling: float = 0.1, - per_slice: bool = False, - alpha: float = 0.05, - cutoff: str = "", -) -> dict[str, FamilyMetrics]: - """Group scores into per-family numbers. Unscorable rows are listed by reason and leave every denominator. - - With a ``cutoff`` each family also reports the same numbers over the rows fixed after it, beside the whole - number and never instead of it: a model that may have read the fix cannot be measured on it the same way as one - that cannot have. Rows the corpus records no date for are counted on their own (Req 10.5).""" - grouped: dict[str, list[PairFamilyScore]] = {} - for score in scores: - grouped.setdefault(f"{score.slice}/{score.family}" if per_slice else score.family, []).append(score) - was_correct = {(item.slice if per_slice else "", item.family, item.pair): item.outcome for item in baseline or ()} - out: dict[str, FamilyMetrics] = {} - for key, group in sorted(grouped.items()): - scorable = [item for item in group if item.outcome != "unscorable"] - reasons: dict[str, int] = {} - for item in group: - if item.outcome == "unscorable": - reason = item.unscorable_reason or "unknown" - reasons[reason] = reasons.get(reason, 0) + 1 - counts = Counter(item.outcome for item in scorable) - total = len(scorable) - detected = counts["pair_correct"] + counts["both_flagged"] - silent = counts["pair_correct"] + counts["both_silent"] - leaks = counts["both_flagged"] + counts["reversed"] - recall = detected / total if total else 0.0 - silence = silent / total if total else 0.0 - leak_rate = leaks / total if total else 0.0 - negative = positive = 0 - for item in scorable: - before = was_correct.get((item.slice if per_slice else "", item.family, item.pair)) - if before is None or before == "unscorable": - continue - if before == "pair_correct" and item.outcome != "pair_correct": - negative += 1 - elif before != "pair_correct" and item.outcome == "pair_correct": - positive += 1 - p_value = sign_test(positive, negative) if baseline is not None else None - recent = [item for item in scorable if is_post_cutoff(item.fix_date, cutoff)] if cutoff else [] - out[key] = FamilyMetrics( - family=group[0].family, - taxonomy_version=taxonomy.version, - scorable=total, - unscorable=reasons, - outcomes={name: count for name, count in sorted(counts.items())}, - recall=recall, - silence=silence, - youden=(recall + silence - 1.0) if total else 0.0, - directional_bias=((counts["both_flagged"] - counts["both_silent"]) / total) if total else 0.0, - leak_rate=leak_rate, - fixed_fpr_recall=recall if leak_rate <= fpr_ceiling else 0.0, - negative_flips=negative, - positive_flips=positive, - p_value=p_value, - reliable_change=bool(p_value is not None and p_value < alpha and positive != negative), - hierarchical_credit=taxonomy.hierarchical, - cutoff=cutoff, - undated=sum(1 for item in group if not item.fix_date) if cutoff else 0, - post_cutoff=_slice_numbers(recent) if cutoff else {}, - ) - return out - - -def _slice_numbers(scores: Sequence[PairFamilyScore]) -> dict[str, object]: - """Recall, silence and Youden over a subset, with the subset's own denominator beside them.""" - total = len(scores) - counts = Counter(item.outcome for item in scores) - recall = (counts["pair_correct"] + counts["both_flagged"]) / total if total else 0.0 - silence = (counts["pair_correct"] + counts["both_silent"]) / total if total else 0.0 - return { - "scorable": total, - "outcomes": {name: count for name, count in sorted(counts.items())}, - "recall": recall, - "silence": silence, - "youden": (recall + silence - 1.0) if total else 0.0, - } - - -def sign_test(better: int, worse: int) -> float: - """Two-sided exact sign test over the pairs that changed. 1.0 when nothing changed, so noise never reads as progress.""" - total = better + worse - if total == 0: - return 1.0 - smaller = min(better, worse) - tail = sum(math.comb(total, index) for index in range(smaller + 1)) / float(2**total) - return float(min(1.0, 2.0 * tail)) - - -__all__ = [ - "CONTEXT_DEPENDENT_FAMILIES", - "FamilyMetrics", - "FamilyOutcome", - "FunctionRanges", - "PairFamilyScore", - "Rung", - "aggregate", - "is_post_cutoff", - "score_pair_family", - "sign_test", - "unscorable_reason", -] diff --git a/src/openultrasast/learning/split.py b/src/openultrasast/learning/split.py deleted file mode 100644 index af3129b..0000000 --- a/src/openultrasast/learning/split.py +++ /dev/null @@ -1,64 +0,0 @@ -"""One rule for who may teach, and one refusal for anything that touches the holdout (Req 5.1, 5.2). - -Every learning path in the tool asks this module the same question, so the split cannot be enforced in -one place and forgotten in another. A teacher is a vendored pair, at a review tier a maintainer stands -behind, on the train split, and scorable. Everything else may be measured; nothing else may teach. - -A refusal is data, not an exception: the path records which holdout pairs a candidate touched and moves -on, so a leak is visible in the run rather than silently accepted. -""" - -from __future__ import annotations - -from collections.abc import Iterable, Sequence -from dataclasses import dataclass - -GATING_TIERS = frozenset({"seeded", "reviewed"}) -TRAIN = "train" -HOLDOUT = "holdout" -REFUSED = "holdout_pair_refused" - - -@dataclass(frozen=True) -class Refusal: - """A candidate that a holdout pair taught, or that touches one.""" - - pairs: tuple[str, ...] - reason: str = REFUSED - - def degradation(self) -> dict[str, object]: - return { - "stage": "learning", - "reason": self.reason, - "pairs": list(self.pairs), - "detail": f"refused: taught by or touching holdout pair(s) {', '.join(self.pairs)}", - } - - -def teachers(cases: Sequence[object]) -> tuple[object, ...]: - """The pairs a learning path may learn from. Everything else is measured, never learned from.""" - return tuple(case for case in cases if is_teacher(case)) - - -def is_teacher(case: object) -> bool: - if not bool(getattr(case, "vendored", True)): - return False - if str(getattr(case, "review_tier", "")) not in GATING_TIERS: - return False - if str(getattr(case, "split", TRAIN)) != TRAIN: - return False - return getattr(case, "unscorable", None) is None - - -def holdout_names(cases: Sequence[object]) -> frozenset[str]: - return frozenset(str(getattr(case, "name", "")) for case in cases if str(getattr(case, "split", "")) == HOLDOUT) - - -def refuse_if_holdout(taught_by: Iterable[str], cases: Sequence[object]) -> Refusal | None: - """None when nothing in ``taught_by`` is a holdout pair; a refusal naming them when something is.""" - held = holdout_names(cases) - touched = tuple(sorted({name for name in taught_by if name in held})) - return Refusal(pairs=touched) if touched else None - - -__all__ = ["GATING_TIERS", "HOLDOUT", "REFUSED", "TRAIN", "Refusal", "holdout_names", "is_teacher", "refuse_if_holdout", "teachers"] diff --git a/src/openultrasast/learning/verifiers.py b/src/openultrasast/learning/verifiers.py deleted file mode 100644 index 5e3063e..0000000 --- a/src/openultrasast/learning/verifiers.py +++ /dev/null @@ -1,140 +0,0 @@ -"""What may raise a claim above suspicion, keyed by family (learning-harness, Req 7.1, 7.3, 7.4). - -A detector's claim is a suspicion until something other than the model agrees with it. What counts as -agreement depends on the family: a canary oracle observed in the sandbox, corroboration from a static -checker, or nothing at all. A family whose verifier does not exist yet reports at suspicion rather than -pretending, and that is the honest default rather than a gap. - -The canary oracle is a function over a sandbox result, so the sandbox keeps deciding what happened and -this module only says what would count. Exit codes are not oracles: a crash and a demonstrated -injection are different evidence, and one family's oracle must not silently answer for another. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Literal, Protocol - -from ..findings import StaticFinding -from ..sandbox import SandboxResult -from ..verification import EvidenceLevel -from .families import Family - -Rung = Literal["suspicion", "static_corroboration", "proven"] -RUNGS: tuple[Rung, ...] = ("suspicion", "static_corroboration", "proven") -VerifierKind = Literal["canary", "static", "none"] - - -@dataclass(frozen=True) -class Verification: - """What a verifier saw, and how high it lets the claim go.""" - - rung: Rung - reason: str - oracle_output: str = "" - - -class Verifier(Protocol): - @property - def kind(self) -> VerifierKind: ... - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: ... - - -def token_oracle(token: str) -> Callable[[SandboxResult], bool]: - """The oracle a canary snippet is written against: the token appears only if the vulnerability fired.""" - - def fired(result: SandboxResult) -> bool: - return token in result.stdout or token in result.stderr - - return fired - - -@dataclass(frozen=True) -class NoVerifier: - """Nothing can confirm this family yet, so the claim stays where the detector left it.""" - - kind: VerifierKind = "none" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - del claim, root, language - return Verification(rung="suspicion", reason="no_verifier") - - -@dataclass(frozen=True) -class StaticVerifier: - """Another checker looked at the same function and agrees the obligation is undischarged.""" - - kind: VerifierKind = "static" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - del language - from ..hunter_tools import PathEscapesRepo, obligations - - if not claim.function_name: - return Verification(rung="suspicion", reason="no_function") - try: - rows = obligations(root, path=claim.path, function=claim.function_name) - except (PathEscapesRepo, OSError): - return Verification(rung="suspicion", reason="checker_unavailable") - if not rows: - return Verification(rung="suspicion", reason="checker_disagrees") - summary = "; ".join(f"{row.get('operation')} missing {row.get('missing')}" for row in rows) - return Verification(rung="static_corroboration", reason="checker_agrees", oracle_output=summary) - - -def verifier_for(family: Family, *, canary: Verifier | None = None) -> Verifier: - """The verifier a family declares. A canary family with no implementation registered stays at suspicion.""" - if family.verifier == "static": - return StaticVerifier() - if family.verifier == "canary": - return canary if canary is not None else NoVerifier(kind="canary") - return NoVerifier() - - -def apply_verification(claim: StaticFinding, verification: Verification) -> StaticFinding: - """A copy of the claim carrying what the verifier said, as a tag. - - The evidence ladder belongs to another boundary and this spec does not change it, so nothing here - writes ``evidence_level``. "proven" is this spec's word, not a rung the ladder defines, and stamping - it onto the field made the project's own verification helpers raise. A caller that wants to move a - claim up the ladder does it through :func:`raise_to`, which can only produce a level the ladder has. - """ - tags = [tag for tag in claim.tags if not tag.startswith("verifier:")] + [f"verifier:{verification.rung}"] - return replace(claim, tags=tags) - - -def evidence_level_for(rung: Rung) -> EvidenceLevel: - """The project's evidence level a verifier rung corresponds to. The only bridge between the two vocabularies.""" - return _LADDER[rung] - - -def raise_to(claim: StaticFinding, rung: Rung) -> StaticFinding: - """A copy of the claim at the ladder level this rung corresponds to. Never a value the ladder lacks.""" - return replace(claim, evidence_level=str(evidence_level_for(rung))) - - -_LADDER: dict[Rung, EvidenceLevel] = { - "suspicion": EvidenceLevel.SUSPICION, - "static_corroboration": EvidenceLevel.STATIC_CORROBORATION, - # A canary that fired in the sandbox demonstrated the vulnerability, which is what this level means here. - "proven": EvidenceLevel.EXPLOIT_DEMONSTRATED, -} - - -__all__ = [ - "RUNGS", - "NoVerifier", - "Rung", - "StaticVerifier", - "Verification", - "Verifier", - "VerifierKind", - "apply_verification", - "evidence_level_for", - "raise_to", - "token_oracle", - "verifier_for", -] diff --git a/src/openultrasast/model/__init__.py b/src/openultrasast/model/__init__.py new file mode 100644 index 0000000..2b15ae3 --- /dev/null +++ b/src/openultrasast/model/__init__.py @@ -0,0 +1,7 @@ +"""The model layer: a deterministic program model that arbitrates detector claims. + +The taxonomy routes work, the candidate enumerator feeds the `suspicion` band, the specs carry the security +vocabularies, and (from group 2) the CPG-backed taint and dominance checks establish the rungs above it. +""" + +from __future__ import annotations diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py new file mode 100644 index 0000000..38a32b8 --- /dev/null +++ b/src/openultrasast/model/audit.py @@ -0,0 +1,166 @@ +"""Module audit: what is load-bearing, what stands alone, and what is orphaned (Req 3). + +The tool outgrew itself — 99 modules and 832 tests at the point this feature began, a large share of them +serving an architecture its own measurements retired. This classifier exists so outgrowth is visible and +checkable rather than a thing someone notices two specs later. + +A module is: + + load_bearing reachable from a gate, the scan CLI, or a surviving requirement + standalone_capability its own entry point and users, not on the scan path + orphaned no importer, no entry point, no requirement — it should not be in the tree + +Pure stdlib, no imports of the package it audits: it reads the import graph with ``ast`` so a broken module +is still classifiable. +""" + +from __future__ import annotations + +import ast +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path + +PACKAGE = "openultrasast" + +# Entry points a module can be reachable from. The three gates are the contract; the CLI is the product. +ROOTS = ("gate", "map_gate", "pair_gate", "cli", "__main__") + +# Modules that stand alone: their own CLI subcommand or external protocol, deliberately off the scan path. +SPEC_OWNED = { + "model.specs": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by", + "model.endpoint": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3", + "model.audit": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan", +} + +STANDALONE = { + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate", + "slot_contract": "the slot/contract protocol stage_processors implements", +} + + +@dataclass(frozen=True) +class ModuleAudit: + module: str + classification: str # load_bearing | standalone_capability | orphaned + reason: str + importers: tuple[str, ...] + + +def module_name(path: Path, root: Path) -> str: + relative = path.relative_to(root).with_suffix("") + parts = [part for part in relative.parts if part != "__init__"] + return ".".join(parts) + + +def import_graph(root: Path) -> dict[str, set[str]]: + """``module -> the modules it imports`` for every module under ``root``, resolved to package-relative names.""" + graph: dict[str, set[str]] = {} + for path in sorted(root.rglob("*.py")): + if "__pycache__" in path.parts: + continue + name = module_name(path, root) + # For a package's `__init__`, the containing package *is* this module, so `from .x import y` resolves to + # `.x`; for a plain module it resolves against the parent. Getting this backwards made every module + # a package re-exports look unimported. + is_package = path.name == "__init__.py" + imported: set[str] = set() + try: + tree = ast.parse(path.read_text()) + except (OSError, SyntaxError): + graph[name] = imported + continue + package_parts = name.split(".") if is_package else name.split(".")[:-1] + package_parts = [part for part in package_parts if part] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level: + base = package_parts[: len(package_parts) - node.level + 1] + target = ".".join([*base, node.module]) if node.module else ".".join(base) + else: + target = node.module or "" + if target.startswith(f"{PACKAGE}."): + target = target[len(PACKAGE) + 1 :] + elif target == PACKAGE: + target = "" + else: + continue + if target: + imported.add(target) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith(f"{PACKAGE}."): + imported.add(alias.name[len(PACKAGE) + 1 :]) + graph[name] = imported + return graph + + +def _reachable(graph: Mapping[str, set[str]], roots: Iterable[str]) -> set[str]: + seen: set[str] = set() + stack = [root for root in roots if root in graph] + while stack: + current = stack.pop() + if current in seen: + continue + seen.add(current) + for target in graph.get(current, ()): + # an import of `a.b` also keeps the package `a` alive + for candidate in (target, target.rsplit(".", 1)[0]): + if candidate in graph and candidate not in seen: + stack.append(candidate) + return seen + + +def audit(root: Path) -> tuple[ModuleAudit, ...]: + """Classify every module under ``root``.""" + graph = import_graph(root) + importers: dict[str, set[str]] = {name: set() for name in graph} + for name, targets in graph.items(): + for target in targets: + for candidate in (target, target.rsplit(".", 1)[0]): + if candidate in importers and candidate != name: + importers[candidate].add(name) + live = _reachable(graph, ROOTS) + results: list[ModuleAudit] = [] + for name in sorted(graph): + if not name: # the package __init__ itself + continue + who = tuple(sorted(importers[name])) + if name in SPEC_OWNED: + results.append(ModuleAudit(name, "load_bearing", SPEC_OWNED[name], who)) + elif name in ROOTS: + results.append(ModuleAudit(name, "load_bearing", "entry point", who)) + elif name in STANDALONE: + results.append(ModuleAudit(name, "standalone_capability", STANDALONE[name], who)) + elif name in live: + results.append(ModuleAudit(name, "load_bearing", f"reachable from {', '.join(ROOTS[:3])} or the CLI", who)) + elif who: + results.append(ModuleAudit(name, "load_bearing", f"imported by {', '.join(who)}", who)) + else: + results.append(ModuleAudit(name, "orphaned", "no importer, no entry point, no surviving requirement", who)) + return tuple(results) + + +def to_dict(results: Iterable[ModuleAudit]) -> dict[str, object]: + rows = list(results) + counts: dict[str, int] = {} + for row in rows: + counts[row.classification] = counts.get(row.classification, 0) + 1 + return { + "modules": len(rows), + "counts": dict(sorted(counts.items())), + "orphaned": [row.module for row in rows if row.classification == "orphaned"], + "standalone": {row.module: row.reason for row in rows if row.classification == "standalone_capability"}, + "rows": [ + {"module": row.module, "classification": row.classification, "reason": row.reason, "importers": list(row.importers)} + for row in rows + ], + } + + +__all__ = ["ModuleAudit", "SPEC_OWNED", "STANDALONE", "audit", "import_graph", "module_name", "to_dict"] diff --git a/src/openultrasast/learning/candidates.py b/src/openultrasast/model/candidates.py similarity index 88% rename from src/openultrasast/learning/candidates.py rename to src/openultrasast/model/candidates.py index e1908be..f2f1ac7 100644 --- a/src/openultrasast/learning/candidates.py +++ b/src/openultrasast/model/candidates.py @@ -15,7 +15,7 @@ the sequence of model calls, are determined by the input alone. from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -263,22 +263,20 @@ def ceiling(cases: Sequence[object], *, taxonomy: object, generator: str = "ir") import tempfile from ..pairs import _materialize_side, _overlay_scan, _quick_scan - from .rounds import _family_of - from .scoring import _spans families: dict[str, dict[str, int]] = {} slices: dict[str, dict[str, int]] = {} gaps: list[tuple[str, str, str]] = [] sites: dict[str, list[int]] = {} for case in cases: - family = _family_of(case, taxonomy) # type: ignore[arg-type] + family = labeled_family(case) slice_name = str(getattr(case, "slice", "") or "") region_path = str(getattr(case, "relpath", "") or "") with tempfile.TemporaryDirectory(prefix="ousast-ceiling-") as scratch: root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") # type: ignore[arg-type] if generator == "ruleset": ranges = _overlay_scan(root).ranges - spans = _spans(case, ranges) + spans = labeled_spans(case, ranges) found = [ finding for finding in _quick_scan(root) @@ -289,7 +287,7 @@ def ceiling(cases: Sequence[object], *, taxonomy: object, generator: str = "ir") labeled = [str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()] result = enumerate_candidates( root, - _Region(region_path, next((name for name in labeled if name), None)), + Region(region_path, next((name for name in labeled if name), None)), family, taxonomy=taxonomy, limit=10**6, # the ceiling is about coverage, never about the batch bound @@ -331,9 +329,45 @@ def _distribution(counts: list[int]) -> dict[str, int]: } +def labeled_family(case: object) -> str: + """The family a pair is labeled with, or ``unknown``. + + The deleted ``learning/rounds.py`` fell back to the LLM classifier here. Nothing does now: an unlabeled + pair is honestly ``unknown`` rather than guessed, which is the same discipline the evidence ladder applies + to a claim the model cannot arbitrate. + """ + for row in getattr(case, "expected", ()) or (): + family = getattr(row, "family", None) + if family: + return str(family) + return "unknown" + + +def labeled_spans( + case: object, ranges: Mapping[str, Sequence[tuple[str, int, int]]], *, function: str = "" +) -> tuple[tuple[str, int, int], ...]: + """The (path, start, end) spans of the labeled functions; ``function`` overrides the label for the fixed side. + + Re-homed from the deleted ``learning/scoring.py`` — the enumerator's ceiling needs it to know which region a + candidate had to fall inside, and it carries none of the K-run machinery that surrounded it. + """ + wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} + wanted.discard("") + found: list[tuple[str, int, int]] = [] + for path, entries in ranges.items(): + for name, start, end in entries: + if name in wanted: + found.append((path, start, end)) + return tuple(found) + + @dataclass(frozen=True) -class _Region: - """The two fields `enumerate_candidates` reads, so the ceiling does not import the detector module.""" +class Region: + """The two fields ``enumerate_candidates`` reads: where to look, and which function bounds the search. + + Public since the detector module that used to own this type was removed with the noise architecture — the + enumerator still needs a name for its input, and callers still need one to construct. + """ path: str function: str | None @@ -347,5 +381,8 @@ __all__ = [ "CandidateKind", "CandidateReport", "CandidateSet", + "Region", "enumerate_candidates", + "labeled_family", + "labeled_spans", ] diff --git a/src/openultrasast/learning/endpoint.py b/src/openultrasast/model/endpoint.py similarity index 96% rename from src/openultrasast/learning/endpoint.py rename to src/openultrasast/model/endpoint.py index a734d30..24c82fa 100644 --- a/src/openultrasast/learning/endpoint.py +++ b/src/openultrasast/model/endpoint.py @@ -154,7 +154,7 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None if configured or key_env: key = os.environ.get(key_env or DEEPSEEK_KEY_ENV, "") base_url = configured or DEEPSEEK_BASE_URL - return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.learning.thinking) + return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.models.thinking) deepseek_key = os.environ.get(DEEPSEEK_KEY_ENV) if deepseek_key: return _deepseek_client( @@ -162,7 +162,7 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), model, provider="deepseek", - thinking=config.learning.thinking, + thinking=config.models.thinking, ) if os.environ.get("OPENROUTER_API_KEY"): try: @@ -175,9 +175,9 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None def resolve_models(config: ResolvedConfig) -> tuple[str, str]: - """(detector model, judge model). `[learning]` wins, then `[models]`, then the shipped defaults.""" - detector = config.learning.detector_model or config.models.hunter or DEFAULT_DETECTOR_MODEL - judge = config.learning.judge_model or config.models.judge or DEFAULT_JUDGE_MODEL + """(detector model, judge model) from `[models]`, then the shipped defaults.""" + detector = config.models.hunter or DEFAULT_DETECTOR_MODEL + judge = config.models.judge or DEFAULT_JUDGE_MODEL return detector, judge diff --git a/src/openultrasast/model/specs.py b/src/openultrasast/model/specs.py new file mode 100644 index 0000000..d8dd24d --- /dev/null +++ b/src/openultrasast/model/specs.py @@ -0,0 +1,156 @@ +"""The security vocabularies the model layer arbitrates with (model-grounded-detection, Req 7.2-7.4). + +Two kinds of knowledge live here, and they are held differently on purpose. + +The **guard vocabulary** is ported verbatim from ``semantic/variants.py``, whose optimisation loop this +feature deleted. Nothing else holds it: it is the closed set of ways a fix discharges a flow, and it tells +the model what a *fixed* twin looks like. Order matters — first match wins. + +The **taint and dominance specs** are *derived* from data that survives: the source/sink/sanitizer facts in +``ruleset/semantic`` and the obligated operations and dischargers in ``ruleset/obligations``. Copying them +here would create a second place to update, so this module reads them through the retained loaders and +groups them by family via the taxonomy's CWE map. + +Read-only by construction (Req 7.4, the verifier boundary): this module exposes no writer, opens no file for +writing, and holds no mutable module state. An LLM or a future optimiser has nothing here to rewrite. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass + +from ..semantic.facts import SemanticFacts, load_facts +from ..semantic.obligations.facts import ObligationFacts, load_obligation_facts +from .taxonomy import FamilyTaxonomy, load_families + +# --- The guard vocabulary (ported verbatim; nothing else holds it) ---------------------------------------- + +GUARD_KINDS = ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") + +# Closed guard classification over the statements a fix added inside the labeled function. +# Order matters: the first matching kind wins. These classify the *fix*, they never detect. +GUARD_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("parameterized_call", re.compile(r"execute\w*\([^,]+,\s*[\(\[]|prepareStatement\(|\?\s*['\"]?\s*[,)]|\$\d\b|%s['\"]\s*,\s*[\(\[]")), + ( + "auth_check", + re.compile( + r"\b(request\.user|req\.user|current_user|request\.state|session\[|is_authenticated|authoriz|require_auth" + r"|login_required|verify_token|check_permission|owner_id\s*[!=]=)" + ), + ), + ( + "allowlist_test", + re.compile( + r"\bnot in\b|\bin\s*[\(\{\[]|\.includes\(|\.has\(|\bstartswith\(|\bstartsWith\(|\bendswith\(|\bendsWith\(" + r"|re\.(?:match|fullmatch)\(|\.test\(|allow_?list|whitelist|\bALLOWED\b|\bsafe_join\(|secure_filename\(" + r"|\brealpath\(|\bnormpath\(|\bresolve\(" + ), + ), + ("bounds_test", re.compile(r"\blen\(|\.length\b|\bsizeof\(|\bstrn?len\(|\bsnprintf\(|\bstrlcpy\(|\bmin\(|\bmax\(|\s(?:<|>|<=|>=)\s")), + ( + "null_test", + re.compile( + r"\bis None\b|\bis not None\b|[!=]= *NULL\b|[!=]==? *null\b|\bnullptr\b|\bif\s*\(\s*!\s*\w|\bif not \w+\s*:" + r"|\bif\s+\w+\s+is\s+None|\?\?|\?\." + ), + ), + ( + "type_change", + re.compile(r"\bint\(|\bparseInt\(|\bNumber\(|\(int\)|\(size_t\)|\(unsigned\b|\bstr\(|\.toString\(|\bstatic_cast<|\bBigInt\("), + ), +) + + +def classify_guard_text(text: str) -> str: + """The guard kind the given statements express, or ``none`` when no known guard is recognised.""" + for kind, pattern in GUARD_PATTERNS: + if pattern.search(text): + return kind + return "none" + + +# --- The specs the CPG queries are parameterised by ------------------------------------------------------ + + +@dataclass(frozen=True) +class TaintSpec: + """What a flow family's source→sink−sanitizer query matches on, for one language.""" + + family: str + language: str + sources: tuple[str, ...] + sinks: tuple[str, ...] + sanitizers: tuple[str, ...] + + +@dataclass(frozen=True) +class DominanceSpec: + """What an absence family's guard-dominates-operation query matches on, for one language. + + ``dischargers`` are the guards that discharge an obligation; the query asks whether one of them dominates + the operation on every path, which is the arbiter for the classes that never crash. + """ + + family: str + language: str + operations: tuple[str, ...] + dischargers: tuple[str, ...] + + +def taint_specs( + *, + language: str, + facts: SemanticFacts | None = None, + taxonomy: FamilyTaxonomy | None = None, +) -> Mapping[str, TaintSpec]: + """One ``TaintSpec`` per flow family that has a sink in this language, grouped through the taxonomy's CWE map. + + Sources and sanitizers are language-wide: a family does not get its own notion of what untrusted input is, + and a sanitizer that breaks one flow breaks it whatever the sink's family. + """ + scoped = (facts if facts is not None else load_facts()).for_language(language) + families = taxonomy if taxonomy is not None else load_families() + sources = tuple(sorted({pattern for fact in scoped.sources for pattern in fact.patterns})) + sanitizers = tuple(sorted({call for fact in scoped.sanitizers for call in fact.calls})) + by_family: dict[str, set[str]] = {} + for sink in scoped.sinks: + family = families.family_of_cwe(sink.cwe) + if family is None: + continue + by_family.setdefault(family.id, set()).update(sink.calls) + return { + family_id: TaintSpec(family=family_id, language=language, sources=sources, sinks=tuple(sorted(calls)), sanitizers=sanitizers) + for family_id, calls in sorted(by_family.items()) + } + + +def dominance_specs(*, language: str, facts: ObligationFacts | None = None) -> Mapping[str, DominanceSpec]: + """One ``DominanceSpec`` per absence family for this language. + + The obligation facts are authored for authorization, so every obligated operation routes to + ``access_control``; the shape generalises when a second absence family is added to the taxonomy. + """ + scoped = (facts if facts is not None else load_obligation_facts()).for_language(language) + if not scoped.operations: + return {} + operations = tuple(sorted({call for fact in scoped.operations for call in fact.calls})) + dischargers = tuple( + sorted( + {call for fact in scoped.dischargers for call in fact.calls} + | {decorator for fact in scoped.dischargers for decorator in fact.decorators} + ) + ) + return {"access_control": DominanceSpec(family="access_control", language=language, operations=operations, dischargers=dischargers)} + + +__all__ = [ + "GUARD_KINDS", + "GUARD_PATTERNS", + "DominanceSpec", + "TaintSpec", + "classify_guard_text", + "dominance_specs", + "taint_specs", +] diff --git a/src/openultrasast/learning/families.py b/src/openultrasast/model/taxonomy.py similarity index 100% rename from src/openultrasast/learning/families.py rename to src/openultrasast/model/taxonomy.py diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index e25b146..dee1536 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -20,7 +20,7 @@ import os import shutil import tempfile import tomllib -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import asdict, dataclass, field, replace from functools import lru_cache from pathlib import Path @@ -98,7 +98,6 @@ _DEFAULT_TIER = { _DEFAULT_PROVENANCE = {"sast": "synthetic", "local": "human", "github": "human", "vfc": "human"} _MANIFEST_MECHANISM = "source_reaches_sink" # cheat-sheet fixtures referenced via expected_from -HunterScan = Callable[[Path], list[StaticFinding]] class CatalogError(ValueError): @@ -173,7 +172,6 @@ class PairOutcome: unadjudicated_vuln: int = 0 unadjudicated_fixed: int = 0 detection_kinds: tuple[str, ...] = () - family: str = "" # the family this pair was scored under (hunter path only) unscorable: str | None = None # why it could not be scored; it then leaves every denominator unresolved_labels: int = 0 # expected rows whose function names no range on the vulnerable side review_tier: str = "advisory" @@ -205,7 +203,6 @@ class PairEvalResult: per_profile: dict[str, PairCorpusMetrics] = field(default_factory=dict) per_tier: dict[str, PairCorpusMetrics] = field(default_factory=dict) per_mechanism: dict[str, PairCorpusMetrics] = field(default_factory=dict) - per_family: dict[str, dict[str, object]] = field(default_factory=dict) # learning-harness Req 3.3: family metrics beside the rest achievable: dict[str, PairCorpusMetrics] = field(default_factory=dict) known_limit: tuple[str, ...] = () loss: dict[str, dict[str, int]] = field(default_factory=dict) @@ -427,15 +424,11 @@ def _review_tier(item: dict[str, object], slice_name: str, name: str) -> tuple[s def evaluate_pair( case: PairCase, *, - hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None, - k_runs: int | None = None, ) -> PairOutcome: with tempfile.TemporaryDirectory(prefix="ousast-pair-") as scratch: vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - if hunter is not None: - return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter, _k_runs(k_runs)) if case.slice in OVERLAY_SLICES: return _evaluate_overlay_pair(case, vuln_root, fix_root, ruleset) return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset) @@ -444,19 +437,15 @@ def evaluate_pair( def evaluate_catalog( cases: Sequence[PairCase], *, - hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None, inventory_sidecar: bool = True, pointers: bool | None = None, - k_runs: int | None = None, ) -> PairEvalResult: """Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan. Pointer pairs (``vendored = false``) are harvested into the cache when ``pointers`` (or OPENULTRASAST_PAIRS_NETWORK=1) allows the network, otherwise skipped with a ``pointer_pair_skipped`` degradation (Req 10.2, 10.3). """ - if hunter is not None: - _k_runs(k_runs) # refuse an impossible K before spending a single call degradations: list[dict[str, object]] = [] ready: list[PairCase] = [] for case in cases: @@ -468,18 +457,11 @@ def evaluate_catalog( cases = tuple(ready) outcomes = tuple(evaluate_pair(case, ruleset=ruleset) for case in cases) scorers: dict[str, dict[str, PairCorpusMetrics]] = {} - hunter_outcomes: list[PairOutcome] = [] # the family scorer runs on the hunter path only for slice_name in sorted({case.slice for case in cases if case.slice in OVERLAY_SLICES}): slice_cases = tuple(case for case in cases if case.slice == slice_name) scorers[slice_name] = {"overlay": _metrics(tuple(item for item in outcomes if item.slice == slice_name))} if inventory_sidecar: scorers[slice_name]["inventory"] = _metrics(tuple(_inventory_only(case, ruleset) for case in slice_cases)) - if hunter is not None: - hunted = tuple(evaluate_pair(case, hunter=hunter, k_runs=k_runs) for case in slice_cases) - hunter_outcomes.extend(hunted) - scorers[slice_name]["hunter"] = _metrics(hunted) - if hunter is None and scorers: - degradations.append({"stage": "pairs", "reason": "hunter_model_unavailable"}) achievable_outcomes = tuple(item for item in outcomes if item.known_limit is None) per_mechanism: dict[str, list[PairOutcome]] = {} for item in outcomes: @@ -503,7 +485,6 @@ def evaluate_catalog( for tier in sorted({item.review_tier for item in outcomes}) }, per_mechanism={mechanism: _metrics(tuple(items)) for mechanism, items in sorted(per_mechanism.items())}, - per_family=_family_metrics(hunter_outcomes), achievable={ slice_name: _metrics(tuple(item for item in achievable_outcomes if item.slice == slice_name)) for slice_name in sorted({item.slice for item in achievable_outcomes}) @@ -514,43 +495,6 @@ def evaluate_catalog( ) -def _family_metrics(outcomes: Sequence[PairOutcome]) -> dict[str, dict[str, object]]: - """Per-family numbers for the pairs the family scorer ran, with unscorable rows out of the denominator.""" - from .learning.families import load_families - from .learning.scoring import PairFamilyScore, aggregate - - scored = [item for item in outcomes if item.family] - if not scored: - return {} - taxonomy = load_families() - scores = [ - PairFamilyScore( - pair=item.name, - family=item.family, - runs=() - if item.unscorable - else ( - ( - ("pair_correct" if item.pair_correct else "both_flagged") - if item.detected_vuln - else ("reversed" if not item.silent_fix else "both_silent") - ), - ), - outcome="unscorable" - if item.unscorable - else ( - ("pair_correct" if item.pair_correct else "both_flagged") - if item.detected_vuln - else ("reversed" if not item.silent_fix else "both_silent") - ), - slice=item.slice, - unscorable_reason=item.unscorable, - ) - for item in scored - ] - return {name: metrics.to_dict() for name, metrics in aggregate(scores, taxonomy=taxonomy).items()} - - def build_pair_signals(outcomes: Sequence[PairOutcome], *, split: str | None = None) -> list[dict[str, object]]: """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms. @@ -737,7 +681,6 @@ def result_payload(result: PairEvalResult) -> dict[str, object]: "per_profile": {name: asdict(metrics) for name, metrics in result.per_profile.items()}, "per_tier": {name: asdict(metrics) for name, metrics in result.per_tier.items()}, "per_mechanism": {name: asdict(metrics) for name, metrics in result.per_mechanism.items()}, - "per_family": dict(result.per_family), "achievable": {name: asdict(metrics) for name, metrics in result.achievable.items()}, "known_limit": list(result.known_limit), "loss": {name: dict(counts) for name, counts in result.loss.items()}, @@ -964,53 +907,6 @@ def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFin return out -def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None, taxonomy: object | None = None) -> HunterScan: - """Run the tool hunter over every file of an isolated pair tree (Req 1.7). - - With a ``taxonomy`` the hunter is asked to name the family of every finding, from the closed list, and the answer - becomes the finding's `family:` tag. The class-aware scorer credits a detection only to a family, so a generalist - that names none can never be scored — it would report recall zero by construction rather than by measurement. - """ - from .complexity.map import Hotspot - from .tool_hunter import _SYSTEM_PROMPT, DEFAULT_MAX_STEPS, run_tool_hunter - - prompt = _SYSTEM_PROMPT - if taxonomy is not None: - ids = ", ".join(family.id for family in taxonomy.families) # type: ignore[attr-defined] - prompt = ( - f'{_SYSTEM_PROMPT}\n\nGive every finding a "family" field, chosen from exactly this list: {ids}. ' - 'Use "unknown" when none of them fits; do not invent a name.' - ) - - def scan(root: Path) -> list[StaticFinding]: - hotspots = [ - Hotspot( - path=target.path, - function_name=None, - score=1.0, - band="high", - signals={}, - rationale="pair corpus function", - test_hint=None, - inventory_finding_ids=(), - ) - for target in _targets(root) - ] - return _with_functions( - root, - run_tool_hunter( - root, - hotspots, - client=client, # type: ignore[arg-type] - model=model, - max_steps=max_steps or DEFAULT_MAX_STEPS, - system_prompt=prompt, - ), - ) - - return scan - - def _inventory_outcome( case: PairCase, vuln_findings: list[StaticFinding], @@ -1146,98 +1042,6 @@ def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path, rule ) -def _k_runs(value: int | None) -> int: - """Req 3.5: three runs per pair per side is the floor, and the default, because one run misses most changes.""" - from .learning.rounds import MIN_K_RUNS - - runs = MIN_K_RUNS if value is None else int(value) - if runs < MIN_K_RUNS: - raise ValueError(f"the hunter path scores at least three runs per pair; {runs} was asked for") - return runs - - -def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan, runs: int) -> PairOutcome: - """Score the model-driven hunter by family and function (learning-harness Req 3.1-3.3, 3.5). - - Detection is a finding of the labeled family inside the labeled function; a finding of another family - or outside it is counted and never called a leak; the words in a finding are never read. A pair that - cannot be scored is reported with its reason and the detector is never run on it. - """ - from .learning.families import load_families - from .learning.scoring import score_pair_family, unscorable_reason - - taxonomy = load_families() - vuln_ranges = _overlay_scan(vuln_root).ranges - reason = unscorable_reason(case, parse_ok=bool(vuln_ranges), ranges=vuln_ranges) - family = _family_of(case, taxonomy) - if reason is not None: - return _hunter_outcome(case, family=family, unscorable=reason) - fix_ranges = _overlay_scan(fix_root).ranges - vuln_runs = [hunter(vuln_root) for _ in range(runs)] - fix_runs = [hunter(fix_root) for _ in range(runs)] - score = score_pair_family(case, family, vuln_runs, fix_runs, ranges=vuln_ranges, taxonomy=taxonomy, fixed_ranges=fix_ranges) - return _hunter_outcome(case, family=family, score=score, vuln=vuln_runs[0], fixed=fix_runs[0]) - - -def _family_of(case: PairCase, taxonomy: object) -> str: - """The family this pair is scored under: its label when it has one, else what the classifier says.""" - from .learning.classify import classify_pair - - for row in case.expected: - label = getattr(row, "family", None) - if label: - return str(label) - answer = classify_pair(case, taxonomy, client=None) # type: ignore[arg-type] - return answer.families[0] if answer.families else "unknown" - - -def _hunter_outcome( - case: PairCase, - *, - family: str, - score: object | None = None, - vuln: Sequence[StaticFinding] = (), - fixed: Sequence[StaticFinding] = (), - unscorable: str | None = None, -) -> PairOutcome: - outcome = getattr(score, "outcome", "unscorable") - detected = outcome in {"pair_correct", "both_flagged"} - silent = outcome in {"pair_correct", "both_silent"} - leaks = tuple(finding.finding_id for finding in fixed) if outcome in {"both_flagged", "reversed"} else () - misses = () if detected else tuple(f"{row.rule_id or '-'}:{row.path}:{row.cwe}" for row in case.expected) - return PairOutcome( - name=case.name, - slice=case.slice, - language=case.language, - origin=case.origin, - detected_vuln=detected, - silent_fix=silent, - pair_correct=outcome == "pair_correct", - vuln_matched=1 if detected else 0, - vuln_expected=len(case.expected), - vuln_findings=len(vuln), - fix_findings=len(fixed), - fix_leaks=len(leaks), - recall=1.0 if detected else 0.0, - misses=misses, - leaks=leaks, - commit_url=case.commit_url, - cve=case.cve, - provenance=case.provenance, - review_tier=case.review_tier, - mechanisms=_mechanisms(case), - known_limit=case.known_limit, - split=case.split, - weak_labels=0, - detection_kinds=("hunter",) if detected else (), - family=family, - unscorable=unscorable or (str(getattr(score, "unscorable_reason", "") or "") or None), - ) - - -# --- matching --------------------------------------------------------------- - - def _adjudicated(records: Sequence[OverlayRecord]) -> bool: return any(record.disposition in {"promote", "demote", "coverage"} for record in records) diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index 0e1b513..5f2637c 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -22,7 +22,6 @@ def write_markdown_report( mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligations: Mapping[str, Mapping[str, object]] | None = None, obligations_summary: Mapping[str, object] | None = None, - learning: Mapping[str, Mapping[str, object]] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications or []) @@ -73,8 +72,6 @@ def write_markdown_report( _append_mechanisms(lines, cited) if cited_obligations: _append_obligations(lines, cited_obligations, obligations_summary) - if learning: - _append_learning(lines, learning) if overlay: _append_overlay(lines, overlay) if complexity_map is not None: @@ -158,7 +155,6 @@ def write_sarif_report( overlay: Sequence[object] | None = None, mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligations: Mapping[str, Mapping[str, object]] | None = None, - learning: Mapping[str, Mapping[str, object]] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) @@ -183,7 +179,6 @@ def write_sarif_report( overlay_by_id.get(finding.finding_id), mechanisms=mechanisms, obligation=(obligations or {}).get(finding.finding_id), - learning=(learning or {}).get(finding.finding_id), ) for finding in findings ], @@ -209,7 +204,6 @@ def write_manifest( provenance: dict[str, object] | None = None, variants: dict[str, object] | None = None, obligations: dict[str, object] | None = None, - learning: dict[str, object] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) @@ -250,8 +244,6 @@ def write_manifest( payload["variants"] = variants if obligations is not None: payload["obligations"] = obligations - if learning: - payload["learning"] = learning path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -295,7 +287,6 @@ def _sarif_result( *, mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligation: Mapping[str, object] | None = None, - learning: Mapping[str, object] | None = None, ) -> dict[str, object]: properties: dict[str, object] = { "finding_id": finding.finding_id, @@ -327,10 +318,6 @@ def _sarif_result( properties["obligation_evidence"] = _strings_of(obligation, "evidence") properties["obligation_known_fix"] = obligation.get("known_fix") properties["obligation_intent"] = obligation.get("intent") - if learning is not None: - properties["family"] = str(learning.get("family", "")) - properties["detector"] = str(learning.get("detector", "")) - properties["verifier"] = str(learning.get("verifier", "")) return { "ruleId": _rule_id(finding), "level": _sarif_level(finding.severity), @@ -486,16 +473,6 @@ def _append_obligations(lines: list[str], cited: Mapping[str, Mapping[str, objec lines.append("") -def _append_learning(lines: list[str], cited: Mapping[str, Mapping[str, object]]) -> None: - """Which family found a claim, which detector version produced it, and what a verifier said about it.""" - lines.extend(["## Classified detectors", "", "Findings from the per-family detectors, with what confirmed them.", ""]) - for finding_id, info in sorted(cited.items()): - lines.append( - f"- `{finding_id}`: family `{info.get('family')}`, detector `{info.get('detector')}`, verifier `{info.get('verifier')}`" - ) - lines.append("") - - def _strings_of(info: Mapping[str, object], key: str) -> list[str]: value = info.get(key) return [str(item) for item in value] if isinstance(value, list | tuple) else [] diff --git a/src/openultrasast/semantic/__init__.py b/src/openultrasast/semantic/__init__.py index ae339f1..d282314 100644 --- a/src/openultrasast/semantic/__init__.py +++ b/src/openultrasast/semantic/__init__.py @@ -3,26 +3,21 @@ from .engines import joern_available, tree_sitter_available from .extra import grammar_for, has_semantic_extra from .facts import FactLoadError, SemanticFacts, load_facts -from .mechanisms import Mechanism, MechanismStore, append_mechanism, order_promotions from .overlay import OverlayError, OverlayRecord, adjudicate, finding_from_coverage, write_overlay from .prove_filter import filter_promoted_hotspots __all__ = [ "FactLoadError", - "Mechanism", - "MechanismStore", "OverlayError", "OverlayRecord", "SemanticFacts", "adjudicate", - "append_mechanism", "filter_promoted_hotspots", "finding_from_coverage", "grammar_for", "has_semantic_extra", "joern_available", "load_facts", - "order_promotions", "tree_sitter_available", "write_overlay", ] diff --git a/src/openultrasast/semantic/ir.py b/src/openultrasast/semantic/ir.py index ffe30b3..543bfc8 100644 --- a/src/openultrasast/semantic/ir.py +++ b/src/openultrasast/semantic/ir.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import re from dataclasses import dataclass @@ -230,3 +231,14 @@ def _unparse(node: ast.AST) -> str: return ast.unparse(node) except Exception: return "" + + +def trailing_name(callee: str) -> str: + """``os.path.join`` -> ``join``; ``Runtime.getRuntime().exec`` -> ``exec``; keeps the last identifier segment. + + Re-homed here from the deleted ``semantic/variants.py``: it is a pure helper over ``CallSite.name`` and has + nothing to do with the mechanism search that used to surround it. + """ + tail = callee.rsplit(".", 1)[-1].rsplit("::", 1)[-1].rsplit("->", 1)[-1] + match = re.search(r"[A-Za-z_$][\w$]*\s*$", tail) + return match.group(0).strip() if match else tail diff --git a/src/openultrasast/semantic/loo.py b/src/openultrasast/semantic/loo.py deleted file mode 100644 index 3c1c67b..0000000 --- a/src/openultrasast/semantic/loo.py +++ /dev/null @@ -1,304 +0,0 @@ -"""Leave-one-out: the pair corpus's own detection rate (corpus-seeded-mechanisms, Req 4). - -For every scorable pair, a temporary mechanism store is seeded from every *other* trusted pair, both excerpts are -materialized and searched for variants, and the pair is scored with the pair rules: detected when a hit lies inside the -labeled function and names the labeled mechanism (or sink), silent when the fixed side yields no hit. Only ``seeded`` and -``reviewed`` pairs teach; ``advisory`` and ``title`` pairs are scored as held-out targets but never seed. Reported, never -gated. - -Obligation-labeled rows (authorization-obligations, Req 7.4) run the obligation checker on both sides with the held-out -store's obligation shapes: detected when a finding tagged with the labeled obligation lies inside the labeled function *and* -carries a ``known_fix`` supplied by the store (the checker's own facts-only finding is not the corpus teaching), silent when -the fixed side carries no such finding; ``found_by`` names the shape record and the pairs that taught it. -""" - -from __future__ import annotations - -import tempfile -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..findings import StaticFinding -from ..pairs import PairCase, _materialize, _targets -from .facts import FactLoadError, SemanticFacts, load_facts -from .functions import named_function_ranges, spans_named -from .mechanisms import Mechanism, MechanismStore, append_from_pair, corpus_mechanisms -from .obligations import check_obligations, findings_to_static, load_obligation_facts, obligation_mechanisms -from .obligations.dominance import OrderDominance -from .seed import Lesson, _parse_language, pair_lessons -from .variant_search import search_tree -from .variants import Shape, trailing_name - - -@dataclass(frozen=True) -class LooOutcome: - pair: str - slice: str - provenance: str - mechanisms: tuple[str, ...] # labeled mechanism ids of the held-out pair - detected: bool - silent: bool - found_by: tuple[tuple[str, tuple[str, ...]], ...] # (mechanism record id, pairs that taught it) for detecting hits - hits_vuln: int - hits_fixed: int - teaches: bool # the pair contributed at least one shape when others were held out - leaked_by: tuple[str, ...] = () # mechanism record ids that hit the fixed side - degradations: tuple[dict[str, object], ...] = () # from variant search on either side - obligations: tuple[str, ...] = () # labeled obligation kinds of the held-out pair (Req 7.4) - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "slice": self.slice, - "provenance": self.provenance, - "mechanisms": list(self.mechanisms), - "detected": self.detected, - "silent": self.silent, - "found_by": [{"mechanism_id": mechanism_id, "pairs": list(pairs)} for mechanism_id, pairs in self.found_by], - "hits_vuln": self.hits_vuln, - "hits_fixed": self.hits_fixed, - "teaches": self.teaches, - "leaked_by": list(self.leaked_by), - "obligations": list(self.obligations), - } - - -@dataclass(frozen=True) -class LooResult: - outcomes: tuple[LooOutcome, ...] - per_slice: dict[str, dict[str, float]] - per_profile: dict[str, dict[str, float]] - per_mechanism: dict[str, dict[str, float]] - skipped: tuple[tuple[str, str], ...] = () - per_obligation_kind: dict[str, dict[str, float]] = field(default_factory=dict) - teaching_pairs: int = 0 - degradations: tuple[dict[str, object], ...] = field(default=()) - - def to_dict(self) -> dict[str, object]: - return { - "per_slice": self.per_slice, - "per_profile": self.per_profile, - "per_mechanism": self.per_mechanism, - "per_obligation_kind": self.per_obligation_kind, - "outcomes": [outcome.to_dict() for outcome in self.outcomes], - "skipped": [{"pair": pair, "reason": reason} for pair, reason in self.skipped], - "teaching_pairs": self.teaching_pairs, - "degradations": list(self.degradations), - } - - -def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = None) -> LooResult: - loaded = facts if facts is not None else _facts() - targets: list[PairCase] = [] - skipped: list[tuple[str, str]] = [] - for case in cases: - if case.unscorable or case.known_limit: - # `PairCase.unscorable` verbatim when the loader computed one: one spelling per corpus fact, and it - # already carries the `known_limit:` prefix when the limit is a maintainer's own word rather than one - # of the vocabulary's. A case built by hand may carry only the declaration. - skipped.append((case.name, case.unscorable or f"known_limit:{case.known_limit}")) - elif not case.vuln_file.is_file() or not case.fixed_file.is_file(): - skipped.append((case.name, "pointer_pair_not_cached" if not case.vendored else "excerpt_missing")) - else: - targets.append(case) - from ..learning.split import GATING_TIERS, holdout_names, is_teacher, refuse_if_holdout - - # learning-harness Req 5.1: every pair is still scored, but only a train-split teacher may seed a store. - lessons = {case.name: pair_lessons(case, loaded) for case in targets if is_teacher(case)} - outcomes = [_hold_out(case, targets, lessons, loaded) for case in targets] - # Req 5.2: the holdout pairs kept out of the teaching set are named, not silently dropped. Non-teachers held - # back for their tier or their unscorable reason are already in `skipped` with that reason. - excluded = [case.name for case in targets if case.name not in lessons] - refusal = refuse_if_holdout(excluded, targets) - refusals = (refusal.degradation(),) if refusal is not None else () - withheld = tuple( - (case.name, "tier" if case.review_tier not in GATING_TIERS else "unscorable") - for case in targets - if case.name in set(excluded) - set(holdout_names(targets)) - ) - skipped.extend((name, f"not_a_teacher:{why}") for name, why in withheld) - return LooResult( - outcomes=tuple(outcomes), - per_slice=_group(outcomes, lambda item: (item.slice,)), - per_profile=_group(outcomes, lambda item: (item.provenance,)), - per_mechanism=_group(outcomes, lambda item: item.mechanisms), - per_obligation_kind=_group(outcomes, lambda item: item.obligations), - skipped=tuple(skipped), - teaching_pairs=sum(1 for shapes in lessons.values() if shapes), - degradations=(*refusals, *_unique_degradations(outcomes)), - ) - - -def _unique_degradations(outcomes: Sequence[LooOutcome]) -> tuple[dict[str, object], ...]: - seen: set[str] = set() - unique: list[dict[str, object]] = [] - for outcome in outcomes: - for item in outcome.degradations: - key = repr(sorted(item.items())) - if key not in seen: - seen.add(key) - unique.append(item) - return tuple(unique) - - -def _hold_out(case: PairCase, targets: Sequence[PairCase], lessons: dict[str, list[Lesson]], facts: SemanticFacts) -> LooOutcome: - with tempfile.TemporaryDirectory(prefix="ousast-loo-") as scratch: - root = Path(scratch) - store = MechanismStore(root / "mechanisms.jsonl") - for teacher in targets: - if teacher.name == case.name: - continue - for lesson in lessons.get(teacher.name, ()): - append_from_pair( - store, - lesson.shape, - summary=lesson.summary, - cwe=lesson.cwe, - pair=teacher.name, - provenance=teacher.provenance, - tier=teacher.review_tier, - ) - scored = score_pair_with_store(case, store, facts, root=root) - return LooOutcome(**{**scored.__dict__, "teaches": bool(lessons.get(case.name))}) - - -def score_pair_with_store(case: PairCase, store: MechanismStore, facts: SemanticFacts, *, root: Path | None = None) -> LooOutcome: - """Search both sides of one pair with a fixed store and score it with the pair rules (shared by LOO and the lever).""" - labeled_mechanisms = tuple(sorted({row.mechanism for row in case.expected if row.mechanism})) - labeled_obligations = tuple(sorted({str(row.obligation) for row in case.expected if getattr(row, "obligation", None)})) - loaded = store.load() - records = {record.id: record for record in corpus_mechanisms(loaded)} - obligation_vuln: list[StaticFinding] = [] - obligation_fix: list[StaticFinding] = [] - with tempfile.TemporaryDirectory(prefix="ousast-loo-pair-") as scratch: - base = root if root is not None else Path(scratch) - vuln_root = _materialize(base / "vuln", case.vuln_file, case.relpath) - fix_root = _materialize(base / "fixed", case.fixed_file, case.relpath) - vuln_search = search_tree(vuln_root, _targets(vuln_root), store, facts, max_mechanisms=len(records) or 1) - fix_search = search_tree(fix_root, _targets(fix_root), store, facts, max_mechanisms=len(records) or 1) - vuln_hits, fix_hits = vuln_search.hits, fix_search.hits - if labeled_obligations: - shapes = obligation_mechanisms(loaded) - obligation_vuln = _obligation_findings(vuln_root, shapes) - obligation_fix = _obligation_findings(fix_root, shapes) - text = case.vuln_file.read_text(errors="ignore") - ranges = named_function_ranges(case.relpath, text, _parse_language(case)) or () - found: list[tuple[str, tuple[str, ...]]] = [] - for finding in obligation_vuln: - record = _known_fix_record(finding, records) - if record is None or finding.line is None or not _inside_label(case, finding.line, ranges): - continue - if any(f"obligation:{kind}" in finding.tags for kind in labeled_obligations): - found.append((record.id, tuple(record.pairs))) - obligation_leaks = tuple( - sorted({record.id for finding in obligation_fix if (record := _known_fix_record(finding, records)) is not None}) - ) - for hit in vuln_hits: - record = records.get(hit.mechanism_id) - if record is None or not _inside_label(case, hit.line, ranges): - continue - if _names_label(case, record, hit.sink_name): - found.append((record.id, tuple(record.pairs))) - return LooOutcome( - pair=case.name, - slice=case.slice, - provenance=case.provenance, - mechanisms=labeled_mechanisms, - detected=bool(found), - silent=not fix_hits and not obligation_leaks, - found_by=tuple(found), - hits_vuln=len(vuln_hits) + len(obligation_vuln), - hits_fixed=len(fix_hits) + len(obligation_fix), - teaches=False, - leaked_by=tuple(sorted({hit.mechanism_id for hit in fix_hits})) + obligation_leaks, - degradations=tuple(vuln_search.degradations) + tuple(fix_search.degradations), - obligations=labeled_obligations, - ) - - -def _obligation_findings(root: Path, shapes: Sequence[object]) -> list[StaticFinding]: - """Function-local obligation findings for one materialized side, with the held-out store's obligation shapes as known fixes.""" - from ..config import ObligationsConfig - from ..mapping import analyze_entry_points - from .ir import parse_file - - targets = _targets(root) - irs: dict[str, tuple[object, str]] = {} - texts: dict[str, str] = {} - for target in targets: - try: - text = (root / target.path).read_text(errors="ignore") - except OSError: - continue - texts[target.path] = text - irs[target.path] = (parse_file(target.path, text, target.language), text) - result = check_obligations( - irs=irs, # type: ignore[arg-type] - entries=analyze_entry_points(root, targets), - facts=load_obligation_facts(), - flow_facts=_facts(), - policy=None, - paths=(), - dominance=OrderDominance(texts=texts), - store_shapes=shapes, - min_siblings=ObligationsConfig().min_siblings, - ) - return findings_to_static(result) - - -def _known_fix_record(finding: StaticFinding, records: dict[str, Mechanism]) -> Mechanism | None: - for tag in finding.tags: - if tag.startswith("mechanism:"): - return records.get(tag.split(":", 1)[1]) - return None - - -def _inside_label(case: PairCase, line: int, ranges: Sequence[tuple[str, int, int]]) -> bool: - functions = {row.function for row in case.expected if row.function} - if not functions: - return True - return any(start <= line <= end for function in functions for start, end in spans_named(ranges, function)) - - -def _names_label(case: PairCase, record: Mechanism, sink_name: str) -> bool: - """The hit names the pair's labeled mechanism, or its labeled sink when the row carries one.""" - shape = Shape.from_dict(record.shape or {}) - for row in case.expected: - if row.mechanism and row.mechanism == shape.mechanism: - return True - if row.sink and trailing_name(row.sink) == sink_name: - return True - return False - - -def _group(outcomes: Sequence[LooOutcome], keys: object) -> dict[str, dict[str, float]]: - buckets: dict[str, list[LooOutcome]] = {} - for outcome in outcomes: - for key in keys(outcome): # type: ignore[operator] - buckets.setdefault(str(key), []).append(outcome) - return {name: _metrics(items) for name, items in sorted(buckets.items())} - - -def _metrics(outcomes: Sequence[LooOutcome]) -> dict[str, float]: - pairs = len(outcomes) - detected = sum(1 for item in outcomes if item.detected) - silent = sum(1 for item in outcomes if item.silent) - recall = detected / pairs if pairs else 0.0 - silence = silent / pairs if pairs else 0.0 - return { - "pairs": pairs, - "detected": detected, - "silent": silent, - "recall": recall, - "silence": silence, - "youden": recall + silence - 1.0 if pairs else 0.0, - "teaching": sum(1 for item in outcomes if item.teaches), - } - - -def _facts() -> SemanticFacts: - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) diff --git a/src/openultrasast/semantic/mechanisms.py b/src/openultrasast/semantic/mechanisms.py deleted file mode 100644 index a2ff748..0000000 --- a/src/openultrasast/semantic/mechanisms.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Append-only mechanism memory. OpenRouter embeddings rank prove budget; never demote.""" - -from __future__ import annotations - -import hashlib -import json -import math -import os -import uuid -from collections.abc import Sequence -from dataclasses import asdict, dataclass -from pathlib import Path - -from ..complexity.map import Hotspot -from ..findings import StaticFinding -from ..index import CodeChunk, EmbeddingClient, VectorIndex, VectorRecord, write_vector_index -from .obligations.shapes import ObligationShape -from .overlay import OverlayRecord -from .variants import Shape - -DEFAULT_MECHANISM_LOG = Path(".openultrasast/mechanisms.jsonl") - - -@dataclass(frozen=True) -class Mechanism: - id: str - summary: str - cwe: str - language: str - tags: tuple[str, ...] - keywords: tuple[str, ...] - what_made_it_exploitable: str - source_finding_id: str = "" - source_repo: str = "" - # corpus-seeded-mechanisms (additive; old rows load with these defaults) - origin: str = "sandbox" # "sandbox" (proven finding) | "corpus" (derived from a trusted pair) - review_tier: str = "" - pairs: tuple[str, ...] = () - shape: dict[str, object] | None = None # Shape.to_dict(); None on sandbox rows (not searchable) - guard: str = "none" - retracted: bool = False # tombstone row appended by the improve lever; load() folds the record away - - -class MechanismStore: - def __init__(self, log_path: Path, cache_path: Path | None = None) -> None: - self.log_path = log_path - self.cache_path = cache_path if cache_path is not None else log_path.with_suffix(".index.json") - - def load(self) -> tuple[Mechanism, ...]: - if not self.log_path.is_file(): - return () - by_id: dict[str, Mechanism] = {} # append-only log; the latest row for an id is its current state - for line in self.log_path.read_text().splitlines(): - if not line.strip(): - continue - payload = json.loads(line) - record = _mechanism_from_payload(payload) - if record.retracted: - by_id.pop(record.id, None) - else: - by_id[record.id] = record - return tuple(by_id.values()) - - def append(self, mechanism: Mechanism) -> None: - self.log_path.parent.mkdir(parents=True, exist_ok=True) - with self.log_path.open("a") as handle: - handle.write(json.dumps(asdict(mechanism), sort_keys=True) + "\n") - - -def append_mechanism( - store: MechanismStore, - *, - summary: str, - cwe: str, - language: str, - tags: Sequence[str], - what_made_it_exploitable: str, - source_finding_id: str = "", - source_repo: str = "", - keywords: Sequence[str] = (), -) -> Mechanism: - mechanism = Mechanism( - id=str(uuid.uuid4()), - summary=summary, - cwe=cwe, - language=language, - tags=tuple(tags), - keywords=tuple(keywords), - what_made_it_exploitable=what_made_it_exploitable, - source_finding_id=source_finding_id, - source_repo=source_repo, - ) - store.append(mechanism) - return mechanism - - -_TIER_RANK = {"seeded": 1, "reviewed": 2} - - -def corpus_mechanism_id(shape: Shape | ObligationShape) -> str: - """Deterministic id from the shape key, so the same shape gets the same record on every machine (lever-addressable).""" - return "corpus:" + hashlib.sha1(shape.key().encode()).hexdigest()[:16] - - -def append_from_pair( - store: MechanismStore, - shape: Shape | ObligationShape, - *, - summary: str, - cwe: str, - pair: str, - provenance: str, - tier: str, -) -> Mechanism: - """Second writer (Req 1.3, 1.4): one record per shape, ``origin = "corpus"``, provenance lists every pair that taught it.""" - mechanism_id = corpus_mechanism_id(shape) - existing = next((item for item in store.load() if item.id == mechanism_id), None) - pairs = tuple(existing.pairs) if existing is not None else () - if pair not in pairs: - pairs = (*pairs, pair) - if existing is not None and _TIER_RANK.get(existing.review_tier, 0) > _TIER_RANK.get(tier, 0): - tier = existing.review_tier # a shape keeps the strongest review it ever had - inherited = tuple(tag for tag in (existing.tags if existing is not None else ()) if tag.startswith(("provenance:", "tier:"))) - if isinstance(shape, Shape): - positions = ", ".join(f"arg{p} ({k})" for p, k in zip(shape.source_positions, shape.source_kinds, strict=True)) - exploitable = f"{positions} reaches {shape.sink_name}/{shape.arity}; the fix added {shape.guard}" - keywords: tuple[str, ...] = (shape.sink_name, shape.guard, *shape.source_kinds) - guard = shape.guard - else: # obligation shape: the discharger the fix carries is the 'guard' the reports show - exploitable = f"{shape.operation_kind} reached without {shape.discharger_kind}; the fix binds {shape.provenance}" - keywords = (shape.operation_kind, shape.discharger_kind, shape.provenance, shape.resource_class) - guard = shape.discharger_kind - record = Mechanism( - id=mechanism_id, - summary=summary, - cwe=cwe, - language=shape.language, - tags=( - shape.mechanism, - f"mechanism:{shape.mechanism}", - f"guard:{guard}", - *sorted({*inherited, f"provenance:{provenance}", f"tier:{tier}"}), - ), - keywords=keywords, - what_made_it_exploitable=exploitable, - source_finding_id="", - source_repo="", - origin="corpus", - review_tier=tier, - pairs=pairs, - shape=shape.to_dict(), - guard=guard, - ) - store.append(record) - return record - - -def corpus_mechanisms(records: Sequence[Mechanism]) -> tuple[Mechanism, ...]: - """Records that variant search may use: corpus origin with a shape.""" - return tuple(item for item in records if item.origin == "corpus" and item.shape is not None) - - -def order_promotions( - hotspots: Sequence[Hotspot], - records: Sequence[OverlayRecord], - findings: Sequence[StaticFinding], - *, - store: MechanismStore | None = None, - client: EmbeddingClient | None = None, - model: str | None = None, -) -> tuple[tuple[Hotspot, ...], str | None]: - """Heuristic order, then optional OpenRouter cosine boost. Never drops a promotion.""" - heuristic = tuple(sorted(hotspots, key=lambda item: (-item.score, item.path, item.function_name or ""))) - if store is None: - return heuristic, None - mechanisms = store.load() - if not mechanisms: - return heuristic, None - if client is None or not model: - return heuristic, "embeddings_unavailable" - overlay_by_id = {record.proposal_id: record for record in records} - finding_by_id = {finding.finding_id: finding for finding in findings} - try: - ranked = _rank_with_embeddings(heuristic, overlay_by_id, finding_by_id, mechanisms, store, client, model) - except Exception: - return heuristic, "embed_failed" - return ranked, None - - -def overlay_text(record: OverlayRecord) -> str: - parts = [record.cwe, record.reason, *record.sources, *record.sinks, *record.sanitizers] - return " ".join(part for part in parts if part) - - -def mechanism_text(mechanism: Mechanism) -> str: - return " ".join( - part - for part in (mechanism.summary, mechanism.what_made_it_exploitable, mechanism.cwe, *mechanism.tags, *mechanism.keywords) - if part - ) - - -def _rank_with_embeddings( - hotspots: tuple[Hotspot, ...], - overlay_by_id: dict[str, OverlayRecord], - finding_by_id: dict[str, StaticFinding], - mechanisms: tuple[Mechanism, ...], - store: MechanismStore, - client: EmbeddingClient, - model: str, -) -> tuple[Hotspot, ...]: - vectors = _mechanism_vectors(mechanisms, store, client, model) - scored: list[tuple[float, Hotspot]] = [] - for hotspot in hotspots: - record = _overlay_for_hotspot(hotspot, overlay_by_id) - finding = _finding_for_hotspot(hotspot, finding_by_id) - language = record.language if record is not None else "" - cwe = record.cwe if record is not None else "" - tags = tuple(finding.tags) if finding is not None else () - filtered = [item for item in mechanisms if _hard_filter(item, language, cwe, tags)] - boost = 0.0 - if record is not None and filtered: - query = client.embed(model=model, inputs=[overlay_text(record)])[0] - for mechanism in filtered: - vector = vectors.get(mechanism.id) - if vector is None: - continue - boost = max(boost, _cosine(query, vector)) - scored.append((hotspot.score + boost, hotspot)) - scored.sort(key=lambda item: (-item[0], item[1].path, item[1].function_name or "")) - return tuple(item[1] for item in scored) - - -def _mechanism_vectors( - mechanisms: tuple[Mechanism, ...], - store: MechanismStore, - client: EmbeddingClient, - model: str, -) -> dict[str, list[float]]: - vectors: dict[str, list[float]] = {} - missing: list[Mechanism] = [] - cached = _read_cache(store.cache_path, model) - for mechanism in mechanisms: - key = f"{mechanism.id}:{model}" - if key in cached: - vectors[mechanism.id] = cached[key] - else: - missing.append(mechanism) - if missing: - embeddings = client.embed(model=model, inputs=[mechanism_text(item) for item in missing]) - for mechanism, embedding in zip(missing, embeddings, strict=True): - vectors[mechanism.id] = embedding - records = [ - VectorRecord( - chunk=CodeChunk( - chunk_id=f"{mechanism.id}:{model}", - namespace="mechanisms", - path=mechanism.id, - language=mechanism.language or "text", - start_line=1, - end_line=1, - text=mechanism_text(mechanism), - metadata={"mechanism_id": mechanism.id, "model": model, "cwe": mechanism.cwe, "language": mechanism.language}, - ), - embedding=vectors[mechanism.id], - ) - for mechanism in mechanisms - if mechanism.id in vectors - ] - index = VectorIndex( - store="json-local", - embedding_model=model, - repo_root=str(store.log_path.parent), - repo_commit=None, - records=records, - ) - write_vector_index(index, store.cache_path) - return vectors - - -def _read_cache(path: Path, model: str) -> dict[str, list[float]]: - if not path.is_file(): - return {} - try: - payload = json.loads(path.read_text()) - except json.JSONDecodeError: - return {} - if str(payload.get("embedding_model")) != model: - return {} - cached: dict[str, list[float]] = {} - for item in payload.get("records", []): - chunk = item.get("chunk", {}) - metadata = chunk.get("metadata", {}) - mechanism_id = str(metadata.get("mechanism_id", "")) - if mechanism_id: - cached[f"{mechanism_id}:{model}"] = [float(value) for value in item.get("embedding", [])] - return cached - - -def _hard_filter(mechanism: Mechanism, language: str, cwe: str, tags: Sequence[str]) -> bool: - if mechanism.language and language and mechanism.language != language: - return False - if mechanism.cwe and cwe and mechanism.cwe != cwe: - return False - tag_ok = not mechanism.tags or not tags or bool(set(mechanism.tags) & set(tags)) - return tag_ok - - -def _overlay_for_hotspot(hotspot: Hotspot, overlay_by_id: dict[str, OverlayRecord]) -> OverlayRecord | None: - for finding_id in hotspot.inventory_finding_ids: - record = overlay_by_id.get(finding_id) - if record is not None: - return record - return None - - -def _finding_for_hotspot(hotspot: Hotspot, finding_by_id: dict[str, StaticFinding]) -> StaticFinding | None: - for finding_id in hotspot.inventory_finding_ids: - finding = finding_by_id.get(finding_id) - if finding is not None: - return finding - return None - - -def _cosine(left: list[float], right: list[float]) -> float: - if len(left) != len(right): - return 0.0 - numerator = sum(a * b for a, b in zip(left, right, strict=True)) - left_norm = math.sqrt(sum(a * a for a in left)) - right_norm = math.sqrt(sum(b * b for b in right)) - if left_norm == 0 or right_norm == 0: - return 0.0 - return numerator / (left_norm * right_norm) - - -def _mechanism_from_payload(payload: dict[str, object]) -> Mechanism: - tags = payload.get("tags") or [] - keywords = payload.get("keywords") or [] - pairs = payload.get("pairs") or [] - shape = payload.get("shape") - if not isinstance(tags, list) or not isinstance(keywords, list) or not isinstance(pairs, list): - raise ValueError("mechanism tags, keywords and pairs must be lists") - return Mechanism( - id=str(payload["id"]), - summary=str(payload.get("summary", "")), - cwe=str(payload.get("cwe", "")), - language=str(payload.get("language", "")), - tags=tuple(str(item) for item in tags), - keywords=tuple(str(item) for item in keywords), - what_made_it_exploitable=str(payload.get("what_made_it_exploitable", "")), - source_finding_id=str(payload.get("source_finding_id", "")), - source_repo=str(payload.get("source_repo", "")), - origin=str(payload.get("origin", "sandbox")), - review_tier=str(payload.get("review_tier", "")), - pairs=tuple(str(item) for item in pairs), - shape=dict(shape) if isinstance(shape, dict) else None, # validated where it is used (search_tree, the lever) - guard=str(payload.get("guard", "none")), - retracted=bool(payload.get("retracted", False)), - ) - - -def embeddings_configured(model: str | None) -> bool: - return bool(model) and bool(os.environ.get("OPENROUTER_API_KEY")) diff --git a/src/openultrasast/semantic/obligations/operations.py b/src/openultrasast/semantic/obligations/operations.py index 58770cb..c5f14a7 100644 --- a/src/openultrasast/semantic/obligations/operations.py +++ b/src/openultrasast/semantic/obligations/operations.py @@ -14,8 +14,7 @@ from collections.abc import Sequence from dataclasses import dataclass from ..facts import SemanticFacts -from ..ir import FileIR, FunctionIR -from ..variants import trailing_name +from ..ir import FileIR, FunctionIR, trailing_name from .facts import DischargerFact, ObligationFacts from .shapes import _call_matches, _constraint_provenance, _is_literal, _operations, _permissive diff --git a/src/openultrasast/semantic/obligations/shapes.py b/src/openultrasast/semantic/obligations/shapes.py index d5a8a47..ffc8c1b 100644 --- a/src/openultrasast/semantic/obligations/shapes.py +++ b/src/openultrasast/semantic/obligations/shapes.py @@ -17,8 +17,7 @@ from dataclasses import dataclass from ..facts import SemanticFacts from ..functions import named_ranges_from_ir -from ..ir import CallSite, FileIR, FunctionIR -from ..variants import trailing_name +from ..ir import CallSite, FileIR, FunctionIR, trailing_name from .facts import DISCHARGER_KINDS, OPERATION_KINDS, PROVENANCE_KINDS, DischargerFact, ObligationFacts, OperationFact FAMILY = "obligation" diff --git a/src/openultrasast/semantic/prove_budget.py b/src/openultrasast/semantic/prove_budget.py deleted file mode 100644 index b0f4e50..0000000 --- a/src/openultrasast/semantic/prove_budget.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Prove-budget ranker: heuristic order plus optional mechanism recall.""" - -from .mechanisms import MechanismStore, append_mechanism, order_promotions -from .prove_filter import filter_promoted_hotspots, promoted_findings - -__all__ = [ - "MechanismStore", - "append_mechanism", - "filter_promoted_hotspots", - "order_promotions", - "promoted_findings", -] diff --git a/src/openultrasast/semantic/seed.py b/src/openultrasast/semantic/seed.py deleted file mode 100644 index c3e039e..0000000 --- a/src/openultrasast/semantic/seed.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Export trusted pairs as mechanism records (corpus-seeded-mechanisms, Req 1). - -Only ``seeded`` and ``reviewed`` pairs seed; ``advisory`` and ``title`` pairs, ``known_limit`` pairs, pointer pairs -without a cached excerpt, and excerpts that fail to parse are skipped with a reason. Runs offline over the vendored -catalog; nothing here fetches, and no code text enters the store. -""" - -from __future__ import annotations - -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..pairs import GATING_TIERS, PairCase -from ..preprocess import LANGUAGE_BY_EXTENSION -from .facts import FactLoadError, SemanticFacts, load_facts -from .ir import FileIR, parse_file -from .mechanisms import MechanismStore, append_from_pair -from .obligations.facts import ObligationFacts, load_obligation_facts -from .obligations.shapes import ABSENCE_MECHANISMS, ObligationShape, derive_obligation, explain_skip -from .variants import Shape, derive_shape - -LANGUAGE_UNSUPPORTED = "variants_language_unsupported" - - -@dataclass(frozen=True) -class ExportReport: - seeded: int # pairs that produced at least one record - records: int # distinct records written (deduped by shape) - skipped: tuple[tuple[str, str], ...] = () # (pair, reason) - degradations: tuple[dict[str, object], ...] = () - seeded_pairs: tuple[str, ...] = field(default=()) - - def to_dict(self) -> dict[str, object]: - return { - "seeded": self.seeded, - "records": self.records, - "skipped": [{"pair": pair, "reason": reason} for pair, reason in self.skipped], - "degradations": list(self.degradations), - "seeded_pairs": list(self.seeded_pairs), - } - - -def export_mechanisms(cases: Sequence[PairCase], store: MechanismStore, *, facts: SemanticFacts | None = None) -> ExportReport: - """Derive one shape per labeled expected row of every trusted pair and append it to ``store`` (deduped by shape).""" - loaded = facts if facts is not None else _facts() - skipped: list[tuple[str, str]] = [] - degradations: list[dict[str, object]] = [] - seeded: list[str] = [] - record_ids: set[str] = set() - unsupported_languages: set[str] = set() - for case in cases: # catalog order; the store lists pairs in the order they taught - reason = _skip_reason(case) - if reason is not None: - skipped.append((case.name, reason)) - continue - vuln_text = _read(case.vuln_file) - fixed_text = _read(case.fixed_file) - language = _parse_language(case) - vuln_ir = parse_file(case.relpath, vuln_text, language) - fixed_ir = parse_file(case.relpath, fixed_text, language) - if not vuln_ir.parse_ok or not fixed_ir.parse_ok: - bad = vuln_ir if not vuln_ir.parse_ok else fixed_ir - if bad.reason == "language_unsupported": - skipped.append((case.name, LANGUAGE_UNSUPPORTED)) - if language not in unsupported_languages: - unsupported_languages.add(language) - degradations.append({"stage": "mechanisms", "reason": LANGUAGE_UNSUPPORTED, "language": language}) - else: - skipped.append((case.name, "parse_failed")) - continue - if not any(row.function and row.mechanism for row in case.expected): - skipped.append((case.name, "no_labeled_function")) - continue - lessons = pair_lessons(case, loaded, vuln_ir=vuln_ir, fixed_ir=fixed_ir, vuln_text=vuln_text, fixed_text=fixed_text) - for lesson in lessons: - record = append_from_pair( - store, - lesson.shape, - summary=lesson.summary, - cwe=lesson.cwe, - pair=case.name, - provenance=case.provenance, - tier=case.review_tier, - ) - record_ids.add(record.id) - if lessons: - seeded.append(case.name) - else: - skipped.append((case.name, _skip_detail(case, vuln_ir, fixed_ir, vuln_text, fixed_text, loaded))) - return ExportReport( - seeded=len(seeded), - records=len(record_ids), - skipped=tuple(skipped), - degradations=tuple(degradations), - seeded_pairs=tuple(seeded), - ) - - -@dataclass(frozen=True) -class Lesson: - """One shape a trusted pair teaches, with the CWE and a text-free summary (a sink shape or an obligation shape).""" - - shape: Shape | ObligationShape - cwe: str - summary: str - - -def pair_lessons( - case: PairCase, - facts: SemanticFacts, - *, - vuln_ir: FileIR | None = None, - fixed_ir: FileIR | None = None, - vuln_text: str | None = None, - fixed_text: str | None = None, -) -> list[Lesson]: - """Shapes derived from every labeled row of a pair (shared by the exporter and leave-one-out). Empty when a side fails to parse.""" - vuln_text = vuln_text if vuln_text is not None else _read(case.vuln_file) - fixed_text = fixed_text if fixed_text is not None else _read(case.fixed_file) - language = _parse_language(case) - vuln_ir = vuln_ir if vuln_ir is not None else parse_file(case.relpath, vuln_text, language) - fixed_ir = fixed_ir if fixed_ir is not None else parse_file(case.relpath, fixed_text, language) - if not vuln_ir.parse_ok or not fixed_ir.parse_ok: - return [] - lessons: list[Lesson] = [] - for row in case.expected: - if not row.function or not row.mechanism: - continue - obligation_kind = getattr(row, "obligation", None) - if obligation_kind or row.mechanism in ABSENCE_MECHANISMS: - obligation = derive_obligation( - vuln_ir, - fixed_ir, - function=row.function, - obligation=obligation_kind, - mechanism=row.mechanism, - facts=_obligation_facts(), - flow_facts=facts, - vuln_text=vuln_text, - fixed_text=fixed_text, - ) - if obligation is not None: - summary = ( - f"{obligation.mechanism}: {obligation.operation_kind} without {obligation.discharger_kind} ({row.cwe}); " - f"fix binds {obligation.provenance}" - ) - lessons.append(Lesson(shape=obligation, cwe=row.cwe, summary=summary)) - continue - # an absence row that teaches no obligation may still teach a sink shape (a weak literal at a sink, say) - shape = derive_shape( - vuln_ir, - fixed_ir, - function=row.function, - sink=row.sink, - line=row.line, - mechanism=row.mechanism, - facts=facts, - vuln_text=vuln_text, - fixed_text=fixed_text, - cwe=row.cwe or None, - ) - if shape is not None: - summary = f"{shape.mechanism}: {'/'.join(shape.source_kinds)} into {shape.sink_name} ({row.cwe}); fix adds {shape.guard}" - lessons.append(Lesson(shape=shape, cwe=row.cwe, summary=summary)) - return lessons - - -def _skip_detail(case: PairCase, vuln_ir: FileIR, fixed_ir: FileIR, vuln_text: str, fixed_text: str, flow_facts: SemanticFacts) -> str: - """Why a parsing, labeled pair taught nothing: the obligation derivation's reason for absence rows, else the sink reason.""" - for row in case.expected: - if row.function and row.mechanism and (getattr(row, "obligation", None) or row.mechanism in ABSENCE_MECHANISMS): - return explain_skip( - vuln_ir, - fixed_ir, - function=row.function, - vuln_text=vuln_text, - fixed_text=fixed_text, - obligation=getattr(row, "obligation", None), - facts=_obligation_facts(), - flow_facts=flow_facts, - ) - return "no_labeled_sink_call_site" - - -_OBLIGATION_FACTS: list[ObligationFacts] = [] - - -def _obligation_facts() -> ObligationFacts: - if not _OBLIGATION_FACTS: - _OBLIGATION_FACTS.append(load_obligation_facts()) - return _OBLIGATION_FACTS[0] - - -def _skip_reason(case: PairCase) -> str | None: - from ..learning.split import is_teacher - - if case.review_tier not in GATING_TIERS: - return f"tier:{case.review_tier}" - if case.unscorable or case.known_limit: - # `PairCase.unscorable` verbatim when the loader computed one, the same string every other path reports: - # `identical_twin` declared in the catalog and `identical_twin` computed from the bytes are one fact, and - # calling either of them `split:train` is simply untrue. A case built by hand may carry only the - # declaration (Req 5.1, 10.3). - return case.unscorable or f"known_limit:{case.known_limit}" - if not is_teacher(case) and case.vendored: - return f"split:{case.split}" - if not case.vuln_file.is_file() or not case.fixed_file.is_file(): - return "pointer_pair_not_cached" if not case.vendored else "excerpt_missing" - return None - - -def _parse_language(case: PairCase) -> str: - by_ext = LANGUAGE_BY_EXTENSION.get(case.vuln_file.suffix.lower()) - if by_ext: - return str(by_ext) - return {"c_cpp": "c"}.get(case.language, case.language) - - -def _read(path: Path) -> str: - try: - return path.read_text(errors="ignore") - except OSError: - return "" - - -def _facts() -> SemanticFacts: - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) diff --git a/src/openultrasast/semantic/variant_search.py b/src/openultrasast/semantic/variant_search.py deleted file mode 100644 index 9c058fe..0000000 --- a/src/openultrasast/semantic/variant_search.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Variant search over a scanned tree and its merge with overlay flows (corpus-seeded-mechanisms, Req 3). - -Depends on the shapes (``variants``), the store (``mechanisms``), the overlay records and ``StaticFinding``; the shape -module itself stays free of those imports. -""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace -from pathlib import Path - -from ..findings import StaticFinding -from ..preprocess import FileTarget -from .facts import SemanticFacts -from .ir import parse_file -from .mechanisms import MechanismStore, corpus_mechanisms -from .overlay import OverlayRecord -from .variants import Shape, VariantHit, match_shapes - -LANGUAGE_UNSUPPORTED = "variants_language_unsupported" - - -@dataclass(frozen=True) -class SearchResult: - hits: tuple[VariantHit, ...] - mechanisms_searched: int - files_searched: int - degradations: tuple[dict[str, object], ...] = () - - -def search_tree( - root: Path, - targets: Sequence[FileTarget], - store: MechanismStore, - facts: SemanticFacts, - *, - max_mechanisms: int, -) -> SearchResult: - """Variant search over the parsed files of a tree with the store's corpus shapes (bounded by ``max_mechanisms``).""" - records = [record for record in corpus_mechanisms(store.load()) if (record.shape or {}).get("family", "sink") == "sink"] - records = records[: max(max_mechanisms, 0)] - shapes: list[Shape] = [] - ids: dict[str, str] = {} - degradations: list[dict[str, object]] = [] - for record in records: - assert record.shape is not None - try: - shape = Shape.from_dict(record.shape) - except (ValueError, TypeError) as exc: # one malformed operator-visible row must not abort the scan - degradations.append( - {"stage": "variants", "reason": "variants_store_row_invalid", "mechanism_id": record.id, "detail": str(exc)[:120]} - ) - continue - shapes.append(shape) - ids[shape.key()] = record.id - if not shapes: - return SearchResult(hits=(), mechanisms_searched=0, files_searched=0, degradations=tuple(degradations)) - hits: list[VariantHit] = [] - files = 0 - unsupported: set[str] = set() - for target in targets: - try: - text = (root / target.path).read_text(errors="ignore") - except OSError: - continue - ir = parse_file(target.path, text, target.language) - if not ir.parse_ok: - if ir.reason == "language_unsupported" and target.language not in unsupported: - unsupported.add(target.language) - degradations.append({"stage": "variants", "reason": "variants_language_unsupported", "language": target.language}) - continue - files += 1 - hits.extend(match_shapes(ir, shapes, facts, mechanism_ids=ids)) - return SearchResult(hits=tuple(hits), mechanisms_searched=len(shapes), files_searched=files, degradations=tuple(degradations)) - - -def hits_to_findings( - hits: Sequence[VariantHit], - records: Sequence[OverlayRecord], - summaries: Mapping[str, tuple[str, tuple[str, ...], str]], -) -> tuple[list[StaticFinding], list[OverlayRecord]]: - """Merge hits into overlay records at the same call site (Req 3.3); the rest become ``suspicion`` findings (Req 3.2).""" - by_site: dict[tuple[str, int | None], list[int]] = {} - for index, record in enumerate(records): - if record.disposition in {"promote", "coverage"} and record.sources and record.sinks: - by_site.setdefault((record.path, record.line), []).append(index) - updated = list(records) - findings: list[StaticFinding] = [] - emitted: set[str] = set() - for hit in hits: - indexes = by_site.get((hit.path, hit.line)) - if indexes: - for index in indexes: - if updated[index].mechanism_id is None: - updated[index] = replace(updated[index], mechanism_id=hit.mechanism_id) - continue - finding_id = f"variant:{hit.mechanism_id}:{hit.path}:{hit.line}" - if finding_id in emitted: - continue - emitted.add(finding_id) - summary, pairs, cwe = summaries.get(hit.mechanism_id, (hit.mechanism_id, (), "")) - taught_by = ", ".join(pairs) if pairs else "the mechanism store" - findings.append( - StaticFinding( - finding_id=finding_id, - path=hit.path, - title=f"Variant of known mechanism: {hit.sink_name}", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale=( - f"{summary}. Structural variant ({hit.source_kind} reaches {hit.sink_name}) of a mechanism learned from " - f"{taught_by}; suspicion until the overlay or the sandbox agrees." - ), - line=hit.line, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["variant", f"mechanism:{hit.mechanism_id}", *([f"cwe:{cwe}"] if cwe else [])], - ranking_priority=0.0, - ) - ) - return findings, updated diff --git a/src/openultrasast/semantic/variants.py b/src/openultrasast/semantic/variants.py deleted file mode 100644 index 56a6a2c..0000000 --- a/src/openultrasast/semantic/variants.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Structural mechanism shapes derived from labeled pairs (corpus-seeded-mechanisms, Req 2). - -A shape is the closed, text-free description of how a known vulnerable call looked: the -sink's trailing callee name, its arity, which argument positions carried source-derived -values and of what kind, and the kind of guard the fix added. Shapes are derived from -``FileIR`` (never from regex over code) and compared structurally, so a variant in another -project matches even when every identifier differs. Stdlib plus ``.ir``, ``.facts`` and -``.functions`` only; never imports the overlay, the scorer, or the store. -""" - -from __future__ import annotations - -import re -from collections.abc import Mapping, Sequence -from dataclasses import dataclass - -from .facts import SemanticFacts, SinkFact -from .functions import named_ranges_from_ir -from .ir import Bind, CallSite, FileIR, FunctionIR - -GUARD_KINDS = ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") -SOURCE_KINDS = ("fact_source", "parameter", "container_read") -_MAX_BIND_DEPTH = 8 - -# Closed guard classification over the statements the fix added inside the labeled function. -# Order matters: the first matching kind wins. These classify the *fix*, they never detect. -_GUARD_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( - ("parameterized_call", re.compile(r"execute\w*\([^,]+,\s*[\(\[]|prepareStatement\(|\?\s*['\"]?\s*[,)]|\$\d\b|%s['\"]\s*,\s*[\(\[]")), - ( - "auth_check", - re.compile( - r"\b(request\.user|req\.user|current_user|request\.state|session\[|is_authenticated|authoriz|require_auth" - r"|login_required|verify_token|check_permission|owner_id\s*[!=]=)" - ), - ), - ( - "allowlist_test", - re.compile( - r"\bnot in\b|\bin\s*[\(\{\[]|\.includes\(|\.has\(|\bstartswith\(|\bstartsWith\(|\bendswith\(|\bendsWith\(" - r"|re\.(?:match|fullmatch)\(|\.test\(|allow_?list|whitelist|\bALLOWED\b|\bsafe_join\(|secure_filename\(" - r"|\brealpath\(|\bnormpath\(|\bresolve\(" - ), - ), - ("bounds_test", re.compile(r"\blen\(|\.length\b|\bsizeof\(|\bstrn?len\(|\bsnprintf\(|\bstrlcpy\(|\bmin\(|\bmax\(|\s(?:<|>|<=|>=)\s")), - ( - "null_test", - re.compile( - r"\bis None\b|\bis not None\b|[!=]= *NULL\b|[!=]==? *null\b|\bnullptr\b|\bif\s*\(\s*!\s*\w|\bif not \w+\s*:" - r"|\bif\s+\w+\s+is\s+None|\?\?|\?\." - ), - ), - ( - "type_change", - re.compile(r"\bint\(|\bparseInt\(|\bNumber\(|\(int\)|\(size_t\)|\(unsigned\b|\bstr\(|\.toString\(|\bstatic_cast<|\bBigInt\("), - ), -) -# Design §Shape: a container read is a subscript or a `.get(` read; source names themselves live in facts, never here. -_CONTAINER_READ = re.compile(r"\[[^\]]+\]|\.get\(") - - -@dataclass(frozen=True) -class Shape: - language: str - sink_name: str # trailing callee segment - arity: int - source_positions: tuple[int, ...] - source_kinds: tuple[str, ...] - guard: str # GUARD_KINDS - mechanism: str # closed vocabulary id from the pair label - - def key(self) -> str: - """Stable identity of the shape; carries no path, line, or literal.""" - positions = ",".join(str(p) for p in self.source_positions) - kinds = ",".join(self.source_kinds) - return f"{self.language}|{self.sink_name}/{self.arity}|pos={positions}|kinds={kinds}|guard={self.guard}|{self.mechanism}" - - def to_dict(self) -> dict[str, object]: - return { - "language": self.language, - "sink_name": self.sink_name, - "arity": self.arity, - "source_positions": list(self.source_positions), - "source_kinds": list(self.source_kinds), - "guard": self.guard, - "mechanism": self.mechanism, - } - - @classmethod - def from_dict(cls, payload: dict[str, object]) -> Shape: - positions = payload.get("source_positions") or [] - kinds = payload.get("source_kinds") or [] - if not isinstance(positions, list) or not isinstance(kinds, list): - raise ValueError("shape positions and kinds must be lists") - guard = str(payload.get("guard", "none")) - if guard not in GUARD_KINDS: - raise ValueError(f"unknown guard kind {guard!r}") - return cls( - language=str(payload.get("language", "")), - sink_name=str(payload.get("sink_name", "")), - arity=int(str(payload.get("arity", 0))), - source_positions=tuple(int(p) for p in positions), - source_kinds=tuple(str(k) for k in kinds), - guard=guard, - mechanism=str(payload.get("mechanism", "other")), - ) - - -def trailing_name(callee: str) -> str: - """``os.path.join`` -> ``join``; ``Runtime.getRuntime().exec`` -> ``exec``; keeps the last identifier segment.""" - tail = callee.rsplit(".", 1)[-1].rsplit("::", 1)[-1].rsplit("->", 1)[-1] - match = re.search(r"[A-Za-z_$][\w$]*\s*$", tail) - return match.group(0).strip() if match else tail - - -def derive_shape( - vuln: FileIR, - fixed: FileIR, - *, - function: str, - sink: str | None, - line: int | None, - mechanism: str, - facts: SemanticFacts, - vuln_text: str = "", - fixed_text: str = "", - cwe: str | None = None, -) -> Shape | None: - """The shape of the labeled sink call in ``function`` on the vulnerable side, with the guard the fixed side added. - - None when either side failed to parse, the function or the labeled call site is not found, or no argument of the - call carries a source (a constant-only call teaches nothing). - """ - if not vuln.parse_ok or not fixed.parse_ok: - return None - target = _function_named(vuln, function, vuln_text) - if target is None: - return None - scoped = facts.for_language(vuln.language) - call = _labeled_call(target, sink, line, scoped, cwe) - if call is None: - return None - kinds = [_source_kind_of_argument(target, call, index, scoped) for index in range(len(call.arg_texts))] - positions = tuple(index for index, kind in enumerate(kinds) if kind is not None) - if not positions: - return None - guard = classify_guard(vuln, fixed, function=function, vuln_text=vuln_text, fixed_text=fixed_text) - return Shape( - language=vuln.language, - sink_name=trailing_name(call.name), - arity=len(call.arg_texts), - source_positions=positions, - source_kinds=tuple(kind for kind in kinds if kind is not None), - guard=guard, - mechanism=mechanism, - ) - - -def classify_guard(vuln: FileIR, fixed: FileIR, *, function: str, vuln_text: str, fixed_text: str) -> str: - """Closed guard kind from the statements inside ``function`` that exist only on the fixed side (``none`` when unknown).""" - before = _function_lines(vuln, function, vuln_text) - after = _function_lines(fixed, function, fixed_text) - added = [line for line in after if line not in set(before)] - for kind, pattern in _GUARD_PATTERNS: - if any(pattern.search(line) for line in added): - return kind - return "none" - - -def source_kind_of_name(function: FunctionIR, name: str, facts: SemanticFacts, *, before_line: int, depth: int = 0) -> str | None: - """How ``name`` came to carry attacker data inside ``function``: a fact source, a parameter, or a container read.""" - if depth > _MAX_BIND_DEPTH: - return None - bind = _last_bind(function, name, before_line) - if bind is not None: - if _source_in(bind.value_text, facts): - return "fact_source" - if _CONTAINER_READ.search(bind.value_text): - return "container_read" - for inner in bind.names: - if inner == name: - continue - kind = source_kind_of_name(function, inner, facts, before_line=bind.line, depth=depth + 1) - if kind is not None: - return kind - if name in function.params: - return "parameter" - return None - - -def _source_kind_of_argument(function: FunctionIR, call: CallSite, index: int, facts: SemanticFacts) -> str | None: - text = call.arg_texts[index] - if _source_in(text, facts): - return "fact_source" - if call.arg_is_constant[index] and not call.arg_names[index]: - return None - if _CONTAINER_READ.search(text) and any(name in function.params for name in call.arg_names[index]): - return "container_read" - found: list[str] = [] - for name in call.arg_names[index]: - kind = source_kind_of_name(function, name, facts, before_line=call.line + 1) - if kind is not None: - found.append(kind) - for preferred in SOURCE_KINDS: - if preferred in found: - return preferred - return None - - -def _source_in(text: str, facts: SemanticFacts) -> bool: - return any(pattern and pattern in text for source in facts.sources for pattern in source.patterns) - - -def _last_bind(function: FunctionIR, name: str, before_line: int) -> Bind | None: - candidates = [bind for bind in function.binds if bind.name == name and bind.line < before_line] - return max(candidates, key=lambda bind: bind.line) if candidates else None - - -def _labeled_call(function: FunctionIR, sink: str | None, line: int | None, facts: SemanticFacts, cwe: str | None) -> CallSite | None: - """The call the label names: line and sink intersected when both exist, else either; else the fact sink in the function. - - Labels that carry only a function and a mechanism (Real-Vuln rows) fall back to the fact-sink call inside the labeled - function, preferring the sink whose CWE matches the row and a call with a non-constant argument. - """ - wanted = trailing_name(sink) if sink else None - on_line = [call for call in function.calls if line is not None and call.line == line] - if on_line and wanted: - exact = [call for call in on_line if trailing_name(call.name) == wanted] - if exact: - return exact[0] - if on_line: - return on_line[0] - if wanted: - for call in function.calls: - if trailing_name(call.name) == wanted: - return call - return None - ranked: list[tuple[int, int, CallSite]] = [] - for order, call in enumerate(function.calls): - fact = _sink_fact_for(call.name, facts) - if fact is None: - continue - cwe_match = 0 if (cwe and fact.cwe == cwe) else 1 - non_constant = 0 if any(not const or names for const, names in zip(call.arg_is_constant, call.arg_names, strict=True)) else 1 - ranked.append((cwe_match + non_constant, order, call)) - ranked.sort(key=lambda item: (item[0], item[1])) - return ranked[0][2] if ranked else None - - -def _sink_fact_for(name: str, facts: SemanticFacts) -> SinkFact | None: - for fact in facts.sinks: - if any(_call_matches(name, call) for call in fact.calls): - return fact - return None - - -def _call_matches(name: str, fact_call: str) -> bool: - """Same rule as the taint walker: exact, dotted suffix, or a bare name equal to the fact's last segment.""" - if name == fact_call or name.endswith("." + fact_call): - return True - return "." not in name and name == fact_call.split(".")[-1] - - -def _function_named(ir: FileIR, function: str, text: str) -> FunctionIR | None: - for item in ir.functions: - if item.name == function: - return item - if text: # labels use declarator names; the parser may name the function differently (or ) - for name, start, _end in named_ranges_from_ir(ir, text): - if name == function: - for item in ir.functions: - if item.start_line == start: - return item - return None - - -def _function_lines(ir: FileIR, function: str, text: str) -> list[str]: - target = _function_named(ir, function, text) - if target is None or not text: - return [] - lines = text.splitlines() - body = lines[target.start_line - 1 : target.end_line] - return [line.strip() for line in body if line.strip()] - - -def shapes_by_sink(shapes: Sequence[Shape]) -> dict[tuple[str, str], list[Shape]]: - """Index for matching: (language, sink_name) -> shapes.""" - index: dict[tuple[str, str], list[Shape]] = {} - for shape in shapes: - index.setdefault((shape.language, shape.sink_name), []).append(shape) - return index - - -@dataclass(frozen=True) -class VariantHit: - path: str - line: int - mechanism_id: str - sink_name: str - source_kind: str - - -def match_shapes( - ir: FileIR, - shapes: Sequence[Shape], - facts: SemanticFacts, - *, - mechanism_ids: Mapping[str, str], -) -> list[VariantHit]: - """Call sites of ``ir`` whose trailing callee name, arity and source-carrying positions match a shape (Req 3.1). - - Any source kind at the shape's positions matches: a variant learned from ``request.args`` still fires when the same - sink takes a parameter. Constant-only arguments never match. ``mechanism_ids`` maps ``Shape.key()`` to the store id. - """ - if not ir.parse_ok: - return [] - index = shapes_by_sink([shape for shape in shapes if shape.language == ir.language]) - if not index: - return [] - scoped = facts.for_language(ir.language) if hasattr(facts, "for_language") else facts - hits: list[VariantHit] = [] - seen: set[tuple[int, str]] = set() - for function in ir.functions: - for call in function.calls: - candidates = index.get((ir.language, trailing_name(call.name))) - if not candidates: - continue - for shape in candidates: - if shape.arity != len(call.arg_texts) or (call.line, shape.key()) in seen: - continue - kinds = [ - _source_kind_of_argument(function, call, position, scoped) - for position in shape.source_positions - if position < len(call.arg_texts) - ] - if len(kinds) != len(shape.source_positions) or any(kind is None for kind in kinds): - continue - seen.add((call.line, shape.key())) - hits.append( - VariantHit( - path=ir.path, - line=call.line, - mechanism_id=mechanism_ids.get(shape.key(), shape.key()), - sink_name=shape.sink_name, - source_kind=str(kinds[0]), - ) - ) - return hits diff --git a/src/openultrasast/vectorstore.py b/src/openultrasast/vectorstore.py deleted file mode 100644 index f5d081e..0000000 --- a/src/openultrasast/vectorstore.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict, dataclass -from pathlib import Path - -from .index import VectorIndex, index_reuse_key, query_vector_index, read_vector_index, write_vector_index -from .preprocess import RepoSnapshot - -SELECTED_LOCAL_VECTOR_STORE = "json-local" - - -@dataclass(frozen=True) -class VectorStoreBakeoffResult: - store: str - metadata_filtering: bool - local_persistence: bool - incremental_keying: bool - export_import: bool - retrieval_quality: bool - selected_default: bool - - @property - def passed(self) -> bool: - return all( - ( - self.metadata_filtering, - self.local_persistence, - self.incremental_keying, - self.export_import, - self.retrieval_quality, - self.selected_default, - ) - ) - - -def bakeoff_json_local(index: VectorIndex, output_path: Path, *, query_embedding: list[float]) -> VectorStoreBakeoffResult: - write_vector_index(index, output_path) - loaded = read_vector_index(output_path) - metadata_matches = query_vector_index(loaded, query_embedding, metadata_filter={"language": "python"}, limit=1) - unfiltered_matches = query_vector_index(loaded, query_embedding, limit=1) - snapshot = RepoSnapshot( - root=loaded.repo_root, - commit=loaded.repo_commit, - file_count=len({record.chunk.path for record in loaded.records}), - languages=_language_counts(loaded), - ) - result = VectorStoreBakeoffResult( - store=SELECTED_LOCAL_VECTOR_STORE, - metadata_filtering=bool(metadata_matches) and metadata_matches[0][1].metadata.get("language") == "python", - local_persistence=output_path.exists() and output_path.stat().st_size > 0, - incremental_keying=index_reuse_key(snapshot, loaded.embedding_model) == index_reuse_key(snapshot, loaded.embedding_model), - export_import=len(loaded.records) == len(index.records) and loaded.embedding_model == index.embedding_model, - retrieval_quality=bool(unfiltered_matches) and unfiltered_matches[0][0] > 0, - selected_default=SELECTED_LOCAL_VECTOR_STORE == "json-local", - ) - return result - - -def bakeoff_payload(result: VectorStoreBakeoffResult) -> dict[str, object]: - payload = asdict(result) - payload["passed"] = result.passed - return payload - - -def _language_counts(index: VectorIndex) -> dict[str, int]: - counts: dict[str, int] = {} - for record in index.records: - counts[record.chunk.language] = counts.get(record.chunk.language, 0) + 1 - return counts diff --git a/tests/test_candidate_enumeration.py b/tests/test_candidate_enumeration.py index ca5277b..7ad2474 100644 --- a/tests/test_candidate_enumeration.py +++ b/tests/test_candidate_enumeration.py @@ -16,7 +16,7 @@ import pytest sys.path.insert(0, str(Path(__file__).parent)) from test_obligation_operations import APP # noqa: E402 -from openultrasast.learning.families import load_families # noqa: E402 +from openultrasast.model.taxonomy import load_families # noqa: E402 from openultrasast.semantic.extra import has_semantic_extra # noqa: E402 INJECTION = ( @@ -33,7 +33,7 @@ INJECTION = ( def _region(path: str = "app.py", function: str | None = "run"): # type: ignore[no-untyped-def] - from openultrasast.learning.detectors import Region + from openultrasast.model.candidates import Region return Region(path=path, function=function) @@ -44,7 +44,7 @@ def _write(tmp_path: Path, name: str, text: str) -> Path: def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", INJECTION) result = enumerate_candidates(root, _region(), "injection", taxonomy=load_families()) @@ -62,7 +62,7 @@ def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path def test_the_family_decides_which_kinds_are_candidates() -> None: - from openultrasast.learning.candidates import FAMILY_SHAPE + from openultrasast.model.candidates import FAMILY_SHAPE taxonomy = load_families() assert set(FAMILY_SHAPE) == {family.id for family in taxonomy.families}, "every family says what carries its bug" @@ -76,7 +76,7 @@ def test_the_family_decides_which_kinds_are_candidates() -> None: @pytest.mark.skipif(not has_semantic_extra(), reason="the guards come from the parsed IR") def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_path: Path) -> None: """An absence bug has no sink. Its candidate is the operation, and what the model needs is what guards it.""" - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) guarded = enumerate_candidates(root, _region(function="guarded"), "access_control", taxonomy=load_families()) @@ -95,7 +95,7 @@ def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_p def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", "def broken(:\n pass\n") result = enumerate_candidates(root, _region(function="broken"), "injection", taxonomy=load_families()) @@ -103,7 +103,7 @@ def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: P def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", "def empty():\n return 1\n") result = enumerate_candidates(root, _region(function="empty"), "injection", taxonomy=load_families()) @@ -111,7 +111,7 @@ def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Pa def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> None: - from openultrasast.learning.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates + from openultrasast.model.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates # one call per line, so the arithmetic is the bound and not the fixture body = "\n".join(f" sink_{index}(value)" for index in range(MAX_CANDIDATES_PER_CALL + 5)) @@ -123,7 +123,7 @@ def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> def test_enumerating_twice_returns_an_identical_sequence(tmp_path: Path) -> None: """The batches, and therefore the sequence of model calls, are determined by the input alone (Req 2.4).""" - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) taxonomy = load_families() @@ -139,7 +139,7 @@ def _order(result): # type: ignore[no-untyped-def] def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) whole = enumerate_candidates(root, _region(function=None), "injection", taxonomy=load_families()) @@ -150,7 +150,7 @@ def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) - @pytest.mark.parametrize("family", ["injection", "path", "deserialization", "untrusted_destination", "config_secrets"]) def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: Path, family: str) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", INJECTION) result = enumerate_candidates(root, _region(), family, taxonomy=load_families()) @@ -164,7 +164,7 @@ def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_path: Path) -> None: from openultrasast.benchmark import ExpectedFinding - from openultrasast.learning.candidates import ceiling + from openultrasast.model.candidates import ceiling from openultrasast.pairs import PairCase def case(name: str, body: str, *, slice_name: str, family: str) -> PairCase: @@ -210,7 +210,7 @@ def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_pa def test_the_ruleset_generator_is_measured_too_so_the_comparison_is_reproducible(tmp_path: Path) -> None: """12.1% against 96.6% is why the ruleset is not the generator. A remembered number is not a measurement.""" - from openultrasast.learning.candidates import ceiling + from openultrasast.model.candidates import ceiling from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_slice, select_vendored cases = [c for c in select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py")) if not c.unscorable][:6] @@ -225,12 +225,12 @@ def test_the_command_runs_offline_and_writes_a_stable_artifact(tmp_path: Path, m for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): monkeypatch.delenv(name, raising=False) - out = tmp_path / "learning" - assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 + out = tmp_path / "model" + assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 artifact = out / "candidate-ceiling.json" assert artifact.is_file() first = artifact.read_bytes() - assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 + assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 assert artifact.read_bytes() == first, "the ceiling is a property of the corpus, not of when it was measured" import json diff --git a/tests/test_degradation_matrix.py b/tests/test_degradation_matrix.py index 48767e6..ad94422 100644 --- a/tests/test_degradation_matrix.py +++ b/tests/test_degradation_matrix.py @@ -44,71 +44,6 @@ def test_the_baseline_records_where_it_came_from() -> None: # --- one optional dependency at a time --------------------------------------- -def test_without_harnessx_the_meta_agent_proposer_proposes_nothing_and_says_why(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning import proposer as proposer_module - from openultrasast.learning.detectors import FamilyConfig - from openultrasast.learning.proposer import FailureFacts, HarnessXProposer - - monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False) - (tmp_path / "injection").mkdir() - facts = FailureFacts( - family="injection", - config=FamilyConfig( - family="injection", version="0", prompt="p", checklist="c", tools=("read_file",), max_steps=1, max_cost_usd=0.1, max_chars=100 - ), - ) - proposer = HarnessXProposer(configs_dir=tmp_path, model="m") - assert proposer.propose(facts) is None - assert proposer.reason == "harnessx_unavailable" - - -def test_without_a_chat_endpoint_the_learning_run_records_it_and_still_writes_its_artifact( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from openultrasast.cli import main - - for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): - monkeypatch.delenv(name, raising=False) - (tmp_path / "a-v.py").write_text("def run():\n return 1\n") - (tmp_path / "a-f.py").write_text("def run():\n return 1\n") - (tmp_path / "pairs.toml").write_text( - '[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n' - 'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n' - ) - printed: list[str] = [] - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: printed.append(" ".join(str(item) for item in args))) - assert main(["learning", "baseline", "--catalog", str(tmp_path / "pairs.toml"), "--out", str(tmp_path / "l"), "--model", "none"]) == 0 - assert any("learning_endpoint_unavailable" in line for line in printed) - assert (tmp_path / "l" / "baseline" / "none" / "report.json").is_file() - - -def test_without_family_configurations_a_scan_records_the_reason_rather_than_the_traceback( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from openultrasast.cli import _run_family_detectors - - class _Settings: - configs_dir = "does/not/exist" - families_path = None - - class _Runtime: - def __init__(self) -> None: - self.state: dict[str, list[dict[str, object]]] = {"degradations": []} - - runtime = _Runtime() - findings, payload, cited = _run_family_detectors( - root=tmp_path, - hotspots=[], - cases=None, - settings=_Settings(), - config=None, - runtime=runtime, # type: ignore[arg-type] - ) - assert (findings, payload, cited) == ([], {}, {}) - assert [item["reason"] for item in runtime.state["degradations"]] == ["learning_configs_unavailable"] - - def test_without_the_network_a_pointer_pair_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: from openultrasast.pairs import evaluate_catalog, load_pair_catalog, select_slice diff --git a/tests/test_fixture_difficulty.py b/tests/test_fixture_difficulty.py deleted file mode 100644 index e5b2012..0000000 --- a/tests/test_fixture_difficulty.py +++ /dev/null @@ -1,117 +0,0 @@ -"""A unit fixture may not be easier than the corpus it claims to cover (learning-harness 4.2, Req 10.3, 10.4). - -Two things a corpus number cannot survive: a byte-identical pair counted as a detector result, and a fixture that -passes because the world it describes is simpler than the one the detector meets. The first is excluded by name; -the second is measured on three axes the corpus actually varies on, and the comparison is recorded. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).parent)) -from test_obligation_operations import APP # noqa: E402 - -from openultrasast.learning.difficulty import DifficultyVector, difficulty_of, hardest, strictly_easier # noqa: E402 -from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored # noqa: E402 - -TWINS = { - "dsvw-dsvw-do-get", - "python-insecure-app-main-try-hack-me", - "pythonssti-main-read-root", - "vampi-books-get-by-title", - "vulnpy-deserialization-do-pickle-load", -} -# The rows the shared obligations fixture claims to cover: the vibe-py absence pairs. -COVERED = ("vampi-books-get-by-title", "vampi-users-update-password", "threatbyte-api-v1-get", "threatbyte-api-v1-delete") -FIXTURE_HANDLERS = ("guarded", "constrained", "constrained_too", "leaky", "from_spec") -REPORT = Path("benchmarks/measurements/2026-09-06-fixture-difficulty.json") - - -# --- identical twins --------------------------------------------------------- - - -def test_every_identical_twin_declares_the_limit_rather_than_only_hashing_to_it() -> None: - """A content hash is evidence; the catalog row is the maintainer standing behind it. - - A re-harvest that changes one byte would otherwise turn a known-unscorable row back into a scored one with no - review, and the number would move for a reason nobody chose.""" - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - for name in sorted(TWINS): - assert cases[name].known_limit == "identical_twin", f"{name} is a twin by content but says nothing in the catalog" - assert cases[name].unscorable == "identical_twin", "the declared limit and the computed one must be one reason, not two" - - -def test_the_twins_leave_every_denominator_with_their_reason_listed() -> None: - from openultrasast.learning.families import load_families - from openultrasast.learning.scoring import PairFamilyScore, aggregate - - cases = [case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if case.name in TWINS] - scores = [ - PairFamilyScore( - pair=case.name, family="injection", slice=case.slice, runs=(), outcome="unscorable", unscorable_reason=case.unscorable - ) - for case in cases - ] - metrics = aggregate(scores, taxonomy=load_families())["injection"] - assert metrics.scorable == 0 - assert metrics.unscorable == {"identical_twin": len(TWINS)} - - -# --- the difficulty check ---------------------------------------------------- - - -def _corpus_vectors() -> list[DifficultyVector]: - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - vectors = [] - for name in COVERED: - case = cases[name] - function = next(row.function for row in case.expected if row.function) - context = [vuln.read_text() for _relpath, vuln, _fixed in case.context_files] - vectors.append(difficulty_of(case.vuln_file.read_text(), function, context=context)) - return vectors - - -def test_the_shared_obligations_fixture_is_not_easier_than_the_rows_it_covers() -> None: - """The check Req 10.4 asks for, on the fixture every obligations test shares.""" - fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS] - corpus = _corpus_vectors() - assert not strictly_easier(fixture, corpus), ( - f"the fixture reaches {hardest(fixture)} where the corpus reaches {hardest(corpus)} " - "(binding hops, registration distance, guard distance)" - ) - - -def test_the_check_catches_a_fixture_that_is_easier_on_one_axis(tmp_path: Path) -> None: - """Mutation of the check itself: drop the corpus-shaped handler and it has to fail.""" - without = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS if name != "from_spec"] - assert strictly_easier(without, _corpus_vectors()) - assert hardest(without)[1] == 0, "every remaining handler carries its route decorator; nothing is registered elsewhere" - - -def test_the_verdict_is_recorded_beside_the_numbers_it_explains() -> None: - assert REPORT.is_file(), "the fixture-difficulty verdict is committed, not recomputed from memory" - payload = json.loads(REPORT.read_text()) - fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS] - corpus = _corpus_vectors() - assert payload["fixture"] == [row.to_dict() for row in fixture] - assert payload["corpus"] == [row.to_dict() for row in corpus] - assert payload["strictly_easier"] is False and payload["verdict"] == "fixture_matches_corpus" - assert payload["axes"] == ["binding_hops", "registration_distance", "guard_distance"] - - -@pytest.mark.parametrize( - ("source", "expected"), - [ - ("@app.route('/x')\ndef h():\n return Book.query.filter_by(user_id=request.user.id).first()\n", (0, 0, 2)), - ("@app.route('/x')\n@login_required\ndef h():\n return 1\n", (0, 0, 0)), - ("def h():\n u = current_user.id\n v = u\n return v\n", (2, 2, 1)), - ("app.add_url_rule('/x', view_func=h)\n\n\ndef h():\n return 1\n", (0, 1, 2)), - ], -) -def test_each_axis_is_read_off_the_code(source: str, expected: tuple[int, int, int]) -> None: - assert difficulty_of(source, "h").axes() == expected diff --git a/tests/test_index.py b/tests/test_index.py index 8b7ed2e..59a9442 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -15,7 +15,6 @@ from openultrasast.index import ( write_vector_index, ) from openultrasast.preprocess import RepoSnapshot, preprocess_repository -from openultrasast.vectorstore import bakeoff_json_local, bakeoff_payload class FakeEmbeddingClient: @@ -103,15 +102,3 @@ def test_retrieval_package_is_bounded_by_character_budget() -> None: assert package.truncated is True -def test_json_local_bakeoff_selects_default_store(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def login(user):\n return user\n") - snapshot, targets = preprocess_repository(repo) - chunks = build_code_chunks(repo, targets) - index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient()) - - result = bakeoff_json_local(index, tmp_path / "vector-index.json", query_embedding=[20.0, 1.0]) - - assert result.passed is True - assert bakeoff_payload(result)["selected_default"] is True diff --git a/tests/test_learning_acceptance.py b/tests/test_learning_acceptance.py deleted file mode 100644 index b0cfdf7..0000000 --- a/tests/test_learning_acceptance.py +++ /dev/null @@ -1,115 +0,0 @@ -"""learning-harness task 3.6: the acceptance rule and the directory revert (offline).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.learning.rounds import NoiseFloor - - -def _floor(family: str, budget: int = 0, *, gates: bool = True) -> NoiseFloor: - return NoiseFloor(family=family, k_runs=5, pairs=10, flaky_pairs=budget, negative_flip_rate=0.0, budget_flips=budget, gates=gates) - - -def _decide(train: float, holdout: float, sweep: dict[str, int], floors: dict[str, NoiseFloor]): # type: ignore[no-untyped-def] - from openultrasast.learning.acceptance import decide - - return decide(target="access_control", train_delta=train, holdout_delta=holdout, sweep=sweep, floors=floors) - - -def test_a_change_that_helps_and_hurts_nobody_is_accepted() -> None: - verdict = _decide(0.1, 0.2, {"injection": 0}, {"injection": _floor("injection")}) - assert verdict.accepted and verdict.reason == "improved" - - -def test_one_side_may_stand_still_but_the_other_must_move() -> None: - assert _decide(0.0, 0.2, {}, {}).accepted - assert _decide(0.2, 0.0, {}, {}).accepted - tie = _decide(0.0, 0.0, {}, {}) - assert not tie.accepted and tie.reason == "tie" # a tie is rejected, or the loop drifts on noise - - -def test_a_decrease_on_either_side_is_rejected() -> None: - train_down = _decide(-0.1, 0.5, {}, {}) - assert not train_down.accepted and train_down.reason == "train_regressed" - holdout_down = _decide(0.5, -0.1, {}, {}) - assert not holdout_down.accepted and holdout_down.reason == "holdout_regressed" - - -def test_another_family_beyond_its_measured_budget_rejects_the_round() -> None: - within = _decide(0.2, 0.2, {"injection": 2}, {"injection": _floor("injection", 2)}) - assert within.accepted # inside the noise this family shows against itself - beyond = _decide(0.2, 0.2, {"injection": 3}, {"injection": _floor("injection", 2)}) - assert not beyond.accepted and beyond.reason == "collateral_regression" - assert beyond.offenders == ("injection",) - - -def test_a_family_that_never_gates_cannot_reject_a_round() -> None: - verdict = _decide(0.2, 0.2, {"memory": 99}, {"memory": _floor("memory", 0, gates=False)}) - assert verdict.accepted and verdict.offenders == () - - -def test_a_family_with_no_measured_floor_gets_no_budget() -> None: - verdict = _decide(0.2, 0.2, {"path": 1}, {}) - assert not verdict.accepted and verdict.offenders == ("path",) # unmeasured is not the same as unlimited - - -def test_a_snapshot_restores_a_family_directory_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.learning.acceptance import DirectorySnapshot - - directory = tmp_path / "access_control" - directory.mkdir() - (directory / "prompt.md").write_text("original\n") - (directory / "family.toml").write_text('version = "0"\n') - snapshot = DirectorySnapshot.of(directory) - (directory / "prompt.md").write_text("changed\n") - (directory / "extra.md").write_text("added by a round\n") - (directory / "family.toml").unlink() - snapshot.restore() - assert (directory / "prompt.md").read_text() == "original\n" - assert (directory / "family.toml").read_text() == 'version = "0"\n' - assert not (directory / "extra.md").exists() # a file the round added is not left behind - assert sorted(path.name for path in directory.iterdir()) == ["family.toml", "prompt.md"] - - -def test_every_other_family_directory_is_untouched_across_a_round(tmp_path: Path) -> None: - """Req 6.2: a round can only write its target, so the freeze is structural rather than promised.""" - from openultrasast.learning.acceptance import DirectorySnapshot, digest_configs - from openultrasast.learning.detectors import write_default_configs - from openultrasast.learning.families import load_families - from openultrasast.learning.proposer import Proposal, apply_proposal - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - before = digest_configs(configs) - snapshot = DirectorySnapshot.of(configs / "access_control") - apply_proposal(Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}), configs / "access_control") - after = digest_configs(configs) - assert after["access_control"] != before["access_control"] - assert {name: value for name, value in after.items() if name != "access_control"} == { - name: value for name, value in before.items() if name != "access_control" - } - snapshot.restore() - assert digest_configs(configs) == before # a rejected round leaves the whole tree as it found it - - -def test_a_bumped_version_is_written_only_on_acceptance(tmp_path: Path) -> None: - from openultrasast.learning.acceptance import bump_version - from openultrasast.learning.detectors import load_family_configs, write_default_configs - from openultrasast.learning.families import load_families - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - bump_version(configs / "access_control", "1") - taxonomy = load_families() - assert load_family_configs(configs, taxonomy)["access_control"].version == "1" - assert load_family_configs(configs, taxonomy)["injection"].version == "0" - - -@pytest.mark.parametrize(("spent", "cap", "expected"), [(0.5, 10.0, True), (10.0, 10.0, True), (10.01, 10.0, False)]) -def test_a_round_over_its_cost_cap_is_reverted(spent: float, cap: float, expected: bool) -> None: - from openultrasast.learning.acceptance import within_budget - - assert within_budget(spent, cap) is expected diff --git a/tests/test_learning_aggregation.py b/tests/test_learning_aggregation.py deleted file mode 100644 index 8561a79..0000000 --- a/tests/test_learning_aggregation.py +++ /dev/null @@ -1,116 +0,0 @@ -"""learning-harness task 2.4: aggregation, noise handling and honest denominators (offline).""" - -from __future__ import annotations - -import json - -from openultrasast.learning.families import load_families -from openultrasast.learning.scoring import FamilyOutcome, PairFamilyScore - - -def _score( - pair: str, outcome: FamilyOutcome, *, family: str = "access_control", slice_name: str = "vibe-py", **extra: object -) -> PairFamilyScore: - runs = () if outcome == "unscorable" else (outcome,) - return PairFamilyScore(pair=pair, family=family, slice=slice_name, runs=runs, outcome=outcome, **extra) # type: ignore[arg-type] - - -def test_rates_use_only_the_scorable_rows_and_the_reasons_are_kept() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - scores = [ - _score("a", "pair_correct"), - _score("b", "both_silent"), - _score("c", "unscorable", unscorable_reason="identical_twin"), - _score("d", "unscorable", unscorable_reason="identical_twin"), - _score("e", "unscorable", unscorable_reason="unsupported_language"), - ] - metrics = aggregate(scores, taxonomy=taxonomy)["access_control"] - assert metrics.scorable == 2 and metrics.unscorable == {"identical_twin": 2, "unsupported_language": 1} - assert metrics.recall == 0.5 and metrics.silence == 1.0 and metrics.youden == 0.5 - assert metrics.outcomes == {"pair_correct": 1, "both_silent": 1} - assert metrics.taxonomy_version == taxonomy.version - json.dumps(metrics.to_dict()) - - -def test_a_family_with_nothing_scorable_reports_no_rate_rather_than_a_perfect_one() -> None: - from openultrasast.learning.scoring import aggregate - - metrics = aggregate([_score("a", "unscorable", unscorable_reason="identical_twin")], taxonomy=load_families())["access_control"] - assert metrics.scorable == 0 and metrics.recall == 0.0 and metrics.silence == 0.0 and metrics.youden == 0.0 - - -def test_directional_bias_separates_flagging_everything_from_saying_everything_is_safe() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - loud = aggregate([_score("a", "both_flagged"), _score("b", "both_flagged")], taxonomy=taxonomy)["access_control"] - quiet = aggregate([_score("a", "both_silent"), _score("b", "both_silent")], taxonomy=taxonomy)["access_control"] - assert loud.directional_bias == 1.0 and quiet.directional_bias == -1.0 - assert loud.youden == quiet.youden == 0.0 # the same Youden, opposite failures: the index is what tells them apart - - -def test_recall_at_a_fixed_false_positive_ceiling_is_forfeited_when_the_family_leaks() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - clean = [_score(str(index), "pair_correct") for index in range(9)] + [_score("x", "both_silent")] - assert aggregate(clean, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"].fixed_fpr_recall == 0.9 - leaky = [_score(str(index), "pair_correct") for index in range(8)] + [_score("y", "both_flagged"), _score("z", "reversed")] - metrics = aggregate(leaky, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"] - assert metrics.recall == 0.9 and metrics.fixed_fpr_recall == 0.0 # a 20% leak rate buys no credit at a 10% ceiling - - -def test_negative_flips_and_the_reliable_change_test_compare_against_a_baseline() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - baseline = [_score(str(index), "pair_correct") for index in range(10)] - same = aggregate(baseline, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert same.negative_flips == 0 and same.reliable_change is False - broke_one = [_score("0", "both_silent")] + [_score(str(index), "pair_correct") for index in range(1, 10)] - one = aggregate(broke_one, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert one.negative_flips == 1 and one.reliable_change is False # one flip in ten is inside the noise - broke_six = [_score(str(index), "both_silent") for index in range(6)] + [_score(str(index), "pair_correct") for index in range(6, 10)] - six = aggregate(broke_six, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert six.negative_flips == 6 and six.reliable_change is True - assert six.p_value is not None and six.p_value < 0.05 - - -def test_metrics_are_reported_per_slice_when_asked() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - scores = [_score("a", "pair_correct", slice_name="vibe-py"), _score("b", "both_silent", slice_name="agent-vfc")] - overall = aggregate(scores, taxonomy=taxonomy) - assert set(overall) == {"access_control"} and overall["access_control"].scorable == 2 - per_slice = aggregate(scores, taxonomy=taxonomy, per_slice=True) - assert set(per_slice) == {"vibe-py/access_control", "agent-vfc/access_control"} - assert per_slice["vibe-py/access_control"].recall == 1.0 and per_slice["agent-vfc/access_control"].recall == 0.0 - - -def test_the_sign_test_is_two_sided_and_symmetric() -> None: - from openultrasast.learning.scoring import sign_test - - assert sign_test(0, 0) == 1.0 - assert sign_test(6, 0) == sign_test(0, 6) < 0.05 - assert sign_test(1, 0) > 0.05 and sign_test(3, 3) == 1.0 - - -def test_every_family_figure_says_when_hierarchical_credit_could_not_apply() -> None: - """Req 3.7: with a flat taxonomy no answer can earn partial credit, and the number must carry that.""" - from openultrasast.learning.families import Family, FamilyTaxonomy - from openultrasast.learning.scoring import aggregate - - flat = load_families() - metrics = aggregate([_score("a", "pair_correct")], taxonomy=flat)["access_control"] - assert metrics.hierarchical_credit is False and metrics.to_dict()["hierarchical_credit"] is False - nested = FamilyTaxonomy( - version="test", - families=( - Family(id="injection", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none"), - Family(id="access_control", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none", parent="injection"), - ), - ) - assert aggregate([_score("a", "pair_correct")], taxonomy=nested)["access_control"].hierarchical_credit is True diff --git a/tests/test_learning_canaries.py b/tests/test_learning_canaries.py deleted file mode 100644 index d7cdfb4..0000000 --- a/tests/test_learning_canaries.py +++ /dev/null @@ -1,197 +0,0 @@ -"""learning-harness task 2.7: canary templates and the second judge (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.config import SandboxConfig -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.regress.safety import check_snippet_safety -from openultrasast.sandbox import SandboxResult -from openultrasast.sandbox.runner import FakeSandboxRunner -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient - -CANARY_FAMILIES = ("injection", "path", "deserialization") - - -def _claim(family: str, function: str = "lookup", line: int = 3) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=function, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}"], - ranking_priority=0.0, - ) - - -def test_every_canary_template_renders_and_passes_the_snippet_safety_check() -> None: - from openultrasast.learning.canaries import render_canary - - for family in CANARY_FAMILIES: - snippet, token = render_canary(family, language="python", path="app.py", function="lookup") - assert token and token in snippet - check_snippet_safety(snippet) # raises when the template would be refused before it ever runs - assert "/workspace" in snippet # the candidate is imported from the read-only source mount - assert render_canary("memory", language="python", path="app.py", function="f") is None - assert render_canary("injection", language="rust", path="a.rs", function="f") is None - - -def test_a_canary_token_is_unique_to_the_experiment_it_names() -> None: - from openultrasast.learning.canaries import render_canary - - first = render_canary("injection", language="python", path="app.py", function="lookup") - second = render_canary("injection", language="python", path="app.py", function="other") - third = render_canary("path", language="python", path="app.py", function="lookup") - assert first and second and third - assert first[1] != second[1] != third[1] and first[1] != third[1] - # the same family, file and function is the same experiment, so it is deliberately the same token - assert render_canary("injection", language="python", path="app.py", function="lookup")[1] == first[1] # type: ignore[index] - - -def test_the_canary_verifier_proves_only_when_the_oracle_fires(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - - (tmp_path / "app.py").write_text("def lookup(value):\n return value\n") - fired = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, f"CANARY {token}", "", False)), limits=SandboxConfig()) - proven = fired.verify(_claim("injection"), root=tmp_path, language="python") - assert proven.rung == "proven" and proven.reason == "oracle_fired" - quiet = CanaryVerifier(runner=_runner(lambda _t: SandboxResult(1, "boom", "Traceback", False)), limits=SandboxConfig()) - unproven = quiet.verify(_claim("injection"), root=tmp_path, language="python") - assert unproven.rung == "suspicion" and unproven.reason == "oracle_quiet" # a crash is not this family's evidence - - -def test_a_family_or_language_with_no_template_stays_at_suspicion(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - - (tmp_path / "app.py").write_text("def lookup(value):\n return value\n") - verifier = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig()) - assert verifier.verify(_claim("memory"), root=tmp_path, language="python").reason == "no_template" - assert verifier.verify(_claim("injection"), root=tmp_path, language="rust").reason == "no_template" - - -def test_the_registry_uses_the_canary_verifier_when_one_is_supplied(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - from openultrasast.learning.verifiers import NoVerifier, verifier_for - - taxonomy = load_families() - canary = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig()) - assert verifier_for(taxonomy.by_id("injection"), canary=canary) is canary - assert isinstance(verifier_for(taxonomy.by_id("access_control"), canary=canary), type(verifier_for(taxonomy.by_id("access_control")))) - assert isinstance(verifier_for(taxonomy.by_id("memory"), canary=canary), NoVerifier) - - -def test_nothing_is_published_as_proven_until_a_second_judge_confirms() -> None: - from openultrasast.learning.canaries import confirm_proven - from openultrasast.learning.verifiers import Verification - - proven = Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc") - agrees = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "the row came back"}))]) - kept = confirm_proven(proven, _claim("injection"), client=agrees, model="judge", snippet="print('x')") - assert kept.rung == "proven" - declines = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": False, "reason": "unrelated output"}))]) - dropped = confirm_proven(proven, _claim("injection"), client=declines, model="judge", snippet="print('x')") - assert dropped.rung == "static_corroboration" and dropped.reason == "judge_declined" - assert confirm_proven(proven, _claim("injection"), client=None, model="", snippet="x").rung == "static_corroboration" - lower = Verification(rung="suspicion", reason="oracle_quiet") - assert confirm_proven(lower, _claim("injection"), client=agrees, model="judge", snippet="x") is lower # nothing to confirm - - -def test_the_judge_sees_the_snippet_and_the_oracle_output_and_never_the_label() -> None: - from openultrasast.learning.canaries import confirm_proven - from openultrasast.learning.verifiers import Verification - - client = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "ok"}))]) - confirm_proven( - Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc"), - _claim("injection"), - client=client, - model="judge", - snippet="print('CANARY abc')", - ) - prompt = json.dumps(client.calls[0]["messages"]) - assert "CANARY abc" in prompt and "print('CANARY abc')" in prompt - assert client.calls[0].get("json_object") is True and client.calls[0]["tools"] == [] - assert "family:injection" not in prompt # the judge is told the claim, not the answer key - - -def test_templates_and_oracle_code_live_outside_every_proposer_write_root(tmp_path: Path) -> None: - from openultrasast.learning import canaries - from openultrasast.learning.detectors import write_default_configs - - write_roots = set(write_default_configs(tmp_path, load_families(), prompt="p", version="0").values()) - module = Path(canaries.__file__).resolve() - assert not any(module.is_relative_to(root.resolve()) for root in write_roots) - assert "site-packages" in str(module) or "src/openultrasast" in str(module) - - -def _runner(reply: object) -> FakeSandboxRunner: - """A sandbox that answers from whatever canary token the snippet actually planted.""" - - class _Runner(FakeSandboxRunner): # type: ignore[misc] - def run(self, spec: object) -> SandboxResult: - body = "\n".join(str(value) for value in (getattr(spec, "scratch_files", {}) or {}).values()) - marker = next((word.strip("'\"") for word in body.split() if "ousast-canary-" in word), "ousast-canary-none") - return reply(marker) # type: ignore[operator] - - return _Runner() - - -VULNERABLE = { - "injection": "import os\n\n\ndef lookup(value):\n os.system(value)\n return 'done'\n", - "path": "def lookup(value):\n with open(value, encoding='utf-8') as handle:\n return handle.read()\n", - "deserialization": "import pickle\n\n\ndef lookup(value):\n return pickle.loads(value)\n", -} -FIXED = { - "injection": ( - "import shlex, subprocess\n\n\ndef lookup(value):\n" - " subprocess.run(['echo', shlex.quote(value)], check=False)\n return 'done'\n" - ), - "path": ( - "from pathlib import Path\n\n\ndef lookup(value):\n" - " name = Path(value).name\n return (Path('/tmp') / name).read_text(encoding='utf-8') if False else ''\n" - ), - "deserialization": "import json\n\n\ndef lookup(value):\n return json.loads(value.decode('utf-8', 'ignore') or '{}')\n", -} - - -@pytest.mark.parametrize("family", CANARY_FAMILIES) -def test_each_canary_fires_on_a_real_bug_and_stays_quiet_on_its_fixed_twin(tmp_path: Path, family: str) -> None: - """The oracle must be a property of the code, not of the harness: run the snippet for real, both sides. - - The injection fixture's fixed twin deliberately echoes its input, which is why the payload carries a - value unrelated to the oracle token: an echoing handler must not be able to satisfy the oracle. - """ - import subprocess - import sys - - from openultrasast.learning.canaries import render_canary - - for side, sources in (("vuln", VULNERABLE), ("fixed", FIXED)): - workspace = tmp_path / side / "workspace" - scratch = tmp_path / side / "scratch" - workspace.mkdir(parents=True) - scratch.mkdir(parents=True) - (workspace / "app.py").write_text(sources[family]) - rendered = render_canary( - family, language="python", path="app.py", function="lookup", workspace=str(workspace), scratch=str(scratch) - ) - assert rendered is not None - snippet, token = rendered - script = scratch / "case.py" - script.write_text(snippet) - completed = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, timeout=60, check=False) - fired = token in completed.stdout or token in completed.stderr - assert fired is (side == "vuln"), f"{family} {side}: {completed.stdout!r} {completed.stderr!r}" diff --git a/tests/test_learning_classifier_report.py b/tests/test_learning_classifier_report.py deleted file mode 100644 index e423435..0000000 --- a/tests/test_learning_classifier_report.py +++ /dev/null @@ -1,115 +0,0 @@ -"""learning-harness task 2.2: the classifier reports on itself and queues disagreements (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored - - -def _case(tmp_path: Path, name: str, *, cwe: str, mechanism: str = "other", family: str | None = None, tier: str = "reviewed") -> PairCase: - (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n") - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}.py", - fixed_file=tmp_path / f"{name}.py", - relpath="app.py", - expected=( - ExpectedFinding(cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function="f", mechanism=mechanism, family=family), - ), - min_recall=1.0, - fix_policy="silent", - review_tier=tier, - reviewer="m" if tier == "reviewed" else "", - ) - - -def test_the_report_covers_the_whole_corpus_and_names_the_unknowns(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG)) - report = measure_classifier(cases, taxonomy, client=None) - assert report.taxonomy_version == taxonomy.version - assert report.classified == len(cases) - assert sum(report.per_tier.values()) == len(cases) and set(report.per_tier) <= {"static", "model", "none"} - assert report.per_family["memory"] >= 150 # the vfc rows land in one family instead of reading `other` - assert report.per_family["access_control"] >= 10 and report.per_family["injection"] >= 40 - assert report.unknown == 2 and report.per_tier["static"] == len(cases) - 2 # tier one places all but two rows - payload = report.to_dict() - json.dumps(payload) - # Task 4.1 declared `family` on the ten absence rows; only the four at a reference tier (`seeded`) count as labels - # a maintainer stands behind, and the six `title`-tier agent rows do not. Agreement over four rows of one family is - # not yet a measurement of the classifier, so the number is recorded, not celebrated. - assert payload["labeled"] == 4 and payload["agreement"] == 1.0 - assert set(report.confusion) == {"access_control"} - - -def test_agreement_uses_the_first_answer_and_the_oracle_uses_any_of_them(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = [ - _case(tmp_path, "hit", cwe="CWE-89", family="injection"), # tier one answers injection: agreement and oracle - _case(tmp_path, "miss", cwe="CWE-89", family="access_control"), # answers injection: neither - ] - report = measure_classifier(cases, taxonomy, client=None) - assert report.labeled == 2 - assert report.agreement == 0.5 and report.oracle_router == 0.5 - assert abs(report.random_router - 1 / 9) < 1e-9 # a uniform pick over the nine real families - assert report.confusion["injection"]["injection"] == 1 and report.confusion["access_control"]["injection"] == 1 - - -def test_every_disagreement_is_queued_once_with_both_answers_and_the_tier(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier, write_review_queue - - taxonomy = load_families() - cases = [_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), _case(tmp_path, "hit", cwe="CWE-89", family="injection")] - report = measure_classifier(cases, taxonomy, client=None) - assert [entry.pair for entry in report.queue] == ["miss"] - entry = report.queue[0] - assert entry.label == "access_control" and entry.classified == ("injection",) and entry.tier == "static" - queue_path = tmp_path / "review-queue.jsonl" - write_review_queue(queue_path, report.queue) - write_review_queue(queue_path, report.queue) # a second measurement must not duplicate the row - rows = [json.loads(line) for line in queue_path.read_text().splitlines() if line.strip()] - assert [row["pair"] for row in rows] == ["miss"] - assert rows[0]["label"] == "access_control" and rows[0]["classified"] == ["injection"] and rows[0]["tier"] == "static" - - -def test_measuring_never_writes_a_label(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - case = _case(tmp_path, "miss", cwe="CWE-89", family="access_control") - measure_classifier([case], taxonomy, client=None) - assert case.expected[0].family == "access_control" # the classifier proposes; the maintainer decides - - -def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = [ - _case(tmp_path, "trusted", cwe="CWE-89", family="access_control", tier="reviewed"), - _case(tmp_path, "untrusted", cwe="CWE-89", family="access_control", tier="title"), - ] - report = measure_classifier(cases, taxonomy, client=None) - assert report.labeled == 1 and report.classified == 2 # an unreviewed label is not a reference answer - assert [entry.pair for entry in report.queue] == ["trusted"] - - -def test_the_report_says_when_hierarchical_credit_could_not_apply(tmp_path: Path) -> None: - """Req 2.6: the shipped taxonomy is flat, so partial credit is inert and every figure must say so.""" - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - report = measure_classifier([_case(tmp_path, "a", cwe="CWE-89", family="injection")], taxonomy, client=None) - assert report.hierarchical_credit is False - assert report.to_dict()["hierarchical_credit"] is False diff --git a/tests/test_learning_classify.py b/tests/test_learning_classify.py deleted file mode 100644 index 25d44fa..0000000 --- a/tests/test_learning_classify.py +++ /dev/null @@ -1,163 +0,0 @@ -"""learning-harness task 2.1: the three-tier auto-classifier (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) - - -def _case(tmp_path: Path, **row: object) -> PairCase: - (tmp_path / "v.py").write_text(APP) - (tmp_path / "f.py").write_text(APP + "# fixed\n") - expected = ExpectedFinding( - cwe=str(row.get("cwe", "")), - vulnerability_class="x", - path="app.py", - evidence="", - function="leaky", - mechanism=str(row.get("mechanism", "other")), - family=row.get("family"), # type: ignore[arg-type] - ) - return PairCase( - name="t", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=(expected,), - min_recall=1.0, - fix_policy="silent", - ) - - -def test_tier_one_decides_from_a_declared_family_a_mechanism_or_a_weakness(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - declared = classify_pair(_case(tmp_path, family="injection", cwe="CWE-639"), taxonomy, client=None) - assert declared.families == ("injection",) and declared.tier == "static" # a declared label wins - mechanism = classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=None) - assert mechanism.families == ("access_control",) and mechanism.tier == "static" - weakness = classify_pair(_case(tmp_path, cwe="CWE-89", mechanism="source_reaches_sink"), taxonomy, client=None) - assert weakness.families == ("injection",) and weakness.tier == "static" # the generic mechanism defers to the weakness - nothing = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=None) - assert nothing.families == () and nothing.tier == "none" # unknown, and no client to ask - - -def test_tier_one_reads_a_finding_from_its_tags_and_its_rule(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_finding - - taxonomy = load_families() - - def finding(finding_id: str, tags: list[str]) -> StaticFinding: - return StaticFinding( - finding_id=finding_id, - path="app.py", - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=12, - function_name="leaky", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=tags, - ranking_priority=0.0, - ) - - tagged = classify_finding(finding("x:app.py:1", ["family:access_control"]), taxonomy, client=None) - assert tagged.families == ("access_control",) and tagged.tier == "static" - obligated = classify_finding(finding("obligation:protected_read:app.py:12:x", ["obligation:protected_read"]), taxonomy, client=None) - assert obligated.families == ("access_control",) - ruled = classify_finding(finding("python-os-command:app.py:3", ["tool-hunter"]), taxonomy, client=None) - assert ruled.families == ("injection",) and ruled.tier == "static" # the rule's weakness names the family - assert classify_finding(finding("tool-hunter:app.py:3", ["tool-hunter"]), taxonomy, client=None).families == () - - -def test_tier_one_classifies_the_shared_handler_without_any_client(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_region - - (tmp_path / "app.py").write_text(APP) - taxonomy = load_families() - leaky = classify_region(tmp_path, "app.py", function="leaky", taxonomy=taxonomy, client=None) - assert leaky.families == ("access_control",) and leaky.tier == "static" - (tmp_path / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n") - injected = classify_region(tmp_path, "flow.py", function="run", taxonomy=taxonomy, client=None) - assert "injection" in injected.families - - -def test_tier_two_asks_the_model_only_when_static_signals_say_nothing(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["output_encoding"]}), mean_logprob=-0.05)]) - answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - assert answered.families == ("output_encoding",) and answered.tier == "model" and answered.confidence == -0.05 - quiet = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}))]) - classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=quiet) - assert quiet.calls == [] # a static answer never spends a model call - - -def test_an_unsure_or_off_vocabulary_answer_becomes_unknown(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - invented = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["sql_injection"]}), mean_logprob=-0.01)]) - assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=invented).families == () - unsure = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}), mean_logprob=-4.0)]) - hedged = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=unsure) - assert hedged.families == () and hedged.tier == "none" and hedged.confidence == -4.0 - broken = ScriptedChatClient([ChatResponse(content="not json at all")]) - assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=broken).families == () - - -def test_the_model_is_asked_in_json_mode_with_the_taxonomy_and_no_thinking(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["path"]}), mean_logprob=-0.1)]) - classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - call = client.calls[0] - prompt = json.dumps(call["messages"]) - assert "json" in prompt.lower() and "access_control" in prompt and "untrusted_destination" in prompt - assert call.get("json_object") is True and call.get("logprobs") is True and call["tools"] == [] - - -def test_multi_label_answers_are_kept_in_taxonomy_order(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["access_control", "injection"]}), mean_logprob=-0.2)]) - answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - assert answered.families == ("injection", "access_control") # taxonomy order, so two runs compare equal - - -def test_a_fabricated_family_label_in_a_catalog_is_rejected_by_name(tmp_path: Path) -> None: - from openultrasast.learning.classify import FamilyLabelError, validate_family_labels - - taxonomy = load_families() - good = _case(tmp_path, family="access_control") - validate_family_labels([good], taxonomy) - with pytest.raises(FamilyLabelError, match="not_a_family"): - validate_family_labels([_case(tmp_path, family="not_a_family")], taxonomy) diff --git a/tests/test_learning_cli.py b/tests/test_learning_cli.py deleted file mode 100644 index b33e45a..0000000 --- a/tests/test_learning_cli.py +++ /dev/null @@ -1,302 +0,0 @@ -"""learning-harness task 3.9: the learning subcommands (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" -CATALOG = ( - '[[pair]]\nname = "{name}"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "{name}-v.py"\nfixed = "{name}-f.py"\n' - 'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "command injection"\npath = "app.py"\nfunction = "run"\n' - 'sink = "system"\nmechanism = "source_reaches_sink"\nfamily = "injection"\n' -) - - -def _detector(monkeypatch: pytest.MonkeyPatch, line: int = 6) -> None: - """A deterministic detector that reads the file it was given and answers differently on the two sides. - - The shipped `scripted` stub answers twice and then goes quiet, and its finding sits at line 3, outside `run`. - Both make every family number zero, and a scoreboard assertion against zeros passes with the detector missing - entirely — which is what these tests used to assert.""" - from openultrasast.learning.endpoint import ChatEndpoint - from openultrasast.tool_hunter import ChatResponse, ToolCall - - class Detector: - def __init__(self) -> None: - self.turn = 0 - - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - del model, tools, options - self.turn += 1 - if self.turn % 2 == 1: - return ChatResponse(tool_calls=(ToolCall(id=f"c{self.turn}", name="grep_repo", arguments={"pattern": "system"}),)) - body = "\n".join(str(message.get("content") or "") for message in messages) - if "os.system" not in body: - return ChatResponse(content="[]") # the fixed side really is different, and the detector sees it - return ChatResponse(content=json.dumps([{"path": "app.py", "line": line, "title": "t", "rationale": "r"}])) - - endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False) - monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (Detector(), endpoint)) - - -@pytest.fixture -def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - for name in ("train-a", "hold-a"): - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - (tmp_path / "pairs.toml").write_text( - CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout") - ) - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - return tmp_path # no chdir: the mechanism vocabulary is found relative to the repository root - - -def test_classify_reports_the_corpus_and_writes_the_queue(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None: - assert main(["learning", "classify", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning")]) == 0 - payload = json.loads((workspace / "learning" / "classifier.json").read_text()) - assert payload["classified"] == 2 and payload["per_family"]["injection"] == 2 - assert payload["hierarchical_credit"] is False - assert "classified 2" in capsys.readouterr().out - - -def test_score_reports_family_numbers_for_a_catalog( - workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch -) -> None: - """The scripted client detects the vulnerable side and is silent on the fix, so this is a real 2/2, not a zero. - - An all-zero scoreboard would satisfy "the family key exists"; it is what a generalist detector scoring every - family produced before round zero ran each family's own configuration.""" - _detector(monkeypatch) - assert main(["learning", "score", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--json"]) == 0 - payload = json.loads(capsys.readouterr().out) - block = payload["families"]["injection"] - assert block["taxonomy_version"] and block["scorable"] == 2 - assert block["outcomes"] == {"pair_correct": 2} and block["recall"] == 1.0 and block["youden"] == 1.0 - - -def test_baseline_writes_the_noise_floor_and_publish_reads_it( - workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch -) -> None: - _detector(monkeypatch) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "scripted", - ] - ) - == 0 - ) - floors = json.loads((workspace / "learning" / "baseline" / "scripted" / "noise-floors.json").read_text()) - assert floors["k_runs"] == 3 and floors["floors"]["injection"]["pairs"] == 2 - report = json.loads((workspace / "learning" / "baseline" / "scripted" / "report.json").read_text()) - # Measured with the injection configuration, not the generalist: `family:unknown` earns no credit against any - # real family, so a baseline built from one shared detector reports a structural zero here. - assert report["metrics"]["injection"]["recall"] == 1.0 - assert report["per_slice"]["vibe-py/injection"]["scorable"] == 2 - capsys.readouterr() - roadmap = workspace / "roadmap.md" - roadmap.write_text("# Roadmap\n\n## Overview\n\ntext\n") - assert ( - main( - ["learning", "publish", "--out", str(workspace / "learning"), "--measurements", str(workspace / "m"), "--roadmap", str(roadmap)] - ) - == 0 - ) - assert "learning-harness:begin" in roadmap.read_text() - assert "| injection |" in roadmap.read_text() - - -def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace: Path) -> None: - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "scripted", - ] - ) - == 0 - ) - assert ( - main( - [ - "learning", - "round", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--family", - "injection", - "--k-runs", - "3", - ] - ) - == 0 - ) - rounds = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()] - assert len(rounds) == 1 and rounds[0]["family"] == "injection" - # Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of - # the enum would have hidden that the round could never propose anything at all. - # The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different - # worlds, and only one of them is a fixable configuration error. - assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal") - assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file() - - -def test_a_recorded_proposal_makes_the_round_do_something(workspace: Path) -> None: - """`--proposals` is what makes a round runnable at all without the optional extra.""" - for command in ( - ["learning", "baseline", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--k-runs", "3"], - ): - assert main([*command, "--model", "scripted"]) == 0 - proposals = workspace / "proposals.json" - proposals.write_text( - json.dumps([{"hypothesis": "ask about quoting", "lever": "checklist", "change": {"checklist.md": "- ask about quoting\n"}}]) - ) - assert ( - main( - [ - "learning", - "round", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--family", - "injection", - "--k-runs", - "3", - "--proposals", - str(proposals), - ] - ) - == 0 - ) - (record,) = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()] - assert record["reason"] != "no_proposal", "a recorded proposal reached the round" - assert json.loads((workspace / "learning" / "rounds" / "1" / "proposal.json").read_text())["lever"] == "checklist" - - -def test_a_missing_endpoint_is_recorded_and_the_rest_still_runs(workspace: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None: # type: ignore[no-untyped-def] - for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): - monkeypatch.delenv(name, raising=False) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "none", - ] - ) - == 0 - ) - out = capsys.readouterr().out - assert "learning_endpoint_unavailable" in out - assert (workspace / "learning" / "baseline" / "none" / "report.json").is_file() # the run still recorded what it could - - -def test_every_subcommand_is_listed_in_help(capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["learning", "--help"]) - out = capsys.readouterr().out - for name in ("classify", "score", "baseline", "round", "publish"): - assert name in out - - -def test_the_pairs_command_scores_k_runs_per_side(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Grepping `--help` for the flag passes with the capability entirely absent, which is what it used to be.""" - from openultrasast import pairs as pairs_module - - calls: list[str] = [] - real = pairs_module.make_hunter_scan - - def counting(client: object, model: str, **kwargs: object): # type: ignore[no-untyped-def] - inner = real(client, model, **kwargs) # type: ignore[arg-type] - - def scan(root: Path): # type: ignore[no-untyped-def] - calls.append(root.name) - return inner(root) - - return scan - - monkeypatch.setattr(pairs_module, "make_hunter_scan", counting) - monkeypatch.setattr("openultrasast.cli.make_hunter_scan", counting) - monkeypatch.setenv("OPENULTRASAST_HUNTER_MODEL", "scripted") - assert main(["pairs", "--catalog", str(workspace / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "4"]) == 0 - assert calls.count("vuln") == 8 and calls.count("fixed") == 8 # two pairs, four runs per side - - -def test_a_baseline_records_what_the_endpoint_actually_spent(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Req 8.3: cost per pair is published beside every number, so the run has to read it off the client. - - The client meters itself; nothing passed the meter to the baseline, so every published cost would have been - 0.0 — and the round's cost cap, which reads the same meter, could never have fired either (Req 9.8).""" - from openultrasast.learning.endpoint import ChatEndpoint - from openultrasast.tool_hunter import ChatResponse - - class Metered: - def __init__(self) -> None: - self.calls = 0 - - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - del model, messages, tools, options - self.calls += 1 - return ChatResponse(content="[]") - - def cost_usd(self) -> float: - return 0.25 * self.calls - - client = Metered() - endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False) - monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (client, endpoint)) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "metered", - ] - ) - == 0 - ) - report = json.loads((workspace / "learning" / "baseline" / "metered" / "report.json").read_text()) - assert client.calls > 0 - assert report["cost_usd"] == pytest.approx(0.25 * client.calls) diff --git a/tests/test_learning_cutoff.py b/tests/test_learning_cutoff.py deleted file mode 100644 index f22fd3d..0000000 --- a/tests/test_learning_cutoff.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Rows fixed after the detector's training cutoff are reported apart (learning-harness 4.3, Req 10.5). - -A model that may have read the fix cannot be measured on it the same way as one that cannot have. The corpus -carries the date at the precision the recipe recorded — a year for every row harvested so far — so a row counts as -post-cutoff only when its *earliest possible* date is after the cutoff. Rows with no date at all are counted -separately rather than quietly folded into either side. -""" - -from __future__ import annotations - -import json -import tomllib -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.learning.families import load_families -from openultrasast.learning.scoring import PairFamilyScore, aggregate -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored - - -def _score(pair: str, outcome: str, *, fix_date: str = "") -> PairFamilyScore: - return PairFamilyScore(pair=pair, family="injection", slice="vibe-py", runs=(outcome,), outcome=outcome, fix_date=fix_date) # type: ignore[arg-type] - - -def test_every_vendored_row_carries_the_date_its_recipe_recorded() -> None: - cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG)) - dated = [case for case in cases if case.fix_date] - assert len(dated) >= 250, "the generator fills the date from the recipe; almost every row has a year" - assert all(len(case.fix_date) in {4, 10} for case in dated), "a date is a year or a full ISO day, never anything else" - by_name = {case.name: case for case in cases} - for slice_name in ("vibe-py", "agent-vfc"): - for recipe in tomllib.loads((Path("benchmarks/pairs") / slice_name / "recipes.toml").read_text())["recipe"]: - case = by_name.get(str(recipe["name"])) - if case is not None and recipe.get("year"): - assert case.fix_date.startswith(str(recipe["year"])) - - -@pytest.mark.parametrize( - ("fix_date", "cutoff", "expected"), - [ - ("2026", "2025-07-01", True), # every day of 2026 is after the cutoff - ("2026", "2026-03-01", False), # a year-precision row that might be before it is not claimed - ("2026-04-02", "2026-03-01", True), - ("2024", "2025-07-01", False), - ("2026-05", "2026-03-01", True), # month precision: the earliest day of May is after 1 March - ("2026-05", "2026-06-01", False), # and before 1 June, so the row is not claimed - ("2025", "2025", False), # a cutoff coarser than a day is refused rather than compared - ("", "2025-07-01", False), - ], -) -def test_a_row_is_post_cutoff_only_when_its_earliest_possible_date_is_after_it(fix_date: str, cutoff: str, expected: bool) -> None: - from openultrasast.learning.scoring import is_post_cutoff - - assert is_post_cutoff(fix_date, cutoff) is expected - - -def test_the_scoreboard_reports_the_post_cutoff_slice_the_cutoff_and_the_undated_count() -> None: - scores = [ - _score("new-correct", "pair_correct", fix_date="2026"), - _score("new-missed", "both_silent", fix_date="2026"), - _score("old-correct", "pair_correct", fix_date="2021"), - _score("undated", "pair_correct"), - ] - block = aggregate(scores, taxonomy=load_families(), cutoff="2025-07-01")["injection"] - assert block.cutoff == "2025-07-01" and block.undated == 1 - assert block.post_cutoff["scorable"] == 2 and block.post_cutoff["recall"] == 0.5 - assert block.scorable == 4, "the slice is reported beside the whole number, never instead of it" - payload = block.to_dict() - json.dumps(payload) - assert payload["cutoff"] == "2025-07-01" and payload["post_cutoff"]["scorable"] == 2 - - -def test_without_a_configured_cutoff_the_scoreboard_says_so_rather_than_inventing_one() -> None: - block = aggregate([_score("a", "pair_correct", fix_date="2026")], taxonomy=load_families())["injection"] - assert block.cutoff == "" and block.post_cutoff == {} and block.undated == 0 - - -def test_the_score_carries_the_date_from_the_pair(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text("def run():\n return 1\n") - case = PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "v.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - fix_date="2026", - ) - score = score_pair_family(case, "injection", [[]], [[]], ranges={"app.py": (("run", 1, 2),)}, taxonomy=load_families()) - assert score.fix_date == "2026" - - -def test_the_published_table_names_the_post_cutoff_slice(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - directory = out / "baseline" / "m" - directory.mkdir(parents=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": "m", - "k_runs": 5, - "cost_usd": 1.0, - "taxonomy_version": "1", - "metrics": { - "injection": { - "family": "injection", - "scorable": 4, - "unscorable": {}, - "recall": 0.5, - "youden": 0.25, - "hierarchical_credit": False, - "cutoff": "2025-07-01", - "undated": 1, - "post_cutoff": {"scorable": 2, "recall": 0.5, "youden": 0.5}, - } - }, - "floors": {}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text("# Roadmap\n") - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "2025-07-01" in report.table - assert "post-cutoff" in report.table.lower() - assert "undated" in report.table.lower() - - -def test_the_command_line_reports_the_slice_the_cutoff_and_the_undated_count(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Req 10.5 says every scoreboard, which includes the one an operator actually reads.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415 - - from openultrasast.cli import main - - for name in ("train-a", "hold-a"): - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - catalog = CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout") - (tmp_path / "pairs.toml").write_text(catalog.replace('review_tier = "seeded"', 'review_tier = "seeded"\nfix_date = "2026"')) - (tmp_path / "openultrasast.toml").write_text('[learning]\nenabled = true\ncutoff_date = "2025-07-01"\n') - _detector(monkeypatch) - captured: list[str] = [] - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: captured.append(" ".join(str(item) for item in args))) - assert ( - main( - [ - "learning", - "score", - "--catalog", - str(tmp_path / "pairs.toml"), - "--out", - str(tmp_path / "learning"), - "--config", - str(tmp_path / "openultrasast.toml"), - ] - ) - == 0 - ) - line = next(item for item in captured if item.startswith("learning score injection")) - assert "post-cutoff(2025-07-01)=2" in line and "undated=0" in line - - -def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`--slice vfc` selected the vfc rows and then `run_baseline` filtered them all out again. - - Its `slices` default is the two web slices, so a baseline over the memory-safety slice — the run Req 8.4 asks - for as a measured, non-gating number — produced an empty report and said nothing about it.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415 - - from openultrasast.cli import main - - (tmp_path / "a-v.py").write_text(VULN) - (tmp_path / "a-f.py").write_text(FIXED) - (tmp_path / "pairs.toml").write_text(CATALOG.format(name="a", split="train").replace('slice = "vibe-py"', 'slice = "vfc"')) - _detector(monkeypatch) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(tmp_path / "pairs.toml"), - "--slice", - "vfc", - "--out", - str(tmp_path / "l"), - "--k-runs", - "3", - "--model", - "m", - ] - ) - == 0 - ) - # A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run - # over the web families. - report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text()) - assert report["slices"] == ["vfc"] - assert report["metrics"]["injection"]["scorable"] == 1 diff --git a/tests/test_learning_detectors.py b/tests/test_learning_detectors.py deleted file mode 100644 index 6043611..0000000 --- a/tests/test_learning_detectors.py +++ /dev/null @@ -1,167 +0,0 @@ -"""learning-harness task 2.5: per-family detector configurations and the family runner (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.learning.families import load_families -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) -REPLY = '[{"path": "app.py", "line": 12, "title": "unconstrained read", "rationale": "no owner check"}]' - - -@pytest.fixture -def repo(tmp_path: Path) -> Path: - (tmp_path / "app.py").write_text(APP) - return tmp_path - - -def _write_config(root: Path, family: str, **overrides: object) -> Path: - directory = root / family - directory.mkdir(parents=True, exist_ok=True) - (directory / "prompt.md").write_text(str(overrides.get("prompt", f"You hunt {family} only."))) - (directory / "checklist.md").write_text(str(overrides.get("checklist", "- did the handler check ownership?"))) - tools = overrides.get("tools", ["read_definition", "obligations"]) - (directory / "family.toml").write_text( - f'version = "{overrides.get("version", "1")}"\ntools = {json.dumps(tools)}\n' - f"max_steps = {overrides.get('max_steps', 3)}\nmax_cost_usd = {overrides.get('max_cost_usd', 0.5)}\n" - f"max_chars = {overrides.get('max_chars', 4000)}\n" - ) - (directory / "hard_negatives.jsonl").write_text(json.dumps({"note": "a guarded handler is not a finding"}) + "\n") - (directory / "counterexamples.jsonl").write_text("") - return directory - - -def test_a_configuration_loads_every_part_it_needs(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs - - _write_config(tmp_path, "access_control") - configs = load_family_configs(tmp_path, load_families()) - config = configs["access_control"] - assert config.family == "access_control" and config.version == "1" - assert config.prompt.startswith("You hunt access_control") and "ownership" in config.checklist - assert config.tools == ("read_definition", "obligations") - assert config.max_steps == 3 and config.max_cost_usd == 0.5 and config.max_chars == 4000 - assert config.hard_negatives[0]["note"].startswith("a guarded handler") - assert config.counterexamples == () - assert load_family_configs(tmp_path, load_families())["access_control"] == config # loading twice is the same config - - -@pytest.mark.parametrize( - ("overrides", "needle"), - [ - ({"prompt": "x" * 9000}, "length"), - ({"tools": ["read_file", "run_shell"]}, "run_shell"), - ({"version": ""}, "version"), - ], - ids=["over_cap_prompt", "unknown_tool", "no_version"], -) -def test_the_loader_refuses_a_broken_configuration_by_name(tmp_path: Path, overrides: dict[str, object], needle: str) -> None: - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "access_control", **overrides) - with pytest.raises(DetectorConfigError, match=needle): - load_family_configs(tmp_path, load_families()) - - -def test_a_directory_that_is_not_a_family_is_refused_by_name(tmp_path: Path) -> None: - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "access_control") - (tmp_path / "sql_injection").mkdir() - (tmp_path / "sql_injection" / "prompt.md").write_text("x") - with pytest.raises(DetectorConfigError, match="sql_injection"): - load_family_configs(tmp_path, load_families()) - - -def test_the_runner_uses_the_family_prompt_tools_and_tags(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector - - configs_dir = tmp_path / "configs" - _write_config(configs_dir, "access_control") - config = load_family_configs(configs_dir, load_families())["access_control"] - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),)), - ChatResponse(content=REPLY), - ] - ) - findings = run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m") - assert [finding.tags for finding in findings] == [["family:access_control", "detector:access_control@1"]] - assert findings[0].evidence_level == "suspicion" - call = client.calls[0] - assert call["messages"][0]["content"] == "You hunt access_control only." - assert [tool["function"]["name"] for tool in call["tools"]] == ["read_definition", "obligations"] - user = str(call["messages"][1]["content"]) - assert "def leaky" in user and "def guarded" in user # the whole file, not an excerpt around a hotspot - assert "ownership" in user and "a guarded handler is not a finding" in user - - -def test_a_region_runs_every_admitted_family_plus_the_generalist(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_region_detectors - - configs_dir = tmp_path / "configs" - for family in ("access_control", "injection", "unknown"): - _write_config(configs_dir, family, tools=["read_file"]) - configs = load_family_configs(configs_dir, load_families()) - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)), - ChatResponse(content=REPLY), - ] - * 4 - ) - region = Region(path="app.py", function="leaky", families=("access_control",)) - findings = run_region_detectors(repo, region, configs, client=client, model="m") - families = sorted({tag for finding in findings for tag in finding.tags if tag.startswith("family:")}) - assert families == ["family:access_control", "family:unknown"] # the admitted family and the generalist, not injection - assert len(client.calls) == 4 # two turns each, and nothing else was run - - -def test_the_runner_never_offers_a_tool_outside_the_curated_set(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector - from openultrasast.tool_hunter import HUNTER_TOOLS - - configs_dir = tmp_path / "configs" - _write_config( - configs_dir, "injection", tools=["read_file", "grep_repo", "find_refs", "read_definition", "entry_points", "flows", "obligations"] - ) - config = load_family_configs(configs_dir, load_families())["injection"] - client = ScriptedChatClient([ChatResponse(content="")]) - run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m") - offered = {tool["function"]["name"] for tool in client.calls[0]["tools"]} - assert offered == {str(tool["function"]["name"]) for tool in HUNTER_TOOLS} - assert "bash" not in offered and "run_shell" not in offered - - -def test_round_zero_can_write_a_configuration_for_every_family(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs, write_default_configs - - taxonomy = load_families() - written = write_default_configs(tmp_path, taxonomy, prompt="You are a security hunter.", version="0") - assert set(written) == {family.id for family in taxonomy.families} - configs = load_family_configs(tmp_path, taxonomy) - assert set(configs) == {family.id for family in taxonomy.families} - assert all(config.version == "0" for config in configs.values()) - assert all(config.prompt.startswith("You are a security hunter.") for config in configs.values()) - assert configs["access_control"].prompt != configs["injection"].prompt # each carries its own family's definition - - -def test_a_configuration_with_no_tools_is_refused_rather_than_reporting_nothing(tmp_path: Path) -> None: - """A detector offered no tools never calls one, and the hunter loop then returns nothing at all.""" - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "injection", tools=[]) - with pytest.raises(DetectorConfigError, match="tools"): - load_family_configs(tmp_path, load_families()) diff --git a/tests/test_learning_journal.py b/tests/test_learning_journal.py deleted file mode 100644 index 6a936c8..0000000 --- a/tests/test_learning_journal.py +++ /dev/null @@ -1,112 +0,0 @@ -"""learning-harness task 2.8: the round journal, the archive and the rejected buffer (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.learning.scoring import PairFamilyScore - - -def _record(round_number: int, family: str, outcome: str, **extra: object) -> object: - from openultrasast.learning.journal import Attribution, RoundRecord - - defaults: dict[str, object] = { - "round": round_number, - "family": family, - "hypothesis": "the checklist never asks about ownership", - "levers": ("checklist",), - "predicted_affected": (family,), - "predicted_at_risk": ("injection",), - "outcome": outcome, - "reason": "", - "target_train_delta": 0.1, - "target_holdout_delta": 0.2, - "sweep": {"injection": 0}, - "attribution": Attribution(flipped_predicted=2, flipped_unpredicted=0, precision=1.0), - "cost_usd": 0.42, - "taxonomy_version": "1", - "config_version": "2", - } - return RoundRecord(**{**defaults, **extra}) # type: ignore[arg-type] - - -def test_a_record_round_trips_through_the_file(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - assert journal.rounds() == () - record = _record(1, "access_control", "accepted") - journal.append(record) - journal.append(_record(2, "injection", "rejected", reason="tie")) - rounds = journal.rounds() - assert [item.round for item in rounds] == [1, 2] and rounds[0] == record - assert rounds[0].attribution.precision == 1.0 and rounds[0].sweep == {"injection": 0} - assert rounds[1].outcome == "rejected" and rounds[1].reason == "tie" - rows = [json.loads(line) for line in (tmp_path / "journal.jsonl").read_text().splitlines() if line.strip()] - assert len(rows) == 2 and rows[0]["hypothesis"].startswith("the checklist") - - -def test_the_journal_is_append_only_and_a_round_number_is_never_reused(tmp_path: Path) -> None: - from openultrasast.learning.journal import JournalError, LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "accepted")) - before = (tmp_path / "journal.jsonl").read_text() - try: - journal.append(_record(1, "injection", "accepted")) - raise AssertionError("a repeated round number must be refused") - except JournalError as error: - assert "1" in str(error) - assert (tmp_path / "journal.jsonl").read_text() == before - assert journal.next_round() == 2 - - -def test_the_rejected_buffer_returns_only_the_target_family(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "rejected", hypothesis="ask about ownership", reason="tie")) - journal.append(_record(2, "injection", "rejected", hypothesis="ask about quoting", reason="tie")) - journal.append(_record(3, "access_control", "accepted", hypothesis="give it the obligations tool")) - assert journal.rejected_buffer("access_control") == ("ask about ownership",) - assert journal.rejected_buffer("injection") == ("ask about quoting",) - assert journal.rejected_buffer("memory") == () - - -def test_the_archive_names_one_winning_configuration_per_pair_and_family(tmp_path: Path) -> None: - from openultrasast.learning.journal import Archive - - archive = Archive(tmp_path / "archive.jsonl") - correct = PairFamilyScore(pair="a", family="access_control", runs=("pair_correct",), outcome="pair_correct") - missed = PairFamilyScore(pair="a", family="access_control", runs=("both_silent",), outcome="both_silent") - archive.record("access_control", "1", correct) - archive.record("access_control", "2", missed) # a later version that loses this pair does not take it - archive.record( - "access_control", "2", PairFamilyScore(pair="b", family="access_control", runs=("pair_correct",), outcome="pair_correct") - ) - assert archive.winners("access_control") == {"a": "1", "b": "2"} - assert archive.winners("injection") == {} - reopened = Archive(tmp_path / "archive.jsonl") - assert reopened.winners("access_control") == {"a": "1", "b": "2"} # survives a reload, like the store it copies - - -def test_a_later_version_takes_a_pair_only_by_winning_it(tmp_path: Path) -> None: - from openultrasast.learning.journal import Archive - - archive = Archive(tmp_path / "archive.jsonl") - archive.record("injection", "1", PairFamilyScore(pair="a", family="injection", runs=("both_silent",), outcome="both_silent")) - assert archive.winners("injection") == {} - archive.record("injection", "2", PairFamilyScore(pair="a", family="injection", runs=("pair_correct",), outcome="pair_correct")) - assert archive.winners("injection") == {"a": "2"} - - -def test_everything_persisted_passes_redaction(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - leaked = "api_key = " + "'" + "A" * 24 + "'" # assembled here so no key-shaped literal sits in the file - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "rejected", hypothesis=f"the prompt should show {leaked}", reason="tie")) - text = (tmp_path / "journal.jsonl").read_text() - assert "A" * 24 not in text and "REDACTED" in text - assert journal.rounds()[0].hypothesis.endswith("'") diff --git a/tests/test_learning_pairs_evidence.py b/tests/test_learning_pairs_evidence.py deleted file mode 100644 index ef7b777..0000000 --- a/tests/test_learning_pairs_evidence.py +++ /dev/null @@ -1,151 +0,0 @@ -"""What the pairs hunter path must prove (learning-harness review round 3, Req 3.2, 3.5). - -Three defects, one test each: K was declared and never threaded, so every stage scored one run; the fixed -side was judged against the *vulnerable* side's function ranges, which differ on most of the corpus; and -the generalist hunter emitted findings with no family tag, which the class-aware scorer can never credit, -so `ousast pairs --hunter` reported recall zero by construction. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase, evaluate_catalog - -# The fix inserts two guard lines above the handler, so `run` starts on a different line on each side. -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = ( - "from flask import request\nimport shlex\nimport os\n\n\n" - "def _safe(value):\n return shlex.quote(value or '')\n\n\n" - "def run():\n os.system(_safe(request.args.get('cmd')))\n" -) - - -def _case(tmp_path: Path, name: str = "p") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", - vulnerability_class="x", - path="app.py", - evidence="", - function="run", - mechanism="other", - family="injection", - ), - ), - min_recall=1.0, - fix_policy="silent", - review_tier="seeded", - ) - - -def _finding(line: int, *, family: str | None = "injection") -> StaticFinding: - return StaticFinding( - finding_id=f"tool-hunter:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["tool-hunter", *([f"family:{family}"] if family else [])], - ranking_priority=0.0, - ) - - -def test_the_k_runs_flag_reaches_the_detector(tmp_path: Path) -> None: - """Req 3.5: three runs per side, per pair. A parameter no caller can set is not a K-run floor.""" - calls: list[str] = [] - - def scan(root: Path) -> list[StaticFinding]: - calls.append(root.name) - return [_finding(6)] if root.name == "vuln" else [] - - evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - assert calls.count("vuln") == 3 and calls.count("fixed") == 3 - - -def test_fewer_than_three_runs_is_refused(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="three runs"): - evaluate_catalog([_case(tmp_path)], hunter=lambda root: [], k_runs=1) - - -def test_the_fixed_side_is_judged_against_the_fixed_side_line_numbers(tmp_path: Path) -> None: - """`run` sits at line 5 on the vulnerable side and line 10 on the fixed side. - - Scoring the fixed side against the vulnerable side's span reads a finding inside the fixed handler as - outside it, so a detector that flags both sides is recorded as silent on the fix — a leak counted as a - clean pass, and always in the direction that flatters the number.""" - - def scan(root: Path) -> list[StaticFinding]: - return [_finding(6)] if root.name == "vuln" else [_finding(11)] - - result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - hunter = result.scorers["vibe-py"]["hunter"] - assert hunter.detected_vuln == 1 - assert hunter.silent_fix == 0, "the finding is inside `run` on the fixed side; that is a leak" - assert hunter.pair_correct == 0 - - -def test_a_generalist_finding_without_a_family_is_never_a_detection(tmp_path: Path) -> None: - """The other half of the same defect: `pairs --hunter` used to feed the scorer untagged findings.""" - - def scan(root: Path) -> list[StaticFinding]: - return [_finding(6, family=None)] if root.name == "vuln" else [] - - result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - assert result.scorers["vibe-py"]["hunter"].detected_vuln == 0 - - -def test_the_hunter_reports_the_family_it_believes_and_that_is_what_is_credited(tmp_path: Path) -> None: - """A generalist that must name the family is answerable; one that names nothing cannot be scored at all.""" - from openultrasast.tool_hunter import _findings_from_content - - findings = _findings_from_content( - tmp_path, - [], - '[{"path": "app.py", "line": 6, "family": "injection", "title": "t", "rationale": "r"}]', - ) - assert findings and "family:injection" in findings[0].tags - untagged = _findings_from_content(tmp_path, [], '[{"path": "app.py", "line": 6, "title": "t"}]') - assert untagged and not [tag for tag in untagged[0].tags if tag.startswith("family:")] - - -def test_the_pair_hunter_asks_for_a_family_from_the_closed_list(tmp_path: Path) -> None: - """The prompt has to name the taxonomy, or the model invents family names and every finding is noise.""" - from openultrasast.pairs import make_hunter_scan - - seen: list[str] = [] - - class Client: - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - seen.append("\n".join(str(message.get("content") or "") for message in messages)) - from openultrasast.tool_hunter import ChatResponse - - return ChatResponse(content="[]", tool_calls=()) - - (tmp_path / "app.py").write_text(VULN) - make_hunter_scan(Client(), "m", taxonomy=load_families())(tmp_path) - prompt = "\n".join(seen) - assert "injection" in prompt and "access_control" in prompt and "untrusted_destination" in prompt - assert '"family"' in prompt diff --git a/tests/test_learning_pairs_path.py b/tests/test_learning_pairs_path.py deleted file mode 100644 index cab5abe..0000000 --- a/tests/test_learning_pairs_path.py +++ /dev/null @@ -1,110 +0,0 @@ -"""learning-harness task 3.2: family scoring on the pairs hunter path (offline).""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.pairs import PairCase, evaluate_pair - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', request.args.get('cmd')], check=False)\n" - - -def _case(tmp_path: Path, *, family: str | None = "injection", unscorable: str | None = None) -> PairCase: - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED) - return PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - unscorable=unscorable, - ) - - -def _finding(family: str, line: int) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:injection@1"], - ranking_priority=0.0, - ) - - -def _hunter(vuln: list[StaticFinding], fixed: list[StaticFinding]): # type: ignore[no-untyped-def] - seen: list[Path] = [] - - def scan(root: Path) -> list[StaticFinding]: - seen.append(root) - return vuln if root.name == "vuln" else fixed - - scan.seen = seen # type: ignore[attr-defined] - return scan - - -def test_the_hunter_path_scores_by_family_and_function_not_by_text(tmp_path: Path) -> None: - right = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [])) - assert right.detected_vuln and right.silent_fix and right.pair_correct - assert right.detection_kinds == ("hunter",) and right.family == "injection" - wrong_family = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("access_control", 6)], [])) - assert not wrong_family.detected_vuln # the right place, the wrong family - loud = StaticFinding(**{**_finding("access_control", 6).__dict__, "title": "CWE-78 command injection in run"}) - assert not evaluate_pair(_case(tmp_path), hunter=_hunter([loud], [])).detected_vuln # the words never count - - -def test_a_finding_of_another_family_on_the_fixed_side_is_not_a_leak(tmp_path: Path) -> None: - outcome = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("access_control", 6)])) - assert outcome.pair_correct and outcome.silent_fix - assert outcome.fix_findings == 1 and outcome.fix_leaks == 0 # counted, never charged - leaked = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("injection", 6)])) - assert leaked.detected_vuln and not leaked.silent_fix and leaked.fix_leaks == 1 - - -def test_an_unscorable_pair_is_reported_as_such_and_never_run(tmp_path: Path) -> None: - hunter = _hunter([_finding("injection", 6)], []) - outcome = evaluate_pair(_case(tmp_path, unscorable="identical_twin"), hunter=hunter) - assert outcome.unscorable == "identical_twin" and not outcome.detected_vuln - assert hunter.seen == [] # type: ignore[attr-defined] - - -def test_a_pair_with_no_family_label_falls_back_to_the_classifier(tmp_path: Path) -> None: - outcome = evaluate_pair(_case(tmp_path, family=None), hunter=_hunter([_finding("injection", 6)], [])) - assert outcome.family == "injection" and outcome.pair_correct # CWE-78 classifies as injection - - -def test_the_overlay_path_is_untouched_by_the_family_scorer(tmp_path: Path) -> None: - overlay = evaluate_pair(_case(tmp_path)) - assert overlay.detection_kinds != ("hunter",) and overlay.family == "" - - -def test_the_scoreboard_carries_the_family_block(tmp_path: Path) -> None: - from openultrasast.pairs import evaluate_catalog, result_payload - - result = evaluate_catalog([_case(tmp_path)], hunter=_hunter([_finding("injection", 6)], []), inventory_sidecar=False) - payload = result_payload(result) - assert "per_family" in payload and payload["per_family"]["injection"]["scorable"] == 1 - assert payload["per_family"]["injection"]["recall"] == 1.0 - assert payload["per_family"]["injection"]["hierarchical_credit"] is False - assert payload["per_family"]["injection"]["taxonomy_version"] diff --git a/tests/test_learning_proposer.py b/tests/test_learning_proposer.py deleted file mode 100644 index 2a0ca2f..0000000 --- a/tests/test_learning_proposer.py +++ /dev/null @@ -1,222 +0,0 @@ -"""learning-harness task 3.5: failure facts, the scripted proposer and the HarnessX proposer (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.learning.detectors import FamilyConfig -from openultrasast.learning.journal import LearningJournal, RoundRecord -from openultrasast.learning.scoring import PairFamilyScore - - -def _config(family: str = "access_control") -> FamilyConfig: - return FamilyConfig( - family=family, - version="0", - prompt="You hunt access control.", - checklist="- did the handler check ownership?", - tools=("read_definition", "obligations"), - max_steps=3, - max_cost_usd=0.5, - max_chars=4000, - ) - - -def _score(pair: str, outcome: str, *, family: str = "access_control") -> PairFamilyScore: - return PairFamilyScore(pair=pair, family=family, runs=(outcome,), outcome=outcome, slice="vibe-py") # type: ignore[arg-type] - - -def test_failure_facts_carry_the_misses_the_leaks_and_the_rejected_buffer(tmp_path: Path) -> None: - from openultrasast.learning.proposer import build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append( - RoundRecord( - round=1, - family="access_control", - hypothesis="ask about ownership", - levers=("checklist",), - predicted_affected=("access_control",), - predicted_at_risk=(), - outcome="rejected", - reason="tie", - ) - ) - scores = [_score("miss-one", "both_silent"), _score("leak-one", "both_flagged"), _score("good", "pair_correct")] - facts = build_failure_facts( - scores, family="access_control", config=_config(), journal=journal, train_pairs={"miss-one", "leak-one", "good"} - ) - assert [item.pair for item in facts.misses] == ["miss-one"] - assert [item.pair for item in facts.leaks] == ["leak-one"] - assert facts.rejected == ("ask about ownership",) - assert facts.config.family == "access_control" and facts.family == "access_control" - - -def test_the_proposer_is_never_handed_a_holdout_pair(tmp_path: Path) -> None: - from openultrasast.learning.proposer import build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - scores = [_score("train-miss", "both_silent"), _score("holdout-miss", "both_silent")] - facts = build_failure_facts(scores, family="access_control", config=_config(), journal=journal, train_pairs={"train-miss"}) - assert [item.pair for item in facts.misses] == ["train-miss"] - assert "holdout-miss" not in json.dumps(facts.to_dict()) - - -def test_the_scripted_proposer_returns_one_change_for_one_family(tmp_path: Path) -> None: - from openultrasast.learning.proposer import Proposal, ScriptedProposer, build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"}) - scripted = ScriptedProposer( - [ - Proposal( - hypothesis="h", - lever="checklist", - change={"checklist.md": "- ask about ownership\n"}, - predicted_affected=("access_control",), - ) - ] - ) - proposal = scripted.propose(facts) - assert proposal is not None and proposal.lever == "checklist" and set(proposal.change) == {"checklist.md"} - assert scripted.propose(facts) is None # a scripted proposer runs out rather than repeating itself - - -@pytest.mark.parametrize( - ("change", "lever", "needle"), - [ - ({"../../etc/passwd": "x"}, "checklist", "outside"), - ({"family.toml": "x", "prompt.md": "y"}, "prompt", "one file"), - ({"secrets.md": "x"}, "checklist", "secrets.md"), - ({"prompt.md": "x" * 9000}, "prompt", "cap"), - ], - ids=["escapes_the_family_directory", "two_files", "unknown_file", "over_the_cap"], -) -def test_a_proposal_that_reaches_beyond_its_family_is_refused_by_name(change: dict[str, str], lever: str, needle: str) -> None: - from openultrasast.learning.proposer import Proposal, refuse_proposal - - refusal = refuse_proposal(Proposal(hypothesis="h", lever=lever, change=change), _config()) # type: ignore[arg-type] - assert refusal is not None and needle in refusal - - -def test_a_well_formed_proposal_is_accepted_and_applies_inside_its_own_directory(tmp_path: Path) -> None: - from openultrasast.learning.proposer import Proposal, apply_proposal, refuse_proposal - - directory = tmp_path / "access_control" - directory.mkdir() - (directory / "checklist.md").write_text("- old\n") - proposal = Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}, predicted_affected=("access_control",)) - assert refuse_proposal(proposal, _config()) is None - apply_proposal(proposal, directory) - assert (directory / "checklist.md").read_text() == "- new\n" - - -def test_the_write_roots_are_one_family_directory_and_exclude_the_checker_and_the_corpus(tmp_path: Path) -> None: - from openultrasast.learning import canaries - from openultrasast.learning.proposer import write_roots - - roots = write_roots(tmp_path / "configs", "access_control") - assert roots == (tmp_path / "configs" / "access_control",) - verifier_module = Path(canaries.__file__).resolve() - assert not any(verifier_module.is_relative_to(root.resolve()) for root in roots) - corpus = Path("benchmarks").resolve() - assert not any(str(corpus).startswith(str(root.resolve())) for root in roots) - - -def test_the_harnessx_proposer_degrades_with_a_reason_when_the_extra_is_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning import proposer as proposer_module - from openultrasast.learning.proposer import HarnessXProposer, build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"}) - monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False) - agent = HarnessXProposer(configs_dir=tmp_path / "configs", model="m") - assert agent.propose(facts) is None - assert agent.reason == "harnessx_unavailable" - assert agent.write_roots("access_control") == (tmp_path / "configs" / "access_control",) - - -# --- the meta-agent proposer (review round 3, task 3.5 finding 1) ------------ - - -def test_the_meta_agent_proposer_reads_its_change_out_of_a_scratch_copy(tmp_path: Path) -> None: - """The meta-agent never writes into the live family directory. - - A proposer that edited the tree directly would have changed the configuration before the round snapshotted - it, so a rejection could not put it back. It gets a copy, and the proposal is the diff of that copy.""" - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - (configs / "injection" / "family.toml").write_text('family = "injection"\nversion = "0"\n') - seen: dict[str, object] = {} - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - seen["workspace"] = workspace - seen["family"] = facts.family - (workspace / "checklist.md").write_text("- ask about quoting\n") - return "ask about quoting" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - proposal = proposer.propose(_facts(tmp_path, "injection")) - assert proposal is not None - assert proposal.lever == "checklist" and proposal.change == {"checklist.md": "- ask about quoting\n"} - assert proposal.hypothesis == "ask about quoting" - assert seen["workspace"] != configs / "injection" - assert (configs / "injection" / "checklist.md").read_text() == "- old\n", "the live tree was edited" - - -def test_the_meta_agent_proposer_refuses_more_than_one_changed_file(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - (configs / "injection" / "prompt.md").write_text("p\n") - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - (workspace / "checklist.md").write_text("- a\n") - (workspace / "prompt.md").write_text("- b\n") - return "two at once" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert "one file" in proposer.reason - - -def test_the_meta_agent_proposer_refuses_a_file_no_lever_owns(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - (workspace / "notes.txt").write_text("x\n") - return "sideways" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert "notes.txt" in proposer.reason - - -def test_the_meta_agent_proposer_says_so_when_it_changed_nothing(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=lambda **_kwargs: "nothing to do") - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert proposer.reason == "no_change_proposed" - - -def _facts(tmp_path: Path, family: str): # type: ignore[no-untyped-def] - from openultrasast.learning.proposer import FailureFacts - - del tmp_path - return FailureFacts(family=family, config=_config(family)) diff --git a/tests/test_learning_publish.py b/tests/test_learning_publish.py deleted file mode 100644 index 07601e2..0000000 --- a/tests/test_learning_publish.py +++ /dev/null @@ -1,209 +0,0 @@ -"""learning-harness task 3.8: publishing every number from its committed artifacts (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -ROADMAP = "# Roadmap\n\n## Overview\n\nSomething about the project.\n\n## Constraints\n\nCore dependencies stay empty.\n" - - -def _baseline(out: Path, model: str, *, recall: float = 1.0, cost: float = 0.5) -> None: - directory = out / "baseline" / model - directory.mkdir(parents=True, exist_ok=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": model, - "k_runs": 5, - "taxonomy_version": "1", - "slices": ["vibe-py"], - "cost_usd": cost, - "floors": {"injection": {"family": "injection", "budget_flips": 1, "negative_flip_rate": 0.1, "k_runs": 5, "gates": True}}, - "metrics": { - "injection": { - "family": "injection", - "taxonomy_version": "1", - "scorable": 10, - "unscorable": {"identical_twin": 2}, - "outcomes": {"pair_correct": round(recall * 10)}, # keep the fixture internally consistent - "recall": recall, - "silence": 1.0, - "youden": recall, - "directional_bias": 0.0, - "leak_rate": 0.0, - "fixed_fpr_recall": recall, - "hierarchical_credit": False, - } - }, - }, - indent=2, - sort_keys=True, - ) - + "\n" - ) - - -def test_publishing_twice_without_new_artifacts_is_byte_identical(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - measurements = tmp_path / "measurements" - first = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - text = roadmap.read_text() - second = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - assert roadmap.read_text() == text - assert first.table == second.table and first.families == second.families - - -def test_every_number_carries_its_denominator_floor_and_cost(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.9, cost=1.8) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - table = report.table - assert "| injection |" in table - assert "10" in table and "identical_twin: 2" in table # scorable and why the rest are not - assert "taxonomy 1" in table and "budget 1" in table - assert "0.20" in table # cost per pair-correct: 1.8 over 9 - assert "hierarchical partial credit could not apply" in table - assert report.command and report.command in roadmap.read_text() - - -def test_a_second_model_appears_beside_the_first(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.9) - _baseline(out, "deepseek-v4-pro", recall=1.0) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "deepseek-v4-flash" in report.table and "deepseek-v4-pro" in report.table - assert set(report.families) == {"injection"} - - -def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - measurements = tmp_path / "m" - measurements.mkdir() - (measurements / "2026-09-06-mechanism-lever-split.json").write_text( - json.dumps( - { - "slice": "vibe-py", - "holdout_pairs": 17, - "before": {"detected": 7, "youden": 0.0588}, - "after": {"detected": 5, "youden": -0.0588}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - stale = "human holdout profile pair_correct 0 -> 6 of 17, Youden 0 -> +0.353, round 2 `no_proposals`." - roadmap.write_text(f"{ROADMAP}\nMeasured 2026-09-05 with the vibe-py candidates: {stale}\n") - report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - text = roadmap.read_text() - assert "-0.0588" in report.table or "-0.059" in report.table - assert "5 of 17" in report.table or "detected 5" in report.table - assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower() - assert text.count("learning-harness:begin") == 1 - # Req 5.3 says *in place of* the earlier one. Leaving the old claim above the new section is how a document - # ends up asserting two different numbers for the same measurement. - assert stale not in text - assert "superseded" in text and "learning-harness" in text - assert "Measured 2026-09-05 with the vibe-py candidates:" in text, "the sentence is corrected, not deleted" - before = roadmap.read_text() - publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - assert roadmap.read_text() == before, "a second publish must be a no-op once the claim is corrected" - - -def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.5) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - _baseline(out, "deepseek-v4-flash", recall=0.9) - publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - text = roadmap.read_text() - assert text.count("learning-harness:begin") == 1 and text.count("learning-harness:end") == 1 - assert "## Overview" in text and "## Constraints" in text # nothing else in the roadmap is disturbed - assert "0.5" not in text.split("learning-harness:begin")[1] # the stale number is gone - - -def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - measurements = tmp_path / "m" - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - written = sorted(path.name for path in measurements.iterdir()) - assert any(name.endswith("-learning-families.json") for name in written) - payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text()) - # keyed by what identifies the run, so two slices of one model cannot collapse into one entry - assert payload["models"]["deepseek-v4-flash (vibe-py)"]["injection"]["scorable"] == 10 - assert payload["command"] - - -def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap) - assert report.families == () and "no baseline" in report.table.lower() - assert "learning-harness:begin" in roadmap.read_text() - - -def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_path: Path) -> None: - """Keyed by model alone, the vibe-py run silently replaced the agent-vfc run and the published table showed - hand-written teaching code as if it were the corpus — the exact failure Req 3 exists to prevent.""" - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - for key, slice_name, recall in (("deepseek-v4-flash-vibe-py", "vibe-py", 0.833), ("deepseek-v4-flash-agent-vfc", "agent-vfc", 0.444)): - directory = out / "baseline" / key - directory.mkdir(parents=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": "deepseek-v4-flash", - "slices": [slice_name], - "k_runs": 5, - "cost_usd": 1.0, - "taxonomy_version": "1", - "metrics": { - "injection": { - "family": "injection", - "scorable": 10, - "unscorable": {}, - "recall": recall, - "silence": 1.0, - "youden": recall, - "hierarchical_credit": False, - "outcomes": {"pair_correct": 5}, - } - }, - "floors": {}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text("# Roadmap\n") - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "vibe-py" in report.table and "agent-vfc" in report.table - assert report.table.count("| injection |") == 2, "one row per slice, not one row that survived" - assert "0.833" in report.table and "0.444" in report.table diff --git a/tests/test_learning_refusals.py b/tests/test_learning_refusals.py deleted file mode 100644 index 16ca7fe..0000000 --- a/tests/test_learning_refusals.py +++ /dev/null @@ -1,94 +0,0 @@ -"""A refusal is data, not a `continue` (learning-harness review round 3, Req 5.1, 5.2). - -`Refusal.degradation()` existed and nothing ever called it, so every path that declined to learn from a pair -declined silently: the mechanism lever skipped a holdout-taught shape without a word, leave-one-out dropped -non-teachers with no record, and the seeder gave a train-split identical twin the reason `split:train`, which -is not true. A leak the run does not mention is a leak nobody reads. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase - -VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str = "train", unscorable: str | None = None, tier: str = "seeded") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier=tier, - unscorable=unscorable, - ) - - -def test_a_train_split_twin_is_refused_for_being_a_twin_not_for_its_split(tmp_path: Path) -> None: - """`split:train` on a train-split row is a lie, and it hides a corpus fact behind a split fact. - - Two vibe-py rows are exactly this: `python-insecure-app-main-try-hack-me` and - `vulnpy-deserialization-do-pickle-load` are train-split identical twins.""" - from openultrasast.semantic.seed import _skip_reason - - assert _skip_reason(_case(tmp_path, "twin", split="train", unscorable="identical_twin")) == "identical_twin" - assert _skip_reason(_case(tmp_path, "held", split="holdout")) == "split:holdout" - assert _skip_reason(_case(tmp_path, "good", split="train")) is None - - -def test_leave_one_out_records_the_pairs_it_would_not_learn_from(tmp_path: Path) -> None: - """Req 5.2: the refusal is recorded with the pair names, on a channel a caller can read.""" - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo([_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")]) - refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"] - assert refusals, "leave-one-out excluded a holdout pair from teaching and said nothing" - assert refusals[0]["pairs"] == ["hold-a"] - assert result.teaching_pairs <= 1 - - -def test_the_mechanism_lever_records_every_shape_a_holdout_pair_taught(tmp_path: Path) -> None: - """The lever used to `continue` past a refused candidate; the run then showed a shorter list with no reason.""" - from openultrasast.improve.evolve import propose_mechanism_edits - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variants import Shape - - cases = [_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")] - candidates = MechanismStore(tmp_path / "candidates.jsonl") - append_from_pair( - candidates, - Shape( - language="python", - sink_name="system", - arity=1, - source_positions=(0,), - source_kinds=("request",), - guard="none", - mechanism="source_reaches_sink", - ), - summary="s", - cwe="CWE-78", - pair="hold-a", - provenance="human", - tier="seeded", - ) - refusals: list[dict[str, object]] = [] - edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl", degradations=refusals) - assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == [] - assert [item["reason"] for item in refusals] == ["holdout_pair_refused"] - assert refusals[0]["pairs"] == ["hold-a"] - json.dumps(refusals) # a degradation has to survive the artifact it is written into diff --git a/tests/test_learning_reliability.py b/tests/test_learning_reliability.py deleted file mode 100644 index abda25d..0000000 --- a/tests/test_learning_reliability.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5). - -The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw -deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and -a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up. - -Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision, -the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether -any of those decisions could be told from a coin flip. -""" - -from __future__ import annotations - -from openultrasast.learning.acceptance import reliability - - -def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None: - result = reliability(better=8, worse=0) - assert result.better == 8 and result.worse == 0 - assert result.p_value < 0.05 and result.reliable is True - - -def test_two_pairs_moving_the_right_way_is_not() -> None: - """n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing.""" - result = reliability(better=2, worse=0) - assert result.p_value == 0.5 and result.reliable is False - - -def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None: - result = reliability(better=0, worse=0) - assert result.p_value == 1.0 and result.reliable is False - - -def test_an_even_split_is_never_reliable_however_large() -> None: - assert reliability(better=20, worse=20).reliable is False - - -def test_the_round_journals_the_reliability_of_its_own_decision() -> None: - from openultrasast.learning.journal import RoundRecord - - record = RoundRecord( - round=1, - family="injection", - hypothesis="h", - levers=("checklist",), - predicted_affected=(), - predicted_at_risk=(), - outcome="accepted", - train_better=8, - train_worse=0, - holdout_better=1, - holdout_worse=0, - ) - payload = record.to_dict() - assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05 - assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so" diff --git a/tests/test_learning_round4_evidence.py b/tests/test_learning_round4_evidence.py deleted file mode 100644 index 3d09877..0000000 --- a/tests/test_learning_round4_evidence.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Three defects the round-4 review found, one test each (learning-harness, Req 3.2, 4.1, 9.1). - -Two of them were the same shape as defects already rejected once: a proposer that cannot propose because the call -is wrong and the error is swallowed, and a K-run floor that turned a documented command into a traceback. The -third is the fixed-side half of the range defect — 28 rows resolve a span on the vulnerable side and none on the -fixed side, so every finding there reads as outside the labeled function and the pair scores as silence. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.harness_ext import has_harnessx -from openultrasast.learning.families import load_families -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored -from openultrasast.semantic.extra import has_semantic_extra - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED_OTHER = "from flask import request\n\n\ndef unrelated():\n return 'ok'\n" - - -def _finding(line: int) -> StaticFinding: - return StaticFinding( - finding_id=f"tool-hunter:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["tool-hunter", "family:injection"], - ranking_priority=0.0, - ) - - -# --- the meta-agent call ----------------------------------------------------- - - -@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable") -def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`MetaAgent(inner_model=...)` wants a `ModelConfig`; a string reaches `self.inner_model.agentic(...)` and dies. - - Every other HarnessX composition site in this repository builds the config properly. Here the mistake was - invisible because a `# type: ignore` hid it from mypy and a bare `except Exception` turned it into the - degradation reason `meta_agent_failed`, which reads exactly like the model declining.""" - from openultrasast.learning import proposer as proposer_module - - seen: dict[str, object] = {} - - class _Agent: - def __init__(self, **kwargs: object) -> None: - seen.update(kwargs) - - async def evolve(self, **kwargs: object) -> Path: - seen["evolve"] = kwargs - workspace = kwargs["current_config"] - assert isinstance(workspace, Path) - (workspace / "checklist.md").write_text("- from the meta agent\n") - return workspace - - monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent) - monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})") - (tmp_path / "injection").mkdir() - (tmp_path / "injection" / "checklist.md").write_text("- old\n") - proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts()) - assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"} - assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config" - assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root" - - -@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable") -def test_a_meta_agent_built_wrongly_is_not_reported_as_a_declined_proposal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """ "We called the agent wrong" and "the agent had nothing to propose" are different facts.""" - from openultrasast.learning import proposer as proposer_module - - def _boom() -> object: - raise TypeError("inner_model must be a ModelConfig") - - monkeypatch.setattr(proposer_module, "_meta_agent_class", _boom) - (tmp_path / "injection").mkdir() - proposer = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m") - with pytest.raises(TypeError, match="ModelConfig"): - proposer.propose(_facts()) - - -def _facts(): # type: ignore[no-untyped-def] - from openultrasast.learning.detectors import FamilyConfig - from openultrasast.learning.proposer import FailureFacts - - return FailureFacts( - family="injection", - config=FamilyConfig( - family="injection", - version="0", - prompt="p", - checklist="- old", - tools=("read_file",), - max_steps=1, - max_cost_usd=0.1, - max_chars=100, - ), - ) - - -# --- the pairs command in its documented form -------------------------------- - - -def test_the_pairs_hunter_path_runs_without_a_k_runs_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """The floor is a default, not a trap: `ousast pairs --hunter` must work as documented.""" - from openultrasast.cli import main - - (tmp_path / "a-v.py").write_text(VULN) - (tmp_path / "a-f.py").write_text(FIXED_OTHER) - (tmp_path / "pairs.toml").write_text( - '[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n' - 'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n' - ) - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - assert main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted"]) == 0 - - -def test_a_k_below_the_floor_names_the_flag_instead_of_raising(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.cli import main - - (tmp_path / "pairs.toml").write_text("") - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - with pytest.raises(SystemExit, match="k-runs"): - main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "1"]) - - -# --- the fixed side of a trap pair ------------------------------------------- - - -def test_a_trap_pair_is_scored_against_the_function_its_fixed_side_really_holds(tmp_path: Path) -> None: - """A Real-Vuln-Benchmark trap takes its fixed side from a different, correctly guarded handler. - - With no span for it, every fixed-side finding counts as outside the labeled function and the pair scores as - silence — the same flattering direction as the range defect, for the 28 rows that resolve one side only.""" - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED_OTHER) - case = PairCase( - name="trap", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection" - ), - ), - min_recall=1.0, - fix_policy="silent", - fix_function="unrelated", - ) - score = score_pair_family( - case, - "injection", - [[_finding(6)]], - [[_finding(5)]], - ranges={"app.py": (("run", 5, 6),)}, - fixed_ranges={"app.py": (("unrelated", 4, 5),)}, - taxonomy=load_families(), - ) - assert score.outcome == "both_flagged", "the finding is inside the fixed side's own handler; that is a leak" - - -def test_a_row_whose_fixed_side_resolves_no_span_says_so_instead_of_scoring_silent(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED_OTHER) - case = PairCase( - name="orphan", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection" - ), - ), - min_recall=1.0, - fix_policy="silent", - ) - score = score_pair_family( - case, - "injection", - [[_finding(6)]], - [[]], - ranges={"app.py": (("run", 5, 6),)}, - fixed_ranges={"app.py": (("unrelated", 4, 5),)}, - taxonomy=load_families(), - ) - assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label:fixed" - - -def test_the_trap_rows_carry_their_fixed_side_function_in_the_catalog() -> None: - """`fix_function` lived only in the recipes; the scorer could not see which handler the fixed side holds.""" - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - assert cases["threatbyte-api-v1-get"].fix_function == "post" - assert cases["threatbyte-api-v1-delete"].fix_function == "post" - assert cases["vampi-users-update-password"].fix_function == "register_user" - assert cases["vampi-books-get-by-title"].fix_function == "", "a row whose two sides are one function names none" - - -@pytest.mark.semantic -@pytest.mark.skipif(not has_semantic_extra(), reason="resolving a span on both sides needs the tree-sitter grammars") -def test_no_vendored_row_resolves_one_side_and_not_the_other() -> None: - """The corpus-wide form of the same defect: 28 rows resolved a vulnerable span and no fixed-side span. - - Every one of them therefore scored the fixed side as silent by construction, whatever the detector said. The - catalog generator now derives `fix_function` from the fixed excerpt itself, so the span exists and the leak is - visible.""" - import tempfile - - from openultrasast.learning.scoring import _spans - from openultrasast.pairs import _materialize_side, _overlay_scan - - orphans = [] - for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)): - if case.unscorable: - continue - with tempfile.TemporaryDirectory() as scratch: - vuln = _materialize_side(Path(scratch) / "vuln", case, side="vuln") - fixed = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - vuln_ranges, fixed_ranges = _overlay_scan(vuln).ranges, _overlay_scan(fixed).ranges - if _spans(case, vuln_ranges) and not _spans(case, fixed_ranges, function=case.fix_function): - orphans.append(case.name) - assert orphans == [], f"rows whose fixed side can only ever score as silence: {orphans}" diff --git a/tests/test_learning_round_evidence.py b/tests/test_learning_round_evidence.py deleted file mode 100644 index 5775f25..0000000 --- a/tests/test_learning_round_evidence.py +++ /dev/null @@ -1,428 +0,0 @@ -"""What a round must prove before it is kept (learning-harness review round 3, Req 4.1, 5.2, 9.5, 9.6, 9.8). - -The round loop was accepting and rejecting on numbers that did not mean what their names said: the -cross-family sweep counted every failing pair rather than the pairs this change broke, unscorable rows -voted against a round they cannot judge, cost was metered once after everything had been paid for, and a -stage that raised left no record at all. Each test here fails on exactly one of those. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.detectors import FamilyConfig, write_default_configs -from openultrasast.learning.families import load_families -from openultrasast.learning.journal import Archive, LearningJournal -from openultrasast.learning.proposer import Proposal, ScriptedProposer -from openultrasast.learning.rounds import NoiseFloor, run_learning_round -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - unscorable=unscorable, - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _proposal(text: str = "- ask about quoting\n") -> Proposal: - return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("train-a",)) - - -@pytest.fixture -def world(tmp_path: Path): # type: ignore[no-untyped-def] - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - (configs / "injection" / "checklist.md").write_text("- old\n") - return { - "configs": configs, - "cases": [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")], - "journal": LearningJournal(tmp_path / "journal.jsonl"), - "archive": Archive(tmp_path / "archive.jsonl"), - "out": tmp_path / "out", - "floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)}, - "root": tmp_path, - } - - -def _run(world, proposal, build, **kwargs): # type: ignore[no-untyped-def] - return run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([proposal] if proposal else []), - scan_factory=build, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - **kwargs, - ) - - -def _injection_improves(config: FamilyConfig): # type: ignore[no-untyped-def] - """Injection detects only once the proposed checklist lands; every other family finds nothing, ever.""" - - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln" or config.family != "injection": - return [] - return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else [] - - return scan - - -# --- the sweep --------------------------------------------------------------- - - -def test_a_family_that_was_already_failing_does_not_reject_the_round(world) -> None: # type: ignore[no-untyped-def] - """Req 9.5 counts *negative flips*, not absolute failures. - - On the real corpus most pairs fail at round zero, so counting failures rejects every round for damage it - did not do — and the harness then reports that it cannot improve.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 0, "access control failed before and after; this change flipped nothing" - assert record.outcome == "accepted" and record.reason == "improved" - - -def test_a_family_this_change_really_broke_still_rejects_the_round(world) -> None: # type: ignore[no-untyped-def] - """The other arm: a pair that was correct before and is not after is a flip, and one is over the budget.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True - return [_finding("injection")] - return [] - - return scan - - record = _run(world, _proposal(), build, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 1 - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_an_unscorable_row_never_votes_in_the_sweep(world) -> None: # type: ignore[no-untyped-def] - """Req 4.1: an unscorable row leaves every denominator, and it certainly does not reject a round.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control", unscorable="identical_twin")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=0, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 0 and record.outcome == "accepted" - - -def test_attribution_names_the_pairs_this_change_flipped_and_no_others(world) -> None: # type: ignore[no-untyped-def] - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - attribution = json.loads((world["out"] / "rounds" / "1" / "attribution.json").read_text()) - assert attribution["flipped_predicted"] == ["hold-a", "train-a"] - assert attribution["flipped_unpredicted"] == [], "a pair that was already failing was never flipped by this round" - - -# --- cost, exceptions, the archive ------------------------------------------ - - -def test_the_cap_stops_a_round_before_it_pays_for_the_next_stage(world) -> None: # type: ignore[no-untyped-def] - """Req 9.8 says stop, revert and record. Metering once, after every stage, can only ever record.""" - stages: list[str] = [] - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - stages.append(f"{config.family}:{root.name}") - return [] - - return scan - - spend = {"usd": 0.0} - - def spent() -> float: - spend["usd"] += 1.0 - return spend["usd"] - - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - record = _run(world, _proposal(), build, cost_cap_usd=0.5, spent_usd=spent, sweep_families=("access_control",)) - assert record.outcome == "reverted_cost" and record.reason == "cost_cap_exceeded" - assert "access_control:vuln" not in stages, "the sweep ran after the cap was already crossed" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_stage_that_raises_is_reverted_journalled_and_re_raised(world) -> None: # type: ignore[no-untyped-def] - """An endpoint failure mid-round is an outcome. Losing the round record loses the money it already spent.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if config.checklist.strip() == "- ask about quoting": - raise RuntimeError("endpoint said no") - return [] - - return scan - - with pytest.raises(RuntimeError, match="endpoint said no"): - _run(world, _proposal(), build) - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()] - assert outcomes == [("reverted", "stage_failed: RuntimeError")] # the exception type is part of the record - assert (world["out"] / "rounds" / "1" / "scores.json").is_file() - - -def test_a_rejected_version_is_archived_so_a_pair_it_alone_wins_is_not_lost(world) -> None: # type: ignore[no-untyped-def] - """Req 9.7: per-pair winners, or the loop collapses onto one lineage and forgets what a rejected try could do. - - This round wins both of its own pairs and is still rejected, because it broke another family. Version 1 is the - only configuration that ever got those two pairs right; dropping it because the round as a whole lost is how a - search collapses onto one lineage.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True - return [_finding("injection")] - return [] - - return scan - - record = _run(world, _proposal(), build, sweep_families=("access_control",)) - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert world["archive"].winners("injection") == {"train-a": "1", "hold-a": "1"} - assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text(), "a rejected round is not versioned" - - -# --- the holdout refusal ----------------------------------------------------- - - -def test_a_proposal_that_names_a_holdout_pair_is_refused_by_name(world) -> None: # type: ignore[no-untyped-def] - """Req 5.2: a candidate that touches a holdout pair is refused and the refusal names the pairs.""" - record = _run( - world, - Proposal( - hypothesis="memorize the answer", - lever="memory", - change={"hard_negatives.jsonl": json.dumps({"pair": "hold-a", "note": "always flag"}) + "\n"}, - predicted_affected=("hold-a",), - ), - _injection_improves, - ) - assert record.outcome == "refused_holdout" - assert "hold-a" in record.reason - assert (world["configs"] / "injection" / "hard_negatives.jsonl").read_text() == "" - - -def test_a_proposal_that_only_names_train_pairs_is_not_refused(world) -> None: # type: ignore[no-untyped-def] - record = _run( - world, - Proposal( - hypothesis="remember this shape", - lever="memory", - change={"hard_negatives.jsonl": json.dumps({"pair": "train-a", "note": "shape"}) + "\n"}, - predicted_affected=("train-a",), - ), - _injection_improves, - ) - assert record.outcome != "refused_holdout" - - -# --- round zero -------------------------------------------------------------- - - -def test_round_zero_measures_each_family_with_its_own_detector(tmp_path: Path) -> None: - """Req 8.2. One detector for every family means the generalist scores them all, and abstention earns no credit - (`related(X, "unknown")` is lateral), so every family but `unknown` baselines at a structural zero.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - seen: list[str] = [] - - def factory(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - seen.append(config.family) - return [_finding(config.family)] if root.name == "vuln" else [] - - return scan - - cases = [_case(tmp_path, "inj", split="train"), _case(tmp_path, "acl", split="train", family="access_control")] - report = run_baseline( - cases, - taxonomy=load_families(), - configs_dir=configs, - scan_factory=factory, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert set(seen) == {"injection", "access_control"}, "each family must be measured with its own configuration" - assert report.metrics["injection"]["recall"] == 1.0 and report.metrics["access_control"]["recall"] == 1.0 - - -def test_round_zero_reports_each_slice_separately(tmp_path: Path) -> None: - """Req 8.4: vibe-py and agent-vfc are different worlds; one pooled number hides which one moved.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - web = _case(tmp_path, "web", split="train") - agent = _case(tmp_path, "agent", split="train") - object.__setattr__(agent, "slice", "agent-vfc") - - def factory(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - return [_finding(config.family)] if root.name == "vuln" else [] - - return scan - - report = run_baseline( - [web, agent], - taxonomy=load_families(), - configs_dir=configs, - scan_factory=factory, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert set(report.per_slice) == {"vibe-py/injection", "agent-vfc/injection"} - payload = json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text()) - assert set(payload["per_slice"]) == {"vibe-py/injection", "agent-vfc/injection"} - - -def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None: - """The floor exists because a single run misses most real changes; a baseline measured at K=1 is a noise floor - of zero and every later round then inherits a budget of nothing.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - with pytest.raises(ValueError, match="three runs"): - run_baseline( - [_case(tmp_path, "inj", split="train")], - taxonomy=load_families(), - configs_dir=configs, - scan_factory=lambda config: lambda root: [], - model="m", - out_dir=tmp_path / "out", - k_runs=1, - ) - - -def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def] - """`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start. - - Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a - provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not - look identical to a model that considered the facts and declined.""" - - class Mute: - reason = "meta_agent_failed: RuntimeError" - - def propose(self, facts): # type: ignore[no-untyped-def] - del facts - return None - - from openultrasast.learning.rounds import run_learning_round - - record = run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=Mute(), - scan_factory=_injection_improves, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - ) - assert record.outcome == "rejected" - assert record.reason == "no_proposal: meta_agent_failed: RuntimeError" - - -def test_a_round_that_dies_before_it_applies_anything_is_still_journalled(world) -> None: # type: ignore[no-untyped-def] - """Measured: round 5 died on a network read during its *first* stage and left no record at all. - - The exception path was wrapped around the stages after the proposal is applied, because those are the ones that - have something to revert. The ones before it had already spent money.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - raise ConnectionError("the body stopped arriving") - - return scan - - with pytest.raises(ConnectionError): - _run(world, _proposal(), build) - outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()] - assert outcomes == [("reverted", "stage_failed: ConnectionError")] - assert (world["out"] / "rounds" / "1").is_dir() - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" diff --git a/tests/test_learning_round_loop.py b/tests/test_learning_round_loop.py deleted file mode 100644 index c3c8892..0000000 --- a/tests/test_learning_round_loop.py +++ /dev/null @@ -1,230 +0,0 @@ -"""learning-harness task 3.7: one evolve round, end to end (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.detectors import FamilyConfig, write_default_configs -from openultrasast.learning.families import load_families -from openultrasast.learning.journal import Archive, LearningJournal -from openultrasast.learning.proposer import Proposal, ScriptedProposer -from openultrasast.learning.rounds import NoiseFloor -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _factory(finds: dict[str, bool]): # type: ignore[no-untyped-def] - """A detector whose behaviour depends on the configuration version it was built from.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - return [_finding(config.family)] if finds.get(config.checklist.strip(), False) else [] - - return scan - - return build - - -def _proposal(text: str) -> Proposal: - return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("injection",)) - - -@pytest.fixture -def world(tmp_path: Path): # type: ignore[no-untyped-def] - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - (configs / "injection" / "checklist.md").write_text("- old\n") - cases = [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")] - return { - "configs": configs, - "cases": cases, - "journal": LearningJournal(tmp_path / "journal.jsonl"), - "archive": Archive(tmp_path / "archive.jsonl"), - "out": tmp_path / "out", - "floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)}, - } - - -def _run(world, proposal: Proposal | None, finds: dict[str, bool], **kwargs): # type: ignore[no-untyped-def] - from openultrasast.learning.rounds import run_learning_round - - return run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([proposal] if proposal else []), - scan_factory=_factory(finds), - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - **kwargs, - ) - - -def test_a_change_that_helps_is_accepted_versioned_and_journalled(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}) - assert record.outcome == "accepted" and record.family == "injection" - assert record.target_train_delta > 0 and record.target_holdout_delta > 0 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- ask about quoting\n" - assert 'version = "1"' in (world["configs"] / "injection" / "family.toml").read_text() - assert [item.outcome for item in world["journal"].rounds()] == ["accepted"] - assert world["archive"].winners("injection") - - -def test_a_change_that_helps_nothing_is_rejected_and_reverted(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- useless\n"), {}) - assert record.outcome == "rejected" and record.reason == "tie" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text() - assert world["journal"].rejected_buffer("injection") == ("ask about quoting",) - - -def test_a_round_is_rejected_when_the_sweep_finds_another_family_below_its_floor(world) -> None: # type: ignore[no-untyped-def] - """The sweep counts pairs this change turned from right to wrong in another family (Req 9.5). - - It used to count every failing pair, which rejected a round for damage it did not do; the pair below is right - before the change and wrong after it, which is the only thing that may reject a round on another family's - behalf. The other arm — a family that was already failing must not reject — lives in - `tests/test_learning_round_evidence.py`. - """ - world["cases"].append(_case(world["configs"].parent, "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True # the injection change lands, and access control stops finding its pair - return [_finding("injection")] - return [] - - return scan - - from openultrasast.learning.rounds import run_learning_round - - record = run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([_proposal("- ask about quoting\n")]), - scan_factory=build, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - sweep_families=("access_control",), - ) - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert record.sweep["access_control"] == 1 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_round_over_its_cost_cap_is_reverted_and_says_so(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}, cost_cap_usd=0.0, spent_usd=lambda: 1.0) - assert record.outcome == "reverted_cost" and record.cost_usd == 1.0 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_proposal_that_reaches_beyond_its_family_never_runs(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, Proposal(hypothesis="h", lever="checklist", change={"../escape.md": "x"}), {}) - assert record.outcome == "rejected" and "outside" in record.reason # the holdout refusal has its own test - assert not (world["configs"].parent / "escape.md").exists() - - -def test_a_round_with_no_proposal_is_recorded_rather_than_skipped(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, None, {}) - assert record.outcome == "rejected" and record.reason == "no_proposal" - assert [item.round for item in world["journal"].rounds()] == [1] - - -def test_the_round_directory_holds_what_a_replay_needs(world) -> None: # type: ignore[no-untyped-def] - record = _run( - world, - _proposal("- ask about quoting\n"), - {"- ask about quoting": True}, - trajectories=lambda: [{"pair": "train-a", "prompt": "api_key = " + "'" + "B" * 24 + "'"}], - ) - directory = world["out"] / "rounds" / "1" - assert (directory / "proposal.json").is_file() and (directory / "scores.json").is_file() - attribution = json.loads((directory / "attribution.json").read_text()) - assert attribution["flipped_predicted"] == ["hold-a", "train-a"] # the round is deterministic; one exact answer - assert attribution["flipped_unpredicted"] == [] - trajectories = (directory / "trajectories.jsonl").read_text() - assert "REDACTED" in trajectories and "B" * 24 not in trajectories - assert record.attribution.precision == 1.0 - - -def test_a_round_number_is_never_reused(world) -> None: # type: ignore[no-untyped-def] - from openultrasast.learning.journal import JournalError - - _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}) - (world["out"] / "rounds" / "2").mkdir(parents=True) - with pytest.raises(JournalError, match="2"): - _run(world, _proposal("- another\n"), {}) - - -def test_every_stage_scores_at_least_three_runs_per_pair(world) -> None: # type: ignore[no-untyped-def] - with pytest.raises(ValueError, match="three"): - _run(world, _proposal("- x\n"), {}, k_runs=2) diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py deleted file mode 100644 index 04e554a..0000000 --- a/tests/test_learning_rounds.py +++ /dev/null @@ -1,377 +0,0 @@ -"""learning-harness task 3.3: round zero and the per-family noise floor (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, family: str = "injection", slice_name: str = "vibe-py", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice=slice_name, - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - review_tier="seeded", - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _is_vulnerable_side(root: Path) -> bool: - """The materialized side, by directory name only. - - An earlier version also matched "-v" anywhere in the path, which the random temporary directory name - sometimes contained, so the detector "found" the bug on the fixed side too and the pair flipped. A - flaky harness is indistinguishable from a flaky detector, which is precisely what a noise floor must - not measure. - """ - return root.name == "vuln" - - -def _steady(root: Path) -> list[StaticFinding]: - return [_finding("injection")] if _is_vulnerable_side(root) else [] - - -def _always(scan): # type: ignore[no-untyped-def] - """Round zero asks for a detector per family; these tests deliberately give every family the same one so the - noise-floor arithmetic is the only thing under test.""" - - def build(_config): # type: ignore[no-untyped-def] - return scan - - return build - - -def _flaky(counter: list[int]): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if not _is_vulnerable_side(root): - return [] - counter.append(1) - return [_finding("injection")] if len(counter) % 2 else [] - - return scan - - -def test_round_zero_writes_a_configuration_for_every_family(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - report = run_baseline( - [_case(tmp_path, "a")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - ) - configs = load_family_configs(tmp_path / "configs", taxonomy) - assert set(configs) == {family.id for family in taxonomy.families} - assert all(config.version == "0" for config in configs.values()) - assert report.model == "m" and report.k_runs == 5 - - -def test_a_steady_detector_has_a_zero_noise_floor_and_a_flaky_one_does_not(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - steady = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "c1", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "o1", - ) - floor = steady.floors["injection"] - assert floor.k_runs == 5 and floor.negative_flip_rate == 0.0 and floor.budget_flips == 0 and floor.gates is True - flaky = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "c2", - scan_factory=_always(_flaky([])), - model="m", - out_dir=tmp_path / "o2", - ) - assert flaky.floors["injection"].negative_flip_rate > 0.0 and flaky.floors["injection"].budget_flips >= 1 - - -def test_the_memory_family_is_measured_but_never_gates(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - report = run_baseline( - [_case(tmp_path, "m", family="memory", slice_name="vfc")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - slices=("vfc",), - ) - assert "memory" in report.floors and report.floors["memory"].gates is False - assert report.metrics["memory"]["scorable"] == 1 - - -def test_the_default_selection_is_the_web_slices(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - cases = [_case(tmp_path, "web", slice_name="vibe-py"), _case(tmp_path, "c", family="memory", slice_name="vfc")] - report = run_baseline( - cases, taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(_steady), model="m", out_dir=tmp_path / "out" - ) - assert set(report.metrics) == {"injection"} and "memory" not in report.floors - - -def test_artifacts_are_keyed_by_model_so_another_model_can_be_compared(tmp_path: Path) -> None: - from openultrasast.learning.rounds import compare_baselines, run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - for index, model in enumerate(("deepseek-v4-flash", "deepseek-v4-pro")): - run_baseline( - [_case(tmp_path, f"a{index}")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model=model, - out_dir=out, - ) - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-pro"] - floors = json.loads((out / "baseline" / "deepseek-v4-flash" / "noise-floors.json").read_text()) - assert floors["model"] == "deepseek-v4-flash" and floors["floors"]["injection"]["k_runs"] == 5 - report = json.loads((out / "baseline" / "deepseek-v4-flash" / "report.json").read_text()) - assert report["metrics"]["injection"]["recall"] == 1.0 and report["taxonomy_version"] - table = compare_baselines(out) - assert set(table) == {"deepseek-v4-flash", "deepseek-v4-pro"} - assert table["deepseek-v4-pro"]["injection"]["youden"] == 1.0 - # a second baseline never rewrites a configuration - assert (tmp_path / "configs" / "injection" / "family.toml").read_text().count("version") == 1 - - -def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - calls: list[Path] = [] - - def counting(root: Path) -> list[StaticFinding]: - calls.append(root) - return _steady(root) - - taxonomy = load_families() - case = _case(tmp_path, "twin") - twin = PairCase(**{**case.__dict__, "unscorable": "identical_twin"}) - report = run_baseline( - [twin], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(counting), model="m", out_dir=tmp_path / "out" - ) - assert report.metrics["injection"]["scorable"] == 0 - assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1} - assert calls == [] - - -def test_round_zero_records_what_it_spent(tmp_path: Path) -> None: - """Publishing puts a cost beside every number, so the baseline has to know what it cost.""" - from openultrasast.learning.rounds import run_baseline - - report = run_baseline( - [_case(tmp_path, "a")], - taxonomy=load_families(), - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - spent_usd=lambda: 1.25, - ) - assert report.cost_usd == 1.25 - assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25 - - -def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_run(tmp_path: Path) -> None: - """One transient error must not cost a run that has already paid for hundreds of calls. - - Round zero is the most expensive thing this harness does. A provider hiccup two thirds of the way through used - to raise straight out of `run_baseline`, and everything measured so far went with it — so the honest response - to a flaky endpoint was to run the whole thing again and hope.""" - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - seen: list[str] = [] - - def build(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - seen.append(root.name) - if len(seen) == 3: - raise TimeoutError("the endpoint hung up") - return _steady(root) - - return scan - - report = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=build, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured" - assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1} - assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file() - - -def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: - """Artifacts are keyed by what identifies the run, and the slice selection is part of that. - - Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model - alone, the second run silently replaces the first and the published table shows one of them as if it were - everything.""" - from openultrasast.learning.rounds import compare_baselines, run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - web = run_baseline( - [_case(tmp_path, "w", slice_name="vibe-py")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - ) - memory = run_baseline( - [_case(tmp_path, "m", family="memory", slice_name="vfc")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - slices=("vfc",), - ) - assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"} - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written - table = compare_baselines(out) - assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"} - assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"] - - -def test_round_zero_keeps_every_pair_it_has_already_paid_for(tmp_path: Path) -> None: - """A four-hour run that is interrupted must not lose four hours of paid calls. - - Measured the hard way: the 176-pair vfc baseline hit its timeout and wrote nothing, because scores were - accumulated in memory and persisted once at the end. An exception was already handled; a timeout, a kill and a - machine going away were not, and they cost the same money.""" - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - scored: list[str] = [] - - def build(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - scored.append(root.name) - # K=3 means three vulnerable-side scans per pair, so this gives up part way through the third pair - if len([item for item in scored if item == "vuln"]) > 6: - raise KeyboardInterrupt("the operator gave up") - return _steady(root) - - return scan - - cases = [_case(tmp_path, name) for name in ("a", "b", "c", "d")] - with pytest.raises(KeyboardInterrupt): - run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=build, - model="m", - out_dir=out, - k_runs=3, - ) - checkpoint = out / "baseline" / "m" / "scores.jsonl" - assert checkpoint.is_file(), "the pairs already measured were not written down" - rows = [json.loads(line) for line in checkpoint.read_text().splitlines() if line.strip()] - assert [row["pair"] for row in rows] == ["a", "b"] - assert all(row["outcome"] == "pair_correct" for row in rows) - - -def test_a_resumed_baseline_does_not_pay_twice(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - cases = [_case(tmp_path, name) for name in ("a", "b")] - run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=out, - k_runs=3, - ) - calls: list[str] = [] - - def counting(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - calls.append(root.name) - return _steady(root) - - return scan - - report = run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=counting, - model="m", - out_dir=out, - k_runs=3, - resume=True, - ) - assert calls == [], "every pair was already measured and paid for" - assert report.metrics["injection"]["scorable"] == 2 diff --git a/tests/test_learning_scan.py b/tests/test_learning_scan.py deleted file mode 100644 index fe7b065..0000000 --- a/tests/test_learning_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -"""learning-harness task 3.4: family detectors, verifiers and tags in a scan (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) - - -def _repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, toml: str, *, configs: bool = True) -> Path: - from openultrasast.learning.detectors import write_default_configs - from openultrasast.learning.families import load_families - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(APP) - (repo / "openultrasast.toml").write_text(toml) - if configs: # round zero writes these; a scan reads them and never creates them - write_default_configs(repo / ".openultrasast/learning/configs", load_families(), prompt="You hunt.", version="0") - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - return repo - - -def _run_dir(repo: Path) -> Path: - return sorted((repo / ".runs").iterdir())[-1] - - -def _scan(repo: Path) -> tuple[list[dict[str, object]], dict[str, object]]: - assert main(["scan", str(repo), "--mode", "standard", "--config", str(repo / "openultrasast.toml")]) == 0 - run_dir = _run_dir(repo) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - manifest = json.loads((run_dir / "manifest.json").read_text()) - return findings, manifest - - -def test_a_standard_scan_tags_findings_with_family_detector_and_verifier(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - findings, manifest = _scan(repo) - detected = [item for item in findings if any(str(tag).startswith("family:") for tag in item["tags"])] - assert detected, "the family detectors ran and their findings are in the report" - tags = set(detected[0]["tags"]) - assert any(tag.startswith("family:") for tag in tags) - assert any(tag.startswith("detector:") for tag in tags) - assert any(tag.startswith("verifier:") for tag in tags) - assert detected[0]["evidence_level"] == "suspicion" # nothing rises without a verifier saying so - block = manifest["learning"] - assert block["taxonomy_version"] and block["round"] is None - assert block["families"] and isinstance(block["classifier"], dict) - assert block["classifier"]["hierarchical_credit"] is False - - -def test_a_missing_family_directory_is_a_recorded_reason_not_a_crash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo( - tmp_path, - monkeypatch, - '[learning]\nenabled = true\nconfigs_dir = "nowhere/configs"\n\n[models]\nhunter = "scripted-model"\n', - configs=False, - ) - findings, manifest = _scan(repo) - assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) - reasons = {str(item.get("reason")) for item in manifest["degradations"]} - assert "learning_configs_unavailable" in reasons - assert "learning" not in manifest or manifest["learning"]["families"] == {} - - -def test_the_feature_is_off_by_configuration_and_leaves_no_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, "[learning]\nenabled = false\n", configs=False) - findings, manifest = _scan(repo) - assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) - assert "learning" not in manifest - - -def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake") - assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0 - run_dir = _run_dir(repo) - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - payload = json.loads((run_dir / "findings.json").read_text())["findings"] - family_findings = {item["finding_id"] for item in payload if any(str(tag).startswith("family:") for tag in item["tags"])} - assert family_findings, "the fixture must actually produce family findings, or this proves nothing" - # A family detector's claim is a suspicion until a verifier speaks; handing it to the sandbox would let an - # unverified model claim drive execution (Req 7.1). - candidates = {str(item.get("finding_id") or item.get("id") or "") for item in verdicts} - assert candidates.isdisjoint(family_findings) - - -def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - _scan(repo) - sarif = json.loads((_run_dir(repo) / "report.sarif").read_text()) - props = [result["properties"] for result in sarif["runs"][0]["results"]] - family_props = [item for item in props if item.get("family")] - assert family_props and all(item.get("detector") and item.get("verifier") for item in family_props) diff --git a/tests/test_learning_scoring.py b/tests/test_learning_scoring.py deleted file mode 100644 index 3c679e6..0000000 --- a/tests/test_learning_scoring.py +++ /dev/null @@ -1,185 +0,0 @@ -"""learning-harness task 2.3: per-pair family scoring (offline).""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase - -RANGES = {"app.py": (("leaky", 10, 13), ("guarded", 4, 7))} - - -def _finding(family: str, line: int, path: str = "app.py") -> StaticFinding: - return StaticFinding( - finding_id=f"detector:{path}:{line}", - path=path, - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@1"], - ranking_priority=0.0, - ) - - -def _case(tmp_path: Path, *, family: str = "access_control", unscorable: str | None = None) -> PairCase: - (tmp_path / "v.py").write_text("x = 1\n") - (tmp_path / "f.py").write_text("x = 2\n") - return PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function="leaky", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - unscorable=unscorable, - ) - - -def test_detection_needs_the_labeled_family_inside_the_labeled_function(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - hit = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert hit.outcome == "pair_correct" and hit.runs == ("pair_correct",) - outside = score_pair_family(case, "access_control", [[_finding("access_control", 5)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert outside.outcome == "both_silent" # the right family in the wrong function is not a detection - other_file = score_pair_family( - case, "access_control", [[_finding("access_control", 12, "other.py")]], [[]], ranges=RANGES, taxonomy=taxonomy - ) - assert other_file.outcome == "both_silent" - wrong_family = score_pair_family(case, "access_control", [[_finding("injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert wrong_family.outcome == "both_silent" and wrong_family.other_findings_vuln == 1 - - -def test_a_fabricated_family_never_counts_and_is_recorded(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[_finding("sql_injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "both_silent" and score.fabricated_vuln == 1 - assert score.other_findings_vuln == 0 # a fabricated id is not another family, it is noise - - -def test_silence_ignores_findings_of_other_families_and_never_calls_them_leaks(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - quiet = score_pair_family( - case, - "access_control", - [[_finding("access_control", 12)]], - [[_finding("injection", 12), _finding("access_control", 5)]], - ranges=RANGES, - taxonomy=taxonomy, - ) - assert quiet.outcome == "pair_correct" and quiet.other_findings_fixed == 2 - leaked = score_pair_family( - case, "access_control", [[_finding("access_control", 12)]], [[_finding("access_control", 13)]], ranges=RANGES, taxonomy=taxonomy - ) - assert leaked.outcome == "both_flagged" - reversed_pair = score_pair_family(case, "access_control", [[]], [[_finding("access_control", 12)]], ranges=RANGES, taxonomy=taxonomy) - assert reversed_pair.outcome == "reversed" - - -def test_the_outcome_of_several_runs_is_their_majority(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - hit = [_finding("access_control", 12)] - score = score_pair_family(case, "access_control", [hit, [], hit], [[], [], []], ranges=RANGES, taxonomy=taxonomy) - assert score.runs == ("pair_correct", "both_silent", "pair_correct") and score.outcome == "pair_correct" - flaky = score_pair_family(case, "access_control", [hit, [], []], [[], [], []], ranges=RANGES, taxonomy=taxonomy) - assert flaky.outcome == "both_silent" and flaky.flips == 1 # a run that disagrees with the majority is a flip - - -def test_an_unscorable_pair_is_never_run_and_keeps_its_reason(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path, unscorable="identical_twin") - score = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "unscorable" and score.unscorable_reason == "identical_twin" and score.runs == () - - -def test_a_pair_with_no_parsed_function_range_is_unscorable_not_a_miss(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy) - assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label" - - -def test_scoring_never_reads_the_text_of_a_finding(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - shouting = _finding("injection", 12) - loud = StaticFinding(**{**shouting.__dict__, "title": "CWE-639 broken object level authorization", "rationale": "IDOR in leaky"}) - score = score_pair_family(_case(tmp_path), "access_control", [[loud]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "both_silent" # the words say access control; the family tag says injection, and only that counts - - -def test_the_rung_is_the_best_verifier_outcome_among_the_detections(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - corroborated = _finding("access_control", 12) - corroborated.tags.append("verifier:static_corroboration") - score = score_pair_family(_case(tmp_path), "access_control", [[corroborated]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "pair_correct" and score.rung == "static_corroboration" - plain = score_pair_family(_case(tmp_path), "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert plain.rung == "suspicion" - - -def test_the_unscorable_reason_tells_a_tooling_gap_from_a_corpus_defect(tmp_path: Path) -> None: - """Req 4.1: an unparsable row is a tooling gap; a row whose labeled function no range names is a corpus defect.""" - from openultrasast.learning.scoring import unscorable_reason - - case = _case(tmp_path) - assert unscorable_reason(case, parse_ok=True, ranges=RANGES) is None - assert unscorable_reason(case, parse_ok=False, ranges={}) == "unsupported_language" - assert unscorable_reason(case, parse_ok=False, ranges=RANGES) == "unsupported_language" # the parse decides, not the ranges - assert unscorable_reason(case, parse_ok=True, ranges={}) == "unresolved_label" - assert unscorable_reason(_case(tmp_path, unscorable="identical_twin"), parse_ok=True, ranges=RANGES) == "identical_twin" - twin = _case(tmp_path, unscorable="known_limit:java hash") - assert unscorable_reason(twin, parse_ok=False, ranges={}) == "known_limit:java hash" # a declared limit wins - - -def test_a_row_whose_registration_context_is_missing_is_named_as_such(tmp_path: Path) -> None: - """Req 4.1: an access-control row whose handler arrived without its registration cannot be scored for that reason.""" - from openultrasast.learning.scoring import unscorable_reason - - case = _case(tmp_path, family="access_control") - assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=("leaky",)) is None - assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=()) == "missing_context" - # only families whose bug shape needs the registration are judged on it - assert unscorable_reason(_case(tmp_path, family="injection"), parse_ok=True, ranges=RANGES, entry_points=()) is None - - -def test_an_unparsable_side_is_scored_unscorable_rather_than_missed(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy, parse_ok=False) - assert score.outcome == "unscorable" and score.unscorable_reason == "unsupported_language" diff --git a/tests/test_learning_split.py b/tests/test_learning_split.py deleted file mode 100644 index c365340..0000000 --- a/tests/test_learning_split.py +++ /dev/null @@ -1,148 +0,0 @@ -"""learning-harness task 3.1: one teacher rule across every learning path (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored - -VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n" - - -def _teaching_case(tmp_path: Path, name: str, *, split: str = "train") -> PairCase: - """A pair that really carries a mechanism shape. - - The trivial `x = 1` fixture below teaches nothing whatever the split says, so a teacher-rule assertion made - against it passes with the rule deleted. This one is the difference between measuring the rule and measuring - an empty corpus.""" - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - ) - - -def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n") - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}.py", - fixed_file=tmp_path / f"{name}.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", function="f", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier=tier, - reviewer="m" if tier == "reviewed" else "", - unscorable=unscorable, - ) - - -def test_only_vendored_trusted_scorable_train_pairs_may_teach(tmp_path: Path) -> None: - from openultrasast.learning.split import teachers - - cases = [ - _case(tmp_path, "train-seeded"), - _case(tmp_path, "train-reviewed", tier="reviewed"), - _case(tmp_path, "holdout-seeded", split="holdout"), - _case(tmp_path, "train-untrusted", tier="title"), - _case(tmp_path, "train-twin", unscorable="identical_twin"), - ] - assert [case.name for case in teachers(cases)] == ["train-seeded", "train-reviewed"] - - -def test_a_candidate_taught_by_a_holdout_pair_is_refused_by_name(tmp_path: Path) -> None: - from openultrasast.learning.split import refuse_if_holdout - - cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")] - assert refuse_if_holdout(["train-one"], cases) is None - refusal = refuse_if_holdout(["train-one", "holdout-one"], cases) - assert refusal is not None and refusal.pairs == ("holdout-one",) - assert refusal.reason == "holdout_pair_refused" - degradation = refusal.degradation() - assert degradation["stage"] == "learning" and degradation["pairs"] == ["holdout-one"] - assert "holdout-one" in str(degradation["detail"]) - - -def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import MechanismStore, corpus_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - cases = select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py")) - holdout = {case.name for case in cases if case.split == "holdout"} - store = MechanismStore(tmp_path / "candidates.jsonl") - report = export_mechanisms(cases, store) - taught_by_holdout = [record.id for record in corpus_mechanisms(store.load()) if set(record.pairs) & holdout] - assert taught_by_holdout == [] - assert report.records > 0 and any(reason.startswith("split:") for _pair, reason in report.skipped) - - -def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None: - """Both arms, because `teaching_pairs <= 1` on a two-pair corpus is satisfied by a rule that does nothing. - - The same two excerpts teach twice when both are on the train split and once when one is held out, and the run - names the pair it refused (Req 5.1, 5.2).""" - from openultrasast.semantic.loo import evaluate_loo - - both_train = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "train-two")]) - assert both_train.teaching_pairs == 2 and both_train.degradations == () - result = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")]) - assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"} # every pair is still scored - assert result.teaching_pairs == 1 - refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"] - assert [item["pairs"] for item in refusals] == [["holdout-one"]] - - -def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None: - """Both arms. The shape is the one the corpus really teaches, taken from the pair itself. - - Asserting only the refusal would pass against a lever that admits nothing at all, and a hand-written shape that - matches no excerpt can never be admitted either, so neither would measure the split rule (Req 5.2).""" - from openultrasast.improve.evolve import propose_mechanism_edits - from openultrasast.semantic.loo import _facts - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.seed import pair_lessons - - cases = [_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")] - (lesson,) = pair_lessons(cases[0], _facts()) - - def store(name: str, pair: str) -> MechanismStore: - path = MechanismStore(tmp_path / name) - append_from_pair(path, lesson.shape, summary=lesson.summary, cwe=lesson.cwe, pair=pair, provenance="human", tier="seeded") - return path - - reasons: list[dict[str, object]] = [] - refused = propose_mechanism_edits(cases, store("refused.jsonl", "holdout-one"), tmp_path / "scan.jsonl", degradations=reasons) - assert [edit.mechanism_id for edit in refused if edit.action == "admit"] == [] - assert [item["pairs"] for item in reasons] == [["holdout-one"]] - admitted = propose_mechanism_edits(cases, store("allowed.jsonl", "train-one"), tmp_path / "scan2.jsonl") - assert [edit.mechanism_id for edit in admitted if edit.action == "admit"], "a train-taught shape is still admissible" - - -def test_the_corrected_lever_number_is_measured_and_committed() -> None: - report = Path("benchmarks/measurements/2026-09-06-mechanism-lever-split.json") - assert report.is_file() - payload = json.loads(report.read_text()) - assert payload["slice"] == "vibe-py" and payload["holdout_pairs"] > 0 - assert set(payload["before"]) >= {"detected", "silent", "youden", "shapes"} - assert set(payload["after"]) >= {"detected", "silent", "youden", "shapes"} - assert payload["after"]["shapes"] < payload["before"]["shapes"] # holdout pairs no longer teach - assert payload["after"]["youden"] <= payload["before"]["youden"] diff --git a/tests/test_learning_thinking.py b/tests/test_learning_thinking.py deleted file mode 100644 index cf2fcfa..0000000 --- a/tests/test_learning_thinking.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3). - -Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys -determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did -not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is -built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.config import load_config - - -def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None: - """The default preserves what round zero was measured under; changing it silently would make the two runs - incomparable while looking like one number.""" - assert load_config(None).learning.thinking is False - path = tmp_path / "openultrasast.toml" - path.write_text("[learning]\nthinking = true\n") - assert load_config(path).learning.thinking is True - - -def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint - - monkeypatch.setenv("DEEPSEEK_API_KEY", "k") - monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False) - path = tmp_path / "openultrasast.toml" - path.write_text("[learning]\nthinking = true\n") - client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] - assert endpoint.thinking is True - assert client._disable_thinking is False # type: ignore[attr-defined] - path.write_text("[learning]\nthinking = false\n") - client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] - assert endpoint.thinking is False and client._disable_thinking is True # type: ignore[attr-defined] - - -def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: - """The condition that produced a number is part of what identifies it.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_rounds import _always, _case, _steady # noqa: PLC0415 - - from openultrasast.learning.families import load_families - from openultrasast.learning.rounds import compare_baselines, run_baseline - - out = tmp_path / "out" - for thinking in (False, True): - run_baseline( - [_case(tmp_path, "a")], - taxonomy=load_families(), - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - thinking=thinking, - ) - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written - assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"} - report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text()) - assert report["thinking"] is True diff --git a/tests/test_learning_verifiers.py b/tests/test_learning_verifiers.py deleted file mode 100644 index ce198fc..0000000 --- a/tests/test_learning_verifiers.py +++ /dev/null @@ -1,145 +0,0 @@ -"""learning-harness task 2.6: the verifier registry and the oracle callback (offline).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result -from openultrasast.sandbox import SandboxResult - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n" - "@app.route('/health')\ndef health():\n return {'ok': True}\n" -) - - -def _claim(function: str, line: int, family: str) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=function, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}"], - ranking_priority=0.0, - ) - - -@pytest.fixture -def repo(tmp_path: Path) -> Path: - (tmp_path / "app.py").write_text(APP) - return tmp_path - - -def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None: - from openultrasast.learning.verifiers import token_oracle - - oracle = token_oracle("CANARY:9f2a") - fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False) - quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False) - assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE - assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE # a crash is not this family's oracle - assert verdict_from_result(quiet).verdict == TRIGGERABLE # without an oracle the exit code still decides - assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired" - - -def test_an_oracle_never_overrides_an_inconclusive_run() -> None: - from openultrasast.learning.verifiers import token_oracle - - oracle = token_oracle("CANARY:9f2a") - timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True) - assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive" - assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive" - assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable" - - -def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None: - from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for - - taxonomy = load_families() - assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier) - assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier) - assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier) - # a canary family with no canary implementation registered yet reports at suspicion rather than pretending - injection = verifier_for(taxonomy.by_id("injection")) - assert isinstance(injection, NoVerifier) and injection.kind == "canary" - - -def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None: - from openultrasast.learning.verifiers import verifier_for - - taxonomy = load_families() - verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python") - assert verification.rung == "suspicion" and verification.reason == "no_verifier" - - -def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None: - from openultrasast.learning.verifiers import verifier_for - - taxonomy = load_families() - verifier = verifier_for(taxonomy.by_id("access_control")) - agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python") - assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output - # the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate - healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python") - assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees" - elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python") - assert elsewhere.rung == "suspicion" - - -def test_a_verification_is_written_onto_the_claim_as_a_tag_and_nothing_else(repo: Path) -> None: - """The evidence ladder belongs to another boundary; a verifier says what it saw with a tag.""" - from openultrasast.learning.verifiers import apply_verification, verifier_for - - taxonomy = load_families() - claim = _claim("leaky", 12, "access_control") - verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python")) - assert "verifier:static_corroboration" in verified.tags - assert verified.evidence_level == "suspicion" and claim.evidence_level == "suspicion" # the ladder is untouched - plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python")) - assert "verifier:suspicion" in plain.tags and plain.tags.count("verifier:suspicion") == 1 - - -def test_a_rung_only_reaches_the_ladder_through_an_explicit_mapping() -> None: - """Every rung must land on a level the project's ladder actually defines, or the report raises later.""" - from openultrasast.learning.verifiers import RUNGS, evidence_level_for - from openultrasast.verification import EvidenceLevel - - assert {rung: evidence_level_for(rung) for rung in RUNGS} == { - "suspicion": EvidenceLevel.SUSPICION, - "static_corroboration": EvidenceLevel.STATIC_CORROBORATION, - "proven": EvidenceLevel.EXPLOIT_DEMONSTRATED, - } - assert all(EvidenceLevel(evidence_level_for(rung).value) for rung in RUNGS) # never a value outside the ladder - - -def test_a_verified_claim_still_passes_the_projects_own_verification_helpers(repo: Path) -> None: - """The bug this pins: stamping a learning rung onto evidence_level raised ValueError in three call sites.""" - from openultrasast.learning.verifiers import Verification, apply_verification, raise_to - from openultrasast.verification import VerificationStatus, is_report_verified, verify_finding - - claim = _claim("lookup", 3, "injection") - proven = apply_verification(claim, Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY x")) - assert "verifier:proven" in proven.tags - assert verify_finding(proven).evidence_level == "suspicion" # would raise ValueError if the rung were stamped on - assert is_report_verified(proven.evidence_level, VerificationStatus.ACCEPTED) is False - raised = raise_to(proven, "proven") - assert raised.evidence_level == "exploit_demonstrated" - assert verify_finding(raised).evidence_level == "exploit_demonstrated" - assert is_report_verified(raised.evidence_level, VerificationStatus.ACCEPTED) is True diff --git a/tests/test_loo.py b/tests/test_loo.py deleted file mode 100644 index 8975adf..0000000 --- a/tests/test_loo.py +++ /dev/null @@ -1,235 +0,0 @@ -"""corpus-seeded-mechanisms task 3.1: leave-one-out is the corpus's own detection rate (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase - -EXEC_A = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute(\"SELECT * FROM t WHERE u = '%s'\" % user)\n" -) -EXEC_A_FIX = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n c.execute('SELECT * FROM t WHERE u = ?', (user,))\n" -) -EXEC_B = ( - "import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n cur.execute('DELETE FROM t WHERE n = ' + name)\n" -) -EXEC_B_FIX = ( - "import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n" - + " cur.execute('DELETE FROM t WHERE n = ?', (name,))\n" -) -SYS_C = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n" -SYS_C_FIX = "import os\nimport subprocess\n\n\ndef ping(host):\n subprocess.run(['ping', host])\n" - - -def _case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="toy", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink" - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance=provenance, - review_tier="seeded", - ) - - -def _toy(tmp_path: Path) -> list[PairCase]: - return [ - _case(tmp_path, "a", EXEC_A, EXEC_A_FIX, function="lookup", cwe="CWE-89"), - _case(tmp_path, "b", EXEC_B, EXEC_B_FIX, function="delete", cwe="CWE-89", provenance="agent"), - _case(tmp_path, "c", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78"), - ] - - -def test_leave_one_out_reports_which_pair_taught_which(tmp_path: Path) -> None: - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo(_toy(tmp_path)) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert set(by_name) == {"a", "b", "c"} - assert by_name["a"].detected and by_name["a"].silent and by_name["a"].found_by # taught by b's execute shape - assert by_name["b"].detected and by_name["b"].silent and any("a" in pairs for _, pairs in by_name["b"].found_by) - assert not by_name["c"].detected and by_name["c"].found_by == () # nobody else teaches os.system: c only tests - slice_metrics = result.per_slice["toy"] - assert slice_metrics["pairs"] == 3 and slice_metrics["detected"] == 2 and slice_metrics["silent"] == 3 - assert abs(slice_metrics["recall"] - 2 / 3) < 1e-9 and slice_metrics["silence"] == 1.0 and abs(slice_metrics["youden"] - 2 / 3) < 1e-9 - assert result.per_profile["agent"]["pairs"] == 1 and result.per_profile["human"]["pairs"] == 2 - assert "source_reaches_sink" in result.per_mechanism - payload = result.to_dict() - assert set(payload) >= {"per_slice", "per_profile", "per_mechanism", "outcomes"} - json.dumps(payload) # artifact-ready - - -def test_leave_one_out_seeds_only_trusted_pairs_and_skips_known_limit(tmp_path: Path) -> None: - from openultrasast.semantic.loo import evaluate_loo - - cases = _toy(tmp_path) - cases[1] = PairCase(**{**cases[1].__dict__, "review_tier": "advisory"}) # b may be tested, never teaches - cases[2] = PairCase(**{**cases[2].__dict__, "known_limit": "engine"}) - result = evaluate_loo(cases) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert set(by_name) == {"a", "b"} # known_limit excluded entirely - assert not by_name["a"].detected # b is advisory: it cannot teach a - assert by_name["b"].detected # a (seeded) teaches b, which is scored as a held-out target - # `b` is excluded from teaching for its tier and says so; without that line a pair that silently stopped - # teaching would look identical to one that never could (Req 5.2). - assert result.skipped == (("c", "known_limit:engine"), ("b", "not_a_teacher:tier")) - - -def test_cli_pairs_loo_writes_the_artifact_offline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - monkeypatch.chdir(Path.cwd()) - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["pairs", "--slice", "vibe-py", "--loo", "--json", "--loo-out", str(tmp_path / "loo.json")]) == 0 - payload = json.loads(buf.getvalue()) - # 35 vendored rows less the five byte-identical twins, which task 4.2 declared unscorable in the catalog: a - # pair whose two sides are the same code cannot be held out against itself. - assert payload["per_slice"]["vibe-py"]["pairs"] == 30 and "found_by" in payload["outcomes"][0] - assert json.loads((tmp_path / "loo.json").read_text()) == payload - - -HELPER_SINK = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n\n\ndef ping(host):\n return run('ping ' + host)\n" -HELPER_SINK_FIX = ( - "import subprocess\n\n\ndef run(cmd):\n subprocess.run(cmd.split())\n\n\ndef ping(host):\n return run('ping ' + host)\n" -) - - -def test_hits_outside_the_labeled_function_or_off_label_never_count(tmp_path: Path) -> None: - """Round-3 mutants: a hit must sit inside the labeled function and name the labeled mechanism or sink.""" - from openultrasast.semantic.loo import evaluate_loo - - teacher = _case(tmp_path, "t", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78") - outside = _case( - tmp_path, "outside", HELPER_SINK, HELPER_SINK_FIX, function="ping", cwe="CWE-78" - ) # sink call lives in `run`, not `ping` - off_label = PairCase( - **{ - **_case(tmp_path, "offlabel", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78").__dict__, - "expected": ( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="ping", mechanism="missing_auth_guard" - ), - ), - } - ) - result = evaluate_loo([teacher, outside, off_label]) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert by_name["outside"].hits_vuln >= 1 and not by_name["outside"].detected - assert by_name["offlabel"].hits_vuln >= 1 and not by_name["offlabel"].detected - - -def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.loo import evaluate_loo - - monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0") - js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n" - (tmp_path / "j-v.js").write_text(js) - (tmp_path / "j-f.js").write_text(js) - case = PairCase( - **{ - **_toy(tmp_path)[0].__dict__, - "name": "j", - "language": "javascript", - "vuln_file": tmp_path / "j-v.js", - "fixed_file": tmp_path / "j-f.js", - "relpath": "app.js", - } - ) - result = evaluate_loo([*_toy(tmp_path), case]) - assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations) - - -# --- authorization-obligations 4.2: leave-one-out per obligation kind ------------------------------------------------- - -_HANDLER = "from flask import request\n\n\n@app.route('/{route}/<key>')\ndef {fn}(key):\n{body}" -OBL_A = _HANDLER.format(route="books", fn="get_book", body=" return Book.query.filter_by(book_title=key).first()\n") -OBL_A_FIX = _HANDLER.format( - route="books", - fn="get_book", - body=" user_id = request.user.id\n return Book.query.filter_by(user_id=user_id, book_title=key).first()\n", -) -OBL_B = _HANDLER.format(route="orders", fn="get_order", body=" return Order.query.filter_by(ref=key).first()\n") -OBL_B_FIX = _HANDLER.format( - route="orders", fn="get_order", body=" owner = current_user.id\n return Order.query.filter_by(owner=owner, ref=key).first()\n" -) -OBL_C = _HANDLER.format( - route="login", fn="login", body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='None')\n return resp\n" -) -OBL_C_FIX = _HANDLER.format( - route="login", - fn="login", - body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='Strict', httponly=1)\n return resp\n", -) - - -def _obligation_case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, obligation: str, mechanism: str) -> PairCase: - case = _case(tmp_path, name, vuln, fixed, function=function, cwe="CWE-639") - row = case.expected[0] - return PairCase( - **{ - **case.__dict__, - "slice": "vibe-py", - "expected": (ExpectedFinding(**{**row.__dict__, "mechanism": mechanism, "obligation": obligation}),), - } - ) - - -def _obligation_toy(tmp_path: Path) -> list[PairCase]: - read = {"obligation": "protected_read", "mechanism": "unconstrained_protected_read"} - return [ - _obligation_case(tmp_path, "a", OBL_A, OBL_A_FIX, function="get_book", **read), - _obligation_case(tmp_path, "b", OBL_B, OBL_B_FIX, function="get_order", **read), - _obligation_case(tmp_path, "c", OBL_C, OBL_C_FIX, function="login", obligation="security_setting", mechanism="permissive_default"), - ] - - -def test_leave_one_out_scores_obligation_rows_per_kind_and_names_the_teaching_pair(tmp_path: Path) -> None: - """Req 7.4: recall, silence and Youden per obligation kind; found_by names the shape record that supplied known_fix.""" - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo(_obligation_toy(tmp_path)) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - a, b, c = by_name["a"], by_name["b"], by_name["c"] - assert a.obligations == ("protected_read",) and c.obligations == ("security_setting",) - assert a.detected and a.silent and [pairs for _, pairs in a.found_by] == [("b",)] # b taught the identity discharger - assert b.detected and b.silent and [pairs for _, pairs in b.found_by] == [("a",)] - assert all(record_id.startswith("corpus:") for record_id, _ in a.found_by) - assert c.teaches and not c.detected and c.found_by == () and c.silent # reached and undischarged, but nobody else teaches the kind - kinds = result.per_obligation_kind - assert kinds["protected_read"]["pairs"] == 2 and kinds["protected_read"]["detected"] == 2 and kinds["protected_read"]["youden"] == 1.0 - assert kinds["security_setting"]["pairs"] == 1 and kinds["security_setting"]["detected"] == 0 - assert "unconstrained_protected_read" in result.per_mechanism # flow mechanisms and obligation kinds sit side by side - payload = result.to_dict() - assert "per_obligation_kind" in payload and payload["outcomes"][0]["obligations"] == ["protected_read"] - json.dumps(payload) - - -def test_obligation_rows_are_not_found_by_the_facts_alone(tmp_path: Path) -> None: - """A finding without a known fix from the held-out store is the checker's own work, not the corpus teaching.""" - from openultrasast.semantic.loo import evaluate_loo - - (only,) = [outcome for outcome in evaluate_loo(_obligation_toy(tmp_path)[:1]).outcomes] - assert only.hits_vuln >= 1 and not only.detected and only.found_by == () # the leaky read is flagged, but no pair taught it diff --git a/tests/test_mechanism_export.py b/tests/test_mechanism_export.py deleted file mode 100644 index 227247b..0000000 --- a/tests/test_mechanism_export.py +++ /dev/null @@ -1,74 +0,0 @@ -"""corpus-seeded-mechanisms: additive store fields and the corpus writer (task 1.2, Req 1.3/1.4).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.semantic.mechanisms import MechanismStore, append_mechanism - - -def _shape(**overrides: object): - from openultrasast.semantic.variants import Shape - - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("fact_source",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return Shape(**base) # type: ignore[arg-type] - - -def test_old_sandbox_rows_load_with_corpus_defaults(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism(store, summary="s", cwe="CWE-78", language="python", tags=("t",), what_made_it_exploitable="x") - # a row written before this spec has no origin/tier/pairs/shape/guard fields at all - raw = json.loads((tmp_path / "mechanisms.jsonl").read_text().splitlines()[0]) - for key in ("origin", "review_tier", "pairs", "shape", "guard"): - raw.pop(key, None) - (tmp_path / "mechanisms.jsonl").write_text(json.dumps(raw) + "\n") - (record,) = store.load() - assert ( - record.origin == "sandbox" and record.review_tier == "" and record.pairs == () and record.shape is None and record.guard == "none" - ) - - -def test_append_from_pair_dedupes_by_shape_and_extends_provenance(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import append_from_pair, corpus_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - first = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") - second = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p2", provenance="agent", tier="seeded") - assert first.id == second.id - assert ( - second.pairs == ("p1", "p2") and second.origin == "corpus" and second.review_tier == "seeded" and second.guard == "allowlist_test" - ) - third = append_from_pair( - store, _shape(guard="none"), summary="request into os.system", cwe="CWE-78", pair="p3", provenance="human", tier="reviewed" - ) - assert third.id != first.id - loaded = store.load() - assert len(loaded) == 2 and {r.id for r in loaded} == {first.id, third.id} # the store is append-only, load folds by id - corpus = corpus_mechanisms(loaded) - assert {r.id for r in corpus} == {first.id, third.id} - assert all(r.shape is not None and r.shape["sink_name"] == "system" for r in corpus) - assert "source_reaches_sink" in first.tags and "mechanism:source_reaches_sink" in first.tags - - -def test_corpus_rows_never_carry_source_text_and_ids_are_deterministic(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import append_from_pair - - a = append_from_pair( - MechanismStore(tmp_path / "a.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="p", provenance="human", tier="seeded" - ) - b = append_from_pair( - MechanismStore(tmp_path / "b.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="q", provenance="human", tier="seeded" - ) - assert a.id == b.id # identity is the shape key, not a random uuid: the lever can name records across machines - text = (tmp_path / "a.jsonl").read_text() - assert "app.py" not in text and "cmd" not in text diff --git a/tests/test_mechanism_lever.py b/tests/test_mechanism_lever.py deleted file mode 100644 index 103ed7b..0000000 --- a/tests/test_mechanism_lever.py +++ /dev/null @@ -1,112 +0,0 @@ -"""corpus-seeded-mechanisms task 4.1: the `mechanisms` lever admits or retracts exporter records, nothing else.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair -from openultrasast.semantic.variants import Shape - - -def _record(store: MechanismStore, **overrides: object): - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("parameter",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return append_from_pair(store, Shape(**base), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") # type: ignore[arg-type] - - -def test_mechanism_edit_is_a_bounded_lever_with_a_stable_key(tmp_path: Path) -> None: - from openultrasast.improve.validator import VALID_LEVERS, MechanismEdit - - assert "mechanisms" in VALID_LEVERS - edit = MechanismEdit(action="admit", mechanism_id="corpus:abc", source="loo", rationale="recovers holdout pair x") - assert edit.lever == "mechanisms" and edit.key() == "mechanisms:admit:corpus:abc" - - -def test_validator_accepts_only_exporter_records_with_closed_guards(tmp_path: Path) -> None: - from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - good = _record(candidates) - validator = EvolveValidator() - by_id = {record.id: record for record in candidates.load()} - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=good.id, source="loo"), by_id) - validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=good.id, source="loo"), by_id) - with pytest.raises(StrictValidationError, match="unknown mechanism"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id="corpus:not-exported", source="loo"), by_id) - with pytest.raises(StrictValidationError, match="action"): - validator.validate_mechanism(MechanismEdit(action="rewrite", mechanism_id=good.id, source="loo"), by_id) - with pytest.raises(StrictValidationError, match="origin"): - from openultrasast.semantic.mechanisms import append_mechanism - - sandbox = append_mechanism(candidates, summary="s", cwe="CWE-78", language="python", tags=(), what_made_it_exploitable="x") - validator.validate_mechanism( - MechanismEdit(action="admit", mechanism_id=sandbox.id, source="loo"), {r.id: r for r in candidates.load()} - ) - # free-form shapes never pass: a record whose shape text is not a closed structure is rejected before it can be admitted - from dataclasses import replace - - forged = replace(good, shape={**(good.shape or {}), "guard": "if user not in ALLOWED"}) - with pytest.raises(StrictValidationError, match="guard"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=forged.id, source="export"), {forged.id: forged}) - forged_text = replace(good, shape={**(good.shape or {}), "sink_name": "os.system(cmd + ' ')"}) - with pytest.raises(StrictValidationError, match="identifier"): - validator.validate_mechanism( - MechanismEdit(action="admit", mechanism_id=forged_text.id, source="export"), {forged_text.id: forged_text} - ) - - -def test_apply_mechanism_edits_admits_into_the_scan_store_and_reverts_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - a = _record(candidates) - b = _record(candidates, sink_name="popen") - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - scan_store.parent.mkdir() - scan_store.write_text("") - before = scan_store.read_bytes() - snapshot = apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=a.id, source="loo")], candidates, scan_store) - admitted = MechanismStore(scan_store).load() - assert [r.id for r in admitted] == [a.id] and admitted[0].origin == "corpus" - apply_mechanism_edits([MechanismEdit(action="retract", mechanism_id=a.id, source="loo")], candidates, scan_store) - assert MechanismStore(scan_store).load() == () # retraction is a tombstone row; load() folds it away - snapshot.restore() - assert scan_store.read_bytes() == before # byte-for-byte revert of the whole round - assert b.id != a.id - - -def test_retract_is_allowed_for_an_admitted_record_that_left_the_candidate_set(tmp_path: Path) -> None: - from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError - - scan = MechanismStore(tmp_path / "scan.jsonl") - stale = _record(scan, sink_name="popen") - validator = EvolveValidator() - with pytest.raises(StrictValidationError, match="unknown mechanism"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=stale.id), {}, admitted={stale.id: stale}) - validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=stale.id), {}, admitted={stale.id: stale}) - - -def test_export_defaults_to_a_candidates_file_and_never_writes_the_scan_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - monkeypatch.chdir(tmp_path) - (tmp_path / "benchmarks").symlink_to(Path(__file__).resolve().parents[1] / "benchmarks") - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["mechanisms", "export", "--slice", "vibe-py", "--json"]) == 0 - payload = __import__("json").loads(buf.getvalue()) - assert payload["store"].endswith("mechanism-candidates.jsonl") - assert not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() diff --git a/tests/test_mechanism_loop.py b/tests/test_mechanism_loop.py deleted file mode 100644 index 558b735..0000000 --- a/tests/test_mechanism_loop.py +++ /dev/null @@ -1,330 +0,0 @@ -"""corpus-seeded-mechanisms task 4.2: the improve loop admits mechanisms that recover holdout pairs and retracts leakers.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding, load_benchmark_manifest -from openultrasast.pairs import PairCase -from openultrasast.ruleset import PatternRule, write_ruleset -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair -from openultrasast.semantic.variants import Shape - -EXEC_VULN = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute('SELECT * FROM t WHERE u = ' + user)\n" -) -EXEC_FIX = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute('SELECT * FROM t WHERE u = ?', (user,))\n" -) -SYS_VULN = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n" -SYS_LEAKY_FIX = ( - "import os\n\n\ndef ping(host):\n log(host)\n os.system('ping ' + host)\n" # the 'fix' still calls the sink: a leaking shape -) - - -def _shape(sink: str, guard: str = "none") -> Shape: - return Shape( - language="python", - sink_name=sink, - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard=guard, - mechanism="source_reaches_sink", - ) - - -def _pair(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="github", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink" - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance=provenance, - split="holdout", - review_tier="reviewed", - reviewer="t", - ) - - -def _round_env(tmp_path: Path) -> dict[str, object]: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def f(x):\n return eval(x)\n") - rd = tmp_path / "ruleset" / "python" - write_ruleset( - rd / "rules.toml", - [PatternRule(rule_id="good-eval", title="e", languages=("python",), cwe="CWE-95", tags=("injection",), pattern=r"\beval\s*\(")], - ) - manifest_path = tmp_path / "bench.toml" - manifest_path.write_text( - 'name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' - '[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n' - 'line = 2\nrule_id = "good-eval"\nsink = "eval"\nevidence = "eval executes input"\n' - ) - return {"target": repo, "manifest": load_benchmark_manifest(manifest_path), "ruleset_dir": tmp_path / "ruleset"} - - -def test_proposer_admits_recovering_records_and_retracts_leakers(tmp_path: Path) -> None: - from openultrasast.improve.evolve import evaluate_mechanism_profiles, propose_mechanism_edits - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - sys_record = append_from_pair( - candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" - ) - holdout = [ - _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), - _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78", provenance="agent"), - ] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - edits = propose_mechanism_edits(holdout, candidates, scan_store) - assert [(e.action, e.mechanism_id) for e in edits] == [("admit", exec_record.id)] # sys leaks on y's fixed side: never proposed - assert "x" in edits[0].rationale - # once a leaking record is admitted (by a human or an earlier round), the proposer asks to retract it - from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits - - apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=sys_record.id)], candidates, scan_store) - edits = propose_mechanism_edits(holdout, candidates, scan_store) - assert ("retract", sys_record.id) in [(e.action, e.mechanism_id) for e in edits] - profiles = evaluate_mechanism_profiles(holdout, scan_store) - assert profiles["agent"]["pairs"] == 1.0 and profiles["agent"]["pair_correct"] == 0.0 # y: detected but leaked - - -def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_round - from openultrasast.improve.validator import MechanismEdit - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - sys_record = append_from_pair( - candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" - ) - holdout = [ - _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), - _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78"), - ] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - ledger = tmp_path / "rule_policy.json" - journal = tmp_path / "journal.json" - outcome = run_round( - env["target"], - env["manifest"], - ledger_path=ledger, - journal_path=journal, - ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] - pair_cases=holdout, - min_holdout_pairs=1, - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - ) - assert outcome.accepted and outcome.reason == "accepted" - assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] - entry = json.loads(journal.read_text())[0] - mech_edits = [e for e in entry["edits"] if e["lever"] == "mechanisms"] - assert mech_edits and mech_edits[0]["mechanism_id"] == exec_record.id and mech_edits[0]["pairs"] == ["teacher-a"] - assert entry["mechanisms_before"]["human"]["pair_correct"] == 0.0 and entry["mechanisms_after"]["human"]["pair_correct"] == 1.0 - before = scan_store.read_bytes() - forced = run_round( - env["target"], - env["manifest"], - ledger_path=ledger, - journal_path=journal, - ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] - pair_cases=holdout, - min_holdout_pairs=1, - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")], - ) - assert not forced.accepted and (forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression")) - assert scan_store.read_bytes() == before # byte-for-byte revert - - -def test_no_gain_admission_is_rejected_and_reverted_mechanism_edits_are_not_reproposed(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_round - from openultrasast.improve.journal import reverted_edit_keys - from openultrasast.improve.validator import MechanismEdit - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - unrelated = append_from_pair( - candidates, _shape("popen"), summary="s", cwe="CWE-78", pair="teacher-c", provenance="human", tier="seeded" - ) - holdout = [_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89")] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - ledger = tmp_path / "rule_policy.json" - journal = tmp_path / "journal.json" - kwargs: dict[str, object] = { - "ledger_path": ledger, - "journal_path": journal, - "ruleset_dir": env["ruleset_dir"], - "pair_cases": holdout, - "min_holdout_pairs": 1, - "mechanism_candidates": candidates.log_path, - "mechanism_store": scan_store, - } - edit = MechanismEdit(action="admit", mechanism_id=unrelated.id, source="export", rationale="scripted: recovers nothing") - outcome = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] - assert not outcome.accepted and outcome.reason == "mechanism_no_gain" - assert not scan_store.exists() or scan_store.read_bytes() == b"" - assert edit.key() in reverted_edit_keys(__import__("json").loads(journal.read_text())) - # the same reverted edit, offered again with a rationale, is filtered before validation (novelty gate) - again = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] - assert again.reason == "no_proposals" and again.mechanism_edits == [] - - -def _holdout_catalog(tmp_path: Path) -> Path: - """A train pair to teach and a holdout pair to recover. - - The catalog used to hold the holdout pair alone, so the only shape available was the one that pair - taught about itself; learning-harness Req 5.1 refuses that, and rightly. - """ - (tmp_path / "x-v.py").write_text(EXEC_VULN) - (tmp_path / "x-f.py").write_text(EXEC_FIX) - (tmp_path / "t-v.py").write_text(EXEC_VULN.replace("lookup", "search").replace("u = ", "name = ")) - (tmp_path / "t-f.py").write_text(EXEC_FIX.replace("lookup", "search").replace("u = ", "name = ")) - row = ( - '[[pair]]\nname = "{name}"\nslice = "github"\nlanguage = "python"\nvuln = "{name[0]}-v.py"\nfixed = "{name[0]}-f.py"\n' - 'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "reviewed"\nreviewer = "t"\n\n' - '[[pair.expected]]\ncwe = "CWE-89"\nclass = "sql injection"\npath = "app.py"\nfunction = "{function}"\n' - 'sink = "execute"\nmechanism = "source_reaches_sink"\n' - ) - catalog = tmp_path / "pairs.toml" - catalog.write_text( - row.replace("{name}", "x").replace("{name[0]}", "x").replace("{split}", "holdout").replace("{function}", "lookup") - + "\n" - + row.replace("{name}", "t").replace("{name[0]}", "t").replace("{split}", "train").replace("{function}", "search") - ) - return catalog - - -def test_ousast_improve_admits_candidates_into_the_scan_store_by_default(tmp_path: Path) -> None: - """Req 5: the lever is reachable from the operator command; export -> candidates -> improve -> mechanisms.jsonl the scan reads.""" - from openultrasast.cli import main - - env = _round_env(tmp_path) - repo = Path(str(env["target"])) - calibration = repo / ".openultrasast" / "calibration" - candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - catalog = _holdout_catalog(tmp_path) - manifest = tmp_path / "bench.toml" - assert ( - main( - ["improve", str(manifest), "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] - ) - == 0 - ) - admitted = MechanismStore(calibration / "mechanisms.jsonl").load() - assert [r.id for r in admitted] == [exec_record.id] - journal = json.loads((calibration / "improve_journal.json").read_text()) - assert any(e["lever"] == "mechanisms" and e["mechanism_id"] == exec_record.id for entry in journal for e in entry["edits"]) - - -def test_run_improvement_threads_mechanism_paths(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_improvement - from openultrasast.pairs import load_pair_catalog - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - cases = load_pair_catalog(_holdout_catalog(tmp_path)) - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - outcomes = run_improvement( - env["target"], - env["manifest"], - ledger_path=tmp_path / "ledger.json", - journal_path=tmp_path / "journal.json", # type: ignore[arg-type] - ruleset_dir=env["ruleset_dir"], - pair_cases=cases, - min_holdout_pairs=1, # type: ignore[arg-type] - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - max_rounds=3, - ) - assert any(o.accepted and o.mechanism_edits for o in outcomes) - assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] - assert outcomes[-1].reason == "no_proposals" # converges once the candidate is admitted - - -def _cli_env(tmp_path: Path) -> tuple[dict[str, object], Path, Path]: - env = _round_env(tmp_path) - calibration = Path(str(env["target"])) / ".openultrasast" / "calibration" - candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") - append_from_pair(candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded") - return env, calibration, _holdout_catalog(tmp_path) - - -def test_dry_run_and_no_mechanisms_never_write_the_scan_store(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] - from openultrasast.cli import main - - env, calibration, catalog = _cli_env(tmp_path) - base = [ - "improve", - str(tmp_path / "bench.toml"), - "--ruleset-dir", - str(env["ruleset_dir"]), - "--pair-catalog", - str(catalog), - "--min-holdout-pairs", - "1", - ] - assert main([*base, "--dry-run"]) == 0 - assert not (calibration / "mechanisms.jsonl").exists() # a dry run mutates nothing under the target - assert main([*base, "--no-mechanisms"]) == 0 - assert not (calibration / "mechanisms.jsonl").exists() - (calibration / "mechanism-candidates.jsonl").unlink() - capsys.readouterr() # the dry run legitimately reports its scratch admission; judge only the candidate-less round below - assert main(base) == 0 - assert not (calibration / "mechanisms.jsonl").exists() # no candidates file: the lever is inert, the round is a plain rule round - out = capsys.readouterr().out - assert "admit " not in out and "retract " not in out - - -def test_documented_default_flow_connects_export_to_improve(tmp_path: Path, monkeypatch, capsys) -> None: # type: ignore[no-untyped-def] - """README flow: `ousast mechanisms export` (cwd default) then `ousast improve --pair-catalog` admits into the target's store.""" - from openultrasast.cli import main - - env = _round_env(tmp_path) - catalog = _holdout_catalog(tmp_path) - monkeypatch.chdir(tmp_path) - # export a candidates file exactly as documented (cwd default), seeded from a tiny trusted catalog of our own - assert main(["mechanisms", "export", "--catalog", str(catalog), "--slice", "all"]) == 0 - exported = tmp_path / ".openultrasast" / "calibration" / "mechanism-candidates.jsonl" - assert exported.is_file() and not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() - # the target is a different directory (the benchmark fixture); improve must still find the cwd candidates by default - assert ( - main( - ["improve", "bench.toml", "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] - ) - == 0 - ) - target_store = Path(str(env["target"])) / ".openultrasast" / "calibration" / "mechanisms.jsonl" - assert MechanismStore(target_store).load() # admitted where the scan of that target reads - out = capsys.readouterr().out - assert "admit corpus:" in out # the accepted round names its mechanism edits diff --git a/tests/test_mechanism_reports.py b/tests/test_mechanism_reports.py deleted file mode 100644 index c19b702..0000000 --- a/tests/test_mechanism_reports.py +++ /dev/null @@ -1,141 +0,0 @@ -"""corpus-seeded-mechanisms task 5.1: reports name the mechanism, the known fix guard, and the pair provenance.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.findings import StaticFinding -from openultrasast.reports import write_markdown_report, write_sarif_report -from openultrasast.semantic.overlay import OverlayRecord - - -def _variant() -> StaticFinding: - return StaticFinding( - finding_id="variant:corpus:abc:app.py:5", - path="app.py", - title="Variant of known mechanism: run_shell", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="parameter into run_shell; fix adds allowlist_test. Structural variant of a mechanism learned from p1, p2.", - line=5, - function_name="handler", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["variant", "mechanism:corpus:abc"], - ranking_priority=0.0, - ) - - -MECHANISMS = { - "corpus:abc": { - "summary": "source_reaches_sink: parameter into run_shell (CWE-78); fix adds allowlist_test", - "guard": "allowlist_test", - "pairs": ["p1", "p2"], - "cwe": "CWE-78", - } -} - - -def test_markdown_names_mechanism_guard_and_provenance_and_keeps_the_suspicion_label(tmp_path: Path) -> None: - path = tmp_path / "report.md" - write_markdown_report([_variant()], path, mechanisms=MECHANISMS) - text = path.read_text() - assert "- Mechanism: `corpus:abc`" in text - assert "source_reaches_sink: parameter into run_shell" in text - assert "- Known fix guard: `allowlist_test`" in text - assert "- Learned from pairs: `p1, p2`" in text - assert "- Evidence level: `suspicion`" in text - assert "## Mechanisms" in text # a section summarising every mechanism the report cites - - -def test_sarif_carries_mechanism_properties_for_variant_and_merged_overlay_findings(tmp_path: Path) -> None: - merged = OverlayRecord( - proposal_id="python-os-command:app.py:9", - path="app.py", - line=9, - disposition="promote", - reason="request reaches os.system", - cwe="CWE-78", - sources=("request",), - sinks=("os.system",), - sanitizers=(), - evidence_level="static_corroboration", - origin="inventory", - engine="python-ast", - language="python", - mechanism_id="corpus:abc", - ) - inventory = StaticFinding( - **{ - **_variant().__dict__, - "finding_id": "python-os-command:app.py:9", - "line": 9, - "tags": ["injection"], - "evidence_level": "static_corroboration", - } - ) - path = tmp_path / "report.sarif" - write_sarif_report([_variant(), inventory], [], path, overlay=[merged], mechanisms=MECHANISMS) - results = json.loads(path.read_text())["runs"][0]["results"] - by_id = {item["properties"]["finding_id"]: item["properties"] for item in results} - variant = by_id["variant:corpus:abc:app.py:5"] - assert ( - variant["mechanism_id"] == "corpus:abc" - and variant["mechanism_guard"] == "allowlist_test" - and variant["mechanism_pairs"] == ["p1", "p2"] - ) - assert variant["evidence_level"] == "suspicion" - merged_props = by_id["python-os-command:app.py:9"] - assert merged_props["mechanism_id"] == "corpus:abc" and merged_props["mechanism_guard"] == "allowlist_test" # from the overlay record - - -def test_reports_without_mechanisms_are_unchanged(tmp_path: Path) -> None: - plain = StaticFinding(**{**_variant().__dict__, "finding_id": "python-os-command:app.py:9", "tags": ["injection"]}) - path = tmp_path / "report.md" - write_markdown_report([plain], path) - text = path.read_text() - assert "Mechanism:" not in text and "## Mechanisms" not in text - - -def test_standard_scan_report_names_the_mechanism_behind_a_variant_finding(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] - from openultrasast.cli import main - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variants import Shape - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n") - shape = Shape( - language="python", - sink_name="run_shell", - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard="allowlist_test", - mechanism="source_reaches_sink", - ) - record = append_from_pair( - MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"), - shape, - summary="parameter into run_shell", - cwe="CWE-78", - pair="p1", - provenance="human", - tier="seeded", - ) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = sorted((repo / ".runs").iterdir())[-1] - report = (run_dir / "report.md").read_text() - assert ( - f"- Mechanism: `{record.id}`" in report - and "- Known fix guard: `allowlist_test`" in report - and "- Learned from pairs: `p1`" in report - ) - sarif = json.loads((run_dir / "report.sarif").read_text()) - props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"]["finding_id"].startswith("variant:")] - assert props and props[0]["mechanism_guard"] == "allowlist_test" and props[0]["evidence_level"] == "suspicion" diff --git a/tests/test_mechanism_seed.py b/tests/test_mechanism_seed.py deleted file mode 100644 index 9bedb2d..0000000 --- a/tests/test_mechanism_seed.py +++ /dev/null @@ -1,112 +0,0 @@ -"""corpus-seeded-mechanisms task 2.1: export trusted pairs as mechanism records (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase, load_pair_catalog, select_slice -from openultrasast.semantic.mechanisms import MechanismStore - -VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FIXED = ( - "from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _case(tmp_path: Path, name: str, *, tier: str, known_limit: str | None = None, vuln: str = VULN, language: str = "python") -> PairCase: - ext = {"python": "py", "javascript": "js"}[language] - (tmp_path / f"{name}-v.{ext}").write_text(vuln) - (tmp_path / f"{name}-f.{ext}").write_text(FIXED if language == "python" else vuln) - return PairCase( - name=name, - slice="vibe-py", - language=language, - origin="test", - vuln_file=tmp_path / f"{name}-v.{ext}", - fixed_file=tmp_path / f"{name}-f.{ext}", - relpath=f"app.{ext}", - expected=( - ExpectedFinding( - cwe="CWE-78", - vulnerability_class="command injection", - path=f"app.{ext}", - evidence="", - sink="os.system", - function="run", - mechanism="source_reaches_sink", - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance="human", - review_tier=tier, - reviewer="t" if tier == "reviewed" else "", - known_limit=known_limit, - ) - - -def test_export_seeds_only_trusted_parsing_pairs_and_reports_every_skip(tmp_path: Path) -> None: - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - cases = [ - _case(tmp_path, "seeded", tier="seeded"), - _case(tmp_path, "reviewed-twin", tier="reviewed"), - _case(tmp_path, "advisory", tier="advisory"), - _case(tmp_path, "title", tier="title"), - _case(tmp_path, "limit", tier="seeded", known_limit="engine cannot see it"), - _case(tmp_path, "broken", tier="seeded", vuln="def run(:\n"), - ] - report = export_mechanisms(cases, store) - records = store.load() - assert len(records) == 1 # seeded and reviewed-twin share one shape: one record, two pairs - (record,) = records - assert record.origin == "corpus" and record.pairs == ("seeded", "reviewed-twin") and record.shape is not None - assert report.seeded == 2 and report.records == 1 - reasons = dict(report.skipped) - assert reasons["advisory"] == "tier:advisory" and reasons["title"] == "tier:title" - assert reasons["limit"].startswith("known_limit") and reasons["broken"] == "parse_failed" - text = (tmp_path / "mechanisms.jsonl").read_text() - assert "request.args" not in text and "app.py" not in text # Req 2.4: no code text in the store - - -def test_export_skips_languages_without_a_grammar_with_a_degradation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.seed import export_mechanisms - - monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0") - store = MechanismStore(tmp_path / "mechanisms.jsonl") - js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n" - report = export_mechanisms([_case(tmp_path, "js", tier="seeded", vuln=js, language="javascript")], store) - assert store.load() == () and dict(report.skipped)["js"] == "variants_language_unsupported" - assert {"stage": "mechanisms", "reason": "variants_language_unsupported", "language": "javascript"} in report.degradations - - -def test_export_over_the_vendored_vibe_py_slice_is_offline_and_deterministic(tmp_path: Path) -> None: - from openultrasast.pairs import select_vendored - from openultrasast.semantic.seed import export_mechanisms - - cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py")) - first = export_mechanisms(cases, MechanismStore(tmp_path / "a.jsonl")) - second = export_mechanisms(cases, MechanismStore(tmp_path / "b.jsonl")) - assert first.records >= 1 and first.records == second.records - assert (tmp_path / "a.jsonl").read_text() == (tmp_path / "b.jsonl").read_text() - assert first.seeded + len(first.skipped) == len(cases) - - -def test_cli_mechanisms_export_writes_the_store_and_prints_the_report(tmp_path: Path) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["mechanisms", "export", "--slice", "vibe-py", "--store", str(tmp_path / "m.jsonl"), "--json"]) == 0 - payload = json.loads(buf.getvalue()) - assert payload["records"] >= 1 and payload["slice"] == "vibe-py" and "skipped" in payload - assert (tmp_path / "m.jsonl").is_file() diff --git a/tests/test_learning_endpoint.py b/tests/test_model_endpoint.py similarity index 85% rename from tests/test_learning_endpoint.py rename to tests/test_model_endpoint.py index fd2c5d6..074c260 100644 --- a/tests/test_learning_endpoint.py +++ b/tests/test_model_endpoint.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import urllib.request +from pathlib import Path from typing import Any import pytest @@ -33,7 +34,7 @@ def _endpoint_env(monkeypatch: pytest.MonkeyPatch, **values: str | None) -> None def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint + from openultrasast.model.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint _endpoint_env(monkeypatch, DEEPSEEK_API_KEY="ds-key", OPENROUTER_API_KEY="or-key", OPENROUTER_BASE_URL="https://openrouter.ai/api/v1") resolved = resolve_chat_endpoint(ResolvedConfig()) @@ -47,7 +48,7 @@ def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint( def test_resolution_order_override_then_scripted_then_openrouter_then_none(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint + from openultrasast.model.endpoint import resolve_chat_endpoint from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient injected = ScriptedChatClient([ChatResponse(content="{}")]) @@ -68,7 +69,7 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint + from openultrasast.model.endpoint import resolve_chat_endpoint _endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key") @@ -81,16 +82,14 @@ def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: p def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: from openultrasast.config import load_config - from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models + from openultrasast.model.endpoint import resolve_chat_endpoint, resolve_models _endpoint_env(monkeypatch, MY_KEY="local-key") monkeypatch.setenv("MY_KEY", "local-key") (tmp_path / "ousast.toml").write_text( - '[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n\n' - "[learning]\nk_runs = 3\nround_cost_cap_usd = 2.5\nminibatch = 4\n" + '[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n' ) config = load_config(tmp_path / "ousast.toml") - assert config.learning.k_runs == 3 and config.learning.round_cost_cap_usd == 2.5 and config.learning.minibatch == 4 resolved = resolve_chat_endpoint(config) assert resolved is not None and resolved[1].base_url == "http://localhost:11434/v1" detector, judge = resolve_models(config) @@ -98,7 +97,7 @@ def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pyte def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_field() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": '{"intent": "public"}', "reasoning_content": "step one"}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -110,7 +109,7 @@ def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_fie def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": ""}, {"content": '{"ok": true}'}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -123,7 +122,7 @@ def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None: def test_log_probabilities_are_requested_only_when_asked_for() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": "{}", "logprobs": {"content": [{"logprob": -0.1}, {"logprob": -0.3}]}}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -133,7 +132,7 @@ def test_log_probabilities_are_requested_only_when_asked_for() -> None: def test_cost_comes_from_the_providers_own_usage_fields() -> None: - from openultrasast.learning.endpoint import ChatEndpoint, price_of + from openultrasast.model.endpoint import ChatEndpoint, price_of flash = ChatEndpoint(provider="deepseek", base_url="https://api.deepseek.com", thinking=False, prices=price_of("deepseek-v4-flash")) cost = flash.cost({"prompt_cache_hit_tokens": 1_000_000, "prompt_cache_miss_tokens": 1_000_000, "completion_tokens": 1_000_000}) @@ -179,7 +178,7 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None `_message_of` returned only `choices[0].message`, so the confidence signal the classifier is supposed to abstain on was always `None` — a knob that reads as "never unsure" whatever the model said.""" - from openultrasast.learning.endpoint import _message_of + from openultrasast.model.endpoint import _message_of payload = { "choices": [ @@ -197,8 +196,29 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None def test_a_reply_without_logprobs_reports_none_rather_than_a_number() -> None: - from openultrasast.learning.endpoint import _message_of + from openultrasast.model.endpoint import _message_of from openultrasast.tool_hunter import chat_response_from_message message = _message_of({"choices": [{"message": {"content": "yes"}}]}) assert chat_response_from_message(message).mean_logprob is None + + +def test_thinking_is_a_measured_condition_on_the_models_section() -> None: + """The knob moved from `[learning]` to `[models]` with the noise architecture's removal. + + Thinking is off by default because that is what the committed baseline was measured under — the provider + silently ignores `temperature` while thinking — not because it is known to be better. + """ + from openultrasast.config import load_config + + assert load_config(None).models.thinking is False + + +def test_the_config_can_turn_thinking_on(tmp_path: Path) -> None: + from openultrasast.config import load_config + + path = tmp_path / "ousast.toml" + path.write_text("[models]\nthinking = true\n") + assert load_config(path).models.thinking is True + path.write_text("[models]\nthinking = false\n") + assert load_config(path).models.thinking is False diff --git a/tests/test_model_specs.py b/tests/test_model_specs.py new file mode 100644 index 0000000..93e212d --- /dev/null +++ b/tests/test_model_specs.py @@ -0,0 +1,101 @@ +"""model-grounded-detection task 1.1: the security vocabularies survive the modules that carried them. + +The guard vocabulary lived in `semantic/variants.py`, whose optimisation loop this feature deletes. The +source/sink/sanitizer facts and the obligation facts live in `ruleset/` data and are retained, so the specs +*derive* from them rather than copying them — a copy would be a second place to update. What must be ported +verbatim is the guard vocabulary, because nothing else holds it. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SPECS = Path("src/openultrasast/model/specs.py") + + +def test_the_guard_vocabulary_is_preserved_verbatim() -> None: + from openultrasast.model.specs import GUARD_KINDS + + assert GUARD_KINDS == ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") + + +def test_the_guard_classifier_still_recognises_each_kind() -> None: + """The patterns classify the *fix*, first match wins — the order is part of the vocabulary.""" + from openultrasast.model.specs import classify_guard_text + + assert classify_guard_text('cur.execute("SELECT * FROM t WHERE id = %s", (uid,))') == "parameterized_call" + assert classify_guard_text("if not current_user.is_authenticated: abort(403)") == "auth_check" + assert classify_guard_text('if name not in ALLOWED: abort(400)') == "allowlist_test" + assert classify_guard_text("if len(buf) > 64: return") == "bounds_test" + assert classify_guard_text("if value is None: return") == "null_test" + assert classify_guard_text("page = int(raw)") == "type_change" + assert classify_guard_text("total = total + 1") == "none" + + +def test_the_seeded_families_and_the_named_gaps_are_both_exact() -> None: + """What the retained fact data can seed, and what it provably cannot — Req 9.2, a gap is *named*. + + `path`, `output_encoding`, `untrusted_destination` and `prototype` have no sink in `ruleset/semantic` + whose CWE routes to them, which is the same hole the entailment ceiling diagnosed ("closed literal sink + table: misses per-family sinks"). Task 4.2 authors those sinks and must update this test when it does — + the set is asserted exactly so a gap can neither appear nor close silently. + """ + from openultrasast.model.specs import dominance_specs, taint_specs + from openultrasast.model.taxonomy import load_families + + taxonomy = load_families() + seeded = set() + for language in ("python", "javascript", "java", "c"): + seeded |= {spec.family for spec in taint_specs(language=language).values()} + seeded |= {spec.family for spec in dominance_specs(language=language).values()} + deferred = {"unknown", "memory"} # the declined bucket and the deferred execution tier + assert seeded == {"injection", "deserialization", "config_secrets", "access_control", "memory"} + gaps = {family.id for family in taxonomy.families} - seeded - deferred + assert gaps == {"path", "output_encoding", "untrusted_destination", "prototype"}, ( + "a family lost or gained a seed without this record being updated" + ) + + +def test_the_injection_taint_spec_carries_sources_sinks_and_sanitizers() -> None: + from openultrasast.model.specs import taint_specs + + spec = taint_specs(language="python")["injection"] + assert any("request.args" in source for source in spec.sources) + assert any(sink in ("eval", "exec", "os.system") for sink in spec.sinks) + assert spec.sanitizers, "a family with no sanitizer model can never distinguish a fixed twin" + + +def test_the_access_control_dominance_spec_carries_operations_and_dischargers() -> None: + from openultrasast.model.specs import dominance_specs + + spec = dominance_specs(language="python")["access_control"] + assert any("filter_by" in operation for operation in spec.operations) + assert spec.dischargers, "an absence family with no discharger model can only ever report suspicion" + + +def test_specs_import_nothing_from_the_removed_packages() -> None: + """Req 1.2: the vocabularies must not depend on the modules this feature deletes.""" + tree = ast.parse(SPECS.read_text()) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + elif isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + forbidden = ("learning", "improve", "variants", "mechanisms", "evolve") + offending = [name for name in imported if any(token in name for token in forbidden)] + assert not offending, f"model/specs.py imports removed code: {offending}" + + +def test_the_specs_module_exposes_no_writer() -> None: + """Req 7.4, the verifier boundary: the models are read-only data no LLM or optimiser can rewrite.""" + tree = ast.parse(SPECS.read_text()) + writers = [ + node.name + for node in tree.body + if isinstance(node, ast.FunctionDef) + and any(node.name.startswith(verb) for verb in ("write", "save", "dump", "store", "append", "update", "set_")) + ] + assert not writers, f"specs must be read-only, found writers: {writers}" + assert "open(" not in SPECS.read_text().replace("# ", ""), "specs must not open a file for writing" diff --git a/tests/test_learning_families.py b/tests/test_model_taxonomy.py similarity index 93% rename from tests/test_learning_families.py rename to tests/test_model_taxonomy.py index 24a9e66..c3510a8 100644 --- a/tests/test_learning_families.py +++ b/tests/test_model_taxonomy.py @@ -14,7 +14,7 @@ GOOD = ( def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None: - from openultrasast.learning.families import FAMILY_IDS, VERIFIER_KINDS, load_families + from openultrasast.model.taxonomy import FAMILY_IDS, VERIFIER_KINDS, load_families taxonomy = load_families() assert len(FAMILY_IDS) == 10 and FAMILY_IDS[-1] == "unknown" @@ -27,7 +27,7 @@ def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None: def test_every_mechanism_id_resolves_to_exactly_one_family() -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families from openultrasast.pairs import load_mechanisms taxonomy = load_families() @@ -43,7 +43,7 @@ def test_every_mechanism_id_resolves_to_exactly_one_family() -> None: def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families taxonomy = load_families() seen: dict[str, str] = {} @@ -63,7 +63,7 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families taxonomy = load_families() assert taxonomy.related("injection", "injection") == "same" @@ -109,7 +109,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared ], ) def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str) -> None: - from openultrasast.learning.families import FamiliesError, load_families + from openultrasast.model.taxonomy import FamiliesError, load_families (tmp_path / "families.toml").write_text(text) with pytest.raises(FamiliesError, match=needle): @@ -117,7 +117,7 @@ def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str def test_the_shipped_file_must_declare_the_whole_closed_set_in_order(tmp_path: Path) -> None: - from openultrasast.learning.families import FamiliesError, load_families + from openultrasast.model.taxonomy import FamiliesError, load_families (tmp_path / "families.toml").write_text(GOOD) with pytest.raises(FamiliesError, match="in that order"): diff --git a/tests/test_module_audit.py b/tests/test_module_audit.py new file mode 100644 index 0000000..b6fc5f8 --- /dev/null +++ b/tests/test_module_audit.py @@ -0,0 +1,45 @@ +"""model-grounded-detection Req 3: outgrowth is visible and checkable, not noticed two specs later.""" + +from __future__ import annotations + +import json +from pathlib import Path + +SRC = Path("src/openultrasast") +MANIFEST = Path("benchmarks/measurements/2026-09-08-module-audit.json") + + +def test_no_module_in_the_tree_is_orphaned() -> None: + from openultrasast.model.audit import audit + + orphans = [row.module for row in audit(SRC) if row.classification == "orphaned"] + assert not orphans, f"orphaned modules must be removed, not carried: {orphans}" + + +def test_every_module_is_classified() -> None: + from openultrasast.model.audit import audit, module_name + + classified = {row.module for row in audit(SRC)} + on_disk = {module_name(path, SRC) for path in SRC.rglob("*.py") if "__pycache__" not in path.parts} + assert on_disk - classified <= {""}, f"unclassified: {sorted(on_disk - classified - {''})}" + + +def test_the_named_subsystems_each_carry_an_explicit_decision() -> None: + """Req 3.3: the five older-phase subsystems get a decision, not silence.""" + from openultrasast.model.audit import audit + + rows = {row.module: row for row in audit(SRC)} + for name in ("fusion", "mcp", "skills", "harness_ext", "hunter_harness"): + assert name in rows, f"{name} vanished without a recorded decision" + assert rows[name].reason, f"{name} has no stated reason" + + +def test_the_manifest_matches_the_tree() -> None: + """A later reader can check the tree against the committed manifest (Req 3.4).""" + from openultrasast.model.audit import audit, to_dict + + assert MANIFEST.is_file(), "the audit manifest is committed evidence, not a local artifact" + committed = json.loads(MANIFEST.read_text()) + current = to_dict(audit(SRC)) + assert committed["counts"] == current["counts"], "the tree drifted from its committed audit" + assert committed["orphaned"] == current["orphaned"] == [] diff --git a/tests/test_obligation_check.py b/tests/test_obligation_check.py index 3344034..a890589 100644 --- a/tests/test_obligation_check.py +++ b/tests/test_obligation_check.py @@ -115,37 +115,24 @@ def test_path_records_gate_reachability_and_a_dominating_hop_discharges() -> Non assert covered.findings # without the hop witness the finding stands -def test_findings_become_suspicion_static_findings_with_known_fix_and_labels(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.obligations.check import findings_to_static - from openultrasast.semantic.obligations.shapes import ObligationShape, obligation_mechanisms +def test_findings_become_suspicion_static_findings_with_labels() -> None: + """The finding-id contract, the suspicion rung and the tags, without the deleted mechanism store. - store = MechanismStore(tmp_path / "m.jsonl") - shape = ObligationShape( - language="python", - operation_kind="protected_read", - discharger_kind="identity_constraint", - provenance="authenticated_context", - resource_class="owned", - mechanism="missing_auth_guard", - ) - record = append_from_pair(store, shape, summary="s", cwe="CWE-639", pair="vampi", provenance="human", tier="seeded") - result = _check(store_shapes=obligation_mechanisms(store.load())) + The original also asserted `known_fix` — a hint the store supplied. The store is gone, so the hint is gone + with it; everything the checker establishes on its own is still pinned here. + """ + from openultrasast.semantic.obligations.check import findings_to_static + + result = _check() leaky = next(f for f in result.findings if f.operation.function == "leaky" and f.missing == "identity_constraint") - assert leaky.known_fix == record.id + guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint") statics = findings_to_static(result) ids = {s.finding_id for s in statics} - guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint") assert f"obligation:protected_read:app.py:{leaky.operation.line}:identity_constraint" in ids # exact id contract assert f"obligation:protected_read:app.py:{guarded.operation.line}:identity_constraint" in ids static = next(s for s in statics if s.line == leaky.operation.line and "discharger:identity_constraint" in s.tags) assert static.evidence_level == "suspicion" - assert { - "obligation:protected_read", - "discharger:identity_constraint", - "obligation_evidence:consistency_violation", - f"mechanism:{record.id}", - } <= set(static.tags) + assert {"obligation:protected_read", "discharger:identity_constraint", "obligation_evidence:consistency_violation"} <= set(static.tags) assert "app.py::constrained" in static.rationale diff --git a/tests/test_obligation_export.py b/tests/test_obligation_export.py deleted file mode 100644 index 7846569..0000000 --- a/tests/test_obligation_export.py +++ /dev/null @@ -1,160 +0,0 @@ -"""authorization-obligations task 1.3: obligation shapes travel through the existing exporter and store.""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase -from openultrasast.semantic.mechanisms import MechanismStore - -VULN = ( - "from flask import request\n\n\ndef get_book(book_title):\n" - " resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n" - " book = Book.query.filter_by(book_title=book_title).first()\n return book\n" -) -FIXED = ( - "from flask import request\n\n\ndef get_book(book_title):\n" - " resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n" - " user = User.query.filter_by(username=resp['sub']).first()\n" - " book = Book.query.filter_by(user=user, book_title=book_title).first()\n return book\n" -) -FLOW_VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FLOW_FIXED = ( - "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - + " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _case( - tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, mechanism: str, tier: str = "seeded", sink: str | None = None -) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function=function, mechanism=mechanism, sink=sink - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance="human", - review_tier=tier, - reviewer="t" if tier == "reviewed" else "", - ) - - -def test_exporter_writes_obligation_rows_deduped_with_provenance_and_skips_untrusted(tmp_path: Path) -> None: - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - cases = [ - _case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard"), - _case( - tmp_path, - "b", - VULN.replace("book", "item"), - FIXED.replace("book", "item"), - function="get_item", - mechanism="missing_auth_guard", - tier="reviewed", - ), - _case(tmp_path, "advisory", VULN, FIXED, function="get_book", mechanism="missing_auth_guard", tier="advisory"), - _case(tmp_path, "flow", FLOW_VULN, FLOW_FIXED, function="run", mechanism="source_reaches_sink", sink="os.system"), - ] - report = export_mechanisms(cases, store) - rows = store.load() - obligations = obligation_mechanisms(rows) - assert len(obligations) == 1 and obligations[0].pairs == ("a", "b") and obligations[0].origin == "corpus" - assert ( - obligations[0].shape is not None - and obligations[0].shape["family"] == "obligation" - and obligations[0].guard == "identity_constraint" - ) - sinks = [r for r in rows if r.shape and r.shape.get("family", "sink") != "obligation"] - assert len(sinks) == 1 and sinks[0].shape is not None and sinks[0].shape["sink_name"] == "system" # flow rows unaffected - assert report.seeded == 3 and dict(report.skipped)["advisory"] == "tier:advisory" - text = (tmp_path / "mechanisms.jsonl").read_text() - assert "book_title" not in text and "resp" not in text and "app.py" not in text - - -def test_absence_pair_without_a_lesson_is_skipped_with_the_derivation_reason(tmp_path: Path) -> None: - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "m.jsonl") - report = export_mechanisms([_case(tmp_path, "same", VULN, VULN, function="get_book", mechanism="missing_auth_guard")], store) - assert store.load() == () and dict(report.skipped)["same"].startswith("no_discharger_added:identity_constraint") - - -def test_sink_search_ignores_obligation_rows(tmp_path: Path) -> None: - from openultrasast.preprocess import preprocess_repository - from openultrasast.semantic.facts import load_facts - from openultrasast.semantic.seed import export_mechanisms - from openultrasast.semantic.variant_search import search_tree - - store = MechanismStore(tmp_path / "m.jsonl") - export_mechanisms([_case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard")], store) - repo = tmp_path / "repo" - repo.mkdir() - (repo / "views.py").write_text(VULN) - _, targets = preprocess_repository(repo) - result = search_tree(repo, targets, store, load_facts(), max_mechanisms=500) - assert result.mechanisms_searched == 0 and result.hits == () and result.degradations == () - - -def test_the_train_split_of_vibe_py_teaches_no_obligation_shape_today(tmp_path: Path) -> None: - """learning-harness Req 5.1 turned this number honest, and the honest number is zero. - - Every vibe-py absence pair that yields an obligation lesson sits on the holdout side, so once only - train-split pairs may teach, the corpus teaches no obligation shape at all. The earlier result - ("at least one obligation shape from vibe-py") was true only because holdout pairs were teaching. - The derivation itself stays covered by the fixture tests above; this pins the corpus gap, which the - absence re-harvest and a maintainer pass over the splits are what close. - """ - from openultrasast.pairs import load_pair_catalog, select_slice, select_vendored - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "m.jsonl") - cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py")) - report = export_mechanisms(cases, store) - assert obligation_mechanisms(store.load()) == () - refused = {pair: reason for pair, reason in report.skipped} - assert refused["vampi-users-update-password"] == "split:holdout" - assert refused["threatbyte-api-v1-get"] == "split:holdout" - # Its own reason, not its split: this row is a byte-identical twin that happens to sit on the holdout side, - # and reporting `split:holdout` would hide a corpus fact behind a split fact (Req 5.2). - assert refused["vampi-books-get-by-title"] == "identical_twin" - flow_rows = [row for row in store.load() if row.shape is not None and row.shape.get("family", "sink") == "sink"] - assert report.records == len(flow_rows) and len(flow_rows) >= 1 # the flow family still teaches from the train split - - -def test_absence_row_without_an_obligation_lesson_still_teaches_its_sink_shape(tmp_path: Path) -> None: - """The fallback that keeps the flow store unchanged: a `missing_auth_guard` row whose twin adds no discharger but whose - labeled sink call carries a source still seeds a sink shape (the threatbyte pairs do exactly this).""" - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - vuln = ( - "import sqlite3\n\n\ndef get(user_id):\n cur = sqlite3.connect('x').cursor()\n" - " cur.execute('SELECT * FROM users WHERE id = ' + user_id)\n return cur.fetchone()\n" - ) - fixed = ( - "import sqlite3\n\n\ndef login(username):\n cur = sqlite3.connect('x').cursor()\n" - " cur.execute('SELECT * FROM users WHERE name = ?', (username,))\n return cur.fetchone()\n" - ) - store = MechanismStore(tmp_path / "m.jsonl") - report = export_mechanisms([_case(tmp_path, "tb", vuln, fixed, function="get", mechanism="missing_auth_guard", sink="execute")], store) - rows = store.load() - assert report.seeded == 1 and obligation_mechanisms(rows) == () - assert len(rows) == 1 and rows[0].shape is not None and rows[0].shape.get("family", "sink") == "sink" - assert rows[0].shape["sink_name"] == "execute" diff --git a/tests/test_pair_corpus.py b/tests/test_pair_corpus.py index cf947f5..cf932ec 100644 --- a/tests/test_pair_corpus.py +++ b/tests/test_pair_corpus.py @@ -370,55 +370,6 @@ def test_known_limit_pairs_are_evaluated_but_excluded_from_achievable(tmp_path: assert set(result.per_mechanism) == {"cross_artifact", "source_reaches_sink"} assert result.loss["sast"]["known_limit"] == 1 assert "overlay" in result.scorers["sast"] and "inventory" in result.scorers["sast"] - assert result.degradations and result.degradations[0]["reason"] == "hunter_model_unavailable" - - -def test_hunter_scorer_credits_a_family_tag_and_never_the_words(tmp_path: Path) -> None: - """learning-harness Req 3.1/3.6 replaced this path's rule: the family tag decides, the rationale never does.""" - from openultrasast.findings import StaticFinding - - src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" - row = ExpectedFinding( - cwe="CWE-95", - vulnerability_class="code injection", - path="app.py", - evidence="", - sink="eval", - function="f", - mechanism="source_reaches_sink", - ) - - def scripted(tags: list[str]): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if "eval(" not in (root / "app.py").read_text(): - return [] - return [ - StaticFinding( - finding_id="tool-hunter:app.py:4", - path="app.py", - title="eval of request input", - severity="high", - confidence="medium", - evidence_level="suspicion", - rationale="CWE-95 eval executes attacker input", - line=4, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=tags, - ranking_priority=1.0, - ) - ] - - return scan - - case = _case(tmp_path, "hunt", src, "def f():\n return 1\n", row) - tagged = evaluate_catalog([case], hunter=scripted(["family:injection"])).scorers["sast"]["hunter"] - assert tagged.pair_correct == 1 and tagged.detected_vuln == 1 and tagged.silent_fix == 1 - # the same finding, the same words, no family tag: not a detection, because the rationale is not evidence - untagged = evaluate_catalog([case], hunter=scripted([])).scorers["sast"]["hunter"] - assert untagged.detected_vuln == 0 and untagged.silent_fix == 1 def test_juliet_strcpy_pair_bodies_differ_and_java_hash_is_known_limit() -> None: diff --git a/tests/test_pair_registration_context.py b/tests/test_pair_registration_context.py index e1fbdac..2e6dfc2 100644 --- a/tests/test_pair_registration_context.py +++ b/tests/test_pair_registration_context.py @@ -17,8 +17,8 @@ from pathlib import Path import pytest -from openultrasast.learning.families import load_families from openultrasast.mapping import analyze_entry_points +from openultrasast.model.taxonomy import load_families from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored from openultrasast.preprocess import preprocess_repository from openultrasast.semantic.extra import has_semantic_extra diff --git a/tests/test_semantic_pipeline.py b/tests/test_semantic_pipeline.py index 493e0e0..b85d3ba 100644 --- a/tests/test_semantic_pipeline.py +++ b/tests/test_semantic_pipeline.py @@ -1,4 +1,4 @@ -"""MAP overlay, prove filter, reports, pair scoring, mechanism memory.""" +"""MAP overlay, prove filter, reports, pair scoring.""" from __future__ import annotations @@ -15,7 +15,7 @@ from openultrasast.gate import LANGUAGE_MANIFESTS from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog from openultrasast.preprocess import preprocess_repository from openultrasast.rank import rank_targets -from openultrasast.semantic import MechanismStore, OverlayRecord, append_mechanism, filter_promoted_hotspots, order_promotions +from openultrasast.semantic import OverlayRecord, filter_promoted_hotspots from openultrasast.semantic.prove_filter import promoted_findings @@ -265,79 +265,3 @@ def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None: assert "coverage" in text -def test_mechanism_order_boosts_paraphrased_match(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism( - store, - summary="request parameter reaches eval", - cwe="CWE-95", - language="python", - tags=["syscall_entry"], - what_made_it_exploitable="dynamic execution of attacker input", - ) - eval_id = "python-unsafe-eval:app.py:1" - hash_id = "python-weak-hash:hash.py:1" - hotspots = (_hotspot(hash_id, score=5.0, path="hash.py"), _hotspot(eval_id, score=5.0, path="app.py")) - records = ( - _record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",), language="python"), - _record(eval_id, "promote", sources=("request",), cwe="CWE-95", sinks=("eval",), language="python"), - ) - findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)] - - class _Client: - def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]: - del model, timeout_seconds - vectors: list[list[float]] = [] - for text in inputs: - if "eval" in text.lower() or "CWE-95" in text: - vectors.append([1.0, 0.0]) - else: - vectors.append([0.0, 1.0]) - return vectors - - ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=_Client(), model="test-embed") - assert degradation is None - assert ordered[0].inventory_finding_ids == (eval_id,) - assert {item.inventory_finding_ids[0] for item in ordered} == {eval_id, hash_id} - - -def test_mechanism_degrades_to_heuristic_without_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism( - store, - summary="request parameter reaches eval", - cwe="CWE-95", - language="python", - tags=["syscall_entry"], - what_made_it_exploitable="dynamic execution of attacker input", - ) - eval_id = "python-unsafe-eval:app.py:1" - hash_id = "python-weak-hash:hash.py:1" - hotspots = (_hotspot(hash_id, score=6.0, path="hash.py"), _hotspot(eval_id, score=3.0, path="app.py")) - records = ( - _record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",)), - _record(eval_id, "promote", sources=("request",)), - ) - findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)] - ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=None, model=None) - assert degradation == "embeddings_unavailable" - assert [item.inventory_finding_ids[0] for item in ordered] == [hash_id, eval_id] - - -def test_mechanisms_are_not_written_from_unadjudicated(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - record = OverlayRecord( - proposal_id="python-unsafe-eval:app.py:1", - path="app.py", - line=1, - disposition="unadjudicated", - reason="parse_failed", - cwe="CWE-95", - sources=(), - sinks=(), - sanitizers=(), - evidence_level="static_corroboration", - ) - assert record.disposition != "promote" - assert store.load() == () diff --git a/tests/test_subtraction_complete.py b/tests/test_subtraction_complete.py new file mode 100644 index 0000000..4ea55cd --- /dev/null +++ b/tests/test_subtraction_complete.py @@ -0,0 +1,81 @@ +"""model-grounded-detection Req 1.3: nothing survives that imports what this feature removed. + +The noise architecture and the mechanism/evolve loop are deleted, not deprecated. This walks the AST of every +surviving module and fails on any import that names one of them — the check that turns a five-thousand-line +deletion from crash-driven discovery into one assertion. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SRC = Path("src/openultrasast") + +REMOVED_MODULES = frozenset( + { + # the noise architecture (the whole learning/ package; taxonomy, candidates and endpoint re-homed to model/) + "learning", + # the mechanism/evolve optimisation loop + "semantic.mechanisms", + "semantic.variants", + "semantic.variant_search", + "semantic.prove_budget", + "semantic.loo", + "semantic.seed", + } +) + + +def _module_names(tree: ast.AST) -> set[str]: + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + elif isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + return names + + +def _names_a_removed_module(imported: str) -> bool: + trimmed = imported.lstrip(".") + return any(trimmed == removed or trimmed.startswith(f"{removed}.") for removed in REMOVED_MODULES) + + +def test_no_surviving_module_imports_a_removed_one() -> None: + offenders: dict[str, list[str]] = {} + for path in sorted(SRC.rglob("*.py")): + named = sorted(name for name in _module_names(ast.parse(path.read_text())) if _names_a_removed_module(name)) + if named: + offenders[str(path)] = named + assert not offenders, f"dangling imports of removed modules: {offenders}" + + +def test_the_removed_modules_are_gone_from_the_tree() -> None: + assert not (SRC / "learning").exists(), "the learning package is removed; taxonomy/candidates/endpoint live under model/" + for relative in ("semantic/mechanisms.py", "semantic/variants.py", "semantic/variant_search.py", + "semantic/prove_budget.py", "semantic/loo.py", "semantic/seed.py"): + assert not (SRC / relative).exists(), f"{relative} still exists" + + +def test_the_mechanism_lever_is_gone_from_the_retained_improve_loop() -> None: + """`improve/` survives with its lever removed, not deleted. + + The design's Removed list named `improve/evolve.py` whole, but the file is a mix: the mechanism lever + (deleted here) and the deterministic rule-status loop that `ousast improve` still runs from benchmark + evidence, which has no LLM in it and is not what this feature supersedes. Deleting the file would have + taken a retained capability with it, so the lever is stripped and the loop stays — asserted by symbol so + the lever cannot creep back. + """ + evolve = (SRC / "improve" / "evolve.py").read_text() + validator = (SRC / "improve" / "validator.py").read_text() + for symbol in ("MechanismEdit", "MechanismStore", "apply_mechanism_edits", "evaluate_mechanism_profiles", + "propose_mechanism_edits", "validate_mechanism", "mechanism_store", "mechanism_candidates"): + assert symbol not in evolve, f"{symbol} survives in improve/evolve.py" + assert symbol not in validator, f"{symbol} survives in improve/validator.py" + assert "def run_improvement(" in evolve and "def propose_status_edits(" in evolve, "the rule-status loop must survive" + + +def test_the_re_homed_modules_are_where_the_model_layer_expects_them() -> None: + for relative in ("model/taxonomy.py", "model/candidates.py", "model/endpoint.py", "model/specs.py"): + assert (SRC / relative).is_file(), f"{relative} is missing" diff --git a/tests/test_variant_scan.py b/tests/test_variant_scan.py deleted file mode 100644 index 76f7963..0000000 --- a/tests/test_variant_scan.py +++ /dev/null @@ -1,130 +0,0 @@ -"""corpus-seeded-mechanisms task 2.3: variant findings in a standard scan, sandbox-eligible, counted in the manifest.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - -VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n" -UNSEEN_SINK = ( - "from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n" # a project helper: no inventory rule, no fact sink -) - - -def _seed(repo: Path) -> str: - from openultrasast.semantic.variants import Shape - - shape = Shape( - language="python", - sink_name="run_shell", - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard="allowlist_test", - mechanism="source_reaches_sink", - ) - store = MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl") - return append_from_pair( - store, shape, summary="parameter into run_shell; fix adds allowlist", cwe="CWE-78", pair="p1", provenance="human", tier="seeded" - ).id - - -def _latest_run(repo: Path) -> Path: - runs = sorted((repo / ".runs").iterdir()) - return runs[-1] - - -def test_standard_scan_emits_variant_findings_and_manifest_counters(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) # only the learned shape can find a project-specific sink - mechanism_id = _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = _latest_run(repo) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - variant = [f for f in findings if f["finding_id"].startswith("variant:")] - assert len(variant) == 1 and variant[0]["finding_id"] == f"variant:{mechanism_id}:app.py:5" - assert variant[0]["evidence_level"] == "suspicion" and f"mechanism:{mechanism_id}" in variant[0]["tags"] - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert manifest["variants"] == {"mechanisms_searched": 1, "files_searched": 1, "findings": 1, "merged_into_overlay": 0} - - -def test_variant_hit_merges_into_an_overlay_flow_at_the_same_call_site(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.variants import Shape - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(VARIANT) # os.system is both an inventory rule and a fact sink: the overlay records the flow - shape = Shape( - language="python", - sink_name="system", - arity=1, - source_positions=(0,), - source_kinds=("fact_source",), - guard="none", - mechanism="source_reaches_sink", - ) - record = append_from_pair( - MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"), - shape, - summary="s", - cwe="CWE-78", - pair="p", - provenance="human", - tier="seeded", - ) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = _latest_run(repo) - overlay = json.loads((run_dir / "overlay.json").read_text())["records"] - assert any(item["mechanism_id"] == record.id and item["line"] == 7 for item in overlay) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - assert not any(f["finding_id"].startswith("variant:") for f in findings) # merged, not duplicated - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert manifest["variants"]["merged_into_overlay"] == 1 and manifest["variants"]["findings"] == 0 - # Req 6.1 end to end: the merged inventory finding names the mechanism in both reports and keeps its own evidence level - report = (run_dir / "report.md").read_text() - assert f"- Mechanism: `{record.id}`" in report and "- Learned from pairs: `p`" in report - sarif = json.loads((run_dir / "report.sarif").read_text()) - props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"].get("mechanism_id") == record.id] - assert props and props[0]["mechanism_guard"] == "none" and props[0]["evidence_level"] != "suspicion" - - -def test_quick_scan_never_searches_variants(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) - _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - assert main(["scan", str(repo), "--mode", "quick"]) == 0 - run_dir = _latest_run(repo) - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert "variants" not in manifest - assert not any(f["finding_id"].startswith("variant:") for f in json.loads((run_dir / "findings.json").read_text())["findings"]) - - -def test_deep_scan_offers_variant_findings_to_the_sandbox_after_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast import cli - from openultrasast.sandbox import FakeSandboxRunner, SandboxResult - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) - mechanism_id = _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False)) - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - assert main(["scan", str(repo), "--mode", "deep"]) == 0 - run_dir = _latest_run(repo) - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]} - assert f"variant:{mechanism_id}:app.py:5" in offered diff --git a/tests/test_variant_search.py b/tests/test_variant_search.py deleted file mode 100644 index 066871e..0000000 --- a/tests/test_variant_search.py +++ /dev/null @@ -1,124 +0,0 @@ -"""corpus-seeded-mechanisms task 2.2: structural variant search over a scanned tree, merged with overlay flows.""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.semantic.facts import load_facts -from openultrasast.semantic.ir import parse_file -from openultrasast.semantic.overlay import OverlayRecord - -PY_FACTS = load_facts() - - -def _shape(**overrides: object): - from openultrasast.semantic.variants import Shape - - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("fact_source",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return Shape(**base) # type: ignore[arg-type] - - -VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n" -PARAM_VARIANT = "import os\n\n\ndef handler(q):\n os.system(q)\n" -CONSTANT = "import os\n\n\ndef handler():\n os.system('ls -la')\n" -ARITY = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q, 1)\n" - - -def test_match_shapes_finds_structural_variants_and_ignores_constants() -> None: - from openultrasast.semantic.variants import match_shapes - - shape = _shape() - hits = match_shapes(parse_file("a.py", VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) - assert [(h.path, h.line, h.mechanism_id, h.sink_name, h.source_kind) for h in hits] == [ - ("a.py", 7, "corpus:abc", "system", "fact_source") - ] - param_hits = match_shapes(parse_file("b.py", PARAM_VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) - assert [(h.line, h.source_kind) for h in param_hits] == [(5, "parameter")] # any source kind at the shape's positions matches - assert match_shapes(parse_file("c.py", CONSTANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == [] - assert match_shapes(parse_file("d.py", ARITY, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == [] - js_shape = _shape(language="javascript", sink_name="exec") - assert match_shapes(parse_file("a.py", VARIANT, "python"), [js_shape], PY_FACTS, mechanism_ids={js_shape.key(): "x"}) == [] - - -def test_hits_become_suspicion_findings_or_merge_into_an_overlay_flow() -> None: - from openultrasast.semantic.variant_search import hits_to_findings - from openultrasast.semantic.variants import VariantHit - - hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="fact_source") - other = VariantHit(path="z.py", line=3, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter") - coverage = OverlayRecord( - proposal_id="overlay-coverage:a.py:7:os.system", - path="a.py", - line=7, - disposition="coverage", - reason="uninventoried sink os.system reached from request", - cwe="CWE-78", - sources=("request",), - sinks=("os.system",), - sanitizers=(), - evidence_level="static_corroboration", - origin="overlay", - engine="python-ast", - language="python", - ) - summaries = {"corpus:abc": ("request into os.system; fix adds allowlist_test", ("p1", "p2"), "CWE-78")} - findings, records = hits_to_findings([hit, other], [coverage], summaries) - assert len(records) == 1 and records[0].mechanism_id == "corpus:abc" and records[0].disposition == "coverage" - assert [f.finding_id for f in findings] == ["variant:corpus:abc:z.py:3"] # the a.py hit merged, no duplicate - (finding,) = findings - assert finding.evidence_level == "suspicion" and "variant" in finding.tags and "mechanism:corpus:abc" in finding.tags - assert "p1" in finding.rationale and finding.line == 3 and finding.path == "z.py" - - -def test_search_tree_uses_the_store_and_respects_the_mechanism_cap(tmp_path: Path) -> None: - from openultrasast.preprocess import preprocess_repository - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variant_search import search_tree - - root = tmp_path / "repo" - root.mkdir() - (root / "views.py").write_text(VARIANT) - (root / "safe.py").write_text(CONSTANT) - store = MechanismStore(tmp_path / "mechanisms.jsonl") - record = append_from_pair(store, _shape(), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") - _, targets = preprocess_repository(root) - result = search_tree(root, targets, store, PY_FACTS, max_mechanisms=500) - assert [(h.path, h.line, h.mechanism_id) for h in result.hits] == [("views.py", 7, record.id)] - assert result.mechanisms_searched == 1 and result.files_searched == 2 - capped = search_tree(root, targets, store, PY_FACTS, max_mechanisms=0) - assert capped.hits == () and capped.mechanisms_searched == 0 - - -def test_hits_never_merge_into_demoted_or_flowless_records() -> None: - from openultrasast.semantic.variant_search import hits_to_findings - from openultrasast.semantic.variants import VariantHit - - hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter") - demoted = OverlayRecord( - proposal_id="python-os-command:a.py:7", - path="a.py", - line=7, - disposition="demote", - reason="constant argument", - cwe="CWE-78", - sources=(), - sinks=("os.system",), - sanitizers=(), - evidence_level="suspicion", - dominating_fact="constant", - origin="inventory", - engine="python-ast", - language="python", - ) - findings, records = hits_to_findings([hit], [demoted], {}) - assert records[0].mechanism_id is None and records[0].disposition == "demote" # Req 3.6: never demote, never re-label a demotion - assert [f.finding_id for f in findings] == ["variant:corpus:abc:a.py:7"] diff --git a/tests/test_variants.py b/tests/test_variants.py deleted file mode 100644 index bc72042..0000000 --- a/tests/test_variants.py +++ /dev/null @@ -1,76 +0,0 @@ -"""corpus-seeded-mechanisms: structural shapes derived from pairs and matched on trees (offline, stdlib ast).""" - -from __future__ import annotations - -from openultrasast.semantic.facts import load_facts -from openultrasast.semantic.ir import parse_file - -VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FIXED = ( - "from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _derive(vuln: str = VULN, fixed: str = FIXED, **overrides: object): - from openultrasast.semantic.variants import derive_shape - - kwargs: dict[str, object] = {"function": "run", "sink": "os.system", "line": None, "mechanism": "source_reaches_sink"} - kwargs.update(overrides) - return derive_shape( - parse_file("app.py", vuln, "python"), - parse_file("app.py", fixed, "python"), - vuln_text=vuln, - fixed_text=fixed, - facts=load_facts(), - **kwargs, - ) - - -def test_derive_shape_from_a_request_to_os_system_pair_with_an_allowlist_fix() -> None: - from openultrasast.semantic.variants import GUARD_KINDS, SOURCE_KINDS - - shape = _derive() - assert shape is not None - assert shape.language == "python" and shape.sink_name == "system" and shape.arity == 1 - assert shape.source_positions == (0,) and shape.source_kinds == ("fact_source",) - assert shape.guard == "allowlist_test" and shape.guard in GUARD_KINDS and shape.mechanism == "source_reaches_sink" - assert set(shape.source_kinds) <= set(SOURCE_KINDS) - - -def test_shape_key_is_stable_and_carries_no_path_line_or_literal() -> None: - a, b = _derive(), _derive() - assert a is not None and b is not None and a.key() == b.key() - key = a.key() - for forbidden in ("app.py", "cmd", "'ls'", "request.args", ":6", ":7"): - assert forbidden not in key - # the same code in another file with different variable names is the same shape - other_vuln = VULN.replace("cmd", "command") - other = _derive(other_vuln, FIXED.replace("cmd", "command")) - assert other is not None and other.key() == key - - -def test_parameter_and_container_read_sources_are_classified() -> None: - vuln = "import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n os.system(cmd + target)\n" - fixed = ( - "import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n if len(target) > 32:\n return None\n" - " os.system(cmd + target)\n" - ) - shape = _derive(vuln, fixed) - assert shape is not None - assert shape.source_positions == (0,) - assert shape.source_kinds in {("parameter",), ("container_read",)} # one argument carrying both; the kind is the first found - assert shape.guard == "bounds_test" - - -def test_unlabeled_or_missing_sink_returns_none_and_guard_defaults_to_none() -> None: - assert _derive(sink="subprocess.run") is None # the label names a sink the function never calls - assert _derive(function="other") is None # no such function - same = _derive(VULN, VULN) - assert same is not None and same.guard == "none" # no statement differs on the fixed side - constant = "import os\n\n\ndef run():\n os.system('ls')\n" - assert _derive(constant, constant) is None # a constant argument carries no source: nothing to learn - - -def test_parse_failed_side_yields_no_shape() -> None: - assert _derive("def run(:\n", FIXED) is None