From 9ed0e45c5f5a4cee82748a57063578c49d8a8dab Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:15:54 +0200 Subject: [PATCH] feat(model-grounded-detection): group 1, subtract the noise architecture 99 source modules -> 81. 832 tests -> 646. The whole learning/ package (16 modules, ~4,100 lines) and six semantic/ modules removed, with the three gates byte-identical to 2026-09-06-gate-baseline.txt at every step. The sequence is what made it safe: vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion third (1.4). Every import break during the deletion was therefore a detachment miss in the importer and was fixed there -- no module was restored to satisfy one. Both misses were the same shape and were fixed as a class: operations.py and shapes.py each imported trailing_name from the deleted variants.py, so it moved to semantic/ir.py, its natural home, and both importers changed in one edit. Ported before deleting: the closed guard vocabulary (verbatim, nothing else holds it) into model/specs.py, which derives TaintSpec and DominanceSpec from the retained ruleset facts rather than copying them. Re-homed: families -> model/taxonomy, candidates -> model/candidates, endpoint -> model/endpoint, plus Region, labeled_family and labeled_spans. labeled_family no longer falls back to the LLM classifier -- an unlabeled pair is honestly unknown. Four deviations, each recorded in the spec notes with its reason. improve/ evolve.py and validator.py were stripped, not deleted: each mixes the mechanism lever (gone) with the deterministic rule-status loop `ousast improve` still runs, which has no LLM in it; a test now asserts the lever's absence by symbol so it cannot creep back. The learning CLI group became `model candidates`, because deleting it outright would have orphaned the retained enumerator by removing its only entry point. thinking moved from [learning] to [models]. A gap named rather than papered over: no retained sink fact routes by CWE to path, output_encoding, untrusted_destination or prototype -- the same hole the entailment ceiling diagnosed. The test asserts the seeded set and the gap set exactly, so task 4.2 must update it when it authors those sinks. The audit found exactly one genuine orphan: vectorstore.py, a bakeoff harness with no production caller and no CLI, alive only through its own test. The five named subsystems each carry an explicit decision. Manifest committed, with a test asserting the tree still matches it. Group 2 (the Joern go/no-go) has not started. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .../specs/model-grounded-detection/spec.json | 10 +- .kiro/specs/model-grounded-detection/tasks.md | 75 +- .../measurements/2026-09-08-module-audit.json | 888 ++++++++++++++++++ src/openultrasast/cli.py | 658 +------------ src/openultrasast/config.py | 44 +- src/openultrasast/improve/evolve.py | 229 +---- src/openultrasast/improve/validator.py | 102 +- src/openultrasast/learning/__init__.py | 13 - src/openultrasast/learning/acceptance.py | 146 --- src/openultrasast/learning/canaries.py | 227 ----- src/openultrasast/learning/classify.py | 399 -------- src/openultrasast/learning/detectors.py | 211 ----- src/openultrasast/learning/difficulty.py | 139 --- src/openultrasast/learning/journal.py | 237 ----- src/openultrasast/learning/proposer.py | 318 ------- src/openultrasast/learning/publish.py | 263 ------ src/openultrasast/learning/rounds.py | 761 --------------- src/openultrasast/learning/scoring.py | 405 -------- src/openultrasast/learning/split.py | 64 -- src/openultrasast/learning/verifiers.py | 140 --- src/openultrasast/model/__init__.py | 7 + src/openultrasast/model/audit.py | 166 ++++ .../{learning => model}/candidates.py | 53 +- .../{learning => model}/endpoint.py | 10 +- src/openultrasast/model/specs.py | 156 +++ .../families.py => model/taxonomy.py} | 0 src/openultrasast/pairs.py | 198 +--- src/openultrasast/reports.py | 23 - src/openultrasast/semantic/__init__.py | 5 - src/openultrasast/semantic/ir.py | 12 + src/openultrasast/semantic/loo.py | 304 ------ src/openultrasast/semantic/mechanisms.py | 363 ------- .../semantic/obligations/operations.py | 3 +- .../semantic/obligations/shapes.py | 3 +- src/openultrasast/semantic/prove_budget.py | 12 - src/openultrasast/semantic/seed.py | 231 ----- src/openultrasast/semantic/variant_search.py | 125 --- src/openultrasast/semantic/variants.py | 347 ------- src/openultrasast/vectorstore.py | 69 -- tests/test_candidate_enumeration.py | 32 +- tests/test_degradation_matrix.py | 65 -- tests/test_fixture_difficulty.py | 117 --- tests/test_index.py | 13 - tests/test_learning_acceptance.py | 115 --- tests/test_learning_aggregation.py | 116 --- tests/test_learning_canaries.py | 197 ---- tests/test_learning_classifier_report.py | 115 --- tests/test_learning_classify.py | 163 ---- tests/test_learning_cli.py | 302 ------ tests/test_learning_cutoff.py | 215 ----- tests/test_learning_detectors.py | 167 ---- tests/test_learning_journal.py | 112 --- tests/test_learning_pairs_evidence.py | 151 --- tests/test_learning_pairs_path.py | 110 --- tests/test_learning_proposer.py | 222 ----- tests/test_learning_publish.py | 209 ----- tests/test_learning_refusals.py | 94 -- tests/test_learning_reliability.py | 56 -- tests/test_learning_round4_evidence.py | 247 ----- tests/test_learning_round_evidence.py | 428 --------- tests/test_learning_round_loop.py | 230 ----- tests/test_learning_rounds.py | 377 -------- tests/test_learning_scan.py | 109 --- tests/test_learning_scoring.py | 185 ---- tests/test_learning_split.py | 148 --- tests/test_learning_thinking.py | 69 -- tests/test_learning_verifiers.py | 145 --- tests/test_loo.py | 235 ----- tests/test_mechanism_export.py | 74 -- tests/test_mechanism_lever.py | 112 --- tests/test_mechanism_loop.py | 330 ------- tests/test_mechanism_reports.py | 141 --- tests/test_mechanism_seed.py | 112 --- ...ing_endpoint.py => test_model_endpoint.py} | 46 +- tests/test_model_specs.py | 101 ++ ...ing_families.py => test_model_taxonomy.py} | 12 +- tests/test_module_audit.py | 45 + tests/test_obligation_check.py | 33 +- tests/test_obligation_export.py | 160 ---- tests/test_pair_corpus.py | 49 - tests/test_pair_registration_context.py | 2 +- tests/test_semantic_pipeline.py | 80 +- tests/test_subtraction_complete.py | 81 ++ tests/test_variant_scan.py | 130 --- tests/test_variant_search.py | 124 --- tests/test_variants.py | 76 -- 86 files changed, 1710 insertions(+), 12158 deletions(-) create mode 100644 benchmarks/measurements/2026-09-08-module-audit.json delete mode 100644 src/openultrasast/learning/__init__.py delete mode 100644 src/openultrasast/learning/acceptance.py delete mode 100644 src/openultrasast/learning/canaries.py delete mode 100644 src/openultrasast/learning/classify.py delete mode 100644 src/openultrasast/learning/detectors.py delete mode 100644 src/openultrasast/learning/difficulty.py delete mode 100644 src/openultrasast/learning/journal.py delete mode 100644 src/openultrasast/learning/proposer.py delete mode 100644 src/openultrasast/learning/publish.py delete mode 100644 src/openultrasast/learning/rounds.py delete mode 100644 src/openultrasast/learning/scoring.py delete mode 100644 src/openultrasast/learning/split.py delete mode 100644 src/openultrasast/learning/verifiers.py create mode 100644 src/openultrasast/model/__init__.py create mode 100644 src/openultrasast/model/audit.py rename src/openultrasast/{learning => model}/candidates.py (88%) rename src/openultrasast/{learning => model}/endpoint.py (96%) create mode 100644 src/openultrasast/model/specs.py rename src/openultrasast/{learning/families.py => model/taxonomy.py} (100%) delete mode 100644 src/openultrasast/semantic/loo.py delete mode 100644 src/openultrasast/semantic/mechanisms.py delete mode 100644 src/openultrasast/semantic/prove_budget.py delete mode 100644 src/openultrasast/semantic/seed.py delete mode 100644 src/openultrasast/semantic/variant_search.py delete mode 100644 src/openultrasast/semantic/variants.py delete mode 100644 src/openultrasast/vectorstore.py delete mode 100644 tests/test_fixture_difficulty.py delete mode 100644 tests/test_learning_acceptance.py delete mode 100644 tests/test_learning_aggregation.py delete mode 100644 tests/test_learning_canaries.py delete mode 100644 tests/test_learning_classifier_report.py delete mode 100644 tests/test_learning_classify.py delete mode 100644 tests/test_learning_cli.py delete mode 100644 tests/test_learning_cutoff.py delete mode 100644 tests/test_learning_detectors.py delete mode 100644 tests/test_learning_journal.py delete mode 100644 tests/test_learning_pairs_evidence.py delete mode 100644 tests/test_learning_pairs_path.py delete mode 100644 tests/test_learning_proposer.py delete mode 100644 tests/test_learning_publish.py delete mode 100644 tests/test_learning_refusals.py delete mode 100644 tests/test_learning_reliability.py delete mode 100644 tests/test_learning_round4_evidence.py delete mode 100644 tests/test_learning_round_evidence.py delete mode 100644 tests/test_learning_round_loop.py delete mode 100644 tests/test_learning_rounds.py delete mode 100644 tests/test_learning_scan.py delete mode 100644 tests/test_learning_scoring.py delete mode 100644 tests/test_learning_split.py delete mode 100644 tests/test_learning_thinking.py delete mode 100644 tests/test_learning_verifiers.py delete mode 100644 tests/test_loo.py delete mode 100644 tests/test_mechanism_export.py delete mode 100644 tests/test_mechanism_lever.py delete mode 100644 tests/test_mechanism_loop.py delete mode 100644 tests/test_mechanism_reports.py delete mode 100644 tests/test_mechanism_seed.py rename tests/{test_learning_endpoint.py => test_model_endpoint.py} (85%) create mode 100644 tests/test_model_specs.py rename tests/{test_learning_families.py => test_model_taxonomy.py} (93%) create mode 100644 tests/test_module_audit.py delete mode 100644 tests/test_obligation_export.py create mode 100644 tests/test_subtraction_complete.py delete mode 100644 tests/test_variant_scan.py delete mode 100644 tests/test_variant_search.py delete mode 100644 tests/test_variants.py diff --git a/.kiro/specs/model-grounded-detection/spec.json b/.kiro/specs/model-grounded-detection/spec.json index b106720..83b4d93 100644 --- a/.kiro/specs/model-grounded-detection/spec.json +++ b/.kiro/specs/model-grounded-detection/spec.json @@ -1,9 +1,9 @@ { "feature_name": "model-grounded-detection", "created_at": "2026-09-08T00:00:00Z", - "updated_at": "2026-09-08T04:00:00Z", + "updated_at": "2026-09-08T10:00:00Z", "language": "en", - "phase": "tasks", + "phase": "implementation", "approvals": { "requirements": { "generated": true, @@ -15,14 +15,14 @@ }, "tasks": { "generated": true, - "approved": false + "approved": true } }, - "ready_for_implementation": false, + "ready_for_implementation": true, "supersedes": [ "learning-harness", "constrained-detector" ], "depends_on": [], - "approval_note": "2026-09-08 corrective brief drafted at the maintainer's direction ('can you produce a corrective spec that supersedes the project and sets a course'). Written after the learning-harness evolve rounds produced zero decisions distinguishable from noise, and after researching Clearwing (/home/mc/Source/clearwing, fork norandom/deepwing), Google Big Sleep, and the DARPA AIxCC 2025 finals. Not approved; requirements not generated. The central recommendation is to adopt Clearwing as the execution substrate rather than rebuild, and to contribute the web/logic detection path (taxonomy + IR candidate enumeration + obligation checker + a differential web oracle) that Clearwing's crash-oracle model does not reach. ; 2026-09-08 renamed execution-grounded-detection -> model-grounded-detection after the maintainer's correction: execution is not possible for all SAST targets and the bugs we own do not crash, so the arbiter is a program model built deterministically from code (IR candidate enumeration, binding-chain dataflow, obligation/guard-dominance) that checks LLM claims, producing a static evidence ladder (suspicion -> model_corroborated -> model_entailed); execution is a deferred tier for memory-safety-in-C only, adopted from Clearwing when a class needs it. ; 2026-09-08 requirements generated at the maintainer's direction ('write the requirements, and include to throw away broken parts. include the reduction of tests to what makes sense. I think there is an outgrowth problem here'): subtraction is Req 1-3 (remove the noise architecture, reduce 832 tests to surviving behavior, audit 99 modules for orphans) and precedes adoption of the Joern CPG arbiter (Req 4-8), the corpus as calibration set (Req 9), and the deferred Clearwing execution tier (Req 10). ; 2026-09-08 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); design generated: subtraction is phase 1 (gate-protected, ~5000 lines removed before any Joern), a Joern spike on the injection family is phase 2 and can end the design before the integration cost is paid. ; 2026-09-08 design approved and tasks generated ('approve the design and generate the tasks. add infos so that Opus can follow the tasks without running into complexity escalations'): tasks carry an agent-guidance preamble with the eighteen-defect lesson, the gate-baseline safety net, the zero-dep capability-detection pattern, and a full coupling map of every learning/mechanism touchpoint so the subtraction is not crash-driven discovery; group 1 is pure gate-protected subtraction, group 2 is a Joern go/no-go that gates groups 3-6." + "approval_note": "2026-09-08 corrective brief drafted at the maintainer's direction ('can you produce a corrective spec that supersedes the project and sets a course'). Written after the learning-harness evolve rounds produced zero decisions distinguishable from noise, and after researching Clearwing (/home/mc/Source/clearwing, fork norandom/deepwing), Google Big Sleep, and the DARPA AIxCC 2025 finals. Not approved; requirements not generated. The central recommendation is to adopt Clearwing as the execution substrate rather than rebuild, and to contribute the web/logic detection path (taxonomy + IR candidate enumeration + obligation checker + a differential web oracle) that Clearwing's crash-oracle model does not reach. ; 2026-09-08 renamed execution-grounded-detection -> model-grounded-detection after the maintainer's correction: execution is not possible for all SAST targets and the bugs we own do not crash, so the arbiter is a program model built deterministically from code (IR candidate enumeration, binding-chain dataflow, obligation/guard-dominance) that checks LLM claims, producing a static evidence ladder (suspicion -> model_corroborated -> model_entailed); execution is a deferred tier for memory-safety-in-C only, adopted from Clearwing when a class needs it. ; 2026-09-08 requirements generated at the maintainer's direction ('write the requirements, and include to throw away broken parts. include the reduction of tests to what makes sense. I think there is an outgrowth problem here'): subtraction is Req 1-3 (remove the noise architecture, reduce 832 tests to surviving behavior, audit 99 modules for orphans) and precedes adoption of the Joern CPG arbiter (Req 4-8), the corpus as calibration set (Req 9), and the deferred Clearwing execution tier (Req 10). ; 2026-09-08 requirements approved by the maintainer in session ('approve the requirements and run the design phase'); design generated: subtraction is phase 1 (gate-protected, ~5000 lines removed before any Joern), a Joern spike on the injection family is phase 2 and can end the design before the integration cost is paid. ; 2026-09-08 design approved and tasks generated ('approve the design and generate the tasks. add infos so that Opus can follow the tasks without running into complexity escalations'): tasks carry an agent-guidance preamble with the eighteen-defect lesson, the gate-baseline safety net, the zero-dep capability-detection pattern, and a full coupling map of every learning/mechanism touchpoint so the subtraction is not crash-driven discovery; group 1 is pure gate-protected subtraction, group 2 is a Joern go/no-go that gates groups 3-6. ; 2026-09-08 tasks approved by the maintainer in session ('approve the tasks and move to 1.) as a whole'); implementing group 1 (subtraction) as a whole." } diff --git a/.kiro/specs/model-grounded-detection/tasks.md b/.kiro/specs/model-grounded-detection/tasks.md index 8719d3b..ea54e92 100644 --- a/.kiro/specs/model-grounded-detection/tasks.md +++ b/.kiro/specs/model-grounded-detection/tasks.md @@ -58,7 +58,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat ## Group 1 — Subtract and audit (gate-protected, no Joern yet) -- [ ] 1.1 Port the security vocabularies out before deleting their modules +- [x] 1.1 Port the security vocabularies out before deleting their modules - Create `src/openultrasast/model/specs.py` and move into it, as data, the knowledge the deleted modules carry: `_GUARD_PATTERNS` and the guard-kind vocabulary from `semantic/variants.py`; the sink token tables (from `benchmarks/pairs/catalog_gen.py::_SINK_TOKENS` and `semantic/facts.py` SinkFacts) as the seed @@ -70,7 +70,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat seed, and that the guard vocabulary is preserved verbatim. `model/specs.py` is read-only data — no LLM and no future optimiser writes it, and a test asserts the module exposes no writer (Req 7.4, the verifier boundary). - _Requirements: 1.2, 7.1, 7.2, 7.3, 7.4_ -- [ ] 1.2 Re-home the three retained modules +- [x] 1.2 Re-home the three retained modules - Move `learning/families.py` → `model/taxonomy.py`, `learning/candidates.py` → `model/candidates.py`, `learning/endpoint.py` → `model/endpoint.py`. Update their imports and every importer. `candidates.py` keeps its behaviour exactly (it is the suspicion-band feeder, Req 8.4) — its tests move with it and stay @@ -80,7 +80,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 4.4, 7.1_ - _Depends: 1.1_ -- [ ] 1.3 Detach the retained paths from the removed code +- [x] 1.3 Detach the retained paths from the removed code - `pairs.py`: remove the hunter-scoring path (`_evaluate_hunter_pair`, `_hunter_outcome`, `_family_metrics`, `PairEvalResult.per_family`, `build_pair_signals` split arg, the `k_runs`/`hunter` plumbing) and its `learning.*` imports; keep `_evaluate_overlay_pair`/`_evaluate_inventory_pair` and the `context_files` @@ -95,7 +95,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 1.3, 1.4, 11.2_ - _Depends: 1.2_ -- [ ] 1.4 Remove the noise architecture and the mechanism/evolve loop +- [x] 1.4 Remove the noise architecture and the mechanism/evolve loop - Delete `learning/{rounds,acceptance,proposer,journal,canaries,verifiers,difficulty,scoring,classify, detectors,split,publish,slices,predicates,judgments}.py` (everything under `learning/` except the three modules re-homed in 1.2, which are already gone from the directory), then remove the now-empty `learning/` @@ -110,7 +110,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 1.1, 1.2, 1.2b, 1.3, 1.4_ - _Depends: 1.3_ -- [ ] 1.5 Reduce the test suite to surviving behaviour +- [x] 1.5 Reduce the test suite to surviving behaviour - For every deleted module, delete the test files that existed only to pin it (`test_learning_*`, `test_mechanism_*`, `test_obligation_*` only where it tested the lever not the checker, `test_pair_*` hunter-scoring cases, etc.) in this same change. For every surviving test, confirm it maps to a surviving @@ -122,7 +122,7 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat - _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5_ - _Depends: 1.4_ -- [ ] 1.6 The module audit manifest +- [x] 1.6 The module audit manifest - Add `model/audit.py` that classifies every `src/**/*.py` as `load_bearing` (reachable from a gate, the scan CLI, or a surviving requirement), `standalone_capability` (its own CLI/users — e.g. `mcp.py`, `skills.py`), or `orphaned` (no importer, no entry, no requirement). Remove the orphans. Give @@ -250,3 +250,66 @@ queries in group 4). The gates, `redaction.py`, `pairs.py` overlay/inventory pat mistake this whole spec was written to stop. - `(P)` parallelism is deliberately absent: the subtraction is sequential by dependency, and the Joern spike gates everything after it. Do not fan out. + +## Implementation Notes — group 1 (2026-09-08) + +Landed as one change. Gate baseline captured before the first edit and re-checked after every step; the three +gates are **byte-identical to `2026-09-06-gate-baseline.txt`** at the end, as Req 1.4 and 11.2 require. + +**What the subtraction cost and bought.** 99 source modules -> 81; 832 tests -> 646 (642 passing, 4 skipped); +the whole `learning/` package (16 modules, ~4,100 lines) plus six `semantic/` modules removed. mypy clean over +82 files, ruff clean, core `dependencies = []` untouched. + +**Sequence that made it safe.** Vocabularies ported first (1.1), importers detached second (1.2-1.3), deletion +third (1.4). Every import break during 1.4 was therefore a detachment miss in the importer, fixed there — no +module was ever restored to satisfy one. Two such misses appeared, both handled as a class rather than an +instance: `semantic/obligations/{operations,shapes}.py` both imported `trailing_name` from the deleted +`variants.py`, so the helper was re-homed to `semantic/ir.py` (its natural place — it is a pure helper over +`CallSite.name`) and both importers updated in the same edit. + +**Deviations from the design, each recorded with its reason.** + +1. *`improve/evolve.py` and `improve/validator.py` were stripped, not deleted.* The design's Removed list named + them whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop + that `ousast improve` still runs from benchmark evidence. That loop has no LLM in it, predates the noise + architecture, and is not what this feature supersedes; deleting the files would have taken a retained + capability with them. `test_subtraction_complete` now asserts the lever's absence **by symbol**, so it + cannot creep back. +2. *The `learning` CLI group became `model candidates`.* Deleting the group outright would have orphaned + `model/candidates.py` — the retained suspicion-band enumerator — by removing its only entry point, and + Req 3 would then have flagged it for deletion. The offline ceiling report survives as `ousast model + candidates`. +3. *`thinking` moved from `[learning]` to `[models]`.* It is a model setting, and `LearningConfig` was removed; + its coverage moved to `test_model_endpoint` rather than being dropped. +4. *Three helpers were re-homed rather than lost:* `Region` (promoted from `_Region`, since the detector module + that owned the type is gone), `labeled_family` and `labeled_spans` (from the deleted `rounds`/`scoring`). + `labeled_family` no longer falls back to the LLM classifier — an unlabeled pair is honestly `unknown`, the + same discipline the evidence ladder applies to a claim the model cannot arbitrate. + +**A measured gap, named rather than papered over (Req 9.2).** Task 1.1's observable asked that every family +have a `TaintSpec` or `DominanceSpec` seed. It cannot today: the retained sink facts have **no sink whose CWE +routes to `path`, `output_encoding`, `untrusted_destination` or `prototype`** — the same hole the entailment +ceiling diagnosed ("closed literal sink table: misses per-family sinks"). Rather than weaken the test, it now +asserts the seeded set *and* the gap set exactly, so neither can change silently, and task 4.2 must update it +when it authors those sinks. + +**Test reduction (Req 2), by cluster.** 34 files deleted whole: 21 `test_learning_*` (rounds, acceptance, +scoring, classify, detectors, proposer, journal, canaries, verifiers, publish, split, cli, scan, pairs_path, +cutoff, reliability, aggregation, classifier_report, pairs_evidence, round evidence x2, thinking), 4 +`test_mechanism_*`, `test_variants`, `test_variant_search`, `test_loo`, `test_obligation_export`, +`test_mechanism_reports`, `test_fixture_difficulty`. Partial trims where a file mixed retained and removed +behaviour: `test_semantic_pipeline` (3 mechanism tests of 12), `test_degradation_matrix` (3 rows for removed +capabilities), `test_pair_corpus` (the hunter scorer), `test_obligation_check` (the `known_fix` assertion, +which depended on the deleted store — the id contract, suspicion rung and tags it also covered are kept), +`test_index` (the bakeoff test). Two files re-homed with their subject: `test_learning_families` -> +`test_model_taxonomy`, `test_learning_endpoint` -> `test_model_endpoint`. + +**The audit (Req 3) found exactly one genuine orphan:** `vectorstore.py`, a store bakeoff harness with no +production caller and no CLI entry, alive only through its own test — removed with it. The five named +subsystems each carry a decision: `mcp`, `skills`, `fusion`, `harness_ext`, `hunter_harness` are all +`standalone_capability` (own entry point or protocol), joined by `stage_processors` and `slot_contract`, which +the zero-dependency guard in `test_gate` pins. Manifest committed at +`benchmarks/measurements/2026-09-08-module-audit.json`; a test asserts the tree still matches it. + +Group 2 (the Joern go/no-go) has not started. `semantic/engines.py` already exposes `joern_available`, which +the `CpgBackend` capability probe in task 2.1 should build on rather than duplicate. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json new file mode 100644 index 0000000..7cefec7 --- /dev/null +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -0,0 +1,888 @@ +{ + "counts": { + "load_bearing": 74, + "standalone_capability": 7 + }, + "generated": "2026-09-08", + "kept_with_a_lever_stripped": { + "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." + }, + "modules": 81, + "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", + "orphaned": [], + "removed_in_this_pass": { + "learning/ (16 modules)": "the noise architecture: rounds, acceptance, proposer, journal, canaries, verifiers, difficulty, scoring, classify, detectors, split, publish. Taxonomy, candidates and endpoint were re-homed to model/ first.", + "semantic/loo.py": "leave-one-out scoring of the mechanism store", + "semantic/mechanisms.py": "the mechanism store and the shape search that fed the optimisation loop", + "semantic/prove_budget.py": "embedding-ordered promotion budget, driven by the mechanism store", + "semantic/seed.py": "the mechanism exporter", + "semantic/variant_search.py": "variant search over the mechanism store", + "semantic/variants.py": "the shape-derivation optimisation surface; its guard vocabulary was ported verbatim to model/specs.py first, and trailing_name re-homed to semantic/ir.py", + "vectorstore.py": "a vector-store bakeoff harness with no production caller and no CLI entry, kept alive only by its own test -- the one genuine orphan the audit found" + }, + "rows": [ + { + "classification": "load_bearing", + "importers": [ + "cli", + "gate", + "improve.evolve", + "map_gate", + "mcp", + "pairs" + ], + "module": "benchmark", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "calibration", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "mcp" + ], + "module": "cli", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity.hints", + "complexity.ledger", + "complexity.map", + "map_gate", + "regress.candidate", + "semantic.prove_filter", + "tool_hunter" + ], + "module": "complexity", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.map" + ], + "module": "complexity.hints", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity", + "complexity.map", + "regress.candidate" + ], + "module": "complexity.ledger", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.hints", + "complexity.ledger", + "map_gate", + "regress.candidate", + "semantic.prove_filter", + "tool_hunter" + ], + "module": "complexity.map", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "complexity", + "complexity.hints", + "complexity.map" + ], + "module": "complexity.signals", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "harness", + "hunter_tools", + "model.endpoint", + "pairs", + "regress.candidate", + "regress.recipes", + "run" + ], + "module": "config", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "benchmark", + "calibration", + "cli", + "complexity.ledger", + "complexity.map", + "complexity.signals", + "fusion", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "rank", + "regress.candidate", + "reports", + "scoring.project_score", + "semantic.obligations.check", + "semantic.overlay", + "semantic.prove_filter", + "stage_processors", + "tool_hunter", + "verification", + "verify_judge" + ], + "module": "findings", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "fusion", + "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "slot_contract" + ], + "module": "harness", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli", + "fusion", + "hunter_harness", + "stage_processors", + "verify_judge" + ], + "module": "harness_ext", + "reason": "HarnessX extension surface; loaded by name when the optional harness is present" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_harness" + ], + "module": "hunter", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "hunter_harness", + "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp" + }, + { + "classification": "load_bearing", + "importers": [ + "tool_hunter" + ], + "module": "hunter_tools", + "reason": "imported by tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve.evolve" + ], + "module": "improve", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve" + ], + "module": "improve.evolve", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve", + "improve.evolve" + ], + "module": "improve.journal", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve", + "improve.evolve" + ], + "module": "improve.validator", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "skills" + ], + "module": "index", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "map_gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "gate", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "preprocess", + "stage_processors" + ], + "module": "mapping", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "cli" + ], + "module": "mcp", + "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "model.specs" + ], + "module": "model", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.audit", + "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "model.candidates", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.endpoint", + "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3" + }, + { + "classification": "load_bearing", + "importers": [], + "module": "model.specs", + "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "model.specs" + ], + "module": "model.taxonomy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "pair_gate", + "reason": "entry point" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve", + "improve.evolve", + "model.candidates", + "pair_gate" + ], + "module": "pairs", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "findings", + "hunter", + "hunter_harness", + "improve.evolve", + "improve.validator", + "scoring.project_score", + "semantic.overlay", + "stage_processors" + ], + "module": "policy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "policy" + ], + "module": "policy.verycode", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "complexity.hints", + "complexity.map", + "complexity.signals", + "findings", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "index", + "map_gate", + "mapping", + "model.candidates", + "pairs", + "rank", + "regress.candidate", + "semantic.overlay", + "skills", + "stage_processors" + ], + "module": "preprocess", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "provenance", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint", + "tool_hunter" + ], + "module": "provider", + "reason": "imported by model.endpoint, tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint", + "tool_hunter" + ], + "module": "provider.openrouter", + "reason": "imported by model.endpoint, tool_hunter" + }, + { + "classification": "load_bearing", + "importers": [ + "calibration", + "cli", + "findings", + "gate", + "hunter", + "hunter_harness", + "hunter_tools", + "improve.evolve", + "map_gate", + "pairs", + "stage_processors" + ], + "module": "rank", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "harness", + "reports", + "semantic.obligations.intent" + ], + "module": "redaction", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "regress.candidate" + ], + "module": "regress", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress" + ], + "module": "regress.candidate", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.recipes", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.safety", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress", + "regress.candidate" + ], + "module": "regress.verdict", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "reports", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "findings", + "hunter", + "hunter_harness", + "improve.evolve", + "improve.validator", + "pairs", + "stage_processors" + ], + "module": "ruleset", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "improve.validator", + "ruleset" + ], + "module": "ruleset.store", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "reports" + ], + "module": "run", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "regress.candidate", + "regress.recipes", + "regress.safety", + "regress.verdict" + ], + "module": "sandbox", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "sandbox" + ], + "module": "sandbox.probe", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "regress.candidate", + "regress.recipes", + "regress.safety", + "sandbox" + ], + "module": "sandbox.runner", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "improve.evolve", + "stage_processors" + ], + "module": "scoring", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "scoring" + ], + "module": "scoring.project_score", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic.cst", + "semantic.engines", + "semantic.functions", + "semantic.ir", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.prove_filter", + "semantic.taint" + ], + "module": "semantic", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.ir" + ], + "module": "semantic.cst", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.overlay" + ], + "module": "semantic.engines", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.cst", + "semantic.engines" + ], + "module": "semantic.extra", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.taint" + ], + "module": "semantic.facts", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "hunter_tools", + "pairs", + "semantic.obligations.shapes", + "semantic.overlay" + ], + "module": "semantic.functions", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "pairs", + "semantic.cst", + "semantic.functions", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.shapes", + "semantic.overlay", + "semantic.taint" + ], + "module": "semantic.ir", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "model.candidates", + "model.specs", + "pairs", + "semantic.obligations.check", + "semantic.obligations.dominance", + "semantic.obligations.intent", + "semantic.obligations.operations", + "semantic.obligations.policy", + "semantic.obligations.shapes", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "semantic.obligations", + "semantic.obligations.intent" + ], + "module": "semantic.obligations.check", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "hunter_tools", + "pairs", + "semantic.obligations.check" + ], + "module": "semantic.obligations.dominance", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.specs", + "pairs", + "semantic.obligations", + "semantic.obligations.check", + "semantic.obligations.operations", + "semantic.obligations.policy", + "semantic.obligations.shapes", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations.facts", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "semantic.obligations.intent", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.candidates", + "semantic.obligations.check", + "semantic.obligations.dominance", + "semantic.obligations.siblings" + ], + "module": "semantic.obligations.operations", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli", + "semantic.obligations.check" + ], + "module": "semantic.obligations.policy", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.obligations", + "semantic.obligations.operations" + ], + "module": "semantic.obligations.shapes", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.obligations.check" + ], + "module": "semantic.obligations.siblings", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic", + "semantic.prove_filter" + ], + "module": "semantic.overlay", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic" + ], + "module": "semantic.prove_filter", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "semantic.overlay" + ], + "module": "semantic.taint", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "standalone_capability", + "importers": [ + "hunter", + "hunter_harness" + ], + "module": "skills", + "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path" + }, + { + "classification": "standalone_capability", + "importers": [ + "stage_processors" + ], + "module": "slot_contract", + "reason": "the slot/contract protocol stage_processors implements" + }, + { + "classification": "standalone_capability", + "importers": [], + "module": "stage_processors", + "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "stages", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "model.endpoint" + ], + "module": "tool_hunter", + "reason": "imported by model.endpoint" + }, + { + "classification": "load_bearing", + "importers": [ + "calibration", + "cli", + "fusion", + "reports", + "verify_judge" + ], + "module": "verification", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + }, + { + "classification": "load_bearing", + "importers": [ + "cli" + ], + "module": "verify_judge", + "reason": "reachable from gate, map_gate, pair_gate or the CLI" + } + ], + "spec": "model-grounded-detection Req 3 (the outgrowth audit)", + "standalone": { + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "slot_contract": "the slot/contract protocol stage_processors implements", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + } +} diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 6d3d65b..e3113a4 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse import json import tempfile -from collections.abc import Callable, Sequence +from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path from time import perf_counter @@ -31,7 +31,6 @@ from .calibration import ( ) from .complexity import map as complexity_map from .complexity.ledger import persist_verdicts -from .complexity.map import Hotspot from .config import ObligationsConfig, load_config, load_dotenv from .findings import StaticFinding, quick_scan_findings, write_findings from .fusion import FusionDecision, fuse_findings_dispatch @@ -42,16 +41,13 @@ from .hunter import run_hunter_pool, write_hunter_trajectories from .hunter_harness import HxScanOrchestrator from .improve import RoundOutcome, run_improvement from .index import build_code_chunks -from .learning.families import FamilyTaxonomy from .mapping import analyze_entry_points, attach_reachability_hints, ingest_sarif, write_entry_points, write_static_hints from .pair_gate import print_pair_metrics from .pairs import ( DEFAULT_CATALOG, SLICE_NAMES, - PairCase, evaluate_catalog, load_pair_catalog, - make_hunter_scan, result_payload, select_profile, select_slice, @@ -61,24 +57,19 @@ from .pairs import ( from .policy import assert_rules_resolve, load_policy from .preprocess import FileTarget, preprocess_repository, write_preprocess_artifact from .provenance import fingerprint -from .provider.openrouter import OpenRouterEmbeddingClient, OpenRouterError from .rank import rank_obligations, rank_targets, write_rankings from .regress import TRIGGERABLE, CandidateVerdict, run_regression, write_verdicts -from .regress.candidate import hotspot_from_finding from .reports import scan_exit_code, write_manifest, write_markdown_report, write_sarif_report from .ruleset import DEFAULT_RULESET_DIR, load_ruleset from .run import ScanRun, create_scan_run from .sandbox import resolve_sandbox_probe, resolve_sandbox_runner from .scoring import build_score_artifact from .semantic import ( - MechanismStore, OverlayRecord, adjudicate, - append_mechanism, filter_promoted_hotspots, finding_from_coverage, load_facts, - order_promotions, write_overlay, ) from .semantic.facts import FactLoadError, SemanticFacts @@ -101,7 +92,7 @@ class ScanOutcome: if TYPE_CHECKING: - from .learning.proposer import Proposer + pass def main(argv: list[str] | None = None) -> int: @@ -154,22 +145,6 @@ def main(argv: list[str] | None = None) -> int: "--profile-tolerance", type=float, default=0.0, help="allowed per-profile drop in pair_correct/Youden before rejecting" ) improve.add_argument("--min-holdout-pairs", type=int, default=5, help="profiles with fewer holdout pairs are reported, not gated") - improve.add_argument( - "--mechanism-candidates", - type=Path, - default=None, - help=( - "exporter candidates the mechanisms lever may admit (default: /.openultrasast/calibration/" - "mechanism-candidates.jsonl, then ./.openultrasast/calibration/mechanism-candidates.jsonl)" - ), - ) - improve.add_argument( - "--mechanism-store", - type=Path, - default=None, - help="scan-time mechanisms.jsonl the lever writes (default: /.openultrasast/calibration/mechanisms.jsonl)", - ) - improve.add_argument("--no-mechanisms", action="store_true", help="disable the mechanisms lever for this run") pairs = subparsers.add_parser( "pairs", @@ -181,72 +156,18 @@ def main(argv: list[str] | None = None) -> int: "--profile", choices=("all", "human", "agent", "mixed", "synthetic"), default="all", help="filter pairs by provenance profile" ) pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split") - pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)") - pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)") pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)") - pairs.add_argument( - "--loo", action="store_true", help="leave-one-out: score each pair against a store seeded from the other trusted pairs" - ) - pairs.add_argument("--loo-out", type=Path, default=None, help="where to write loo.json (default: reports/loo.json)") pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON") - pairs.add_argument( - "--k-runs", type=int, default=None, help="runs per pair on the hunter path; never fewer than three, and three by default" - ) subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration") - mechanisms = subparsers.add_parser("mechanisms", help="maintainer: mechanism memory seeded from trusted pairs") - mechanisms_sub = mechanisms.add_subparsers(dest="mechanisms_command", required=True) - export = mechanisms_sub.add_parser("export", help="derive mechanism records from seeded/reviewed pairs (offline)") - export.add_argument("--slice", choices=SLICE_NAMES, default="all") - export.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - export.add_argument( - "--store", - type=Path, - default=Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl", - help="append-only candidates file the improve lever admits from (the scan reads mechanisms.jsonl next to it; never written here)", - ) - export.add_argument("--json", action="store_true") - - learning = subparsers.add_parser("learning", help="maintainer: the classified detector set and its rounds") - learning_sub = learning.add_subparsers(dest="learning_command", required=True) - for name, help_text in ( - ("classify", "classify every labeled pair and report the classifier against itself"), - ("score", "score a catalog by family, with the denominator every number was computed over"), - ("baseline", "round zero: clone one detector into every family and measure each against itself"), - ("candidates", "offline: what a candidate-driven detector could find, per family and per slice"), - ("round", "one evolve round: propose one change for one family and let the evidence decide"), - ("publish", "regenerate every published number from the artifacts that produced it"), - ): - command = learning_sub.add_parser(name, help=help_text) - command.add_argument("--out", type=Path, default=Path(".openultrasast/learning"), help="where artifacts are read and written") - if name != "publish": - command.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) - command.add_argument("--slice", choices=SLICE_NAMES, default="all") - command.add_argument("--json", action="store_true") - if name == "baseline": - command.add_argument( - "--resume", - action="store_true", - help="keep the pairs a previous run of this baseline already measured and paid for", - ) - if name == "candidates": - command.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites") - if name in {"baseline", "round", "score"}: - command.add_argument("--k-runs", type=int, default=None, help="runs per pair; never fewer than three") - command.add_argument("--model", default=None, help="detector model; artifacts are keyed by it") - command.add_argument("--config", type=Path, default=Path("openultrasast.toml")) - if name == "round": - command.add_argument("--family", required=True) - command.add_argument("--cost-cap-usd", type=float, default=None) - command.add_argument( - "--proposals", - type=Path, - default=None, - help="a JSON list of proposals to try in order; without it the round asks the meta-agent", - ) - if name == "publish": - command.add_argument("--measurements", type=Path, default=Path("benchmarks/measurements")) - command.add_argument("--roadmap", type=Path, default=Path(".kiro/steering/roadmap.md")) + model_cmd = subparsers.add_parser("model", help="maintainer: the model layer that arbitrates detector claims") + model_sub = model_cmd.add_subparsers(dest="model_command", required=True) + candidates = model_sub.add_parser("candidates", help="offline: what the candidate enumerator can reach, per family and per slice") + candidates.add_argument("--out", type=Path, default=Path(".openultrasast/model"), help="where artifacts are written") + candidates.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + candidates.add_argument("--slice", choices=SLICE_NAMES, default="all") + candidates.add_argument("--generator", choices=("ir", "ruleset"), default="ir", help="which pass enumerates the candidate sites") + candidates.add_argument("--json", action="store_true") args = parser.parse_args(argv) if args.command == "scan": @@ -268,9 +189,6 @@ def main(argv: list[str] | None = None) -> int: pair_catalog=None if args.no_pair_gate else args.pair_catalog, profile_tolerance=args.profile_tolerance, min_holdout_pairs=args.min_holdout_pairs, - mechanism_candidates=args.mechanism_candidates, - mechanism_store=args.mechanism_store, - mechanisms_enabled=not args.no_mechanisms, ) if args.command == "pairs": return _pairs( @@ -279,17 +197,10 @@ def main(argv: list[str] | None = None) -> int: json_out=args.json, profile=args.profile, split=args.split, - hunter=args.hunter, - hunter_model=args.hunter_model, pointers=args.pointers, - loo=args.loo, - loo_out=args.loo_out, - k_runs=args.k_runs, ) - if args.command == "mechanisms": - return _mechanisms_export(args.catalog, args.slice, args.store, json_out=args.json) - if args.command == "learning": - return _learning(args) + if args.command == "model": + return _model_candidates(args) if args.command == "mcp": from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional @@ -478,12 +389,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut built_map = None overlay_records: list[OverlayRecord] = [] wrote_overlay = False - variants_payload: dict[str, object] | None = None - variant_findings: list[StaticFinding] = [] - mechanisms_cited: dict[str, dict[str, object]] = {} obligations_payload: dict[str, object] | None = None - learning_payload: dict[str, object] | None = None - learning_cited: dict[str, dict[str, object]] = {} obligations_cited: dict[str, dict[str, object]] = {} if Stage.MAP in plan.requested: built_map = runtime.run_stage( @@ -517,19 +423,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if extra: findings = findings + extra write_findings(findings, findings_path) - if config.variants.enabled: - variant_findings, overlay_records, variants_payload, mechanisms_cited = _search_variants( - root=run.target, - targets=targets, - overlay_records=overlay_records, - facts=facts_or_error, - max_mechanisms=config.variants.max_mechanisms, - runtime=runtime, - ) - if variant_findings: - findings = findings + variant_findings - write_findings(findings, findings_path) - write_overlay(overlay_records, overlay_path) if config.obligations.enabled: obligation_findings, obligations_payload, obligations_cited = _check_obligations( root=run.target, @@ -543,18 +436,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if obligation_findings: findings = rank_obligations(findings + obligation_findings) write_findings(findings, findings_path) - if config.learning.enabled: - family_findings, learning_payload, learning_cited = _run_family_detectors( - root=run.target, - hotspots=built_map.hotspots if built_map is not None else (), - cases=None, - settings=config.learning, - config=config, - runtime=runtime, - ) - if family_findings: - findings = findings + family_findings - write_findings(findings, findings_path) plan = record_completed(plan, Stage.MAP) if not hunter_model: runtime.state["degradations"].append(skip_as_degradation(Stage.MAP, "hunter_model_unavailable")) @@ -588,33 +469,11 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut sandbox = resolve_sandbox_runner() hotspots = built_map.hotspots if built_map is not None else () hotspots = filter_promoted_hotspots(hotspots, overlay_records, findings) - store = MechanismStore(run.target / CALIBRATION_DIR / "mechanisms.jsonl") - embed_client = None - embed_model = config.embeddings.model - if embed_model: - try: - embed_client = OpenRouterEmbeddingClient.from_env() - except OpenRouterError: - embed_client = None - hotspots, budget_degradation = order_promotions( - hotspots, - overlay_records, - findings, - store=store, - client=embed_client, - model=embed_model, - ) - if budget_degradation: - runtime.state["degradations"].append({"stage": "prove_budget", "reason": budget_degradation}) promoted = [ finding for finding in findings if finding.finding_id in {record.proposal_id for record in overlay_records if record.disposition == "promote"} ] - # Variant findings are candidates after the promotions (Req 3.4): same safety check, same cap, lowest rank. - variant_ids = {finding.finding_id for finding in variant_findings} - promoted = promoted + [finding for finding in findings if finding.finding_id in variant_ids] - hotspots = hotspots + tuple(_hotspot_from_variant(finding) for finding in findings if finding.finding_id in variant_ids) cwe_by_rule_id = {rule.rule_id: rule.cwe for rule in ruleset} rule_cwe_by_finding = {finding.finding_id: cwe_by_rule_id.get(finding.finding_id.split(":", 1)[0], "") for finding in promoted} languages_by_path = {target.path: target.language for target in targets} @@ -640,7 +499,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut if proven_ids and any(finding.finding_id.startswith("obligation:") for finding in findings): findings = rank_obligations(findings, proven_ids=proven_ids) write_findings(findings, findings_path) - _append_proven_mechanisms(store, records, overlay_records, findings, run.target) verdict_records = records wrote_verdicts = True plan = record_completed(plan, Stage.REGRESS) @@ -655,10 +513,8 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut complexity_map=built_map, verdicts=verdict_records if wrote_verdicts else None, overlay=overlay_records if wrote_overlay else None, - mechanisms=mechanisms_cited or None, obligations=obligations_cited or None, obligations_summary=obligations_payload or None, - learning=learning_cited or None, ), ) runtime.run_stage( @@ -668,9 +524,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut verifications, sarif_path, overlay=overlay_records if wrote_overlay else None, - mechanisms=mechanisms_cited or None, obligations=obligations_cited or None, - learning=learning_cited or None, ), ) runtime.run_stage( @@ -699,9 +553,7 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut complexity=complexity_payload, worth_fixing=worth_fixing_payload, provenance=provenance.to_dict(), - variants=variants_payload, obligations=obligations_payload, - learning=learning_payload, ), ) runtime.finish(status="succeeded") @@ -716,35 +568,6 @@ def _run_scan(path: Path, config_path: Path, mode: str, fail_on: str) -> ScanOut ) -def _append_proven_mechanisms( - store: MechanismStore, - verdicts: tuple[CandidateVerdict, ...], - overlay_records: list[OverlayRecord], - findings: list[StaticFinding], - target: Path, -) -> None: - overlay_by_id = {record.proposal_id: record for record in overlay_records} - finding_by_id = {finding.finding_id: finding for finding in findings} - for verdict in verdicts: - if verdict.verdict != TRIGGERABLE: - continue - for finding_id in verdict.inventory_finding_ids: - record = overlay_by_id.get(finding_id) - if record is None or record.disposition != "promote": - continue - finding = finding_by_id.get(finding_id) - append_mechanism( - store, - summary=record.reason, - cwe=record.cwe, - language=record.language or verdict.language, - tags=finding.tags if finding is not None else (), - what_made_it_exploitable=record.reason, - source_finding_id=finding_id, - source_repo=str(target), - ) - - def _attach_tool_hunter( *, root: Path, @@ -900,9 +723,6 @@ def _improve( pair_catalog: Path | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, - mechanisms_enabled: bool = True, ) -> int: if not manifest_path.exists() or not manifest_path.is_file(): raise SystemExit(f"benchmark manifest is not a file: {manifest_path}") @@ -913,19 +733,6 @@ def _improve( pair_cases = select_split(load_pair_catalog(pair_catalog), "holdout") manifest = load_benchmark_manifest(manifest_path) target = resolve_benchmark_source(manifest_path, manifest) - # corpus-seeded-mechanisms Req 5: the lever runs when a candidates file exists (exporter output) and the pair gate is on. - # Candidates: an explicit path, else the target's calibration dir, else the working directory's (where `mechanisms export` - # writes by default). The store always lives where a scan of this target reads it unless overridden. - calibration = target / CALIBRATION_DIR - candidates_path: Path | None = mechanism_candidates - if candidates_path is None: - for candidate in (calibration / "mechanism-candidates.jsonl", Path(CALIBRATION_DIR) / "mechanism-candidates.jsonl"): - if candidate.is_file(): - candidates_path = candidate - break - if not mechanisms_enabled or not pair_cases or candidates_path is None or not candidates_path.is_file(): - candidates_path = None - store_path = mechanism_store if mechanism_store is not None else calibration / "mechanisms.jsonl" policy = load_policy() # The loop writes its accepted ledger where `scan`/`benchmark` read it, so a @@ -935,11 +742,6 @@ def _improve( if dry_run: ledger_path = Path(scratch) / "rule_policy.json" journal_path = Path(scratch) / "improve_journal.json" - if candidates_path is not None: # a dry run must not mutate the scan-time store either - scratch_store = Path(scratch) / "mechanisms.jsonl" - if store_path.is_file(): - scratch_store.write_bytes(store_path.read_bytes()) - store_path = scratch_store else: ledger_path = ledger or default_ledger journal_path = journal or ledger_path.with_name("improve_journal.json") @@ -957,8 +759,6 @@ def _improve( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, - mechanism_candidates=candidates_path, - mechanism_store=store_path if candidates_path is not None else None, ) _print_improve_outcomes(outcomes, manifest_path, target, ledger_path, dry_run=dry_run) return 0 @@ -971,30 +771,13 @@ def _pairs( json_out: bool, profile: str = "all", split: str = "all", - hunter: bool = False, - hunter_model: str | None = None, pointers: bool = False, - loo: bool = False, - loo_out: Path | None = None, - k_runs: int | None = None, ) -> int: if not catalog.exists() or not catalog.is_file(): raise SystemExit(f"pair catalog is not a file: {catalog}") cases = select_split(select_profile(select_slice(load_pair_catalog(catalog), slice_name), profile), split) - if loo: - return _pairs_loo(cases, loo_out or Path("reports") / "loo.json", json_out=json_out) - scan = None - if hunter: - config_path = Path("openultrasast.toml") - config = load_config(config_path if config_path.exists() else None) - model = hunter_model or config.models.hunter or "" - client = tool_hunter.resolve_hunter_client() - if model and client is not None: - from .learning.families import load_families - - scan = make_hunter_scan(client, model, taxonomy=load_families()) try: - result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None, k_runs=k_runs) + result = evaluate_catalog(cases, pointers=True if pointers else None) except ValueError as exc: raise SystemExit(f"--k-runs: {exc}") from exc if json_out: @@ -1005,50 +788,6 @@ def _pairs( return 0 -def _search_variants( - *, - root: Path, - targets: list[FileTarget], - overlay_records: list[OverlayRecord], - facts: SemanticFacts | FactLoadError, - max_mechanisms: int, - runtime: HarnessRuntime, -) -> tuple[list[StaticFinding], list[OverlayRecord], dict[str, object], dict[str, dict[str, object]]]: - """Third MAP proposer (corpus-seeded-mechanisms Req 3): shapes from the mechanism store, merged with overlay flows. - - Also returns the mechanisms the findings cite (summary, guard, pairs, cwe) for the reports (Req 6.1). - """ - from .semantic.mechanisms import corpus_mechanisms - from .semantic.variant_search import hits_to_findings, search_tree - - store = MechanismStore(root / CALIBRATION_DIR / "mechanisms.jsonl") - empty: dict[str, object] = {"mechanisms_searched": 0, "files_searched": 0, "findings": 0, "merged_into_overlay": 0} - if isinstance(facts, FactLoadError): - return [], overlay_records, empty, {} - result = runtime.run_stage("variants", lambda: search_tree(root, targets, store, facts, max_mechanisms=max_mechanisms)) - for degradation in result.degradations: - runtime.state["degradations"].append(degradation) - records_by_id = {record.id: record for record in corpus_mechanisms(store.load())} - summaries = {record.id: (record.summary, record.pairs, record.cwe) for record in records_by_id.values()} - findings, records = hits_to_findings(result.hits, overlay_records, summaries) - merged = sum(1 for before, after in zip(overlay_records, records, strict=True) if before.mechanism_id is None and after.mechanism_id) - payload: dict[str, object] = { - "mechanisms_searched": result.mechanisms_searched, - "files_searched": result.files_searched, - "findings": len(findings), - "merged_into_overlay": merged, - } - cited_ids = {tag.split(":", 1)[1] for finding in findings for tag in finding.tags if tag.startswith("mechanism:")} | { - str(record.mechanism_id) for record in records if record.mechanism_id - } - cited: dict[str, dict[str, object]] = { - mechanism_id: {"summary": record.summary, "guard": record.guard, "pairs": list(record.pairs), "cwe": record.cwe} - for mechanism_id, record in records_by_id.items() - if mechanism_id in cited_ids - } - return findings, records, payload, cited - - def _check_obligations( *, root: Path, @@ -1065,7 +804,7 @@ def _check_obligations( enter the sandbox candidate set. """ from .semantic.ir import parse_file - from .semantic.obligations import load_obligation_facts, obligation_mechanisms + from .semantic.obligations import load_obligation_facts from .semantic.obligations.check import check_obligations, findings_to_static from .semantic.obligations.dominance import OrderDominance from .semantic.obligations.policy import PolicyError, load_declared_policy @@ -1089,7 +828,6 @@ def _check_obligations( policy = load_declared_policy(policy_path) except PolicyError as exc: runtime.state["degradations"].append({"stage": "obligations", "reason": "policy_invalid", "detail": str(exc)[:200]}) - store = MechanismStore(root / CALIBRATION_DIR / "mechanisms.jsonl") result = runtime.run_stage( "obligations", lambda: check_obligations( @@ -1100,7 +838,7 @@ def _check_obligations( policy=policy, paths=(), dominance=OrderDominance(texts=texts), - store_shapes=obligation_mechanisms(store.load()), + store_shapes=(), # known-fix hints came from the deleted mechanism store min_siblings=settings.min_siblings, ), ) @@ -1138,292 +876,6 @@ def _check_obligations( return statics, payload, cited -def _run_family_detectors( - *, - root: Path, - hotspots: Sequence[object], - cases: object, - settings: object, - config: object, - runtime: HarnessRuntime, -) -> tuple[list[StaticFinding], dict[str, object], dict[str, dict[str, object]]]: - """Fifth MAP proposer (learning-harness Req 6.3, 7.1): one detector per admitted family, plus the generalist. - - Every claim passes through its family's verifier and carries what produced it. Nothing rises above - suspicion here: the verifier writes a tag, and the evidence ladder is another boundary's to move. - """ - del cases - from .learning.classify import classify_region - from .learning.detectors import DetectorConfigError, Region, load_family_configs, run_region_detectors - from .learning.endpoint import resolve_chat_endpoint, resolve_models - from .learning.families import FamiliesError, load_families - from .learning.verifiers import apply_verification, verifier_for - - empty: dict[str, dict[str, object]] = {} - configs_dir = root / str(getattr(settings, "configs_dir", ".openultrasast/learning/configs")) - try: - taxonomy = load_families(Path(p) if (p := getattr(settings, "families_path", None)) else None) - configs = load_family_configs(configs_dir, taxonomy) - except (FamiliesError, DetectorConfigError) as exc: - runtime.state["degradations"].append({"stage": "learning", "reason": "learning_configs_unavailable", "detail": str(exc)[:200]}) - return [], {}, empty - if not configs: - runtime.state["degradations"].append( - {"stage": "learning", "reason": "learning_configs_unavailable", "detail": f"no family configurations under {configs_dir}"} - ) - return [], {}, empty - resolved = resolve_chat_endpoint(config) # type: ignore[arg-type] - if resolved is None: - runtime.state["degradations"].append({"stage": "learning", "reason": "learning_endpoint_unavailable"}) - return [], {}, empty - client, endpoint = resolved - model = resolve_models(config)[0] # type: ignore[arg-type] - findings: list[StaticFinding] = [] - per_family: dict[str, int] = {} - per_tier: dict[str, int] = {} - abstained = 0 - seen: set[str] = set() - for hotspot in hotspots: - path = str(getattr(hotspot, "path", "")) - function = getattr(hotspot, "function_name", None) - key = f"{path}::{function}" - if not path or key in seen: - continue - seen.add(key) - answer = classify_region(root, path, function=function, taxonomy=taxonomy, client=None) - per_tier[answer.tier] = per_tier.get(answer.tier, 0) + 1 - abstained += 0 if answer.families else 1 - region = Region(path=path, function=str(function) if function else None, families=answer.families) - - def _run(region: Region = region) -> list[StaticFinding]: - return run_region_detectors(root, region, configs, client=client, model=model) - - produced = runtime.run_stage("learning", _run) - for finding in produced: - family_id = next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), "unknown") - verification = verifier_for(taxonomy.by_id(family_id)).verify(finding, root=root, language="python") - findings.append(apply_verification(finding, verification)) - per_family[family_id] = per_family.get(family_id, 0) + 1 - payload: dict[str, object] = { - "taxonomy_version": taxonomy.version, - "families": per_family, - # What the classifier did on *this* scan. Scoring it needs labels a maintainer stands behind, which a scan - # of someone else's repository does not have, so the block reports the routing rather than an accuracy. - "classifier": { - "regions": len(seen), - "per_tier": dict(sorted(per_tier.items())), - "abstained": abstained, - # Req 2.6/3.7: say out loud that no family declares a parent at this taxonomy version, so the - # partial credit for a parent-child relation never applied to any number here. - "hierarchical_credit": taxonomy.hierarchical, - }, - "round": None, - "endpoint": endpoint.provider, - "regions": len(seen), - } - cited: dict[str, dict[str, object]] = { - finding.finding_id: { - "family": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")), ""), - "detector": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("detector:")), ""), - "verifier": next((tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("verifier:")), ""), - } - for finding in findings - } - return findings, payload, cited - - -def _learning(args: argparse.Namespace) -> int: - """The maintainer's entry points into the classified detector set. Everything degrades with a reason.""" - from .learning.classify import measure_classifier, write_review_queue - from .learning.families import load_families - from .learning.publish import publish - - out: Path = args.out - taxonomy = load_families() - if args.learning_command == "publish": - report = publish(learning_dir=out, measurements_dir=args.measurements, roadmap=args.roadmap) - print(f"learning publish: {len(report.families)} families over {len(report.models)} model(s) -> {args.roadmap}") - for artifact in report.artifacts: - print(f" artifact {artifact}") - return 0 - cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice)) - if args.learning_command == "candidates": - # No model is called anywhere in this command. The ceiling is a property of the corpus and the enumerator, - # so it can end this feature before a cent is spent on it (constrained-detector task 1.3). - from .learning.candidates import ceiling - - coverage = ceiling(cases, taxonomy=taxonomy, generator=args.generator) - out.mkdir(parents=True, exist_ok=True) - payload = coverage.to_dict() - (out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - for family, block in sorted(coverage.per_family.items()): - print( - f"learning candidates {family}: {block['with_candidate']}/{block['pairs']} " - f"= {_as_float(block['ceiling']):.1%} ceiling, sites {block['sites']}" - ) - for slice_name, block in sorted(coverage.per_slice.items()): - print(f" slice {slice_name}: {_as_float(block['ceiling']):.1%} over {block['pairs']} pairs") - return 0 - if args.learning_command == "classify": - measured = measure_classifier(cases, taxonomy, client=None) - out.mkdir(parents=True, exist_ok=True) - payload = measured.to_dict() - (out / "classifier.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - written = write_review_queue(out / "review-queue.jsonl", measured.queue) - if args.json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - print(f"learning classify: classified {measured.classified}, unknown {measured.unknown}, queued {written}") - return 0 - return _learning_run(args, cases, taxonomy) - - -def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: FamilyTaxonomy) -> int: - """score, baseline and round: the three that need a detector, and degrade with a reason without one.""" - from .learning.detectors import load_family_configs, write_default_configs - from .learning.endpoint import resolve_chat_endpoint, resolve_models - from .learning.journal import Archive, LearningJournal - from .learning.rounds import DEFAULT_SLICES, MIN_K_RUNS, load_noise_floors, run_baseline, run_learning_round - from .learning.scoring import aggregate - from .tool_hunter import _SYSTEM_PROMPT - - out: Path = args.out - config = load_config(args.config if args.config.exists() else None) - k_runs = args.k_runs or max(config.learning.k_runs, MIN_K_RUNS) - model = args.model or resolve_models(config)[0] - configs_dir = out / "configs" - if not configs_dir.exists(): - write_default_configs(configs_dir, taxonomy, prompt=_SYSTEM_PROMPT, version="0") # type: ignore[arg-type] - resolved = resolve_chat_endpoint(config) - if resolved is None: - print("learning: learning_endpoint_unavailable; scoring what can be scored without a detector") - client = resolved[0] if resolved else None - # The client meters itself; a run that never reads the meter publishes a cost of zero and can never trip its - # own cap (Req 8.3, 9.8). - meter = getattr(client, "cost_usd", None) - spent: Callable[[], float] | None = meter if callable(meter) else None - configs = load_family_configs(configs_dir, taxonomy) # type: ignore[arg-type] - - def factory(family_config: object) -> Callable[[Path], list[StaticFinding]]: - from .learning.detectors import Region, run_family_detector - - def scan(root: Path) -> list[StaticFinding]: - if client is None: - return [] - region = Region(path=_first_source(root), function=None) - return run_family_detector(root, region, family_config, client=client, model=model) # type: ignore[arg-type] - - return scan - - if args.learning_command == "score": - from .learning.rounds import score_case - - scores = [ - score_case(case, factory(configs[_label(case, taxonomy)]), taxonomy=taxonomy, runs=k_runs, family=_label(case, taxonomy)) - for case in cases - if _label(case, taxonomy) in configs - ] - cutoff = config.learning.cutoff_date or "" - families = {name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, cutoff=cutoff).items()} - if args.json: - print(json.dumps({"families": families}, indent=2, sort_keys=True)) - else: - for name, block in sorted(families.items()): - raw = block.get("post_cutoff") - recent: dict[str, object] = raw if isinstance(raw, dict) else {} - tail = ( - f" post-cutoff({cutoff})={recent.get('scorable', 0)} undated={block['undated']}" - if cutoff - else " post-cutoff=none configured" - ) - print( - f"learning score {name}: scorable={block['scorable']} recall={block['recall']:.3f} youden={block['youden']:.3f}{tail}" - ) - return 0 - if args.learning_command == "baseline": - report = run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=configs_dir, - scan_factory=factory, - model=model, - out_dir=out, - k_runs=k_runs, - cutoff=config.learning.cutoff_date or "", - # `--slice` already chose the rows; without this the baseline's own default filters them out again and - # a run over the memory-safety slice reports nothing at all (Req 8.4). - slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES, - spent_usd=spent, - thinking=config.learning.thinking, - resume=args.resume, - ) - print(f"learning baseline cost: ${report.cost_usd:.4f}") - print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}") - return 0 - journal = LearningJournal(out / "journal.jsonl") - record = run_learning_round( - cases, - family=args.family, - taxonomy=taxonomy, - configs_dir=configs_dir, - proposer=_proposer(args, configs_dir, model, config.harnessx.provider), - scan_factory=factory, - model=model, - journal=journal, - archive=Archive(out / "archive.jsonl"), - floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES, config.learning.thinking), - out_dir=out, - cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, - minibatch=config.learning.minibatch, - k_runs=k_runs, - spent_usd=spent, - # Every other family the corpus actually carries, so a change that reaches beyond its own directory is - # measured rather than assumed impossible (Req 9.5). - sweep_families=tuple(sorted({_label(case, taxonomy) for case in cases} & set(configs) - {args.family})), - ) - print(f"learning round {record.round} {record.family}: {record.outcome} ({record.reason}) cost ${record.cost_usd:.4f}") - return 0 - - -def _as_float(value: object) -> float: - return float(value) if isinstance(value, int | float) else 0.0 - - -def _label(case: PairCase, taxonomy: object) -> str: - """The family that routes this pair: the maintainer's label when there is one, else the classifier's answer. - - Reading only the declared label would send every unlabeled row to the generalist, whose findings earn no credit - against any real family, so a scoreboard built that way reports structural zeros (Req 2.1, 8.2).""" - from .learning.rounds import _family_of - - return _family_of(case, taxonomy) # type: ignore[arg-type] - - -def _proposer(args: argparse.Namespace, configs_dir: Path, model: str, provider: str = "anthropic") -> Proposer: - """`--proposals FILE` runs a recorded sequence offline; without it the round asks the meta-agent.""" - from .learning.proposer import HarnessXProposer, Proposal, ScriptedProposer - - path = getattr(args, "proposals", None) - if path is None: - return HarnessXProposer(configs_dir=configs_dir, model=model, provider=provider) - rows = json.loads(Path(path).read_text(encoding="utf-8")) - return ScriptedProposer( - [ - Proposal( - hypothesis=str(row["hypothesis"]), - lever=row["lever"], - change=dict(row["change"]), - predicted_affected=tuple(row.get("predicted_affected", ())), - predicted_at_risk=tuple(row.get("predicted_at_risk", ())), - ) - for row in rows - ] - ) - - def _first_source(root: Path) -> str: from .preprocess import preprocess_repository @@ -1431,60 +883,12 @@ def _first_source(root: Path) -> str: return targets[0].path if targets else "." -def _hotspot_from_variant(finding: StaticFinding) -> Hotspot: - return hotspot_from_finding(finding, rationale="variant of a known mechanism (suspicion; sandbox may raise)") - - -def _pairs_loo(cases: Sequence[PairCase], out: Path, *, json_out: bool) -> int: - """Req 4: the corpus's own detection rate; written as an artifact, never a gate.""" - from .semantic.loo import evaluate_loo - - result = evaluate_loo(select_vendored(cases)) - payload = result.to_dict() - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - if json_out: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - for slice_name, metrics in result.per_slice.items(): - print( - f"loo {slice_name}: pairs={metrics['pairs']} detected={metrics['detected']} silent={metrics['silent']} " - f"youden={metrics['youden']:+.3f} teaching={metrics['teaching']}" - ) - print(f"artifact={out}") - return 0 - - -def _mechanisms_export(catalog: Path, slice_name: str, store_path: Path, *, json_out: bool) -> int: - from .semantic.mechanisms import MechanismStore - from .semantic.seed import export_mechanisms - - if not catalog.is_file(): - raise SystemExit(f"pair catalog is not a file: {catalog}") - cases = select_vendored(select_slice(load_pair_catalog(catalog), slice_name)) - report = export_mechanisms(cases, MechanismStore(store_path)) - payload = {"slice": slice_name, "store": str(store_path), "pairs": len(cases), **report.to_dict()} - if json_out: - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - print(f"mechanisms export slice={slice_name} pairs={len(cases)} seeded={report.seeded} records={report.records} -> {store_path}") - for pair, reason in report.skipped: - print(f" skip {pair}: {reason}") - return 0 - - def _print_improve_outcomes(outcomes: list[RoundOutcome], manifest_path: Path, target: Path, ledger_path: Path, *, dry_run: bool) -> None: print(f"manifest={manifest_path}") print(f"target={target}") accepted = [o for o in outcomes if o.accepted] for outcome in outcomes: - edits = ( - ", ".join( - [f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits] - + [f"{m.action} {m.mechanism_id}" for m in outcome.mechanism_edits] - ) - or "-" - ) + edits = ", ".join(f"{e.rule_id}:{e.from_status}->{e.to_status}" for e in outcome.edits) or "-" print( f"round {outcome.round}: {outcome.reason} | " f"recall {outcome.recall_before:.2%}->{outcome.recall_after:.2%} " @@ -1506,6 +910,36 @@ def _load_scan_findings(run_dir: Path) -> list[StaticFinding]: return load_findings(run_dir / "findings.json") +def _model_candidates(args: argparse.Namespace) -> int: + """What the candidate enumerator can reach, per family and per slice. No model is called anywhere here. + + The ceiling is a property of the corpus and the enumerator, so it can end a design before a cent is spent + on it — which is what it did for the pattern-ruleset generator (12.1% against the IR's 96.6%). + """ + from .model.candidates import ceiling + from .model.taxonomy import load_families + + cases = select_vendored(select_slice(load_pair_catalog(args.catalog), args.slice)) + coverage = ceiling(cases, taxonomy=load_families(), generator=args.generator) + out: Path = args.out + out.mkdir(parents=True, exist_ok=True) + payload = coverage.to_dict() + (out / "candidate-ceiling.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + for family, block in sorted(coverage.per_family.items()): + ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0 + print( + f"model candidates {family}: {block['with_candidate']}/{block['pairs']} " + f"= {ceiling_value:.1%} ceiling, sites {block['sites']}" + ) + for slice_name, block in sorted(coverage.per_slice.items()): + ceiling_value = float(block["ceiling"]) if isinstance(block["ceiling"], int | float) else 0.0 + print(f" slice {slice_name}: {ceiling_value:.1%} over {block['pairs']} pairs") + return 0 + + def _index(path: Path, config_path: Path, chunk_lines: int) -> int: if not path.exists() or not path.is_dir(): raise SystemExit(f"index path is not a directory: {path}") diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index b47e00d..69d2253 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -36,6 +36,11 @@ class ModelConfig: judge: str | None = None # second, independent judgement before anything is published as proven chat_base_url: str | None = None # chat endpoint; independent of the embedding endpoint chat_api_key_env: str | None = None # environment variable holding that endpoint's key + # Whether the chat model reasons before answering. Off by default because that is what the committed + # baseline was measured under: the provider silently ignores `temperature` while thinking, so thinking was + # disabled to make `temperature: 0` mean something. It did not — the measured run-to-run disagreement came + # from the agentic path, not the decoder — which is why the model layer arbitrates instead of averaging. + thinking: bool = False @dataclass(frozen=True) @@ -138,26 +143,6 @@ class VariantsConfig: max_mechanisms: int = 500 -@dataclass(frozen=True) -class LearningConfig: - # learning-harness: the classified detector set and its rounds. - enabled: bool = False # off until round zero has written the family configurations this reads - families_path: str | None = None # taxonomy file; the shipped ruleset when unset - configs_dir: str = ".openultrasast/learning/configs" - k_runs: int = 5 # runs per pair; a single run misses most real changes - round_cost_cap_usd: float = 10.0 - minibatch: int = 8 # train pairs in a round's first stage - detector_model: str | None = None - judge_model: str | None = None - cutoff_date: str | None = None # rows fixed after this date are reported as a separate slice - # Whether the detector model reasons before answering. Off by default because that is what round zero was - # measured under, not because it is known to be better: the provider silently ignores `temperature` while - # thinking, and thinking was disabled to make `temperature: 0` mean something. It did not — the measured - # disagreement between runs comes from the agentic path, not the decoder — so which mode wins is a question - # for a second baseline rather than for a default (Req 8.3). - thinking: bool = False - - @dataclass(frozen=True) class ObligationsConfig: # authorization-obligations: the obligation checker in MAP (standard/deep only). @@ -183,7 +168,6 @@ class ResolvedConfig: regress: RegressConfig = RegressConfig() variants: VariantsConfig = VariantsConfig() obligations: ObligationsConfig = ObligationsConfig() - learning: LearningConfig = LearningConfig() runs_dir: str = ".openultrasast/runs" @@ -210,7 +194,6 @@ def load_config(config_path: Path | None = None) -> ResolvedConfig: regress=_load_regress(data.get("regress", {})), variants=_load_variants(data.get("variants", {})), obligations=_load_obligations(data.get("obligations", {})), - learning=_load_learning(data.get("learning", {})), runs_dir=os.environ.get("OPENULTRASAST_RUNS_DIR", ".openultrasast/runs"), ) @@ -238,6 +221,7 @@ def _load_models(value: object) -> ModelConfig: judge=_string(data.get("judge")), chat_base_url=_string(data.get("chat_base_url")), chat_api_key_env=_string(data.get("chat_api_key_env")), + thinking=bool(data.get("thinking", False)), ) @@ -385,22 +369,6 @@ def _load_variants(value: object) -> VariantsConfig: ) -def _load_learning(value: object) -> LearningConfig: - data = _section(value) - return LearningConfig( - enabled=bool(data.get("enabled", LearningConfig.enabled)), - families_path=_string(data.get("families_path")), - configs_dir=_string(data.get("configs_dir")) or LearningConfig.configs_dir, - k_runs=_int_value(data.get("k_runs"), LearningConfig.k_runs), - round_cost_cap_usd=_float_value(data.get("round_cost_cap_usd"), LearningConfig.round_cost_cap_usd), - minibatch=_int_value(data.get("minibatch"), LearningConfig.minibatch), - detector_model=_string(data.get("detector_model")), - judge_model=_string(data.get("judge_model")), - cutoff_date=_string(data.get("cutoff_date")), - thinking=bool(data.get("thinking", False)), - ) - - def _load_obligations(value: object) -> ObligationsConfig: data = _section(value) return ObligationsConfig( diff --git a/src/openultrasast/improve/evolve.py b/src/openultrasast/improve/evolve.py index bca4470..e64a11c 100644 --- a/src/openultrasast/improve/evolve.py +++ b/src/openultrasast/improve/evolve.py @@ -26,9 +26,8 @@ from ..preprocess import preprocess_repository from ..rank import rank_targets from ..ruleset import DEFAULT_RULESET_DIR, PatternRule, load_ruleset, read_rule_ledger, write_rule_ledger from ..scoring import build_score_artifact -from ..semantic.facts import SemanticFacts from .journal import append_round, load_journal, next_round_index, reverted_edit_keys -from .validator import EvolveValidator, MechanismEdit, RuleStatusEdit, StrictValidationError, apply_mechanism_edits, edits_to_ledger +from .validator import EvolveValidator, RuleStatusEdit, StrictValidationError, edits_to_ledger @dataclass(frozen=True) @@ -46,9 +45,6 @@ class RoundOutcome: matched_before: int = 0 matched_after: int = 0 profile_regressions: list[str] = field(default_factory=list) - mechanism_edits: list[MechanismEdit] = field(default_factory=list) - mechanisms_before: dict[str, dict[str, float]] = field(default_factory=dict) - mechanisms_after: dict[str, dict[str, float]] = field(default_factory=dict) profiles_under_minimum: list[str] = field(default_factory=list) per_profile_before: dict[str, dict[str, float]] = field(default_factory=dict) per_profile_after: dict[str, dict[str, float]] = field(default_factory=dict) @@ -89,121 +85,6 @@ def evaluate_profiles( } -def evaluate_mechanism_profiles(pair_cases: Sequence[object], store_path: Path) -> dict[str, dict[str, float]]: - """Per-profile pair metrics of variant search with the given scan-time store over the holdout pairs (vendored, gating tiers). - - Split-blind for the same reason as `evaluate_profiles`: the store is the thing the teacher rule governs, and this - function only reads it (learning-harness Req 5.1).""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore - - cases = list(select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS)) - store = MechanismStore(store_path) - by_profile: dict[str, list[tuple[bool, bool]]] = {} - for case in cases: - outcome = score_pair_with_store(case, store, _facts()) - by_profile.setdefault(case.provenance, []).append((outcome.detected, outcome.silent)) - result: dict[str, dict[str, float]] = {} - for profile, items in sorted(by_profile.items()): - pairs = len(items) - correct = sum(1 for detected, silent in items if detected and silent) - recall = sum(1 for detected, _ in items if detected) / pairs - silence = sum(1 for _, silent in items if silent) / pairs - result[profile] = { - "pairs": float(pairs), - "pair_correct": float(correct), - "pair_pass_rate": correct / pairs, - "youden": recall + silence - 1.0, - } - return result - - -def propose_mechanism_edits( - pair_cases: Sequence[object], - candidates: object, - store_path: Path, - degradations: list[dict[str, object]] | None = None, -) -> list[MechanismEdit]: - """Req 5.3: admit a candidate that recovers a currently missed holdout pair without leaking; retract an admitted leaker.""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore, corpus_mechanisms - - cases_all = [case for case in pair_cases if isinstance(case, PairCase)] - cases = list(select_tier(select_vendored(cases_all), GATING_TIERS)) - if not cases: - return [] - facts = _facts() - scan = MechanismStore(store_path) - admitted = {record.id: record for record in corpus_mechanisms(scan.load())} - current = {case.name: score_pair_with_store(case, scan, facts) for case in cases} - edits: list[MechanismEdit] = [] - leakers = sorted({mechanism_id for outcome in current.values() for mechanism_id in outcome.leaked_by if mechanism_id in admitted}) - for mechanism_id in leakers: - leaked = sorted(name for name, outcome in current.items() if mechanism_id in outcome.leaked_by) - edits.append( - MechanismEdit( - action="retract", mechanism_id=mechanism_id, source="loo", rationale=f"leaks on the fixed side of {', '.join(leaked)}" - ) - ) - missed = [case for case in cases if not current[case.name].detected] - from ..learning.split import refuse_if_holdout - - for record in corpus_mechanisms(candidates.load()): # type: ignore[attr-defined] - if record.id in admitted or not missed: - continue - refusal = refuse_if_holdout(record.pairs, cases_all) - if refusal is not None: - # learning-harness Req 5.2: a shape a holdout pair taught can never recover that pair, and the run - # says which pairs it was rather than showing a shorter list with no reason. - if degradations is not None: - degradations.append(refusal.degradation()) - continue - with tempfile.TemporaryDirectory(prefix="ousast-lever-") as scratch: - trial = MechanismStore(Path(scratch) / "trial.jsonl") - trial.append(record) - recovered = [case.name for case in missed if score_pair_with_store(case, trial, facts).detected] - leaks = [case.name for case in cases if not score_pair_with_store(case, trial, facts).silent] - if recovered and not leaks: - edits.append( - MechanismEdit( - action="admit", - mechanism_id=record.id, - source="loo", - rationale=f"recovers holdout {', '.join(recovered)}; taught by {', '.join(record.pairs)}", - ) - ) - return edits - - -def _facts() -> SemanticFacts: - from ..semantic.facts import FactLoadError, load_facts - - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) - - -def _leaking_admissions(edits: Sequence[MechanismEdit], pair_cases: Sequence[object], store_path: Path) -> list[str]: - """Admitted record ids that hit the fixed side of any holdout pair with the store as it stands after the edits.""" - from ..pairs import GATING_TIERS, PairCase, select_tier, select_vendored - from ..semantic.loo import score_pair_with_store - from ..semantic.mechanisms import MechanismStore - - admitted = {edit.mechanism_id for edit in edits if edit.action == "admit"} - if not admitted: - return [] - cases = select_tier(select_vendored([case for case in pair_cases if isinstance(case, PairCase)]), GATING_TIERS) - store = MechanismStore(store_path) - facts = _facts() - leaking: set[str] = set() - for case in cases: - leaking |= set(score_pair_with_store(case, store, facts).leaked_by) & admitted - return sorted(leaking) - - def profile_regressions( before: dict[str, dict[str, float]], after: dict[str, dict[str, float]], @@ -278,9 +159,6 @@ def run_round( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, - scripted_mechanism_edits: Sequence[MechanismEdit] | None = None, ) -> RoundOutcome: policy = policy if policy is not None else load_policy() rounds = load_journal(journal_path) @@ -295,46 +173,18 @@ def run_round( ruleset_by_id = {rule.rule_id: rule for rule in before_rules} edits = propose_status_edits(before.result.metrics.per_rule, ruleset_by_id, current_ledger, blocked) - # corpus-seeded-mechanisms Req 5: the `mechanisms` lever proposes from the holdout pairs, gated like every other edit. - mechanism_edits: list[MechanismEdit] = [] refusals: list[dict[str, object]] = [] - candidates_store = None - if mechanism_candidates is not None and mechanism_store is not None and pair_cases: - from ..semantic.mechanisms import MechanismStore - - candidates_store = MechanismStore(mechanism_candidates) - mechanism_edits = ( - list(scripted_mechanism_edits) - if scripted_mechanism_edits is not None - else propose_mechanism_edits(pair_cases, candidates_store, mechanism_store, degradations=refusals) - ) - blocked_mechanisms = {edit.key() for edit in mechanism_edits} & blocked - mechanism_edits = [edit for edit in mechanism_edits if edit.key() not in blocked_mechanisms] - if not edits and not mechanism_edits: + if not edits: return _outcome(round_index, accepted=False, reason="no_proposals", edits=[], before=before, after=before, degradations=refusals) validator = EvolveValidator() try: for edit in edits: validator.validate(edit, ruleset_by_id, policy) - if mechanism_edits and candidates_store is not None and mechanism_store is not None: - from ..semantic.mechanisms import MechanismStore - - by_id = {record.id: record for record in candidates_store.load()} - admitted_by_id = {record.id: record for record in MechanismStore(mechanism_store).load()} - for mechanism_edit in mechanism_edits: - validator.validate_mechanism(mechanism_edit, by_id, admitted=admitted_by_id) except StrictValidationError as exc: - _record(journal_path, round_index, edits, "rejected", before, before, str(exc), mechanism_edits=mechanism_edits) + _record(journal_path, round_index, edits, "rejected", before, before, str(exc)) return _outcome( - round_index, - accepted=False, - reason=f"validation_failed: {exc}", - edits=edits, - before=before, - after=before, - mechanism_edits=mechanism_edits, - degradations=refusals, + round_index, accepted=False, reason=f"validation_failed: {exc}", edits=edits, before=before, after=before, degradations=refusals ) candidate = edits_to_ledger(edits, current_ledger) @@ -365,39 +215,10 @@ def run_round( regressed, small = profile_regressions(profiles_before, profiles_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs) if regressed: accepted = False - mechanisms_before: dict[str, dict[str, float]] = {} - mechanisms_after: dict[str, dict[str, float]] = {} - mechanism_reason = "" - if accepted and mechanism_edits and candidates_store is not None and mechanism_store is not None and pair_cases: - # The lever changes what variant search finds, so its gate is the same per-profile clause over the holdout pairs - # scored with the store before and after; an admission that leaks or lowers any profile is reverted byte for byte. - mechanisms_before = evaluate_mechanism_profiles(pair_cases, mechanism_store) - snapshot = apply_mechanism_edits(mechanism_edits, candidates_store, mechanism_store) - mechanisms_after = evaluate_mechanism_profiles(pair_cases, mechanism_store) - mech_regressed, mech_small = profile_regressions( - mechanisms_before, mechanisms_after, tolerance=profile_tolerance, min_pairs=min_holdout_pairs - ) - leaking = _leaking_admissions(mechanism_edits, pair_cases, mechanism_store) - improved = any( - mechanisms_after[p]["pair_correct"] > mechanisms_before.get(p, {}).get("pair_correct", 0.0) for p in mechanisms_after - ) or any(edit.action == "retract" for edit in mechanism_edits) - if mech_regressed or leaking or not improved: - snapshot.restore() - accepted = False - regressed = regressed + mech_regressed - if leaking: - mechanism_reason = f"mechanism_leak:{leaking[0]}" # an admission that fires on a holdout fixed side (Req 5.3) - elif mech_regressed: - mechanism_reason = f"profile_regression:{mech_regressed[0]}" - else: - mechanism_reason = "mechanism_no_gain" - small = sorted(set(small) | set(mech_small)) if accepted: if edits: write_rule_ledger(ledger_path, candidate) reason = "accepted" - elif mechanism_reason: - reason = mechanism_reason elif regressed: reason = f"profile_regression:{regressed[0]}" # every regressed profile is listed in profile_regressions else: @@ -412,20 +233,9 @@ def run_round( reason, regressed, small, - mechanism_edits=mechanism_edits, - mechanisms_before=mechanisms_before, - mechanisms_after=mechanisms_after, - candidates=candidates_store, ) outcome = _outcome( - round_index, - accepted=accepted, - reason=reason, - edits=edits, - before=before, - after=after, - mechanism_edits=mechanism_edits, - degradations=refusals, + round_index, accepted=accepted, reason=reason, edits=edits, before=before, after=after, degradations=refusals ) return RoundOutcome( **{ @@ -434,8 +244,6 @@ def run_round( "profiles_under_minimum": small, "per_profile_before": profiles_before, "per_profile_after": profiles_after, - "mechanisms_before": mechanisms_before, - "mechanisms_after": mechanisms_after, } ) @@ -454,8 +262,6 @@ def run_improvement( pair_cases: Sequence[object] | None = None, profile_tolerance: float = 0.0, min_holdout_pairs: int = 5, - mechanism_candidates: Path | None = None, - mechanism_store: Path | None = None, ) -> list[RoundOutcome]: """Run improvement rounds until convergence (no new proposals) or ``max_rounds``.""" outcomes: list[RoundOutcome] = [] @@ -472,8 +278,6 @@ def run_improvement( pair_cases=pair_cases, profile_tolerance=profile_tolerance, min_holdout_pairs=min_holdout_pairs, - mechanism_candidates=mechanism_candidates, - mechanism_store=mechanism_store, ) outcomes.append(outcome) if outcome.reason == "no_proposals": @@ -542,7 +346,6 @@ def _outcome( edits: list[RuleStatusEdit], before: _Eval, after: _Eval, - mechanism_edits: list[MechanismEdit] | None = None, degradations: list[dict[str, object]] | None = None, ) -> RoundOutcome: return RoundOutcome( @@ -550,7 +353,6 @@ def _outcome( accepted=accepted, reason=reason, edits=edits, - mechanism_edits=list(mechanism_edits or []), degradations=list(degradations or []), recall_before=before.recall, recall_after=after.recall, @@ -573,14 +375,7 @@ def _record( reason: str = "", profile_regressions: list[str] | None = None, profiles_under_minimum: list[str] | None = None, - mechanism_edits: Sequence[MechanismEdit] = (), - mechanisms_before: dict[str, dict[str, float]] | None = None, - mechanisms_after: dict[str, dict[str, float]] | None = None, - candidates: object | None = None, ) -> None: - pairs_by_id: dict[str, list[str]] = {} - if candidates is not None: - pairs_by_id = {record.id: list(record.pairs) for record in candidates.load()} # type: ignore[attr-defined] append_round( journal_path, { @@ -592,21 +387,7 @@ def _record( "edits": [ {"key": e.key(), "lever": e.lever, "rule_id": e.rule_id, "from": e.from_status, "to": e.to_status, "rationale": e.rationale} for e in edits - ] - + [ - { - "key": m.key(), - "lever": m.lever, - "action": m.action, - "mechanism_id": m.mechanism_id, - "source": m.source, - "pairs": pairs_by_id.get(m.mechanism_id, []), - "rationale": m.rationale, - } - for m in mechanism_edits ], - "mechanisms_before": mechanisms_before or {}, - "mechanisms_after": mechanisms_after or {}, "recall_before": round(before.recall, 4), "recall_after": round(after.recall, 4), "fp_before": round(before.fp_rate, 4), diff --git a/src/openultrasast/improve/validator.py b/src/openultrasast/improve/validator.py index 807518e..7440d27 100644 --- a/src/openultrasast/improve/validator.py +++ b/src/openultrasast/improve/validator.py @@ -8,11 +8,9 @@ impossible; the validator additionally enforces status/bounds/staging/resolution from __future__ import annotations -import json import re -from collections.abc import Mapping, Sequence -from dataclasses import asdict, dataclass -from pathlib import Path +from collections.abc import Mapping +from dataclasses import dataclass from ..policy import CwePolicy from ..ruleset import PatternRule @@ -62,40 +60,6 @@ class PolicyConstantEdit: return f"policy:{self.name}:{self.from_value}->{self.to_value}" -@dataclass(frozen=True) -class MechanismEdit: - """A `mechanisms`-lever edit (corpus-seeded-mechanisms Req 5): admit or retract one exporter record by id. - - The loop never writes a shape; it may only move a record the exporter derived from a trusted pair into or out of - the scan-time store. Free text is confined to ``rationale``. - """ - - action: str # admit | retract - mechanism_id: str - source: str = "loo" # loo | export - rationale: str = "" - lever: str = "mechanisms" - - def key(self) -> str: - return f"mechanisms:{self.action}:{self.mechanism_id}" - - -@dataclass(frozen=True) -class StoreSnapshot: - """Bytes of the scan-time mechanism store before a round; ``restore`` is the byte-for-byte revert.""" - - path: Path - content: bytes | None # None when the file did not exist - - def restore(self) -> None: - if self.content is None: - if self.path.exists(): - self.path.unlink() - return - self.path.parent.mkdir(parents=True, exist_ok=True) - self.path.write_bytes(self.content) - - @dataclass(frozen=True) class EvolveBounds: k_range: tuple[float, float] = (20.0, 200.0) @@ -113,49 +77,9 @@ class EvolveValidator: self._validate_rule(edit, ruleset_by_id, policy) elif isinstance(edit, PolicyConstantEdit): self._validate_policy(edit) - elif isinstance(edit, MechanismEdit): - raise StrictValidationError("mechanism edits need the exporter candidate set: use validate_mechanism", kind="mechanism_change") else: raise StrictValidationError(f"unknown edit type {type(edit).__name__}", kind="rule_change") - def validate_mechanism( - self, edit: MechanismEdit, candidates: Mapping[str, object], admitted: Mapping[str, object] | None = None - ) -> None: - """Req 5.2: only records the exporter derived from trusted pairs, with a closed guard and identifier-only shape text. - - A retraction may also name a record that is admitted in the scan store but no longer in the candidate set. - """ - from ..semantic.variants import GUARD_KINDS, SOURCE_KINDS - - if edit.action not in MECHANISM_ACTIONS: - raise StrictValidationError(f"mechanism edit action {edit.action!r} is not admit or retract", kind="mechanism_change") - record = candidates.get(edit.mechanism_id) - if record is None and edit.action == "retract" and admitted is not None: - record = admitted.get(edit.mechanism_id) - if record is None: - raise StrictValidationError( - f"unknown mechanism {edit.mechanism_id!r}: not in the exporter candidate set", kind="mechanism_change" - ) - if getattr(record, "origin", "") != "corpus": - raise StrictValidationError( - f"{edit.mechanism_id}: origin {getattr(record, 'origin', '')!r} is not corpus (only exporter records)", - kind="mechanism_change", - ) - shape = getattr(record, "shape", None) - if not isinstance(shape, dict): - raise StrictValidationError(f"{edit.mechanism_id}: record carries no shape", kind="mechanism_change") - if str(shape.get("guard")) not in GUARD_KINDS or str(getattr(record, "guard", "")) not in GUARD_KINDS: - raise StrictValidationError( - f"{edit.mechanism_id}: guard {shape.get('guard')!r} is not a closed guard kind", kind="mechanism_change" - ) - for field_name in ("sink_name", "language", "mechanism"): - value = str(shape.get(field_name, "")) - if not _IDENTIFIER.match(value.replace("-", "_")): - raise StrictValidationError(f"{edit.mechanism_id}: {field_name} {value!r} is not an identifier", kind="mechanism_change") - kinds = shape.get("source_kinds") or [] - if not isinstance(kinds, list) or any(str(kind) not in SOURCE_KINDS for kind in kinds): - raise StrictValidationError(f"{edit.mechanism_id}: source kinds {kinds!r} are not closed kinds", kind="mechanism_change") - def _validate_rule(self, edit: RuleStatusEdit, ruleset_by_id: Mapping[str, PatternRule], policy: Mapping[str, CwePolicy]) -> None: if edit.to_status not in VALID_STATUS: raise StrictValidationError(f"invalid status {edit.to_status!r} for {edit.rule_id}") @@ -178,25 +102,6 @@ class EvolveValidator: raise StrictValidationError(f"MIN_SCORE={edit.to_value} out of bounds {self._bounds.min_score_range}", kind="policy_change") -def apply_mechanism_edits(edits: Sequence[MechanismEdit], candidates: object, scan_store: Path) -> StoreSnapshot: - """Apply admit/retract edits to the scan-time store; returns the snapshot whose ``restore`` reverts byte for byte. - - Admission appends the exporter record unchanged; retraction appends a tombstone row (``retracted = true``) so the - log stays append-only and ``MechanismStore.load`` folds the record away. - """ - snapshot = StoreSnapshot(path=scan_store, content=scan_store.read_bytes() if scan_store.exists() else None) - by_id = {record.id: record for record in candidates.load()} # type: ignore[attr-defined] - scan_store.parent.mkdir(parents=True, exist_ok=True) - with scan_store.open("a") as handle: - for edit in edits: - record = by_id[edit.mechanism_id] - payload = asdict(record) - if edit.action == "retract": - payload["retracted"] = True - handle.write(json.dumps(payload, sort_keys=True) + "\n") - return snapshot - - def edits_to_ledger(edits: list[RuleStatusEdit], base: Mapping[str, dict[str, object]]) -> dict[str, dict[str, object]]: """Overlay rule-status edits onto a copy of the base ledger.""" ledger: dict[str, dict[str, object]] = {key: dict(value) for key, value in base.items()} @@ -211,9 +116,6 @@ __all__ = [ "EvolveBounds", "EvolveValidator", "MECHANISM_ACTIONS", - "MechanismEdit", - "StoreSnapshot", - "apply_mechanism_edits", "PolicyConstantEdit", "RuleStatusEdit", "StrictValidationError", diff --git a/src/openultrasast/learning/__init__.py b/src/openultrasast/learning/__init__.py deleted file mode 100644 index 12f7d6f..0000000 --- a/src/openultrasast/learning/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""The learning harness: classified detectors improved one family at a time (learning-harness spec). - -Layers left to right, and nothing imports rightward: ``families`` -> ``classify`` -> ``scoring`` -> -``split`` -> ``detectors`` and ``verifiers`` -> ``rounds`` -> ``publish``. ``pairs`` and ``benchmark`` -never import this package; the corpus stores a family label as an opaque string and the classifier -validates it. -""" - -from __future__ import annotations - -from .families import FAMILY_IDS, VERIFIER_KINDS, FamiliesError, Family, FamilyTaxonomy, load_families - -__all__ = ["FAMILY_IDS", "VERIFIER_KINDS", "FamiliesError", "Family", "FamilyTaxonomy", "load_families"] diff --git a/src/openultrasast/learning/acceptance.py b/src/openultrasast/learning/acceptance.py deleted file mode 100644 index 46a1ebe..0000000 --- a/src/openultrasast/learning/acceptance.py +++ /dev/null @@ -1,146 +0,0 @@ -"""When a round is kept, and how it is undone (learning-harness, Req 6.2, 9.5). - -A round is accepted only when the family it targets does not go backwards on either the train or the -held-out pairs, at least one of them moves forward, and no other family flips more pairs than that -family showed against itself in round zero. A tie is a rejection: accepting one would let the loop -wander on noise and call it progress. - -A family with no measured floor gets no budget. Unmeasured is not the same as unlimited, and the -alternative is a family that never ran in round zero quietly absorbing every regression. - -Undoing is a directory restored byte for byte, files the round added removed, files it deleted put back. -A revert that only rewrites what it can see is not a revert. -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path - -from .rounds import NoiseFloor - - -@dataclass(frozen=True) -class Verdict: - accepted: bool - reason: str - offenders: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return {"accepted": self.accepted, "reason": self.reason, "offenders": list(self.offenders)} - - -def decide( - *, - target: str, - train_delta: float, - holdout_delta: float, - sweep: Mapping[str, int], - floors: Mapping[str, NoiseFloor], -) -> Verdict: - """The acceptance rule, in one place, so no caller can soften it.""" - if train_delta < 0: - return Verdict(accepted=False, reason="train_regressed") - if holdout_delta < 0: - return Verdict(accepted=False, reason="holdout_regressed") - if train_delta == 0 and holdout_delta == 0: - return Verdict(accepted=False, reason="tie") - offenders = tuple( - family for family, flips in sorted(sweep.items()) if family != target and _gates(family, floors) and flips > _budget(family, floors) - ) - if offenders: - return Verdict(accepted=False, reason="collateral_regression", offenders=offenders) - return Verdict(accepted=True, reason="improved") - - -@dataclass(frozen=True) -class Reliability: - """Whether a stage's movement could be told from a coin flip.""" - - better: int - worse: int - p_value: float - reliable: bool - - def to_dict(self) -> dict[str, object]: - return {"better": self.better, "worse": self.worse, "p_value": self.p_value, "reliable": self.reliable} - - -def reliability(*, better: int, worse: int, alpha: float = 0.05) -> Reliability: - """The two-sided sign test over the pairs that changed, for the record rather than for the decision. - - `decide` compares raw deltas, as Req 9.5 specifies. On a detector whose measured floor says half its pairs - disagree with themselves between runs, a raw delta is mostly noise — which is what makes a deterministic - decoder look necessary. It is not: the alternative is an acceptance rule that models the score as noisy, the - way reflective-evolution and Bayesian prompt optimizers do, and keeps per-instance winners on a frontier - instead of demanding a single global improvement. Changing the rule is a maintainer's call, so every round - records this beside its decision and the journal can be read to answer the question with data. - """ - from .scoring import sign_test - - p_value = sign_test(better, worse) - return Reliability(better=better, worse=worse, p_value=p_value, reliable=bool(p_value < alpha and better != worse)) - - -def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool: - floor = floors.get(family) - return floor.gates if floor is not None else True - - -def _budget(family: str, floors: Mapping[str, NoiseFloor]) -> int: - floor = floors.get(family) - return floor.budget_flips if floor is not None else 0 - - -def within_budget(spent_usd: float, cap_usd: float) -> bool: - """A round may spend up to its cap; crossing it reverts the round rather than finishing it.""" - return spent_usd <= cap_usd - - -@dataclass(frozen=True) -class DirectorySnapshot: - """Every file of one directory as it was, so a rejected round leaves nothing behind.""" - - directory: Path - files: tuple[tuple[str, bytes], ...] - - @staticmethod - def of(directory: Path) -> DirectorySnapshot: - entries = [(path.relative_to(directory).as_posix(), path.read_bytes()) for path in sorted(directory.rglob("*")) if path.is_file()] - return DirectorySnapshot(directory=directory, files=tuple(entries)) - - def restore(self) -> None: - kept = {name for name, _ in self.files} - for path in sorted(self.directory.rglob("*"), reverse=True): - if path.is_file() and path.relative_to(self.directory).as_posix() not in kept: - path.unlink() - for name, content in self.files: - target = self.directory / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(content) - - -def digest_configs(configs_dir: Path) -> dict[str, str]: - """One digest per family directory, so an untouched family can be shown to be untouched.""" - digests: dict[str, str] = {} - for directory in sorted(path for path in configs_dir.iterdir() if path.is_dir()) if configs_dir.is_dir() else []: - digest = hashlib.sha256() - for path in sorted(directory.rglob("*")): - if path.is_file(): - digest.update(path.relative_to(directory).as_posix().encode()) - digest.update(path.read_bytes()) - digests[directory.name] = digest.hexdigest() - return digests - - -def bump_version(directory: Path, version: str) -> None: - """Write the accepted version into this family's settings, and only this family's.""" - settings = directory / "family.toml" - lines = settings.read_text(encoding="utf-8").splitlines() - rewritten = [f'version = "{version}"' if line.strip().startswith("version") else line for line in lines] - settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8") - - -__all__ = ["DirectorySnapshot", "Reliability", "Verdict", "bump_version", "decide", "digest_configs", "reliability", "within_budget"] diff --git a/src/openultrasast/learning/canaries.py b/src/openultrasast/learning/canaries.py deleted file mode 100644 index 82aea36..0000000 --- a/src/openultrasast/learning/canaries.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Canary snippets and the second judge (learning-harness, Req 7.2, 7.5). - -A canary is how a family says "the vulnerability fired": a snippet plants a marker only the -vulnerability can reveal, and the token appearing in the sandbox output is the oracle. A marker file an -injected command writes, a file under the scratch mount that only a traversal reaches, an object that -only an unsafe load reconstructs. The token is derived from the family, file and function, so one -family's oracle can never answer for another. - -The injection canary confirms untrusted input reaching an interpreter: command, code and template -execution. A query-language injection whose payload never reaches a shell is not confirmed by it and -stays a suspicion, which is the honest answer rather than a false negative dressed as a verdict. - -Nothing reaches `proven` on a machine's say-so alone: a second, independent judgement reads the snippet -and what the sandbox printed and either confirms or drops the claim back to corroboration. Templates -and oracle code live in the package, outside every directory a round may write. -""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass -from pathlib import Path - -from ..config import SandboxConfig -from ..findings import StaticFinding -from ..regress.candidate import DEFAULT_IMAGES, RegressionRunner -from ..regress.verdict import TRIGGERABLE -from ..sandbox.runner import SCRATCH_MOUNT, WORKSPACE_MOUNT, SandboxRunner -from ..tool_hunter import ChatClient -from .verifiers import Verification, VerifierKind, token_oracle - -_TOKEN_PREFIX = "ousast-canary-" -_JUDGE_SYSTEM = ( - "You check one security claim against what a sandbox actually printed. " - 'Answer with JSON only: {"confirms": true|false, "reason": ""}. ' - "Confirm only when the printed output can be explained by the claimed vulnerability and by nothing else." -) - - -def canary_token(family: str, path: str, function: str) -> str: - """The token the oracle looks for. Unique to the experiment, so one canary can never answer for another.""" - digest = hashlib.sha256(f"{family}|{path}|{function}".encode()).hexdigest()[:12] - return f"{_TOKEN_PREFIX}{digest}" - - -def payload_token(family: str, path: str, function: str) -> str: - """The value carried *inside* an injected payload, deliberately unrelated to the oracle token. - - A handler that merely echoes its input would otherwise print the oracle token and satisfy the oracle - without the vulnerability firing at all. The snippet prints the oracle token only after seeing this - value somewhere the payload alone could not have put it. - """ - digest = hashlib.sha256(f"payload|{family}|{path}|{function}".encode()).hexdigest()[:12] - return f"ousast-payload-{digest}" - - -def render_canary( - family: str, *, language: str, path: str, function: str, workspace: str = WORKSPACE_MOUNT, scratch: str = SCRATCH_MOUNT -) -> tuple[str, str] | None: - """The snippet that plants this family's canary and the token that proves it fired, or None. - - ``workspace`` and ``scratch`` default to the sandbox mounts; a test points them at real directories so - the snippet can be run for what it actually does rather than for what a fake runner says it does. - """ - template = _TEMPLATES.get((family, language)) - if template is None: - return None - token = canary_token(family, path, function) - module = path.replace("\\", "/") - snippet = template.format( - token=token, - payload=payload_token(family, path, function), - workspace=workspace, - scratch=scratch, - module=module, - function=function, - ) - return snippet, token - - -@dataclass(frozen=True) -class CanaryVerifier: - """Runs a family's canary in the sandbox and lets only its own token count as proof.""" - - runner: SandboxRunner - limits: SandboxConfig - images: dict[str, str] | None = None - kind: VerifierKind = "canary" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - family = next((tag.split(":", 1)[1] for tag in claim.tags if tag.startswith("family:")), "") - rendered = render_canary(family, language=language, path=claim.path, function=claim.function_name or "") - if rendered is None: - return Verification(rung="suspicion", reason="no_template") - snippet, token = rendered - images = self.images or dict(DEFAULT_IMAGES) - verdict = RegressionRunner(self.runner).run_recipe( - language, snippet, images.get(language, "ousast-missing-image"), self.limits, repo_root=root, oracle=token_oracle(token) - ) - if verdict.verdict == TRIGGERABLE: - return Verification(rung="proven", reason="oracle_fired", oracle_output=token) - return Verification(rung="suspicion", reason="oracle_quiet" if verdict.reason == "oracle_quiet" else verdict.reason) - - -def confirm_proven( - verification: Verification, claim: StaticFinding, *, client: ChatClient | None, model: str, snippet: str -) -> Verification: - """Req 7.5: a claim is published as proven only when a second, independent judgement agrees.""" - if verification.rung != "proven": - return verification - if client is None or not model: - return Verification(rung="static_corroboration", reason="judge_unavailable", oracle_output=verification.oracle_output) - question = ( - f"Claim: {claim.title}. {claim.rationale[:400]}\n" - f"Location: {claim.path} line {claim.line}, function {claim.function_name}.\n\n" - f"Snippet that was run:\n{snippet}\n\nWhat the sandbox printed:\n{verification.oracle_output}\n\n" - "Does that output demonstrate the claim? Answer as JSON." - ) - messages: list[dict[str, object]] = [{"role": "system", "content": _JUDGE_SYSTEM}, {"role": "user", "content": question}] - try: - response = client.complete(model=model, messages=messages, tools=[], json_object=True) # type: ignore[call-arg] - except TypeError: - response = client.complete(model=model, messages=messages, tools=[]) - except Exception: # noqa: BLE001 — a judge that cannot answer must not promote a claim - return Verification(rung="static_corroboration", reason="judge_unavailable", oracle_output=verification.oracle_output) - if _confirms(response.content): - return verification - return Verification(rung="static_corroboration", reason="judge_declined", oracle_output=verification.oracle_output) - - -def _confirms(content: str | None) -> bool: - if not isinstance(content, str) or "{" not in content: - return False - try: - payload = json.loads(content[content.index("{") : content.rindex("}") + 1]) - except (ValueError, json.JSONDecodeError): - return False - return isinstance(payload, dict) and payload.get("confirms") is True - - -# Templates are data, not prose: each plants a marker only its own vulnerability can reveal, imports the -# candidate from the read-only source mount, and prints the token on that path alone. A snippet that -# cannot reach the function prints nothing, which is a suspicion, not a proof. -_PYTHON_INJECTION = """# canary: untrusted input reaches an interpreter, which writes a marker only it could write. -# The payload carries a different value from the oracle token, so a handler that merely echoes its input -# cannot satisfy the oracle; only something that executed the payload can put PLANTED into the marker. -import sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} -MARKER = {scratch!r} + "/canary-injection.txt" -payload = "x; python3 -c \\"open(%r,'w').write(%r)\\"" % (MARKER, PLANTED) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - if callable(handler): - try: - handler(payload) - except Exception as inner: - print("handler raised:", type(inner).__name__) - import os - if os.path.exists(MARKER): - with open(MARKER, encoding="utf-8") as handle: - if PLANTED in handle.read(): - print("CANARY", CANARY) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_PYTHON_PATH = """# canary: a traversal reads a file the handler was never meant to open -import sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} -MARKER = {scratch!r} + "/canary-path.txt" -with open(MARKER, "w", encoding="utf-8") as handle: - handle.write(CANARY) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - result = handler(MARKER) if callable(handler) else None - if CANARY in str(result): - print("CANARY", CANARY) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_PYTHON_DESERIALIZATION = """# canary: an unsafe load reconstructs an object that announces itself -import pickle, sys -sys.path.insert(0, {workspace!r}) -CANARY = {token!r} -PLANTED = {payload!r} - - -class Marker: - def __reduce__(self): - return (print, ("CANARY", CANARY)) - - -blob = pickle.dumps(Marker()) -try: - import importlib.util - spec = importlib.util.spec_from_file_location("candidate", {workspace!r} + "/" + {module!r}) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - handler = getattr(module, {function!r}, None) - if callable(handler): - handler(blob) -except Exception as error: - print("canary could not reach the function:", type(error).__name__) -""" - -_TEMPLATES: dict[tuple[str, str], str] = { - ("injection", "python"): _PYTHON_INJECTION, - ("path", "python"): _PYTHON_PATH, - ("deserialization", "python"): _PYTHON_DESERIALIZATION, -} - - -__all__ = ["CanaryVerifier", "canary_token", "confirm_proven", "payload_token", "render_canary"] diff --git a/src/openultrasast/learning/classify.py b/src/openultrasast/learning/classify.py deleted file mode 100644 index 802b082..0000000 --- a/src/openultrasast/learning/classify.py +++ /dev/null @@ -1,399 +0,0 @@ -"""The auto-classifier: which families a pair, a finding or a region belongs to (learning-harness, Req 2). - -Three tiers, cheapest first. Tier one decides from what the repository already knows: a declared family -label, an obligation kind, a mechanism id, the rule a finding came from, or the labeled weakness. Tier -two asks the model one closed question in JSON mode, with thinking disabled so temperature applies, and -abstains when the answer is off-vocabulary or the reply's mean token log-probability is below the -threshold. Tier three is abstention. Every answer records which tier decided it, and answers are -multi-label because one function can owe more than one family. - -The classifier proposes; it never writes a label. It also owns validation of the labels the corpus -carries, because the corpus stores a family as an opaque string and this module owns the taxonomy. -""" - -from __future__ import annotations - -import json -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import Literal - -from ..findings import StaticFinding -from ..tool_hunter import ChatClient -from .families import FamilyTaxonomy - -Tier = Literal["static", "model", "none"] -MIN_CONFIDENCE = -1.0 # mean token log-probability; below this the model is guessing, so we abstain -_SYSTEM = ( - "You classify a piece of code or a labeled bug into vulnerability families for a security tool. " - 'Answer with JSON only, of the form {"families": ["", ...]}, using ids from the list you are given. ' - "Use an empty list when none of them fits. Do not explain and do not invent an id." -) - - -class FamilyLabelError(ValueError): - """Raised when a catalog row carries a family id outside the taxonomy.""" - - -@dataclass(frozen=True) -class Classification: - families: tuple[str, ...] # empty means unknown - tier: Tier - confidence: float | None = None # mean token log-probability when the model answered - - -def classify_pair( - case: object, taxonomy: FamilyTaxonomy, *, client: ChatClient | None, model: str = "", use_declared: bool = True -) -> Classification: - """The families a labeled pair belongs to: its declared label, else its mechanism, else its weakness. - - ``use_declared`` is False when the classifier is being measured against those same labels, so that - agreement is not the classifier reading back the answer it is scored on. - """ - families = _static_families_of_pair(case, taxonomy, use_declared=use_declared) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _pair_question(case)) - - -def classify_finding(finding: StaticFinding, taxonomy: FamilyTaxonomy, *, client: ChatClient | None, model: str = "") -> Classification: - """The families a finding belongs to: its own family tag, else what produced it, else the model.""" - families = _static_families_of_finding(finding, taxonomy) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _finding_question(finding)) - - -def classify_region( - root: Path, path: str, *, function: str | None, taxonomy: FamilyTaxonomy, client: ChatClient | None, model: str = "" -) -> Classification: - """The families a code region belongs to, from what the static tools already see in it.""" - families = _static_families_of_region(root, path, function, taxonomy) - if families: - return Classification(families=families, tier="static") - return _ask(client, model, taxonomy, _region_question(root, path, function)) - - -def validate_family_labels(cases: Sequence[object], taxonomy: FamilyTaxonomy) -> None: - """Every family a catalog row declares must be in the taxonomy. The corpus stores labels; this module owns them.""" - known = {family.id for family in taxonomy.families} - for case in cases: - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if label and label not in known: - name = getattr(case, "name", "?") - raise FamilyLabelError(f"pair {name}: unknown family {label!r}; expected one of {', '.join(sorted(known))}") - - -def _static_families_of_pair(case: object, taxonomy: FamilyTaxonomy, *, use_declared: bool = True) -> tuple[str, ...]: - declared: list[str] = [] - derived: list[str] = [] - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if use_declared and label and any(family.id == label for family in taxonomy.families): - declared.append(str(label)) - continue - obligation = getattr(row, "obligation", None) - if obligation: - derived.append("access_control") - mechanism = getattr(row, "mechanism", None) - by_mechanism = taxonomy.family_of_mechanism(str(mechanism)) if mechanism else None - if by_mechanism is not None and by_mechanism.id != "unknown": - derived.append(by_mechanism.id) - continue - by_cwe = taxonomy.family_of_cwe(str(getattr(row, "cwe", "") or "")) - if by_cwe is not None and by_cwe.id != "unknown": - derived.append(by_cwe.id) - return _ordered(declared or derived, taxonomy) - - -def _static_families_of_finding(finding: StaticFinding, taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - known = {family.id for family in taxonomy.families} - tagged = [tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:") and tag.split(":", 1)[1] in known] - if tagged: - return _ordered(tagged, taxonomy) - found: list[str] = [] - for tag in finding.tags: - if tag.startswith("obligation:") or tag.startswith("discharger:"): - found.append("access_control") - elif tag.startswith("mechanism:"): - family = taxonomy.family_of_mechanism(tag.split(":", 1)[1]) - if family is not None and family.id != "unknown": - found.append(family.id) - if finding.finding_id.startswith("obligation:"): - found.append("access_control") - rule_family = taxonomy.family_of_cwe(_rule_cwe(finding.finding_id.split(":", 1)[0])) - if rule_family is not None and rule_family.id != "unknown": - found.append(rule_family.id) - return _ordered(found, taxonomy) - - -def _static_families_of_region(root: Path, path: str, function: str | None, taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - from ..hunter_tools import PathEscapesRepo, flows, obligations - - found: list[str] = [] - if function: - try: - if obligations(root, path=path, function=function): - found.append("access_control") - for record in flows(root, path=path, function=function): - family = taxonomy.family_of_cwe(str(record.get("cwe") or "")) - if family is not None and family.id != "unknown": - found.append(family.id) - except (PathEscapesRepo, OSError): - return () - return _ordered(found, taxonomy) - - -def _ask(client: ChatClient | None, model: str, taxonomy: FamilyTaxonomy, question: str) -> Classification: - if client is None: - return Classification(families=(), tier="none") - catalog = "\n".join(f"- {family.id}: {family.description}" for family in taxonomy.families) - messages: list[dict[str, object]] = [ - {"role": "system", "content": _SYSTEM}, - {"role": "user", "content": f"Families:\n{catalog}\n\n{question}\n\nAnswer as JSON."}, - ] - try: - try: - response = client.complete(model=model, messages=messages, tools=[], json_object=True, logprobs=True) # type: ignore[call-arg] - except TypeError: # a client without the adapter's extras - response = client.complete(model=model, messages=messages, tools=[]) - except Exception: # noqa: BLE001 — the classifier is advisory; any failed call abstains, including in the fallback - return Classification(families=(), tier="none") - confidence = response.mean_logprob - if confidence is not None and confidence < MIN_CONFIDENCE: - return Classification(families=(), tier="none", confidence=confidence) - families = _ordered(_parse(response.content), taxonomy) - if not families: - return Classification(families=(), tier="none", confidence=confidence) - return Classification(families=families, tier="model", confidence=confidence) - - -def _parse(content: str | None) -> list[str]: - if not isinstance(content, str) or not content.strip(): - return [] - try: - payload = json.loads(content[content.index("{") : content.rindex("}") + 1]) - except (ValueError, json.JSONDecodeError): - return [] - families = payload.get("families") if isinstance(payload, dict) else None - if not isinstance(families, list): - return [] - return [str(item) for item in families if isinstance(item, str)] - - -def _ordered(names: Sequence[str], taxonomy: FamilyTaxonomy) -> tuple[str, ...]: - """Taxonomy order, de-duplicated, unknown ids dropped, so two runs compare equal.""" - wanted = set(names) - return tuple(family.id for family in taxonomy.families if family.id in wanted and family.id != "unknown") - - -def _rule_cwe(rule_id: str) -> str: - from ..ruleset import DEFAULT_RULESET_DIR, load_ruleset - - for rule in load_ruleset(DEFAULT_RULESET_DIR): - if rule.rule_id == rule_id: - return rule.cwe - return "" - - -def _pair_question(case: object) -> str: - row = next(iter(getattr(case, "expected", ()) or ()), None) - parts = [f"Labeled bug in {getattr(case, 'relpath', '?')}"] - if row is not None: - parts.append( - f"function {getattr(row, 'function', '?')}, weakness {getattr(row, 'cwe', '?')}, mechanism {getattr(row, 'mechanism', '?')}" - ) - return "Which families does it belong to? " + "; ".join(parts) - - -def _finding_question(finding: StaticFinding) -> str: - return f"Which families does this finding belong to? {finding.title}. {finding.rationale[:400]}" - - -def _region_question(root: Path, path: str, function: str | None) -> str: - from ..hunter_tools import PathEscapesRepo, read_definition - - try: - found = read_definition(root, function or "", max_chars=2000) if function else None - except (PathEscapesRepo, OSError): - found = None - body = str(found["text"]) if found else "" - return f"Which families does this code belong to?\n{path}::{function}\n{body}" - - -@dataclass(frozen=True) -class QueueEntry: - """One disagreement between a maintainer label and the classifier, for a human to settle.""" - - pair: str - label: str - classified: tuple[str, ...] - tier: Tier - confidence: float | None = None - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "label": self.label, - "classified": list(self.classified), - "tier": self.tier, - "confidence": self.confidence, - } - - -@dataclass(frozen=True) -class ClassifierReport: - """How the classifier did on its own: coverage over the whole corpus, agreement where a reference exists. - - ``labeled`` is how many rows carry a maintainer family label. Agreement, the confusion matrix and both - baselines are computed over those rows only, so an unlabeled corpus reports coverage and says the - reference set is empty rather than inventing a score. - """ - - taxonomy_version: str - classified: int - labeled: int - agreement: float # credit for the classifier's first answer - oracle_router: float # credit for the best of its answers - random_router: float # a uniform pick over the real families - confusion: dict[str, dict[str, int]] = field(default_factory=dict) - per_tier: dict[str, int] = field(default_factory=dict) - per_family: dict[str, int] = field(default_factory=dict) - unknown: int = 0 - queue: tuple[QueueEntry, ...] = () - hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 2.6) - - def to_dict(self) -> dict[str, object]: - return { - "taxonomy_version": self.taxonomy_version, - "classified": self.classified, - "labeled": self.labeled, - "agreement": self.agreement, - "oracle_router": self.oracle_router, - "random_router": self.random_router, - "confusion": {label: dict(sorted(answers.items())) for label, answers in sorted(self.confusion.items())}, - "per_tier": dict(sorted(self.per_tier.items())), - "per_family": dict(sorted(self.per_family.items())), - "unknown": self.unknown, - "hierarchical_credit": self.hierarchical_credit, - "queue": [entry.to_dict() for entry in self.queue], - } - - -def measure_classifier( - cases: Sequence[object], - taxonomy: FamilyTaxonomy, - *, - client: ChatClient | None, - model: str = "", - reference_tiers: Sequence[str] = ("seeded", "reviewed"), -) -> ClassifierReport: - """Classify every case and score the classifier against the labels a maintainer stands behind.""" - rows = list(cases) - real = [family.id for family in taxonomy.families if family.id != "unknown"] - per_tier: dict[str, int] = {} - per_family: dict[str, int] = {} - confusion: dict[str, dict[str, int]] = {} - queue: list[QueueEntry] = [] - unknown = 0 - credits: list[float] = [] - oracles: list[float] = [] - for case in rows: - answer = classify_pair(case, taxonomy, client=client, model=model, use_declared=False) - per_tier[answer.tier] = per_tier.get(answer.tier, 0) + 1 - for family_id in answer.families: - per_family[family_id] = per_family.get(family_id, 0) + 1 - if not answer.families: - unknown += 1 - label = _reference_label(case, taxonomy, reference_tiers) - if label is None: - continue - first = answer.families[0] if answer.families else "unknown" - credits.append(_credit(taxonomy, label, (first,))) - oracles.append(_credit(taxonomy, label, answer.families or ("unknown",))) - confusion.setdefault(label, {})[first] = confusion.setdefault(label, {}).get(first, 0) + 1 - if credits[-1] < 1.0: - queue.append( - QueueEntry( - pair=str(getattr(case, "name", "?")), - label=label, - classified=answer.families, - tier=answer.tier, - confidence=answer.confidence, - ) - ) - labeled = len(credits) - return ClassifierReport( - taxonomy_version=taxonomy.version, - classified=len(rows), - labeled=labeled, - agreement=sum(credits) / labeled if labeled else 0.0, - oracle_router=sum(oracles) / labeled if labeled else 0.0, - random_router=(1.0 / len(real)) if real and labeled else 0.0, - confusion=confusion, - per_tier=per_tier, - per_family=per_family, - unknown=unknown, - queue=tuple(queue), - hierarchical_credit=taxonomy.hierarchical, - ) - - -def write_review_queue(path: Path, entries: Sequence[QueueEntry]) -> int: - """Append disagreements a human has not seen yet; a pair already queued is never duplicated.""" - path.parent.mkdir(parents=True, exist_ok=True) - seen: set[str] = set() - if path.exists(): - for line in path.read_text(encoding="utf-8").splitlines(): - if line.strip(): - try: - seen.add(str(json.loads(line).get("pair", ""))) - except json.JSONDecodeError: - continue - written = 0 - with path.open("a", encoding="utf-8") as handle: - for entry in entries: - if entry.pair in seen: - continue - handle.write(json.dumps(entry.to_dict(), sort_keys=True) + "\n") - seen.add(entry.pair) - written += 1 - return written - - -def _reference_label(case: object, taxonomy: FamilyTaxonomy, reference_tiers: Sequence[str]) -> str | None: - """The family a maintainer stands behind for this pair, or None. Only trusted tiers are a reference.""" - if str(getattr(case, "review_tier", "")) not in reference_tiers: - return None - known = {family.id for family in taxonomy.families} - for row in getattr(case, "expected", ()) or (): - label = getattr(row, "family", None) - if label and str(label) in known: - return str(label) - return None - - -def _credit(taxonomy: FamilyTaxonomy, label: str, answers: Sequence[str]) -> float: - """1.0 for the same family, half for a parent or child, nothing for a lateral or fabricated answer.""" - best = 0.0 - for answer in answers: - relation = taxonomy.related(label, answer) - best = max(best, 1.0 if relation == "same" else 0.5 if relation == "parent_child" else 0.0) - return best - - -__all__ = [ - "MIN_CONFIDENCE", - "Classification", - "ClassifierReport", - "FamilyLabelError", - "QueueEntry", - "classify_finding", - "classify_pair", - "classify_region", - "measure_classifier", - "validate_family_labels", - "write_review_queue", -] diff --git a/src/openultrasast/learning/detectors.py b/src/openultrasast/learning/detectors.py deleted file mode 100644 index 5763cc5..0000000 --- a/src/openultrasast/learning/detectors.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Per-family detector configurations and the runner that uses them (learning-harness, Req 6.1-6.5). - -A detector is a configuration, not code: a system prompt, an evidence checklist, the curated tools it -may call, its step and cost budget, and the hard negatives and counterexamples earlier rounds left it. -Round zero writes one directory per family from the same prompt, so the baseline is controlled, and a -round rewrites exactly one directory and bumps that family's version. Every other family is frozen by -construction, because a round can only touch the directory it targets. - -The runner gives a detector the whole file rather than an excerpt around a hotspot, since context loss -is a documented cause of misjudging patched code, and tags each finding with exactly one family and the -configuration version that produced it, so a finding can always be traced back to what made it. -""" - -from __future__ import annotations - -import json -import tomllib -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from pathlib import Path - -from ..findings import StaticFinding -from ..tool_hunter import HUNTER_TOOLS, ChatClient, run_tool_hunter -from .families import FamilyTaxonomy - -MAX_PROMPT_CHARS = 8000 # prompt plus checklist; unbounded prompts are the documented signature of overfitting -GENERALIST = "unknown" # the family every region also runs, so a bug outside the taxonomy is still looked for -CURATED_TOOLS = tuple(str(schema["function"]["name"]) for schema in HUNTER_TOOLS) # type: ignore[index] -_REQUIRED = ("prompt.md", "family.toml") - - -class DetectorConfigError(ValueError): - """Raised when a detector configuration is missing, over its cap, or names something outside the closed sets.""" - - -@dataclass(frozen=True) -class FamilyConfig: - family: str - version: str - prompt: str - checklist: str - tools: tuple[str, ...] - max_steps: int - max_cost_usd: float - max_chars: int - hard_negatives: tuple[Mapping[str, object], ...] = () - counterexamples: tuple[Mapping[str, object], ...] = () - - @property - def detector_id(self) -> str: - return f"{self.family}@{self.version}" - - -@dataclass(frozen=True) -class Region: - """A place a detector is pointed at: one file, optionally one function, and the families admitted for it.""" - - path: str - function: str | None = None - families: tuple[str, ...] = () - - -def load_family_configs(configs_dir: Path, taxonomy: FamilyTaxonomy) -> dict[str, FamilyConfig]: - """Every configuration under ``configs_dir``. A directory that is not a family, or a broken one, fails loud.""" - known = {family.id for family in taxonomy.families} - configs: dict[str, FamilyConfig] = {} - for directory in sorted(path for path in configs_dir.iterdir() if path.is_dir()) if configs_dir.is_dir() else []: - if directory.name not in known: - raise DetectorConfigError(f"{directory}: {directory.name!r} is not a family; expected one of {', '.join(sorted(known))}") - configs[directory.name] = _load_one(directory) - return configs - - -def _load_one(directory: Path) -> FamilyConfig: - for name in _REQUIRED: - if not (directory / name).is_file(): - raise DetectorConfigError(f"{directory}: {name} is required") - try: - settings = tomllib.loads((directory / "family.toml").read_text(encoding="utf-8")) - except tomllib.TOMLDecodeError as exc: - raise DetectorConfigError(f"{directory / 'family.toml'}: not valid TOML: {exc}") from exc - version = str(settings.get("version", "")).strip() - if not version: - raise DetectorConfigError(f"{directory}: version is required") - prompt = (directory / "prompt.md").read_text(encoding="utf-8").strip() - checklist = _read_optional(directory / "checklist.md") - if len(prompt) + len(checklist) > MAX_PROMPT_CHARS: - raise DetectorConfigError( - f"{directory}: prompt and checklist length {len(prompt) + len(checklist)} exceeds the cap of {MAX_PROMPT_CHARS}" - ) - tools = tuple(str(name) for name in settings.get("tools", []) or ()) - if not tools: - raise DetectorConfigError(f"{directory}: tools is required and must not be empty; a detector with no tools reports nothing") - unknown_tools = [name for name in tools if name not in CURATED_TOOLS] - if unknown_tools: - raise DetectorConfigError(f"{directory}: unknown tool(s) {', '.join(unknown_tools)}; expected from {', '.join(CURATED_TOOLS)}") - return FamilyConfig( - family=directory.name, - version=version, - prompt=prompt, - checklist=checklist, - tools=tools, - max_steps=int(settings.get("max_steps", 4)), - max_cost_usd=float(settings.get("max_cost_usd", 0.5)), - max_chars=int(settings.get("max_chars", 4000)), - hard_negatives=_read_jsonl(directory / "hard_negatives.jsonl"), - counterexamples=_read_jsonl(directory / "counterexamples.jsonl"), - ) - - -def run_family_detector(root: Path, region: Region, config: FamilyConfig, *, client: ChatClient, model: str) -> list[StaticFinding]: - """One family's detector over one region. Findings carry that family and the configuration that found them.""" - return run_tool_hunter( - root, - [], - client=client, - model=model, - max_steps=config.max_steps, - system_prompt=config.prompt, - user_prompt=_question(region, config), - context_files=(region.path,), - tools=config.tools, - max_chars=config.max_chars, - tags=(f"family:{config.family}", f"detector:{config.detector_id}"), - ) - - -def run_region_detectors( - root: Path, region: Region, configs: Mapping[str, FamilyConfig], *, client: ChatClient, model: str -) -> list[StaticFinding]: - """Every family the classifier admitted for this region, plus the generalist (Req 6.3).""" - wanted = [family for family in region.families if family in configs and family != GENERALIST] - if GENERALIST in configs: - wanted.append(GENERALIST) - findings: list[StaticFinding] = [] - for family in wanted: - findings.extend(run_family_detector(root, region, configs[family], client=client, model=model)) - return findings - - -def write_default_configs( - configs_dir: Path, taxonomy: FamilyTaxonomy, *, prompt: str, version: str, tools: Sequence[str] | None = None -) -> dict[str, Path]: - """Round zero: one directory per family from the same prompt, so the baseline is controlled (Req 8.1).""" - written: dict[str, Path] = {} - for family in taxonomy.families: - directory = configs_dir / family.id - directory.mkdir(parents=True, exist_ok=True) - (directory / "prompt.md").write_text(f"{prompt}\n\nFamily: {family.id}. {family.description}\n", encoding="utf-8") - (directory / "checklist.md").write_text("- name the file, the line and why the code is reachable\n", encoding="utf-8") - selected = list(tools) if tools is not None else list(CURATED_TOOLS) - (directory / "family.toml").write_text( - f'version = "{version}"\ntools = {json.dumps(selected)}\nmax_steps = 4\nmax_cost_usd = 0.5\nmax_chars = 4000\n', - encoding="utf-8", - ) - for name in ("hard_negatives.jsonl", "counterexamples.jsonl"): - if not (directory / name).exists(): - (directory / name).write_text("", encoding="utf-8") - written[family.id] = directory - return written - - -def _question(region: Region, config: FamilyConfig) -> str: - where = f"{region.path}::{region.function}" if region.function else region.path - parts = [f"Look for {config.family} problems in {where}. The whole file follows."] - if config.checklist: - parts.append(f"Evidence checklist:\n{config.checklist}") - if config.hard_negatives: - parts.append("Known non-issues, do not report these:\n" + "\n".join(_line(item) for item in config.hard_negatives)) - if config.counterexamples: - parts.append("Worked examples:\n" + "\n".join(_line(item) for item in config.counterexamples)) - parts.append("Reply with a JSON array of objects with path, line, title and rationale, after using a tool.") - return "\n\n".join(parts) - - -def _line(item: Mapping[str, object]) -> str: - return "- " + "; ".join(f"{key}: {value}" for key, value in sorted(item.items())) - - -def _read_optional(path: Path) -> str: - return path.read_text(encoding="utf-8").strip() if path.is_file() else "" - - -def _read_jsonl(path: Path) -> tuple[Mapping[str, object], ...]: - if not path.is_file(): - return () - rows: list[Mapping[str, object]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - payload = json.loads(line) - except json.JSONDecodeError as exc: - raise DetectorConfigError(f"{path}: not valid JSON lines: {exc}") from exc - if isinstance(payload, dict): - rows.append(payload) - return tuple(rows) - - -__all__ = [ - "CURATED_TOOLS", - "GENERALIST", - "MAX_PROMPT_CHARS", - "DetectorConfigError", - "FamilyConfig", - "Region", - "load_family_configs", - "run_family_detector", - "run_region_detectors", - "write_default_configs", -] diff --git a/src/openultrasast/learning/difficulty.py b/src/openultrasast/learning/difficulty.py deleted file mode 100644 index 8b4b273..0000000 --- a/src/openultrasast/learning/difficulty.py +++ /dev/null @@ -1,139 +0,0 @@ -"""How hard a piece of code makes an absence bug to see (learning-harness, Req 10.4). - -A unit fixture that is easier than the corpus is worse than no fixture: the checker passes on it, the number -looks like capability, and the corpus rows it claims to cover keep failing for reasons nobody is looking at. The -three dimensions below are the ones the corpus actually varies on, measured on the vulnerable side: - -- ``binding_hops`` — how many assignments sit between the identity source and the constrained use. `filter_by( - user_id=request.user.id)` is nothing to trace; `resp = token_validator(...)` then `User.query.filter_by( - username=resp['sub'])` then `filter_by(user=user, ...)` is two. -- ``registration_distance`` — 0 when the route decorator sits on the handler, 1 when a statement elsewhere in the - same file registers it, 2 when the registration lives in another document entirely (connexion's `operationId`, - an express router). Two is where the entry-point mapper stops being able to name the handler at all. -- ``guard_distance`` — 0 when a decorator carries the guard, 1 when the guard is a call in the body, 2 when the - handler carries no visible guard. A decorator is a pattern; a call in the body has to be understood. - -Higher is harder on every axis, so "strictly easier" is: no dimension harder, at least one easier. -""" - -from __future__ import annotations - -import ast -import re -from collections.abc import Sequence -from dataclasses import dataclass - -IDENTITY_SOURCES = ("current_user", "request.user", "session", "g.user", "auth", "token", "jwt", "claims", "sub", "principal") -GUARD_TOKENS = ("login_required", "token_required", "requires_auth", "authenticate", "authorize", "token_validator", "current_user") -_ROUTE_DECORATOR = re.compile(r"\.(route|get|post|put|patch|delete)\b") - - -@dataclass(frozen=True) -class DifficultyVector: - """One handler's difficulty. Higher is harder on every axis.""" - - function: str - binding_hops: int - registration_distance: int - guard_distance: int - - def axes(self) -> tuple[int, int, int]: - return (self.binding_hops, self.registration_distance, self.guard_distance) - - def to_dict(self) -> dict[str, object]: - return { - "function": self.function, - "binding_hops": self.binding_hops, - "registration_distance": self.registration_distance, - "guard_distance": self.guard_distance, - } - - -def difficulty_of(source: str, function: str, *, context: Sequence[str] = ()) -> DifficultyVector: - """The vector for one Python handler. ``context`` is any document that registers it from outside the file.""" - try: - tree = ast.parse(source) - except SyntaxError: - return DifficultyVector(function=function, binding_hops=0, registration_distance=2, guard_distance=2) - node = _function(tree, function) - if node is None: - return DifficultyVector(function=function, binding_hops=0, registration_distance=2, guard_distance=2) - decorators = [ast.unparse(item) for item in node.decorator_list] + _class_decorators(tree, node) - return DifficultyVector( - function=function, - binding_hops=_binding_hops(node), - registration_distance=_registration_distance(source, tree, function, decorators, context), - guard_distance=_guard_distance(node, decorators), - ) - - -def hardest(vectors: Sequence[DifficultyVector]) -> tuple[int, int, int]: - """The per-axis maximum over a set of handlers: what the set can be asked to cope with.""" - if not vectors: - return (0, 0, 0) - return tuple(max(row.axes()[index] for row in vectors) for index in range(3)) # type: ignore[return-value] - - -def strictly_easier(fixture: Sequence[DifficultyVector], corpus: Sequence[DifficultyVector]) -> bool: - """True when the fixture set reaches no further than the corpus on any axis and falls short on at least one. - - Per axis and over the whole set, because a suite covers a bug shape with several handlers between them: one - fixture handler may carry the long binding chain and another the missing guard. What must not happen is a suite - whose hardest case on some axis is easier than the corpus rows its tests claim to cover.""" - if not corpus: - return False - limit, reach = hardest(corpus), hardest(fixture) - return all(reach[index] <= limit[index] for index in range(3)) and any(reach[index] < limit[index] for index in range(3)) - - -def _function(tree: ast.AST, name: str) -> ast.FunctionDef | ast.AsyncFunctionDef | None: - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name: - return node - return None - - -def _class_decorators(tree: ast.AST, node: ast.AST) -> list[str]: - for klass in ast.walk(tree): - if isinstance(klass, ast.ClassDef) and any(child is node for child in klass.body): - return [ast.unparse(item) for item in klass.decorator_list] - return [] - - -def _binding_hops(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: - """Assignments that carry the identity forward before it constrains anything.""" - tainted: set[str] = set() - hops = 0 - for statement in ast.walk(node): - if not isinstance(statement, ast.Assign): - continue - text = ast.unparse(statement.value) - if any(token in text for token in IDENTITY_SOURCES) or any(name in text for name in tainted): - hops += 1 - for target in statement.targets: - tainted.update(part.id for part in ast.walk(target) if isinstance(part, ast.Name)) - return hops - - -def _registration_distance(source: str, tree: ast.AST, function: str, decorators: Sequence[str], context: Sequence[str]) -> int: - if any(_ROUTE_DECORATOR.search(item) for item in decorators): - return 0 - mention = re.compile(rf"(? int: - if any(token in item.lower() for item in decorators for token in GUARD_TOKENS): - return 0 - body = ast.unparse(node).lower() - return 1 if any(token in body for token in GUARD_TOKENS) else 2 - - -__all__ = ["GUARD_TOKENS", "IDENTITY_SOURCES", "DifficultyVector", "difficulty_of", "hardest", "strictly_easier"] diff --git a/src/openultrasast/learning/journal.py b/src/openultrasast/learning/journal.py deleted file mode 100644 index a14f808..0000000 --- a/src/openultrasast/learning/journal.py +++ /dev/null @@ -1,237 +0,0 @@ -"""The round journal, the per-family archive and the rejected buffer (learning-harness, Req 9.3, 9.6, 9.7, 11.3). - -The journal is append-only, one JSON line per round, and a round number is never reused, so the record -of what was tried cannot be quietly rewritten by the thing that is trying. Every round writes a line -whether it was accepted or not: a rejection is evidence too, and the rejected hypotheses are what stop -the proposer from offering the same idea twice. - -The archive keeps, per family, which configuration version wins which pair, rather than one champion. -Keeping only the best-overall candidate throws away a version that is the only one solving some pair, -which is how a search collapses onto one lineage. A later version takes a pair only by winning it. - -Nothing is written without passing redaction, because a trajectory or a hypothesis may quote the code -it was reasoning about. -""" - -from __future__ import annotations - -import json -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..redaction import redact_secrets -from .scoring import PairFamilyScore - -Outcome = str # accepted | rejected | reverted_cost | refused_holdout - - -def _as_int(value: object) -> int: - return int(value) if isinstance(value, int | float | str) and not isinstance(value, bool) else 0 - - -def _as_float(value: object) -> float: - return float(value) if isinstance(value, int | float | str) and not isinstance(value, bool) else 0.0 - - -def _as_strings(value: object) -> tuple[str, ...]: - return tuple(str(item) for item in value) if isinstance(value, list | tuple) else () - - -def _as_sweep(value: object) -> dict[str, int]: - return {str(key): _as_int(item) for key, item in value.items()} if isinstance(value, Mapping) else {} - - -class JournalError(ValueError): - """Raised when a round would overwrite history.""" - - -@dataclass(frozen=True) -class Attribution: - """Whether the round moved what it said it would move, and what it moved by accident.""" - - flipped_predicted: int = 0 - flipped_unpredicted: int = 0 - precision: float = 0.0 - - def to_dict(self) -> dict[str, object]: - return { - "flipped_predicted": self.flipped_predicted, - "flipped_unpredicted": self.flipped_unpredicted, - "precision": self.precision, - } - - @staticmethod - def from_dict(payload: Mapping[str, object]) -> Attribution: - return Attribution( - flipped_predicted=_as_int(payload.get("flipped_predicted")), - flipped_unpredicted=_as_int(payload.get("flipped_unpredicted")), - precision=_as_float(payload.get("precision")), - ) - - -@dataclass(frozen=True) -class RoundRecord: - round: int - family: str - hypothesis: str - levers: tuple[str, ...] - predicted_affected: tuple[str, ...] - predicted_at_risk: tuple[str, ...] - outcome: Outcome - reason: str = "" - target_train_delta: float = 0.0 - target_holdout_delta: float = 0.0 - sweep: dict[str, int] = field(default_factory=dict) # family -> negative flips in the cross-family sweep - attribution: Attribution = Attribution() - cost_usd: float = 0.0 - taxonomy_version: str = "" - config_version: str = "" - # Pairs that moved each way in each stage, so the record can say whether the decision was distinguishable - # from noise. The decision itself is Req 9.5's raw-delta rule; this is the evidence beside it. - train_better: int = 0 - train_worse: int = 0 - holdout_better: int = 0 - holdout_worse: int = 0 - - def to_dict(self) -> dict[str, object]: - return { - "round": self.round, - "family": self.family, - "hypothesis": self.hypothesis, - "levers": list(self.levers), - "predicted_affected": list(self.predicted_affected), - "predicted_at_risk": list(self.predicted_at_risk), - "outcome": self.outcome, - "reason": self.reason, - "target_train_delta": self.target_train_delta, - "target_holdout_delta": self.target_holdout_delta, - "sweep": dict(sorted(self.sweep.items())), - "attribution": self.attribution.to_dict(), - "cost_usd": self.cost_usd, - "taxonomy_version": self.taxonomy_version, - "config_version": self.config_version, - **{f"train_{key}": value for key, value in self._reliability(self.train_better, self.train_worse).items()}, - **{f"holdout_{key}": value for key, value in self._reliability(self.holdout_better, self.holdout_worse).items()}, - } - - @staticmethod - def _reliability(better: int, worse: int) -> dict[str, object]: - from .acceptance import reliability - - result = reliability(better=better, worse=worse) - return {"better": result.better, "worse": result.worse, "p_value": result.p_value, "reliable": result.reliable} - - @staticmethod - def from_dict(payload: Mapping[str, object]) -> RoundRecord: - attribution = payload.get("attribution") - return RoundRecord( - round=_as_int(payload.get("round")), - family=str(payload.get("family", "")), - hypothesis=str(payload.get("hypothesis", "")), - levers=_as_strings(payload.get("levers")), - predicted_affected=_as_strings(payload.get("predicted_affected")), - predicted_at_risk=_as_strings(payload.get("predicted_at_risk")), - outcome=str(payload.get("outcome", "")), - reason=str(payload.get("reason", "")), - target_train_delta=_as_float(payload.get("target_train_delta")), - target_holdout_delta=_as_float(payload.get("target_holdout_delta")), - sweep=_as_sweep(payload.get("sweep")), - attribution=Attribution.from_dict(attribution if isinstance(attribution, Mapping) else {}), - cost_usd=_as_float(payload.get("cost_usd")), - taxonomy_version=str(payload.get("taxonomy_version", "")), - config_version=str(payload.get("config_version", "")), - ) - - -class LearningJournal: - """Append-only history of rounds. What was tried, what it predicted, and what actually happened.""" - - def __init__(self, path: Path) -> None: - self.path = path - - def append(self, record: RoundRecord) -> None: - existing = {item.round for item in self.rounds()} - if record.round in existing: - raise JournalError(f"round {record.round} is already journaled; history is append-only") - self.path.parent.mkdir(parents=True, exist_ok=True) - line = redact_secrets(json.dumps(record.to_dict(), sort_keys=True)) - with self.path.open("a", encoding="utf-8") as handle: - handle.write(line + "\n") - - def rounds(self) -> tuple[RoundRecord, ...]: - return tuple(RoundRecord.from_dict(payload) for payload in _read_jsonl(self.path)) - - def next_round(self) -> int: - rounds = self.rounds() - return max((item.round for item in rounds), default=0) + 1 - - def rejected_buffer(self, family: str) -> tuple[str, ...]: - """Hypotheses this family already tried and lost, so the proposer is not handed the same idea twice.""" - return tuple(item.hypothesis for item in self.rounds() if item.family == family and item.outcome != "accepted" and item.hypothesis) - - -class Archive: - """Per family, which configuration version wins which pair. Never one champion.""" - - def __init__(self, path: Path) -> None: - self.path = path - - def record(self, family: str, config_version: str, score: PairFamilyScore) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "family": family, - "config_version": config_version, - "pair": score.pair, - "outcome": score.outcome, - "flips": score.flips, - } - with self.path.open("a", encoding="utf-8") as handle: - handle.write(redact_secrets(json.dumps(payload, sort_keys=True)) + "\n") - - def winners(self, family: str) -> dict[str, str]: - """pair -> the earliest configuration version that scored it correct. - - Earliest, not latest: the point of the archive is that a version which solves a pair keeps its - claim to it, so a later version that loses the pair cannot quietly take it off the board. - """ - winners: dict[str, str] = {} - for payload in _read_jsonl(self.path): - if str(payload.get("family")) != family or str(payload.get("outcome")) != "pair_correct": - continue - winners.setdefault(str(payload.get("pair")), str(payload.get("config_version"))) - return winners - - -def _read_jsonl(path: Path) -> list[Mapping[str, object]]: - if not path.is_file(): - return [] - rows: list[Mapping[str, object]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(payload, dict): - rows.append(payload) - return rows - - -def summarize(records: Sequence[RoundRecord]) -> dict[str, object]: - """What the rounds add up to: how often a prediction landed, and what it cost.""" - accepted = [item for item in records if item.outcome == "accepted"] - predicted = sum(item.attribution.flipped_predicted for item in records) - unpredicted = sum(item.attribution.flipped_unpredicted for item in records) - return { - "rounds": len(records), - "accepted": len(accepted), - "flipped_predicted": predicted, - "flipped_unpredicted": unpredicted, - "prediction_precision": predicted / (predicted + unpredicted) if predicted + unpredicted else 0.0, - "cost_usd": round(sum(item.cost_usd for item in records), 4), - } - - -__all__ = ["Archive", "Attribution", "JournalError", "LearningJournal", "Outcome", "RoundRecord", "summarize"] diff --git a/src/openultrasast/learning/proposer.py b/src/openultrasast/learning/proposer.py deleted file mode 100644 index 7881657..0000000 --- a/src/openultrasast/learning/proposer.py +++ /dev/null @@ -1,318 +0,0 @@ -"""What a round proposes, and who may propose it (learning-harness, Req 9.1, 9.2, 5.4, 7.6). - -A proposer is handed facts, never scores it is about to be judged on: the misses and leaks of one family -on the train split, what each run actually did, and the hypotheses this family already tried and lost. -Self-diagnosis from free narrative confabulates; structured facts are what make a proposal answerable. - -A proposal is one hypothesis, one lever, and one file inside one family's directory. The narrowness is -the safety property: a round can only touch what it targets, so every other family is frozen by -construction rather than by promise, and the verifier code and the corpus stay outside every write root. -""" - -from __future__ import annotations - -import json -import shutil -import tempfile -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import Literal, Protocol - -from ..harness_ext import has_harnessx -from .detectors import MAX_PROMPT_CHARS, FamilyConfig -from .journal import LearningJournal -from .scoring import PairFamilyScore - -Lever = Literal["tool", "checklist", "counterexample", "memory", "prompt"] -LEVERS: tuple[Lever, ...] = ("tool", "checklist", "counterexample", "memory", "prompt") -# The only files a round may write, one per lever. Anything else is a proposal reaching beyond its family. -EDITABLE_FILES: dict[Lever, str] = { - "tool": "family.toml", - "checklist": "checklist.md", - "counterexample": "counterexamples.jsonl", - "memory": "hard_negatives.jsonl", - "prompt": "prompt.md", -} - - -@dataclass(frozen=True) -class PairFact: - """What one pair did, in terms a proposer can act on.""" - - pair: str - outcome: str - runs: tuple[str, ...] = () - other_findings: int = 0 - rung: str = "suspicion" - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "outcome": self.outcome, - "runs": list(self.runs), - "other_findings": self.other_findings, - "rung": self.rung, - } - - -@dataclass(frozen=True) -class FailureFacts: - family: str - config: FamilyConfig - misses: tuple[PairFact, ...] = () - leaks: tuple[PairFact, ...] = () - rejected: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "config_version": self.config.version, - "tools": list(self.config.tools), - "misses": [item.to_dict() for item in self.misses], - "leaks": [item.to_dict() for item in self.leaks], - "rejected": list(self.rejected), - } - - -@dataclass(frozen=True) -class Proposal: - hypothesis: str - lever: Lever - change: Mapping[str, str] # file name inside the family directory -> its new contents - predicted_affected: tuple[str, ...] = () - predicted_at_risk: tuple[str, ...] = () - - def to_dict(self) -> dict[str, object]: - return { - "hypothesis": self.hypothesis, - "lever": self.lever, - "change": dict(self.change), - "predicted_affected": list(self.predicted_affected), - "predicted_at_risk": list(self.predicted_at_risk), - } - - -class Proposer(Protocol): - def propose(self, facts: FailureFacts) -> Proposal | None: ... - - -def build_failure_facts( - scores: Sequence[PairFamilyScore], - *, - family: str, - config: FamilyConfig, - journal: LearningJournal, - train_pairs: set[str], -) -> FailureFacts: - """The facts of one family's failures on the train split. A holdout pair never appears here (Req 5.4).""" - mine = [item for item in scores if item.family == family and item.pair in train_pairs and item.outcome != "unscorable"] - return FailureFacts( - family=family, - config=config, - misses=tuple(_fact(item, item.other_findings_vuln) for item in mine if item.outcome in {"both_silent", "reversed"}), - leaks=tuple(_fact(item, item.other_findings_fixed) for item in mine if item.outcome in {"both_flagged", "reversed"}), - rejected=journal.rejected_buffer(family), - ) - - -def refuse_proposal(proposal: Proposal, config: FamilyConfig) -> str | None: - """None when the proposal is one legal change to its own family; otherwise why it is refused.""" - if proposal.lever not in LEVERS: - return f"unknown lever {proposal.lever!r}; expected one of {', '.join(LEVERS)}" - if len(proposal.change) != 1: - return f"a round changes one file, not {len(proposal.change)}" - name, content = next(iter(proposal.change.items())) - if "/" in name or "\\" in name or name.startswith("."): - return f"{name!r} reaches outside the family directory" - if name != EDITABLE_FILES[proposal.lever]: - return f"{name!r} is not the file the {proposal.lever!r} lever writes ({EDITABLE_FILES[proposal.lever]})" - if name == "prompt.md" and len(content) + len(config.checklist) > MAX_PROMPT_CHARS: - return f"the proposed prompt exceeds the cap of {MAX_PROMPT_CHARS} characters" - if name == "checklist.md" and len(content) + len(config.prompt) > MAX_PROMPT_CHARS: - return f"the proposed checklist exceeds the cap of {MAX_PROMPT_CHARS} characters" - return None - - -def apply_proposal(proposal: Proposal, directory: Path) -> None: - """Write the one file, inside the one directory. Callers snapshot first so a rejection can revert.""" - directory.mkdir(parents=True, exist_ok=True) - for name, content in proposal.change.items(): - (directory / name).write_text(content, encoding="utf-8") - - -def write_roots(configs_dir: Path, family: str) -> tuple[Path, ...]: - """Everything a round may write: exactly one family's directory, and never the verifiers or the corpus.""" - return (configs_dir / family,) - - -@dataclass -class ScriptedProposer: - """A fixed sequence of proposals, for tests and for the first offline rounds.""" - - proposals: list[Proposal] = field(default_factory=list) - used: int = 0 - - def propose(self, facts: FailureFacts) -> Proposal | None: - del facts - if self.used >= len(self.proposals): - return None - self.used += 1 - return self.proposals[self.used - 1] - - -MetaAgentCall = Callable[..., str] - - -@dataclass -class HarnessXProposer: - """The meta-agent, with write access limited to a copy of the family directory it is targeting. - - It never edits the live tree. The round snapshots the family directory *after* a proposal exists, so a proposer - that wrote in place would have changed the configuration before there was anything to restore, and a rejection - could not put it back. The agent works in a scratch copy and the proposal is the diff of that copy, which also - means the one-file rule and the lever whitelist are enforced on what the agent actually did rather than on what - it claimed (Req 9.1, 9.2, 5.4). - """ - - configs_dir: Path - model: str - # The vendor the meta-agent talks to. Hard-coding anthropic here sent it to a provider with no key configured - # while the detector was talking to DeepSeek, and the round journalled that as "no proposal". - provider: str = "anthropic" - reason: str = "" - agent: MetaAgentCall | None = None # injected in tests; None composes MetaAgent behind harness_ext - - def write_roots(self, family: str) -> tuple[Path, ...]: - return write_roots(self.configs_dir, family) - - def propose(self, facts: FailureFacts) -> Proposal | None: - call = self.agent - if call is None: - if not has_harnessx(): - self.reason = "harnessx_unavailable" - return None - try: - call = _meta_agent_call(self.provider) - except Exception: # noqa: BLE001 — a proposer that cannot start proposes nothing and says so - self.reason = "harnessx_unavailable" - return None - family_dir = self.configs_dir / facts.family - with tempfile.TemporaryDirectory(prefix="ousast-propose-") as scratch: - workspace = Path(scratch) / facts.family - shutil.copytree(family_dir, workspace) - before = _directory_text(workspace) - try: - hypothesis = call(workspace=workspace, facts=facts, model=self.model) - except (TypeError, AttributeError, ImportError): - # "we called the agent wrong" is not "the agent declined", and folding one into the other is how a - # broken composition reads as a quiet round for as long as nobody looks. - raise - except Exception as exc: # noqa: BLE001 — an agent that failed proposed nothing, and the round says why - self.reason = f"meta_agent_failed: {type(exc).__name__}" - return None - after = _directory_text(workspace) - changed = sorted(name for name in set(before) | set(after) if before.get(name) != after.get(name)) - if not changed: - self.reason = "no_change_proposed" - return None - if len(changed) > 1: - self.reason = f"a round changes one file, not {len(changed)}: {', '.join(changed)}" - return None - name = changed[0] - lever: Lever | None = next((key for key, filename in EDITABLE_FILES.items() if filename == name), None) - if lever is None or name not in after: - self.reason = f"{name!r} is not a file any lever writes" - return None - self.reason = "" - return Proposal( - hypothesis=str(hypothesis or "").strip() or f"meta-agent edit to {name}", - lever=lever, - change={name: after[name]}, - predicted_affected=tuple(item.pair for item in facts.misses), - predicted_at_risk=tuple(item.pair for item in facts.leaks), - ) - - -def _directory_text(directory: Path) -> dict[str, str]: - """Every readable file directly inside ``directory``, by name. Nested paths are not a lever's business.""" - out: dict[str, str] = {} - for path in sorted(directory.iterdir()) if directory.is_dir() else []: - if path.is_file(): - try: - out[path.name] = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - return out - - -def _meta_agent_class() -> type: - """`MetaAgent` behind `harness_ext`. A seam, so the argument contract can be pinned without the extra.""" - from ..harness_ext import require_harnessx - - require_harnessx() - from harnessx.meta_harness.agent import MetaAgent # type: ignore[import-not-found] - - return MetaAgent # type: ignore[no-any-return] - - -def _model_config(model: str, provider: str = "anthropic") -> object: - """`MetaAgent(inner_model=...)` takes a `ModelConfig`, never a model name; the name goes to the provider.""" - from ..harness_ext import build_provider, require_harnessx - - require_harnessx() - from harnessx.core.model_config import ModelConfig # type: ignore[import-not-found] - - return ModelConfig(main=build_provider(model, provider)) - - -def _meta_agent_call(provider: str = "anthropic") -> MetaAgentCall: - """Compose `MetaAgent.evolve` behind `harness_ext`, writing only inside the scratch copy. - - `evolve` is a coroutine and returns a HarnessX config path; what this proposer takes from it is the state of the - workspace afterwards, so the return value is only used for the hypothesis line. - """ - from ..harness_ext import require_harnessx - - require_harnessx() - - def call(*, workspace: Path, facts: FailureFacts, model: str) -> str: - import asyncio - - trajectories = workspace.parent / "trajectories" - trajectories.mkdir(parents=True, exist_ok=True) - (trajectories / "facts.json").write_text(facts_prompt(facts), encoding="utf-8") - agent = _meta_agent_class()(inner_model=_model_config(model, provider), allowed_write_roots=(workspace,)) - output = workspace.parent / "meta-out" - output.mkdir(parents=True, exist_ok=True) - asyncio.run(agent.evolve(current_config=workspace, trajectories_dir=trajectories, output_dir=output)) - return f"meta-agent round for {facts.family}" - - return call - - -def _fact(score: PairFamilyScore, other: int) -> PairFact: - return PairFact(pair=score.pair, outcome=score.outcome, runs=score.runs, other_findings=other, rung=score.rung) - - -def facts_prompt(facts: FailureFacts) -> str: - """The facts as the proposer sees them: structured, and free of anything it will be judged on.""" - return json.dumps(facts.to_dict(), indent=2, sort_keys=True) - - -__all__ = [ - "EDITABLE_FILES", - "LEVERS", - "FailureFacts", - "HarnessXProposer", - "Lever", - "PairFact", - "Proposal", - "Proposer", - "ScriptedProposer", - "apply_proposal", - "build_failure_facts", - "facts_prompt", - "refuse_proposal", - "write_roots", -] diff --git a/src/openultrasast/learning/publish.py b/src/openultrasast/learning/publish.py deleted file mode 100644 index afa9550..0000000 --- a/src/openultrasast/learning/publish.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Every published number, regenerated from the artifacts that produced it (learning-harness, Req 11.1, 11.2, 4.5, 5.3). - -A number in a roadmap that nobody can regenerate is a claim, not a measurement. This writes them all -from one entry point, out of the artifacts committed beside them, and puts the denominator next to each: -how many pairs could be scored, why the rest could not, which taxonomy version scored them, the noise -the family shows against itself, and what the run cost per pair it got right. - -Running it twice without new artifacts changes nothing, and the roadmap section is replaced in place -rather than appended, so the file cannot accumulate stale numbers beside fresh ones. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import UTC, datetime -from pathlib import Path - -BEGIN = "" -END = "" -COMMAND = "uv run ousast learning publish" - - -@dataclass(frozen=True) -class Baseline: - """One model's round-zero artifact, read back.""" - - model: str - slices: tuple[str, ...] = () - metrics: dict[str, Mapping[str, object]] = field(default_factory=dict) - floors: dict[str, Mapping[str, object]] = field(default_factory=dict) - cost_usd: float = 0.0 - k_runs: int = 0 - - -@dataclass(frozen=True) -class PublishReport: - command: str - table: str - families: tuple[str, ...] = () - models: tuple[str, ...] = () - artifacts: tuple[Path, ...] = field(default_factory=tuple) - - -def publish(*, learning_dir: Path, measurements_dir: Path, roadmap: Path, date: str | None = None) -> PublishReport: - """Regenerate the family numbers, write the artifact, and replace the roadmap section in place.""" - stamp = date or datetime.now(UTC).strftime("%Y-%m-%d") - models = _baselines(learning_dir) - families = tuple(sorted({family for baseline in models.values() for family in baseline.metrics})) - lever = _lever(measurements_dir) - table = _table(models, families, lever) - artifacts: list[Path] = [] - if models: - measurements_dir.mkdir(parents=True, exist_ok=True) - artifact = measurements_dir / f"{stamp}-learning-families.json" - artifact.write_text( - json.dumps( - { - "generated": stamp, - "command": COMMAND, - "models": {name: dict(baseline.metrics) for name, baseline in sorted(models.items())}, - "floors": {name: dict(baseline.floors) for name, baseline in sorted(models.items())}, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - artifacts.append(artifact) - _replace_section(roadmap, table) - _supersede(roadmap) - return PublishReport(command=COMMAND, table=table, families=families, models=tuple(sorted(models)), artifacts=tuple(artifacts)) - - -def _label(model: str, slices: tuple[str, ...]) -> str: - """What identifies a run in the published table: the model, and the slice when it is not the default pair.""" - return f"{model} ({', '.join(slices)})" if slices else model - - -def _baselines(learning_dir: Path) -> dict[str, Baseline]: - root = learning_dir / "baseline" - found: dict[str, Baseline] = {} - for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []: - report = directory / "report.json" - if not report.is_file(): - continue - payload = json.loads(report.read_text(encoding="utf-8")) - # Keyed by model *and* slice. Keyed by model alone, a second slice silently replaced the first and the - # published table showed one of them as if it were the corpus (Req 3.1, 3.2). - model = str(payload.get("model", directory.name)) - slices = tuple(str(item) for item in (payload.get("slices") or ())) - found[_label(model, slices)] = Baseline( - model=model, - slices=slices, - metrics={str(name): block for name, block in (payload.get("metrics") or {}).items() if isinstance(block, Mapping)}, - floors={str(name): block for name, block in (payload.get("floors") or {}).items() if isinstance(block, Mapping)}, - cost_usd=float(payload.get("cost_usd", 0.0) or 0.0), - k_runs=int(payload.get("k_runs", 0) or 0), - ) - return found - - -def _lever(measurements_dir: Path) -> Mapping[str, object] | None: - path = measurements_dir / "2026-09-06-mechanism-lever-split.json" - if not path.is_file(): - return None - payload = json.loads(path.read_text(encoding="utf-8")) - return payload if isinstance(payload, dict) else None - - -def _table(models: Mapping[str, Baseline], families: tuple[str, ...], lever: Mapping[str, object] | None) -> str: - if not models: - return ( - f"Generated by `{COMMAND}`.\n\nNo baseline has been recorded yet, so there is nothing to publish. " - "Run round zero first; a number without the run that produced it is a claim, not a measurement.\n" - ) - lines = [ - f"Generated by `{COMMAND}` from the artifacts committed beside these numbers.", - "", - ( - "| model | slice | family | scorable | unscorable | recall | silence | Youden " - "| at fixed FPR | noise floor | cost per correct pair |" - ), - "|---|---|---|---|---|---|---|---|---|---|---|", - ] - for _label_key, baseline in sorted(models.items()): - model = baseline.model - slice_name = ", ".join(baseline.slices) or "all" - for family in families: - row = baseline.metrics.get(family) - if row is None: - continue - outcomes = row.get("outcomes") - raw_correct = outcomes.get("pair_correct") if isinstance(outcomes, Mapping) else None - correct = int(raw_correct) if isinstance(raw_correct, int | float) else 0 - floor = baseline.floors.get(family) - raw_budget = floor.get("budget_flips") if floor is not None else None - budget = int(raw_budget) if isinstance(raw_budget, int | float) else 0 - cost = baseline.cost_usd - lines.append( - f"| {model} | {slice_name} | {family} | {row.get('scorable', 0)} (taxonomy {row.get('taxonomy_version', '')}) " - f"| {_reasons(row)} | {_num(row.get('recall'))} | {_num(row.get('silence'))} | {_num(row.get('youden'))} " - f"| {_num(row.get('fixed_fpr_recall'))} | budget {budget} | {_cost(cost, correct)} |" - ) - lines += ["", _credit_note(models), ""] - cutoff = _cutoff_note(models, families) - if cutoff: - lines += [cutoff, ""] - if lever is not None: - lines += [_lever_note(lever), ""] - return "\n".join(lines) - - -def _cutoff_note(models: Mapping[str, Baseline], families: tuple[str, ...]) -> str: - """The post-cutoff slice, its cutoff and the undated count (Req 10.5). - - A model that may have read the fix cannot be measured on it the same way as one that cannot have, so the rows - fixed after the detector's training cutoff are reported apart. Rows the corpus records no date for are in - neither slice, and their number is here rather than folded silently into the whole.""" - rows = [ - (model, family, baseline.metrics[family]) - for model, baseline in sorted(models.items()) - for family in families - if family in baseline.metrics and str(baseline.metrics[family].get("cutoff") or "") - ] - if not rows: - return "No detector cutoff is configured, so no post-cutoff slice is reported; every figure above is over the whole corpus." - cutoff = str(rows[0][2].get("cutoff")) - lines = [ - f"Post-cutoff slice, fixed after {cutoff} and therefore beyond the detector's training data. " - "A row counts only when its earliest possible date is after the cutoff, so a year-precision date can " - "under-claim this slice but never over-claim it.", - "", - "| model | family | post-cutoff scorable | post-cutoff recall | post-cutoff Youden | undated |", - "|---|---|---|---|---|---|", - ] - for model, family, row in rows: - block = row.get("post_cutoff") - block = block if isinstance(block, Mapping) else {} - lines.append( - f"| {model} | {family} | {block.get('scorable', 0)} | {_num(block.get('recall'))} " - f"| {_num(block.get('youden'))} | {row.get('undated', 0)} |" - ) - return "\n".join(lines) - - -def _credit_note(models: Mapping[str, Baseline]) -> str: - hierarchical = any(bool(row.get("hierarchical_credit")) for baseline in models.values() for row in baseline.metrics.values()) - if hierarchical: - return "A parent-child answer earned partial credit in these figures." - return "No family in this taxonomy declares a parent, so hierarchical partial credit could not apply to any figure above." - - -def _lever_note(lever: Mapping[str, object]) -> str: - before = lever.get("before") - after = lever.get("after") - before_map = before if isinstance(before, Mapping) else {} - after_map = after if isinstance(after, Mapping) else {} - pairs = lever.get("holdout_pairs", 0) - return ( - f"Mechanism lever on the {lever.get('slice', '')} holdout: detected {after_map.get('detected')} of {pairs} at Youden " - f"{after_map.get('youden')} once only train-split pairs teach, against detected {before_map.get('detected')} at Youden " - f"{before_map.get('youden')} when every pair taught. The difference is what holdout pairs stopped teaching." - ) - - -def _reasons(row: Mapping[str, object]) -> str: - reasons = row.get("unscorable") or {} - if not isinstance(reasons, Mapping) or not reasons: - return "none" - return ", ".join(f"{name}: {count}" for name, count in sorted(reasons.items())) - - -def _num(value: object) -> str: - return f"{float(value):.3f}" if isinstance(value, int | float) else "-" - - -def _cost(cost: float, correct: int) -> str: - return f"{cost / correct:.2f}" if cost and correct else "-" - - -# Claims the split correction supersedes, as they were written before holdout pairs stopped teaching. Req 5.3 says -# the corrected number is published *in place of* the earlier one; a document that carries both asserts two numbers -# for one measurement and the reader has no way to tell which run is current. -_SUPERSEDED = ( - re.compile(r"human holdout profile pair_correct \d+ -> \d+ of \d+, Youden [-+]?\d+ -> [-+]?[\d.]+(?:, round \d+ `[^`]+`)?\."), - re.compile(r"The lever recovered \d+ of \d+ holdout pairs at Youden [-+][\d.]+\."), -) -_SUPERSEDED_NOTE = ( - "superseded by the learning-harness section below: that number was measured with holdout pairs teaching " - "(see `benchmarks/measurements/2026-09-06-mechanism-lever-split.json`)." -) - - -def _supersede(roadmap: Path) -> None: - """Rewrite every superseded lever claim to point at the regenerated section. Idempotent: the pattern is gone - afterwards, so a second run changes nothing.""" - if not roadmap.is_file(): - return - text = roadmap.read_text(encoding="utf-8") - updated = text - for pattern in _SUPERSEDED: - updated = pattern.sub(_SUPERSEDED_NOTE, updated) - if updated != text: - roadmap.write_text(updated, encoding="utf-8") - - -def _replace_section(roadmap: Path, table: str) -> None: - body = f"{BEGIN}\n\n## Classified detector numbers\n\n{table.rstrip()}\n\n{END}\n" - text = roadmap.read_text(encoding="utf-8") if roadmap.is_file() else "" - if BEGIN in text and END in text: - head, _, rest = text.partition(BEGIN) - _, _, tail = rest.partition(END) - roadmap.write_text(head + body + tail.lstrip("\n"), encoding="utf-8") - return - roadmap.parent.mkdir(parents=True, exist_ok=True) - roadmap.write_text((text.rstrip("\n") + "\n\n" if text else "") + body, encoding="utf-8") - - -__all__ = ["BEGIN", "COMMAND", "END", "Baseline", "PublishReport", "publish"] diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py deleted file mode 100644 index 89b8bd0..0000000 --- a/src/openultrasast/learning/rounds.py +++ /dev/null @@ -1,761 +0,0 @@ -"""Round zero and the per-family noise floor (learning-harness, Req 8). - -Before anything learns, every family gets the same detector and is measured against itself. Running a -pair several times at the same settings does not give the same answer: the chat provider offers no seed, -so the spread between runs is real and unavoidable. That spread is the family's noise floor, and it -becomes the budget a later round must stay inside before a cross-family change counts as a regression. -Choosing a budget instead of measuring one would be a guess dressed as a threshold. - -Artifacts are keyed by detector model, so a second model is baselined by changing the model and running -again. Configurations are never touched by a model swap, which is what makes the two numbers comparable -and what makes reconsidering the model cheap. -""" - -from __future__ import annotations - -import json -import math -import re -import tempfile -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..findings import StaticFinding -from ..pairs import PairCase -from .classify import classify_pair -from .detectors import FamilyConfig, write_default_configs -from .families import FamilyTaxonomy -from .journal import Archive, Attribution, LearningJournal, RoundRecord -from .proposer import Proposal, Proposer -from .scoring import FamilyMetrics, PairFamilyScore, aggregate, score_pair_family - -DEFAULT_SLICES = ("vibe-py", "agent-vfc") # the web families, where real-world code and dangerous fruit coincide -NON_GATING_FAMILIES = frozenset({"memory"}) # measured on the vendored slice, never allowed to fail a round -DEFAULT_K_RUNS = 5 -Scan = Callable[[Path], list[StaticFinding]] - - -@dataclass(frozen=True) -class NoiseFloor: - """How much a family disagrees with itself between runs, and the budget that follows from it.""" - - family: str - k_runs: int - pairs: int - flaky_pairs: int - negative_flip_rate: float - budget_flips: int - gates: bool = True - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "k_runs": self.k_runs, - "pairs": self.pairs, - "flaky_pairs": self.flaky_pairs, - "negative_flip_rate": self.negative_flip_rate, - "budget_flips": self.budget_flips, - "gates": self.gates, - } - - -@dataclass(frozen=True) -class BaselineReport: - model: str - k_runs: int - taxonomy_version: str - slices: tuple[str, ...] - thinking: bool = False # the condition that produced these numbers, so two modes cannot be read as one - floors: dict[str, NoiseFloor] = field(default_factory=dict) - metrics: dict[str, dict[str, object]] = field(default_factory=dict) - # `/`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one - # pooled number hides which of them moved (Req 8.4). - per_slice: dict[str, dict[str, object]] = field(default_factory=dict) - scores: tuple[PairFamilyScore, ...] = () - cost_usd: float = 0.0 - - def to_dict(self) -> dict[str, object]: - return { - "model": self.model, - "k_runs": self.k_runs, - "taxonomy_version": self.taxonomy_version, - "slices": list(self.slices), - "thinking": self.thinking, - "cost_usd": self.cost_usd, - "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())}, - "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())}, - "per_slice": {name: dict(block) for name, block in sorted(self.per_slice.items())}, - } - - -def run_baseline( - cases: Sequence[PairCase], - *, - taxonomy: FamilyTaxonomy, - configs_dir: Path, - scan_factory: Callable[[FamilyConfig], Scan], - model: str, - out_dir: Path, - slices: Sequence[str] = DEFAULT_SLICES, - k_runs: int = DEFAULT_K_RUNS, - prompt: str | None = None, - spent_usd: Callable[[], float] | None = None, - cutoff: str = "", - thinking: bool = False, - resume: bool = False, -) -> BaselineReport: - """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts. - - The factory is the point. One shared `scan` would score every family with one detector, and since the generalist - tags its findings `family:unknown` — which earns no credit against any real family, by the taxonomy's own - decision that abstention is not a partial answer — every family but `unknown` would baseline at a structural - zero and the noise floors would all be nil (Req 8.2). - """ - from ..tool_hunter import _SYSTEM_PROMPT - from .detectors import load_family_configs - - if k_runs < MIN_K_RUNS: - raise ValueError(f"round zero scores at least three runs per pair; {k_runs} was asked for") - if not configs_dir.exists(): - write_default_configs(configs_dir, taxonomy, prompt=prompt or _SYSTEM_PROMPT, version="0") - configs = load_family_configs(configs_dir, taxonomy) - selected = [case for case in cases if case.slice in set(slices)] - # Every pair is written down as it is measured. Round zero over the vfc slice is 1760 detector runs and hours of - # paid calls; accumulating in memory and persisting once at the end means a timeout, a kill or a machine going - # away costs all of it. An exception already cost only one pair; nothing else did. - checkpoint = out_dir / "baseline" / _artifact_key(model, slices, thinking) / "scores.jsonl" - done = _checkpointed(checkpoint) if resume else {} - if not resume and checkpoint.exists(): - checkpoint.unlink() - scores = [] - for case in selected: - name = _family_of(case, taxonomy) - config = configs.get(name) - if config is None: - continue # a family with no configuration is not measured rather than measured with someone else's - if case.name in done: - scores.append(done[case.name]) - continue - try: - score = score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name) - scores.append(score) - _append_score(checkpoint, score) - except Exception: # noqa: BLE001 — a provider hiccup costs one pair, not the whole run - # Round zero is the most expensive thing this harness does. Raising here threw away everything already - # measured and paid for, so the only honest response to a flaky endpoint was to run it all again. - unreachable = PairFamilyScore( - pair=case.name, - family=name, - slice=case.slice, - fix_date=case.fix_date, - runs=(), - outcome="unscorable", - unscorable_reason="detector_unreachable", - ) - scores.append(unreachable) - _append_score(checkpoint, unreachable) - metrics = aggregate(scores, taxonomy=taxonomy, cutoff=cutoff) - floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()} - report = BaselineReport( - model=model, - k_runs=k_runs, - taxonomy_version=taxonomy.version, - slices=tuple(slices), - thinking=thinking, - floors=floors, - metrics={name: block.to_dict() for name, block in metrics.items()}, - per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()}, - scores=tuple(scores), - cost_usd=spent_usd() if spent_usd is not None else 0.0, - ) - _write(out_dir, report) - return report - - -def _append_score(path: Path, score: PairFamilyScore) -> None: - """One line per pair, flushed, so an interrupted run keeps what it measured.""" - path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "pair": score.pair, - "family": score.family, - "slice": score.slice, - "fix_date": score.fix_date, - "outcome": score.outcome, - "runs": list(score.runs), - "flips": score.flips, - "other_findings_vuln": score.other_findings_vuln, - "other_findings_fixed": score.other_findings_fixed, - "fabricated_vuln": score.fabricated_vuln, - "fabricated_fixed": score.fabricated_fixed, - "rung": score.rung, - "unscorable_reason": score.unscorable_reason, - } - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(payload, sort_keys=True) + "\n") - handle.flush() - - -def _checkpointed(path: Path) -> dict[str, PairFamilyScore]: - """What a previous run of this exact baseline already measured and paid for.""" - if not path.is_file(): - return {} - out: dict[str, PairFamilyScore] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - try: - row = json.loads(line) - except json.JSONDecodeError: - continue # a line half-written when the run died is not a measurement - out[str(row["pair"])] = PairFamilyScore( - pair=str(row["pair"]), - family=str(row["family"]), - slice=str(row.get("slice", "")), - fix_date=str(row.get("fix_date", "")), - runs=tuple(row.get("runs", ())), - outcome=row["outcome"], - flips=int(row.get("flips", 0)), - other_findings_vuln=int(row.get("other_findings_vuln", 0)), - other_findings_fixed=int(row.get("other_findings_fixed", 0)), - fabricated_vuln=int(row.get("fabricated_vuln", 0)), - fabricated_fixed=int(row.get("fabricated_fixed", 0)), - rung=row.get("rung", "suspicion"), - unscorable_reason=row.get("unscorable_reason"), - ) - return out - - -def score_case(case: PairCase, scan: Scan, *, taxonomy: FamilyTaxonomy, runs: int, family: str | None = None) -> PairFamilyScore: - """Run one detector over both sides of one pair ``runs`` times and score it for its family. - - A pair the corpus already marked unscorable is never run: spending model calls on a row that cannot - be scored buys nothing and would put its noise into a floor it does not belong in. - """ - from ..pairs import _materialize_side, _overlay_scan - - name = family or _family_of(case, taxonomy) - if case.unscorable: - return PairFamilyScore( - pair=case.name, - family=name, - runs=(), - outcome="unscorable", - slice=case.slice, - fix_date=case.fix_date, - unscorable_reason=case.unscorable, - ) - with tempfile.TemporaryDirectory(prefix="ousast-round-") as scratch: - vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") - fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - ranges = _overlay_scan(vuln_root).ranges - fixed_ranges = _overlay_scan(fix_root).ranges - vuln_runs = [scan(vuln_root) for _ in range(max(runs, 1))] - fix_runs = [scan(fix_root) for _ in range(max(runs, 1))] - return score_pair_family( - case, name, vuln_runs, fix_runs, ranges=ranges, taxonomy=taxonomy, parse_ok=bool(ranges), fixed_ranges=fixed_ranges - ) - - -def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]: - """Every baseline written under ``out_dir``, model by model, so two detectors can be read side by side.""" - root = out_dir / "baseline" - table: dict[str, dict[str, dict[str, object]]] = {} - for directory in sorted(path for path in root.iterdir() if path.is_dir()) if root.is_dir() else []: - report = directory / "report.json" - if not report.is_file(): - continue - payload = json.loads(report.read_text(encoding="utf-8")) - model = str(payload.get("model", directory.name)) - slices = tuple(str(item) for item in (payload.get("slices") or ())) - conditions = [*(() if not payload.get("thinking") else ("thinking",)), *(() if slices == DEFAULT_SLICES else slices)] - label = f"{model} ({', '.join(conditions)})" if conditions else model - table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} - return table - - -def _floor(family: str, metrics: FamilyMetrics, scores: Sequence[PairFamilyScore], k_runs: int) -> NoiseFloor: - """A family's own disagreement between runs, as a rate and as a whole number of pairs.""" - scorable = [item for item in scores if item.outcome != "unscorable"] - flaky = sum(1 for item in scorable if item.flips) - rate = flaky / len(scorable) if scorable else 0.0 - return NoiseFloor( - family=family, - k_runs=k_runs, - pairs=metrics.scorable, - flaky_pairs=flaky, - negative_flip_rate=rate, - budget_flips=math.ceil(rate * metrics.scorable) if metrics.scorable else 0, - gates=family not in NON_GATING_FAMILIES, - ) - - -def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str: - for row in case.expected: - if row.family: - return str(row.family) - answer = classify_pair(case, taxonomy, client=None) - return answer.families[0] if answer.families else "unknown" - - -def _artifact_key(model: str, slices: Sequence[str], thinking: bool = False) -> str: - """What identifies a baseline run: the detector model, and the slice selection when it is not the default. - - Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and - the published table shows one of them as if it were everything (Req 8.4).""" - parts = [_slug(model)] - if thinking: - parts.append("thinking") - if tuple(slices) != DEFAULT_SLICES: - parts.append(_slug("-".join(sorted(slices)))) - return "-".join(parts) - - -def _write(out_dir: Path, report: BaselineReport) -> None: - directory = out_dir / "baseline" / _artifact_key(report.model, report.slices, report.thinking) - directory.mkdir(parents=True, exist_ok=True) - payload = report.to_dict() - (directory / "noise-floors.json").write_text( - json.dumps({"model": report.model, "k_runs": report.k_runs, "floors": payload["floors"]}, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - (directory / "report.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def _slug(model: str) -> str: - return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed" - - -def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = (), thinking: bool = False) -> dict[str, NoiseFloor]: - """The budgets a later round must stay inside, as round zero measured them for this model.""" - path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES, thinking) / "noise-floors.json" - if not path.is_file(): - return {} - payload = json.loads(path.read_text(encoding="utf-8")) - floors: dict[str, NoiseFloor] = {} - for name, block in (payload.get("floors") or {}).items(): - if isinstance(block, Mapping): - floors[str(name)] = NoiseFloor( - family=str(block.get("family", name)), - k_runs=int(block.get("k_runs", DEFAULT_K_RUNS) or DEFAULT_K_RUNS), - pairs=int(block.get("pairs", 0) or 0), - flaky_pairs=int(block.get("flaky_pairs", 0) or 0), - negative_flip_rate=float(block.get("negative_flip_rate", 0.0) or 0.0), - budget_flips=int(block.get("budget_flips", 0) or 0), - gates=bool(block.get("gates", True)), - ) - return floors - - -MIN_K_RUNS = 3 # Req 3.5: a single run misses most real changes, so no stage may score with fewer - - -def run_learning_round( - cases: Sequence[PairCase], - *, - family: str, - taxonomy: FamilyTaxonomy, - configs_dir: Path, - proposer: Proposer, - scan_factory: Callable[[FamilyConfig], Scan], - model: str, - journal: LearningJournal, - archive: Archive, - floors: Mapping[str, NoiseFloor], - out_dir: Path, - cost_cap_usd: float = 10.0, - minibatch: int = 8, - k_runs: int = MIN_K_RUNS, - sweep_families: Sequence[str] = (), - spent_usd: Callable[[], float] | None = None, - trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None, -) -> RoundRecord: - """One round: propose one change for one family, then let the evidence decide whether it stays. - - Staged so that the cheap answer comes first: a train minibatch, then the family's held-out pairs, - then every other family. Each stage scores several runs per pair, because a single run misses most - real changes. The proposer never sees a held-out pair, and the family directory is snapshotted before - the change so any outcome other than acceptance leaves the tree exactly as it was found. - """ - from .acceptance import DirectorySnapshot, bump_version, decide, within_budget - from .detectors import load_family_configs - from .proposer import apply_proposal, build_failure_facts, refuse_proposal - from .split import is_teacher, refuse_if_holdout - - if k_runs < MIN_K_RUNS: - raise ValueError(f"a stage scores at least three runs per pair; {k_runs} was asked for") - round_number = journal.next_round() # type: ignore[attr-defined] - directory = out_dir / "rounds" / str(round_number) - if directory.exists(): - from .journal import JournalError - - raise JournalError(f"round {round_number} already has a directory; history is append-only") - configs = load_family_configs(configs_dir, taxonomy) - config = configs[family] - train = [case for case in cases if is_teacher(case) and _family_of(case, taxonomy) == family] - holdout = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == family] - batch = train[:minibatch] - spent = 0.0 - - def _score(rows: Sequence[PairCase], scan: Scan, name: str) -> list[PairFamilyScore]: - return [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows] - - def _over_cap() -> bool: - """Req 9.8: cost is read after every stage, so a round stops rather than finishing and then complaining.""" - nonlocal spent - spent = spent_usd() if spent_usd is not None else 0.0 - return not within_budget(spent, cost_cap_usd) - - record = _round_record(round_number, family, None, taxonomy, config) - try: - before_scan = scan_factory(config) - before_train = _score(batch, before_scan, family) - before_hold = _score(holdout, before_scan, family) - except Exception as exc: - # These stages run before anything is applied, so there is nothing to revert — but they have already spent - # money, and a round that leaves no record is a round nobody can account for. Measured: round 5 died on a - # network read during its first stage and vanished. - _finish( - journal, - directory, - record, - outcome="reverted", - reason=f"stage_failed: {type(exc).__name__}", - cost=spent_usd() if spent_usd is not None else 0.0, - trajectories=trajectories, - ) - raise - if _over_cap(): - return _finish( - journal, directory, record, outcome="reverted_cost", reason="cost_cap_exceeded", cost=spent, trajectories=trajectories - ) - facts = build_failure_facts(before_train, family=family, config=config, journal=journal, train_pairs={case.name for case in train}) - proposal = proposer.propose(facts) # type: ignore[attr-defined] - record = _round_record(round_number, family, proposal, taxonomy, config) - if proposal is None: - # `no_proposal` covers two different worlds: a proposer that considered the facts and declined, and one - # that could not start at all. Losing that distinction hides a fixable configuration error as a quiet round. - why = str(getattr(proposer, "reason", "") or "") - return _finish( - journal, - directory, - record, - outcome="rejected", - reason=f"no_proposal: {why}" if why else "no_proposal", - cost=spent, - trajectories=trajectories, - ) - refusal = refuse_proposal(proposal, config) - if refusal is not None: - return _finish( - journal, directory, record, outcome="rejected", reason=refusal, proposal=proposal, cost=spent, trajectories=trajectories - ) - leak = refuse_if_holdout(_pairs_named_by(proposal, cases), cases) - if leak is not None: - # Req 5.2: a change a holdout pair taught, or that names one, is refused by name rather than measured. - return _finish( - journal, - directory, - record, - outcome="refused_holdout", - reason=leak.degradation()["detail"], # type: ignore[arg-type] - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - # The sweep's "before" is measured with the unchanged configurations, so what it later reports is what this - # round flipped and not what was already failing (Req 9.5). - before_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs) - if _over_cap(): - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - snapshot = DirectorySnapshot.of(configs_dir / family) - apply_proposal(proposal, configs_dir / family) - version = str(int(config.version) + 1) if config.version.isdigit() else f"{config.version}+1" - try: - after_config = load_family_configs(configs_dir, taxonomy)[family] - after_scan = scan_factory(after_config) - after_train = _score(batch, after_scan, family) - after_hold = _score(holdout, after_scan, family) - for score in after_train + after_hold: - archive.record(family, version, score) # type: ignore[attr-defined] - if _over_cap(): - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - scores=after_train + after_hold, - trajectories=trajectories, - ) - after_sweep = _sweep_scores(cases, taxonomy, configs, scan_factory, sweep_families, family, k_runs) - sweep, flipped_unpredicted = _negative_flips(before_sweep, after_sweep) - train_delta = _delta(before_train, after_train) - holdout_delta = _delta(before_hold, after_hold) - flipped = _flipped(before_train + before_hold, after_train + after_hold) - train_moves = _moves(before_train, after_train) - holdout_moves = _moves(before_hold, after_hold) - attribution = Attribution( - flipped_predicted=len(flipped), - flipped_unpredicted=len(flipped_unpredicted), - precision=len(flipped) / (len(flipped) + len(flipped_unpredicted)) if flipped or flipped_unpredicted else 0.0, - ) - if _over_cap(): - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="reverted_cost", - reason="cost_cap_exceeded", - proposal=proposal, - cost=spent, - scores=after_train + after_hold, - trajectories=trajectories, - ) - verdict = decide(target=family, train_delta=train_delta, holdout_delta=holdout_delta, sweep=sweep, floors=floors) # type: ignore[arg-type] - if not verdict.accepted: - snapshot.restore() - return _finish( - journal, - directory, - record, - outcome="rejected", - reason=verdict.reason, - proposal=proposal, - cost=spent, - train_delta=train_delta, - holdout_delta=holdout_delta, - sweep=sweep, - attribution=attribution, - moves=(train_moves, holdout_moves), - flipped=flipped, - unpredicted=flipped_unpredicted, - scores=after_train + after_hold, - trajectories=trajectories, - ) - bump_version(configs_dir / family, version) - return _finish( - journal, - directory, - record, - outcome="accepted", - reason=verdict.reason, - proposal=proposal, - cost=spent, - train_delta=train_delta, - holdout_delta=holdout_delta, - sweep=sweep, - attribution=attribution, - moves=(train_moves, holdout_moves), - flipped=flipped, - unpredicted=flipped_unpredicted, - scores=after_train + after_hold, - version=version, - trajectories=trajectories, - ) - except Exception as exc: - # A stage that raises has already spent money and already changed the tree. Both facts belong in the - # journal before the exception leaves, or the next round starts from a state nothing describes. - snapshot.restore() - _finish( - journal, - directory, - record, - outcome="reverted", - reason=f"stage_failed: {type(exc).__name__}", - proposal=proposal, - cost=spent, - trajectories=trajectories, - ) - raise - - -def _pairs_named_by(proposal: Proposal, cases: Sequence[PairCase]) -> tuple[str, ...]: - """Every corpus pair this proposal names, in its predictions or in the file it writes. - - A memory entry that quotes a held-out pair is the loop learning the answer instead of the shape, and it looks - exactly like a legitimate hard negative until the holdout number is read as if it meant something.""" - haystack = "\n".join([*proposal.predicted_affected, *proposal.predicted_at_risk, *proposal.change.values()]) - # Substring first, regex only for the handful that survive it: the corpus is a few hundred rows and a round - # runs this once, but a per-round sweep of one regex per row is the kind of thing that stops being free. - named = [ - name - for case in cases - if (name := str(getattr(case, "name", ""))) and name in haystack and re.search(rf"(? dict[str, dict[str, str]]: - """Every other family's held-out pairs, family by family, as `{family: {pair: outcome}}`. - - Called once before the change and once after it. Scoring both sides is the only way to say which pairs *this* - round broke: on a real corpus most pairs are failing already, and counting failures would reject every round - for damage it did not do (Req 9.5). Unscorable rows are dropped here rather than counted as failures (Req 4.1). - """ - outcomes: dict[str, dict[str, str]] = {} - for name in families: - if name == target or name not in configs: - continue - rows = [case for case in cases if case.split == "holdout" and _family_of(case, taxonomy) == name] - scan = scan_factory(configs[name]) - scored = [score_case(case, scan, taxonomy=taxonomy, runs=k_runs, family=name) for case in rows] - outcomes[name] = {item.pair: item.outcome for item in scored if item.outcome != "unscorable"} - return outcomes - - -def _negative_flips(before: Mapping[str, Mapping[str, str]], after: Mapping[str, Mapping[str, str]]) -> tuple[dict[str, int], list[str]]: - """Pairs that were right before this round and are not right after it, per family and by name.""" - flips: dict[str, int] = {} - broken: list[str] = [] - for name, was in before.items(): - now = after.get(name, {}) - lost = sorted(pair for pair, outcome in was.items() if outcome == "pair_correct" and now.get(pair) != "pair_correct") - flips[name] = len(lost) - broken.extend(lost) - return flips, broken - - -def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> float: - """The change in how many pairs came out right, as a fraction of the pairs that could be scored.""" - scorable = [item for item in before if item.outcome != "unscorable"] - if not scorable: - return 0.0 - was = sum(1 for item in before if item.outcome == "pair_correct") - now = sum(1 for item in after if item.outcome == "pair_correct") - return (now - was) / len(scorable) - - -def _moves(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> tuple[int, int]: - """(pairs that became correct, pairs that stopped being correct) — what the sign test is run over.""" - was = {item.pair: item.outcome for item in before if item.outcome != "unscorable"} - better = worse = 0 - for item in after: - previous = was.get(item.pair) - if previous is None or item.outcome == "unscorable": - continue - if previous != "pair_correct" and item.outcome == "pair_correct": - better += 1 - elif previous == "pair_correct" and item.outcome != "pair_correct": - worse += 1 - return better, worse - - -def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]: - """Pairs this round turned right, by name and in a stable order.""" - was = {item.pair: item.outcome for item in before} - return sorted(item.pair for item in after if item.outcome == "pair_correct" and was.get(item.pair) not in {"pair_correct", None}) - - -def _round_record( - round_number: int, family: str, proposal: Proposal | None, taxonomy: FamilyTaxonomy, config: FamilyConfig -) -> dict[str, object]: - return { - "round": round_number, - "family": family, - "hypothesis": proposal.hypothesis if proposal is not None else "", - "levers": (proposal.lever,) if proposal is not None else (), - "predicted_affected": tuple(proposal.predicted_affected) if proposal is not None else (), - "predicted_at_risk": tuple(proposal.predicted_at_risk) if proposal is not None else (), - "taxonomy_version": taxonomy.version, - "config_version": config.version, - } - - -def _finish( - journal: LearningJournal, - directory: Path, - base: dict[str, object], - *, - outcome: str, - reason: str, - proposal: Proposal | None = None, - cost: float = 0.0, - train_delta: float = 0.0, - holdout_delta: float = 0.0, - sweep: Mapping[str, int] | None = None, - attribution: Attribution | None = None, - moves: tuple[tuple[int, int], tuple[int, int]] = ((0, 0), (0, 0)), - flipped: Sequence[str] = (), - unpredicted: Sequence[str] = (), - scores: Sequence[PairFamilyScore] = (), - version: str | None = None, - trajectories: Callable[[], Sequence[Mapping[str, object]]] | None = None, -) -> RoundRecord: - """Write the round directory and journal the outcome. Every outcome is journalled, not just the good ones.""" - from ..redaction import redact_secrets - - record = RoundRecord( - **base, # type: ignore[arg-type] - outcome=outcome, - reason=reason, - target_train_delta=train_delta, - target_holdout_delta=holdout_delta, - sweep=dict(sweep or {}), - attribution=attribution or Attribution(), - cost_usd=cost, - train_better=moves[0][0], - train_worse=moves[0][1], - holdout_better=moves[1][0], - holdout_worse=moves[1][1], - ) - if version is not None: - record = RoundRecord(**{**record.__dict__, "config_version": version}) - directory.mkdir(parents=True, exist_ok=True) - if proposal is not None: - (directory / "proposal.json").write_text( - redact_secrets(json.dumps(proposal.to_dict(), indent=2, sort_keys=True)) + "\n", - encoding="utf-8", - ) - (directory / "scores.json").write_text( - json.dumps([{"pair": item.pair, "family": item.family, "outcome": item.outcome} for item in scores], indent=2, sort_keys=True) - + "\n", - encoding="utf-8", - ) - (directory / "attribution.json").write_text( - json.dumps({"flipped_predicted": list(flipped), "flipped_unpredicted": list(unpredicted)}, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - rows = list(trajectories() if trajectories is not None else ()) - (directory / "trajectories.jsonl").write_text( - "".join(redact_secrets(json.dumps(row, sort_keys=True)) + "\n" for row in rows), encoding="utf-8" - ) - journal.append(record) # type: ignore[attr-defined] - return record - - -__all__ = [ - "DEFAULT_K_RUNS", - "MIN_K_RUNS", - "run_learning_round", - "DEFAULT_SLICES", - "NON_GATING_FAMILIES", - "BaselineReport", - "NoiseFloor", - "Scan", - "compare_baselines", - "load_noise_floors", - "run_baseline", - "score_case", -] diff --git a/src/openultrasast/learning/scoring.py b/src/openultrasast/learning/scoring.py deleted file mode 100644 index b324e36..0000000 --- a/src/openultrasast/learning/scoring.py +++ /dev/null @@ -1,405 +0,0 @@ -"""Class-aware, pair-wise scoring (learning-harness, Req 3, 4.1, 4.5). - -A pair is scored once per family. Detection means a finding whose family tag relates to the labeled -family, sitting inside the labeled function; the words in a finding are never read, because a text -token was exactly the proxy that made the old hunter number meaningless. Silence means no such finding -on the fixed side: findings of other families and findings outside the labeled function are counted -separately and are never called leaks. A pair that cannot be scored keeps its reason and is never run. - -Every evaluation runs a pair K times, because a single run misses most real changes. The recorded -outcome is the majority of the runs, and the runs that disagree with it are the pair's own flakiness. -""" - -from __future__ import annotations - -import math -from collections import Counter -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from typing import Literal - -from ..findings import StaticFinding -from .families import FamilyTaxonomy - -FamilyOutcome = Literal["pair_correct", "both_flagged", "both_silent", "reversed", "unscorable"] -# Families whose bug shape is the absence of a check on a reached handler: without the registration that -# makes the handler reachable, the pair says nothing either way, which is a corpus defect with its own name. -CONTEXT_DEPENDENT_FAMILIES = frozenset({"access_control"}) - - -def is_post_cutoff(fix_date: str, cutoff: str) -> bool: - """True when the *earliest possible* date of ``fix_date`` is after ``cutoff`` (Req 10.5). - - The corpus records dates at the precision it has: a full ISO day for a few rows, a year for almost all of them. - A year-precision row counts only when the whole year is after the cutoff, so a coarse date can under-claim the - clean slice but never over-claim it — and over-claiming is the one that would turn contamination into a result. - """ - if not fix_date or len(cutoff) != 10: - return False # a cutoff coarser than a day is the one over-claim path there is, so it is refused outright - earliest = fix_date if len(fix_date) == 10 else f"{fix_date}-01-01"[:10] - return earliest > cutoff - - -Rung = Literal["suspicion", "static_corroboration", "proven"] -_RUNGS: tuple[Rung, ...] = ("suspicion", "static_corroboration", "proven") -FunctionRanges = Mapping[str, Sequence[tuple[str, int, int]]] - - -@dataclass(frozen=True) -class PairFamilyScore: - """What one pair says about one family.""" - - pair: str - family: str - runs: tuple[FamilyOutcome, ...] - outcome: FamilyOutcome - slice: str = "" - other_findings_vuln: int = 0 # right place, another family: not a detection, never a leak - other_findings_fixed: int = 0 - fabricated_vuln: int = 0 # a family id outside the taxonomy: noise, and worth naming - fabricated_fixed: int = 0 - flips: int = 0 # runs that disagreed with the majority - rung: Rung = "suspicion" - unscorable_reason: str | None = None - fix_date: str = "" # the date the corpus recorded, at whatever precision it has (Req 10.5) - - -def score_pair_family( - case: object, - family: str, - vuln_runs: Sequence[Sequence[StaticFinding]], - fixed_runs: Sequence[Sequence[StaticFinding]], - *, - ranges: FunctionRanges, - taxonomy: FamilyTaxonomy, - parse_ok: bool = True, - entry_points: Sequence[str] | None = None, - fixed_ranges: FunctionRanges | None = None, -) -> PairFamilyScore: - """Score one pair for one family over K runs of the detector on each side. - - ``fixed_ranges`` is the *fixed* side's parse. A fix that adds a guard above the handler moves its first line, - so judging the fixed side against the vulnerable side's span reads a finding inside the fixed handler as - outside it: a leak recorded as silence, and always in the direction that flatters the number. - """ - name = str(getattr(case, "name", "?")) - slice_name = str(getattr(case, "slice", "") or "") - reason = unscorable_reason(case, parse_ok=parse_ok, ranges=ranges, entry_points=entry_points, family=family) - if reason is not None: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason=reason, - ) - spans = _spans(case, ranges) - if fixed_ranges is None: - fixed_spans = spans - else: - # A trap pair's fixed side holds a different, correctly guarded handler; `fix_function` names it. Without a - # span there, every fixed-side finding reads as outside the labeled function and the pair scores as silence. - fixed_spans = _spans(case, fixed_ranges, function=str(getattr(case, "fix_function", "") or "")) - if not fixed_spans: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason="unresolved_label:fixed", - ) - outcomes: list[FamilyOutcome] = [] - other_vuln = other_fixed = fabricated_vuln = fabricated_fixed = 0 - rung: Rung = "suspicion" - for index in range(max(len(vuln_runs), len(fixed_runs))): - vuln = list(vuln_runs[index]) if index < len(vuln_runs) else [] - fixed = list(fixed_runs[index]) if index < len(fixed_runs) else [] - hits_vuln = _hits(vuln, family, spans, taxonomy) - hits_fixed = _hits(fixed, family, fixed_spans, taxonomy) - other_vuln += _others(vuln, family, spans, taxonomy) - other_fixed += _others(fixed, family, fixed_spans, taxonomy) - fabricated_vuln += _fabricated(vuln, taxonomy) - fabricated_fixed += _fabricated(fixed, taxonomy) - rung = _best_rung(rung, hits_vuln) - outcomes.append(_outcome(bool(hits_vuln), bool(hits_fixed))) - if not outcomes: - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=(), - outcome="unscorable", - unscorable_reason="no_runs", - ) - majority = Counter(outcomes).most_common(1)[0][0] - return PairFamilyScore( - pair=name, - family=family, - slice=slice_name, - fix_date=str(getattr(case, "fix_date", "") or ""), - runs=tuple(outcomes), - outcome=majority, - other_findings_vuln=other_vuln, - other_findings_fixed=other_fixed, - fabricated_vuln=fabricated_vuln, - fabricated_fixed=fabricated_fixed, - flips=sum(1 for item in outcomes if item != majority), - rung=rung, - ) - - -def _outcome(detected: bool, leaked: bool) -> FamilyOutcome: - if detected and not leaked: - return "pair_correct" - if detected and leaked: - return "both_flagged" - return "reversed" if leaked else "both_silent" - - -def unscorable_reason( - case: object, - *, - parse_ok: bool = True, - ranges: FunctionRanges, - entry_points: Sequence[str] | None = None, - family: str = "", -) -> str | None: - """Why this pair cannot be scored, or None. The reasons have different owners, so they must not be conflated. - - A maintainer-declared limit wins over anything computed. An unparsable side is a tooling gap - (``unsupported_language``), a labeled function no parsed range names is a corpus labelling defect - (``unresolved_label``), and a handler that arrived without the registration that makes it reachable is a - harvest defect (``missing_context``), which only matters for families whose bug shape is a missing check. - """ - declared = getattr(case, "unscorable", None) - if declared: - return str(declared) - if not parse_ok: - return "unsupported_language" - if not _spans(case, ranges): - return "unresolved_label" - labeled_family = family or next( - (str(getattr(row, "family", "") or "") for row in getattr(case, "expected", ()) or () if getattr(row, "family", None)), "" - ) - if entry_points is not None and labeled_family in CONTEXT_DEPENDENT_FAMILIES: - labeled = {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} - if not (labeled & set(entry_points)): - return "missing_context" - return None - - -def _spans(case: object, ranges: FunctionRanges, *, function: str = "") -> tuple[tuple[str, int, int], ...]: - """The (path, start, end) spans of the labeled functions. ``function`` overrides the label for the fixed side.""" - wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} - wanted.discard("") - found: list[tuple[str, int, int]] = [] - for path, entries in ranges.items(): - for name, start, end in entries: - if name in wanted: - found.append((path, start, end)) - return tuple(found) - - -def _inside(finding: StaticFinding, spans: Sequence[tuple[str, int, int]]) -> bool: - return finding.line is not None and any(path == finding.path and start <= finding.line <= end for path, start, end in spans) - - -def _families_of(finding: StaticFinding) -> tuple[str, ...]: - return tuple(tag.split(":", 1)[1] for tag in finding.tags if tag.startswith("family:")) - - -def _relation(finding: StaticFinding, family: str, taxonomy: FamilyTaxonomy) -> str: - relations = [taxonomy.related(family, candidate) for candidate in _families_of(finding)] - for wanted in ("same", "parent_child", "lateral"): - if wanted in relations: - return wanted - return "fabricated" if relations else "none" - - -def _hits( - findings: Sequence[StaticFinding], family: str, spans: Sequence[tuple[str, int, int]], taxonomy: FamilyTaxonomy -) -> list[StaticFinding]: - return [finding for finding in findings if _inside(finding, spans) and _relation(finding, family, taxonomy) in {"same", "parent_child"}] - - -def _others(findings: Sequence[StaticFinding], family: str, spans: Sequence[tuple[str, int, int]], taxonomy: FamilyTaxonomy) -> int: - return sum(1 for finding in findings if _relation(finding, family, taxonomy) == "lateral" or not _inside(finding, spans)) - - -def _fabricated(findings: Sequence[StaticFinding], taxonomy: FamilyTaxonomy) -> int: - known = {item.id for item in taxonomy.families} - return sum(1 for finding in findings if any(candidate not in known for candidate in _families_of(finding))) - - -def _best_rung(current: Rung, hits: Sequence[StaticFinding]) -> Rung: - best = current - for finding in hits: - for tag in finding.tags: - if tag.startswith("verifier:"): - candidate = tag.split(":", 1)[1] - if candidate in _RUNGS and _RUNGS.index(candidate) > _RUNGS.index(best): # type: ignore[arg-type] - best = candidate # type: ignore[assignment] - return best - - -@dataclass(frozen=True) -class FamilyMetrics: - """One family's numbers, with the denominator they were computed over and why rows fell out of it.""" - - family: str - taxonomy_version: str - scorable: int - unscorable: dict[str, int] = field(default_factory=dict) - outcomes: dict[str, int] = field(default_factory=dict) - recall: float = 0.0 # flagged the vulnerable side - silence: float = 0.0 # said nothing on the fixed side - youden: float = 0.0 - directional_bias: float = 0.0 # positive: flags both sides; negative: silent on both. Youden cannot tell them apart. - leak_rate: float = 0.0 - fixed_fpr_recall: float = 0.0 # recall, forfeited entirely when the leak rate exceeds the ceiling - negative_flips: int = 0 # pairs that were correct in the baseline and are not now - positive_flips: int = 0 - p_value: float | None = None # two-sided sign test over the discordant pairs - reliable_change: bool = False - hierarchical_credit: bool = False # False: the taxonomy is flat, so no answer could earn partial credit (Req 3.7) - cutoff: str = "" # the detector cutoff these numbers were split on, or "" when none is configured (Req 10.5) - undated: int = 0 # rows the corpus records no date for; they are in neither slice and are counted here - post_cutoff: dict[str, object] = field(default_factory=dict) # the same numbers over rows fixed after the cutoff - - def to_dict(self) -> dict[str, object]: - return { - "family": self.family, - "taxonomy_version": self.taxonomy_version, - "scorable": self.scorable, - "unscorable": dict(sorted(self.unscorable.items())), - "outcomes": dict(sorted(self.outcomes.items())), - "recall": self.recall, - "silence": self.silence, - "youden": self.youden, - "directional_bias": self.directional_bias, - "leak_rate": self.leak_rate, - "fixed_fpr_recall": self.fixed_fpr_recall, - "negative_flips": self.negative_flips, - "positive_flips": self.positive_flips, - "p_value": self.p_value, - "reliable_change": self.reliable_change, - "hierarchical_credit": self.hierarchical_credit, - "cutoff": self.cutoff, - "undated": self.undated, - "post_cutoff": dict(self.post_cutoff), - } - - -def aggregate( - scores: Sequence[PairFamilyScore], - *, - taxonomy: FamilyTaxonomy, - baseline: Sequence[PairFamilyScore] | None = None, - fpr_ceiling: float = 0.1, - per_slice: bool = False, - alpha: float = 0.05, - cutoff: str = "", -) -> dict[str, FamilyMetrics]: - """Group scores into per-family numbers. Unscorable rows are listed by reason and leave every denominator. - - With a ``cutoff`` each family also reports the same numbers over the rows fixed after it, beside the whole - number and never instead of it: a model that may have read the fix cannot be measured on it the same way as one - that cannot have. Rows the corpus records no date for are counted on their own (Req 10.5).""" - grouped: dict[str, list[PairFamilyScore]] = {} - for score in scores: - grouped.setdefault(f"{score.slice}/{score.family}" if per_slice else score.family, []).append(score) - was_correct = {(item.slice if per_slice else "", item.family, item.pair): item.outcome for item in baseline or ()} - out: dict[str, FamilyMetrics] = {} - for key, group in sorted(grouped.items()): - scorable = [item for item in group if item.outcome != "unscorable"] - reasons: dict[str, int] = {} - for item in group: - if item.outcome == "unscorable": - reason = item.unscorable_reason or "unknown" - reasons[reason] = reasons.get(reason, 0) + 1 - counts = Counter(item.outcome for item in scorable) - total = len(scorable) - detected = counts["pair_correct"] + counts["both_flagged"] - silent = counts["pair_correct"] + counts["both_silent"] - leaks = counts["both_flagged"] + counts["reversed"] - recall = detected / total if total else 0.0 - silence = silent / total if total else 0.0 - leak_rate = leaks / total if total else 0.0 - negative = positive = 0 - for item in scorable: - before = was_correct.get((item.slice if per_slice else "", item.family, item.pair)) - if before is None or before == "unscorable": - continue - if before == "pair_correct" and item.outcome != "pair_correct": - negative += 1 - elif before != "pair_correct" and item.outcome == "pair_correct": - positive += 1 - p_value = sign_test(positive, negative) if baseline is not None else None - recent = [item for item in scorable if is_post_cutoff(item.fix_date, cutoff)] if cutoff else [] - out[key] = FamilyMetrics( - family=group[0].family, - taxonomy_version=taxonomy.version, - scorable=total, - unscorable=reasons, - outcomes={name: count for name, count in sorted(counts.items())}, - recall=recall, - silence=silence, - youden=(recall + silence - 1.0) if total else 0.0, - directional_bias=((counts["both_flagged"] - counts["both_silent"]) / total) if total else 0.0, - leak_rate=leak_rate, - fixed_fpr_recall=recall if leak_rate <= fpr_ceiling else 0.0, - negative_flips=negative, - positive_flips=positive, - p_value=p_value, - reliable_change=bool(p_value is not None and p_value < alpha and positive != negative), - hierarchical_credit=taxonomy.hierarchical, - cutoff=cutoff, - undated=sum(1 for item in group if not item.fix_date) if cutoff else 0, - post_cutoff=_slice_numbers(recent) if cutoff else {}, - ) - return out - - -def _slice_numbers(scores: Sequence[PairFamilyScore]) -> dict[str, object]: - """Recall, silence and Youden over a subset, with the subset's own denominator beside them.""" - total = len(scores) - counts = Counter(item.outcome for item in scores) - recall = (counts["pair_correct"] + counts["both_flagged"]) / total if total else 0.0 - silence = (counts["pair_correct"] + counts["both_silent"]) / total if total else 0.0 - return { - "scorable": total, - "outcomes": {name: count for name, count in sorted(counts.items())}, - "recall": recall, - "silence": silence, - "youden": (recall + silence - 1.0) if total else 0.0, - } - - -def sign_test(better: int, worse: int) -> float: - """Two-sided exact sign test over the pairs that changed. 1.0 when nothing changed, so noise never reads as progress.""" - total = better + worse - if total == 0: - return 1.0 - smaller = min(better, worse) - tail = sum(math.comb(total, index) for index in range(smaller + 1)) / float(2**total) - return float(min(1.0, 2.0 * tail)) - - -__all__ = [ - "CONTEXT_DEPENDENT_FAMILIES", - "FamilyMetrics", - "FamilyOutcome", - "FunctionRanges", - "PairFamilyScore", - "Rung", - "aggregate", - "is_post_cutoff", - "score_pair_family", - "sign_test", - "unscorable_reason", -] diff --git a/src/openultrasast/learning/split.py b/src/openultrasast/learning/split.py deleted file mode 100644 index af3129b..0000000 --- a/src/openultrasast/learning/split.py +++ /dev/null @@ -1,64 +0,0 @@ -"""One rule for who may teach, and one refusal for anything that touches the holdout (Req 5.1, 5.2). - -Every learning path in the tool asks this module the same question, so the split cannot be enforced in -one place and forgotten in another. A teacher is a vendored pair, at a review tier a maintainer stands -behind, on the train split, and scorable. Everything else may be measured; nothing else may teach. - -A refusal is data, not an exception: the path records which holdout pairs a candidate touched and moves -on, so a leak is visible in the run rather than silently accepted. -""" - -from __future__ import annotations - -from collections.abc import Iterable, Sequence -from dataclasses import dataclass - -GATING_TIERS = frozenset({"seeded", "reviewed"}) -TRAIN = "train" -HOLDOUT = "holdout" -REFUSED = "holdout_pair_refused" - - -@dataclass(frozen=True) -class Refusal: - """A candidate that a holdout pair taught, or that touches one.""" - - pairs: tuple[str, ...] - reason: str = REFUSED - - def degradation(self) -> dict[str, object]: - return { - "stage": "learning", - "reason": self.reason, - "pairs": list(self.pairs), - "detail": f"refused: taught by or touching holdout pair(s) {', '.join(self.pairs)}", - } - - -def teachers(cases: Sequence[object]) -> tuple[object, ...]: - """The pairs a learning path may learn from. Everything else is measured, never learned from.""" - return tuple(case for case in cases if is_teacher(case)) - - -def is_teacher(case: object) -> bool: - if not bool(getattr(case, "vendored", True)): - return False - if str(getattr(case, "review_tier", "")) not in GATING_TIERS: - return False - if str(getattr(case, "split", TRAIN)) != TRAIN: - return False - return getattr(case, "unscorable", None) is None - - -def holdout_names(cases: Sequence[object]) -> frozenset[str]: - return frozenset(str(getattr(case, "name", "")) for case in cases if str(getattr(case, "split", "")) == HOLDOUT) - - -def refuse_if_holdout(taught_by: Iterable[str], cases: Sequence[object]) -> Refusal | None: - """None when nothing in ``taught_by`` is a holdout pair; a refusal naming them when something is.""" - held = holdout_names(cases) - touched = tuple(sorted({name for name in taught_by if name in held})) - return Refusal(pairs=touched) if touched else None - - -__all__ = ["GATING_TIERS", "HOLDOUT", "REFUSED", "TRAIN", "Refusal", "holdout_names", "is_teacher", "refuse_if_holdout", "teachers"] diff --git a/src/openultrasast/learning/verifiers.py b/src/openultrasast/learning/verifiers.py deleted file mode 100644 index 5e3063e..0000000 --- a/src/openultrasast/learning/verifiers.py +++ /dev/null @@ -1,140 +0,0 @@ -"""What may raise a claim above suspicion, keyed by family (learning-harness, Req 7.1, 7.3, 7.4). - -A detector's claim is a suspicion until something other than the model agrees with it. What counts as -agreement depends on the family: a canary oracle observed in the sandbox, corroboration from a static -checker, or nothing at all. A family whose verifier does not exist yet reports at suspicion rather than -pretending, and that is the honest default rather than a gap. - -The canary oracle is a function over a sandbox result, so the sandbox keeps deciding what happened and -this module only says what would count. Exit codes are not oracles: a crash and a demonstrated -injection are different evidence, and one family's oracle must not silently answer for another. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Literal, Protocol - -from ..findings import StaticFinding -from ..sandbox import SandboxResult -from ..verification import EvidenceLevel -from .families import Family - -Rung = Literal["suspicion", "static_corroboration", "proven"] -RUNGS: tuple[Rung, ...] = ("suspicion", "static_corroboration", "proven") -VerifierKind = Literal["canary", "static", "none"] - - -@dataclass(frozen=True) -class Verification: - """What a verifier saw, and how high it lets the claim go.""" - - rung: Rung - reason: str - oracle_output: str = "" - - -class Verifier(Protocol): - @property - def kind(self) -> VerifierKind: ... - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: ... - - -def token_oracle(token: str) -> Callable[[SandboxResult], bool]: - """The oracle a canary snippet is written against: the token appears only if the vulnerability fired.""" - - def fired(result: SandboxResult) -> bool: - return token in result.stdout or token in result.stderr - - return fired - - -@dataclass(frozen=True) -class NoVerifier: - """Nothing can confirm this family yet, so the claim stays where the detector left it.""" - - kind: VerifierKind = "none" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - del claim, root, language - return Verification(rung="suspicion", reason="no_verifier") - - -@dataclass(frozen=True) -class StaticVerifier: - """Another checker looked at the same function and agrees the obligation is undischarged.""" - - kind: VerifierKind = "static" - - def verify(self, claim: StaticFinding, *, root: Path, language: str) -> Verification: - del language - from ..hunter_tools import PathEscapesRepo, obligations - - if not claim.function_name: - return Verification(rung="suspicion", reason="no_function") - try: - rows = obligations(root, path=claim.path, function=claim.function_name) - except (PathEscapesRepo, OSError): - return Verification(rung="suspicion", reason="checker_unavailable") - if not rows: - return Verification(rung="suspicion", reason="checker_disagrees") - summary = "; ".join(f"{row.get('operation')} missing {row.get('missing')}" for row in rows) - return Verification(rung="static_corroboration", reason="checker_agrees", oracle_output=summary) - - -def verifier_for(family: Family, *, canary: Verifier | None = None) -> Verifier: - """The verifier a family declares. A canary family with no implementation registered stays at suspicion.""" - if family.verifier == "static": - return StaticVerifier() - if family.verifier == "canary": - return canary if canary is not None else NoVerifier(kind="canary") - return NoVerifier() - - -def apply_verification(claim: StaticFinding, verification: Verification) -> StaticFinding: - """A copy of the claim carrying what the verifier said, as a tag. - - The evidence ladder belongs to another boundary and this spec does not change it, so nothing here - writes ``evidence_level``. "proven" is this spec's word, not a rung the ladder defines, and stamping - it onto the field made the project's own verification helpers raise. A caller that wants to move a - claim up the ladder does it through :func:`raise_to`, which can only produce a level the ladder has. - """ - tags = [tag for tag in claim.tags if not tag.startswith("verifier:")] + [f"verifier:{verification.rung}"] - return replace(claim, tags=tags) - - -def evidence_level_for(rung: Rung) -> EvidenceLevel: - """The project's evidence level a verifier rung corresponds to. The only bridge between the two vocabularies.""" - return _LADDER[rung] - - -def raise_to(claim: StaticFinding, rung: Rung) -> StaticFinding: - """A copy of the claim at the ladder level this rung corresponds to. Never a value the ladder lacks.""" - return replace(claim, evidence_level=str(evidence_level_for(rung))) - - -_LADDER: dict[Rung, EvidenceLevel] = { - "suspicion": EvidenceLevel.SUSPICION, - "static_corroboration": EvidenceLevel.STATIC_CORROBORATION, - # A canary that fired in the sandbox demonstrated the vulnerability, which is what this level means here. - "proven": EvidenceLevel.EXPLOIT_DEMONSTRATED, -} - - -__all__ = [ - "RUNGS", - "NoVerifier", - "Rung", - "StaticVerifier", - "Verification", - "Verifier", - "VerifierKind", - "apply_verification", - "evidence_level_for", - "raise_to", - "token_oracle", - "verifier_for", -] diff --git a/src/openultrasast/model/__init__.py b/src/openultrasast/model/__init__.py new file mode 100644 index 0000000..2b15ae3 --- /dev/null +++ b/src/openultrasast/model/__init__.py @@ -0,0 +1,7 @@ +"""The model layer: a deterministic program model that arbitrates detector claims. + +The taxonomy routes work, the candidate enumerator feeds the `suspicion` band, the specs carry the security +vocabularies, and (from group 2) the CPG-backed taint and dominance checks establish the rungs above it. +""" + +from __future__ import annotations diff --git a/src/openultrasast/model/audit.py b/src/openultrasast/model/audit.py new file mode 100644 index 0000000..38a32b8 --- /dev/null +++ b/src/openultrasast/model/audit.py @@ -0,0 +1,166 @@ +"""Module audit: what is load-bearing, what stands alone, and what is orphaned (Req 3). + +The tool outgrew itself — 99 modules and 832 tests at the point this feature began, a large share of them +serving an architecture its own measurements retired. This classifier exists so outgrowth is visible and +checkable rather than a thing someone notices two specs later. + +A module is: + + load_bearing reachable from a gate, the scan CLI, or a surviving requirement + standalone_capability its own entry point and users, not on the scan path + orphaned no importer, no entry point, no requirement — it should not be in the tree + +Pure stdlib, no imports of the package it audits: it reads the import graph with ``ast`` so a broken module +is still classifiable. +""" + +from __future__ import annotations + +import ast +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path + +PACKAGE = "openultrasast" + +# Entry points a module can be reachable from. The three gates are the contract; the CLI is the product. +ROOTS = ("gate", "map_gate", "pair_gate", "cli", "__main__") + +# Modules that stand alone: their own CLI subcommand or external protocol, deliberately off the scan path. +SPEC_OWNED = { + "model.specs": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by", + "model.endpoint": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3", + "model.audit": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan", +} + +STANDALONE = { + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate", + "slot_contract": "the slot/contract protocol stage_processors implements", +} + + +@dataclass(frozen=True) +class ModuleAudit: + module: str + classification: str # load_bearing | standalone_capability | orphaned + reason: str + importers: tuple[str, ...] + + +def module_name(path: Path, root: Path) -> str: + relative = path.relative_to(root).with_suffix("") + parts = [part for part in relative.parts if part != "__init__"] + return ".".join(parts) + + +def import_graph(root: Path) -> dict[str, set[str]]: + """``module -> the modules it imports`` for every module under ``root``, resolved to package-relative names.""" + graph: dict[str, set[str]] = {} + for path in sorted(root.rglob("*.py")): + if "__pycache__" in path.parts: + continue + name = module_name(path, root) + # For a package's `__init__`, the containing package *is* this module, so `from .x import y` resolves to + # `.x`; for a plain module it resolves against the parent. Getting this backwards made every module + # a package re-exports look unimported. + is_package = path.name == "__init__.py" + imported: set[str] = set() + try: + tree = ast.parse(path.read_text()) + except (OSError, SyntaxError): + graph[name] = imported + continue + package_parts = name.split(".") if is_package else name.split(".")[:-1] + package_parts = [part for part in package_parts if part] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level: + base = package_parts[: len(package_parts) - node.level + 1] + target = ".".join([*base, node.module]) if node.module else ".".join(base) + else: + target = node.module or "" + if target.startswith(f"{PACKAGE}."): + target = target[len(PACKAGE) + 1 :] + elif target == PACKAGE: + target = "" + else: + continue + if target: + imported.add(target) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith(f"{PACKAGE}."): + imported.add(alias.name[len(PACKAGE) + 1 :]) + graph[name] = imported + return graph + + +def _reachable(graph: Mapping[str, set[str]], roots: Iterable[str]) -> set[str]: + seen: set[str] = set() + stack = [root for root in roots if root in graph] + while stack: + current = stack.pop() + if current in seen: + continue + seen.add(current) + for target in graph.get(current, ()): + # an import of `a.b` also keeps the package `a` alive + for candidate in (target, target.rsplit(".", 1)[0]): + if candidate in graph and candidate not in seen: + stack.append(candidate) + return seen + + +def audit(root: Path) -> tuple[ModuleAudit, ...]: + """Classify every module under ``root``.""" + graph = import_graph(root) + importers: dict[str, set[str]] = {name: set() for name in graph} + for name, targets in graph.items(): + for target in targets: + for candidate in (target, target.rsplit(".", 1)[0]): + if candidate in importers and candidate != name: + importers[candidate].add(name) + live = _reachable(graph, ROOTS) + results: list[ModuleAudit] = [] + for name in sorted(graph): + if not name: # the package __init__ itself + continue + who = tuple(sorted(importers[name])) + if name in SPEC_OWNED: + results.append(ModuleAudit(name, "load_bearing", SPEC_OWNED[name], who)) + elif name in ROOTS: + results.append(ModuleAudit(name, "load_bearing", "entry point", who)) + elif name in STANDALONE: + results.append(ModuleAudit(name, "standalone_capability", STANDALONE[name], who)) + elif name in live: + results.append(ModuleAudit(name, "load_bearing", f"reachable from {', '.join(ROOTS[:3])} or the CLI", who)) + elif who: + results.append(ModuleAudit(name, "load_bearing", f"imported by {', '.join(who)}", who)) + else: + results.append(ModuleAudit(name, "orphaned", "no importer, no entry point, no surviving requirement", who)) + return tuple(results) + + +def to_dict(results: Iterable[ModuleAudit]) -> dict[str, object]: + rows = list(results) + counts: dict[str, int] = {} + for row in rows: + counts[row.classification] = counts.get(row.classification, 0) + 1 + return { + "modules": len(rows), + "counts": dict(sorted(counts.items())), + "orphaned": [row.module for row in rows if row.classification == "orphaned"], + "standalone": {row.module: row.reason for row in rows if row.classification == "standalone_capability"}, + "rows": [ + {"module": row.module, "classification": row.classification, "reason": row.reason, "importers": list(row.importers)} + for row in rows + ], + } + + +__all__ = ["ModuleAudit", "SPEC_OWNED", "STANDALONE", "audit", "import_graph", "module_name", "to_dict"] diff --git a/src/openultrasast/learning/candidates.py b/src/openultrasast/model/candidates.py similarity index 88% rename from src/openultrasast/learning/candidates.py rename to src/openultrasast/model/candidates.py index e1908be..f2f1ac7 100644 --- a/src/openultrasast/learning/candidates.py +++ b/src/openultrasast/model/candidates.py @@ -15,7 +15,7 @@ the sequence of model calls, are determined by the input alone. from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -263,22 +263,20 @@ def ceiling(cases: Sequence[object], *, taxonomy: object, generator: str = "ir") import tempfile from ..pairs import _materialize_side, _overlay_scan, _quick_scan - from .rounds import _family_of - from .scoring import _spans families: dict[str, dict[str, int]] = {} slices: dict[str, dict[str, int]] = {} gaps: list[tuple[str, str, str]] = [] sites: dict[str, list[int]] = {} for case in cases: - family = _family_of(case, taxonomy) # type: ignore[arg-type] + family = labeled_family(case) slice_name = str(getattr(case, "slice", "") or "") region_path = str(getattr(case, "relpath", "") or "") with tempfile.TemporaryDirectory(prefix="ousast-ceiling-") as scratch: root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") # type: ignore[arg-type] if generator == "ruleset": ranges = _overlay_scan(root).ranges - spans = _spans(case, ranges) + spans = labeled_spans(case, ranges) found = [ finding for finding in _quick_scan(root) @@ -289,7 +287,7 @@ def ceiling(cases: Sequence[object], *, taxonomy: object, generator: str = "ir") labeled = [str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()] result = enumerate_candidates( root, - _Region(region_path, next((name for name in labeled if name), None)), + Region(region_path, next((name for name in labeled if name), None)), family, taxonomy=taxonomy, limit=10**6, # the ceiling is about coverage, never about the batch bound @@ -331,9 +329,45 @@ def _distribution(counts: list[int]) -> dict[str, int]: } +def labeled_family(case: object) -> str: + """The family a pair is labeled with, or ``unknown``. + + The deleted ``learning/rounds.py`` fell back to the LLM classifier here. Nothing does now: an unlabeled + pair is honestly ``unknown`` rather than guessed, which is the same discipline the evidence ladder applies + to a claim the model cannot arbitrate. + """ + for row in getattr(case, "expected", ()) or (): + family = getattr(row, "family", None) + if family: + return str(family) + return "unknown" + + +def labeled_spans( + case: object, ranges: Mapping[str, Sequence[tuple[str, int, int]]], *, function: str = "" +) -> tuple[tuple[str, int, int], ...]: + """The (path, start, end) spans of the labeled functions; ``function`` overrides the label for the fixed side. + + Re-homed from the deleted ``learning/scoring.py`` — the enumerator's ceiling needs it to know which region a + candidate had to fall inside, and it carries none of the K-run machinery that surrounded it. + """ + wanted = {function} if function else {str(getattr(row, "function", "") or "") for row in getattr(case, "expected", ()) or ()} + wanted.discard("") + found: list[tuple[str, int, int]] = [] + for path, entries in ranges.items(): + for name, start, end in entries: + if name in wanted: + found.append((path, start, end)) + return tuple(found) + + @dataclass(frozen=True) -class _Region: - """The two fields `enumerate_candidates` reads, so the ceiling does not import the detector module.""" +class Region: + """The two fields ``enumerate_candidates`` reads: where to look, and which function bounds the search. + + Public since the detector module that used to own this type was removed with the noise architecture — the + enumerator still needs a name for its input, and callers still need one to construct. + """ path: str function: str | None @@ -347,5 +381,8 @@ __all__ = [ "CandidateKind", "CandidateReport", "CandidateSet", + "Region", "enumerate_candidates", + "labeled_family", + "labeled_spans", ] diff --git a/src/openultrasast/learning/endpoint.py b/src/openultrasast/model/endpoint.py similarity index 96% rename from src/openultrasast/learning/endpoint.py rename to src/openultrasast/model/endpoint.py index a734d30..24c82fa 100644 --- a/src/openultrasast/learning/endpoint.py +++ b/src/openultrasast/model/endpoint.py @@ -154,7 +154,7 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None if configured or key_env: key = os.environ.get(key_env or DEEPSEEK_KEY_ENV, "") base_url = configured or DEEPSEEK_BASE_URL - return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.learning.thinking) + return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.models.thinking) deepseek_key = os.environ.get(DEEPSEEK_KEY_ENV) if deepseek_key: return _deepseek_client( @@ -162,7 +162,7 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), model, provider="deepseek", - thinking=config.learning.thinking, + thinking=config.models.thinking, ) if os.environ.get("OPENROUTER_API_KEY"): try: @@ -175,9 +175,9 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None def resolve_models(config: ResolvedConfig) -> tuple[str, str]: - """(detector model, judge model). `[learning]` wins, then `[models]`, then the shipped defaults.""" - detector = config.learning.detector_model or config.models.hunter or DEFAULT_DETECTOR_MODEL - judge = config.learning.judge_model or config.models.judge or DEFAULT_JUDGE_MODEL + """(detector model, judge model) from `[models]`, then the shipped defaults.""" + detector = config.models.hunter or DEFAULT_DETECTOR_MODEL + judge = config.models.judge or DEFAULT_JUDGE_MODEL return detector, judge diff --git a/src/openultrasast/model/specs.py b/src/openultrasast/model/specs.py new file mode 100644 index 0000000..d8dd24d --- /dev/null +++ b/src/openultrasast/model/specs.py @@ -0,0 +1,156 @@ +"""The security vocabularies the model layer arbitrates with (model-grounded-detection, Req 7.2-7.4). + +Two kinds of knowledge live here, and they are held differently on purpose. + +The **guard vocabulary** is ported verbatim from ``semantic/variants.py``, whose optimisation loop this +feature deleted. Nothing else holds it: it is the closed set of ways a fix discharges a flow, and it tells +the model what a *fixed* twin looks like. Order matters — first match wins. + +The **taint and dominance specs** are *derived* from data that survives: the source/sink/sanitizer facts in +``ruleset/semantic`` and the obligated operations and dischargers in ``ruleset/obligations``. Copying them +here would create a second place to update, so this module reads them through the retained loaders and +groups them by family via the taxonomy's CWE map. + +Read-only by construction (Req 7.4, the verifier boundary): this module exposes no writer, opens no file for +writing, and holds no mutable module state. An LLM or a future optimiser has nothing here to rewrite. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass + +from ..semantic.facts import SemanticFacts, load_facts +from ..semantic.obligations.facts import ObligationFacts, load_obligation_facts +from .taxonomy import FamilyTaxonomy, load_families + +# --- The guard vocabulary (ported verbatim; nothing else holds it) ---------------------------------------- + +GUARD_KINDS = ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") + +# Closed guard classification over the statements a fix added inside the labeled function. +# Order matters: the first matching kind wins. These classify the *fix*, they never detect. +GUARD_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("parameterized_call", re.compile(r"execute\w*\([^,]+,\s*[\(\[]|prepareStatement\(|\?\s*['\"]?\s*[,)]|\$\d\b|%s['\"]\s*,\s*[\(\[]")), + ( + "auth_check", + re.compile( + r"\b(request\.user|req\.user|current_user|request\.state|session\[|is_authenticated|authoriz|require_auth" + r"|login_required|verify_token|check_permission|owner_id\s*[!=]=)" + ), + ), + ( + "allowlist_test", + re.compile( + r"\bnot in\b|\bin\s*[\(\{\[]|\.includes\(|\.has\(|\bstartswith\(|\bstartsWith\(|\bendswith\(|\bendsWith\(" + r"|re\.(?:match|fullmatch)\(|\.test\(|allow_?list|whitelist|\bALLOWED\b|\bsafe_join\(|secure_filename\(" + r"|\brealpath\(|\bnormpath\(|\bresolve\(" + ), + ), + ("bounds_test", re.compile(r"\blen\(|\.length\b|\bsizeof\(|\bstrn?len\(|\bsnprintf\(|\bstrlcpy\(|\bmin\(|\bmax\(|\s(?:<|>|<=|>=)\s")), + ( + "null_test", + re.compile( + r"\bis None\b|\bis not None\b|[!=]= *NULL\b|[!=]==? *null\b|\bnullptr\b|\bif\s*\(\s*!\s*\w|\bif not \w+\s*:" + r"|\bif\s+\w+\s+is\s+None|\?\?|\?\." + ), + ), + ( + "type_change", + re.compile(r"\bint\(|\bparseInt\(|\bNumber\(|\(int\)|\(size_t\)|\(unsigned\b|\bstr\(|\.toString\(|\bstatic_cast<|\bBigInt\("), + ), +) + + +def classify_guard_text(text: str) -> str: + """The guard kind the given statements express, or ``none`` when no known guard is recognised.""" + for kind, pattern in GUARD_PATTERNS: + if pattern.search(text): + return kind + return "none" + + +# --- The specs the CPG queries are parameterised by ------------------------------------------------------ + + +@dataclass(frozen=True) +class TaintSpec: + """What a flow family's source→sink−sanitizer query matches on, for one language.""" + + family: str + language: str + sources: tuple[str, ...] + sinks: tuple[str, ...] + sanitizers: tuple[str, ...] + + +@dataclass(frozen=True) +class DominanceSpec: + """What an absence family's guard-dominates-operation query matches on, for one language. + + ``dischargers`` are the guards that discharge an obligation; the query asks whether one of them dominates + the operation on every path, which is the arbiter for the classes that never crash. + """ + + family: str + language: str + operations: tuple[str, ...] + dischargers: tuple[str, ...] + + +def taint_specs( + *, + language: str, + facts: SemanticFacts | None = None, + taxonomy: FamilyTaxonomy | None = None, +) -> Mapping[str, TaintSpec]: + """One ``TaintSpec`` per flow family that has a sink in this language, grouped through the taxonomy's CWE map. + + Sources and sanitizers are language-wide: a family does not get its own notion of what untrusted input is, + and a sanitizer that breaks one flow breaks it whatever the sink's family. + """ + scoped = (facts if facts is not None else load_facts()).for_language(language) + families = taxonomy if taxonomy is not None else load_families() + sources = tuple(sorted({pattern for fact in scoped.sources for pattern in fact.patterns})) + sanitizers = tuple(sorted({call for fact in scoped.sanitizers for call in fact.calls})) + by_family: dict[str, set[str]] = {} + for sink in scoped.sinks: + family = families.family_of_cwe(sink.cwe) + if family is None: + continue + by_family.setdefault(family.id, set()).update(sink.calls) + return { + family_id: TaintSpec(family=family_id, language=language, sources=sources, sinks=tuple(sorted(calls)), sanitizers=sanitizers) + for family_id, calls in sorted(by_family.items()) + } + + +def dominance_specs(*, language: str, facts: ObligationFacts | None = None) -> Mapping[str, DominanceSpec]: + """One ``DominanceSpec`` per absence family for this language. + + The obligation facts are authored for authorization, so every obligated operation routes to + ``access_control``; the shape generalises when a second absence family is added to the taxonomy. + """ + scoped = (facts if facts is not None else load_obligation_facts()).for_language(language) + if not scoped.operations: + return {} + operations = tuple(sorted({call for fact in scoped.operations for call in fact.calls})) + dischargers = tuple( + sorted( + {call for fact in scoped.dischargers for call in fact.calls} + | {decorator for fact in scoped.dischargers for decorator in fact.decorators} + ) + ) + return {"access_control": DominanceSpec(family="access_control", language=language, operations=operations, dischargers=dischargers)} + + +__all__ = [ + "GUARD_KINDS", + "GUARD_PATTERNS", + "DominanceSpec", + "TaintSpec", + "classify_guard_text", + "dominance_specs", + "taint_specs", +] diff --git a/src/openultrasast/learning/families.py b/src/openultrasast/model/taxonomy.py similarity index 100% rename from src/openultrasast/learning/families.py rename to src/openultrasast/model/taxonomy.py diff --git a/src/openultrasast/pairs.py b/src/openultrasast/pairs.py index e25b146..dee1536 100644 --- a/src/openultrasast/pairs.py +++ b/src/openultrasast/pairs.py @@ -20,7 +20,7 @@ import os import shutil import tempfile import tomllib -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import asdict, dataclass, field, replace from functools import lru_cache from pathlib import Path @@ -98,7 +98,6 @@ _DEFAULT_TIER = { _DEFAULT_PROVENANCE = {"sast": "synthetic", "local": "human", "github": "human", "vfc": "human"} _MANIFEST_MECHANISM = "source_reaches_sink" # cheat-sheet fixtures referenced via expected_from -HunterScan = Callable[[Path], list[StaticFinding]] class CatalogError(ValueError): @@ -173,7 +172,6 @@ class PairOutcome: unadjudicated_vuln: int = 0 unadjudicated_fixed: int = 0 detection_kinds: tuple[str, ...] = () - family: str = "" # the family this pair was scored under (hunter path only) unscorable: str | None = None # why it could not be scored; it then leaves every denominator unresolved_labels: int = 0 # expected rows whose function names no range on the vulnerable side review_tier: str = "advisory" @@ -205,7 +203,6 @@ class PairEvalResult: per_profile: dict[str, PairCorpusMetrics] = field(default_factory=dict) per_tier: dict[str, PairCorpusMetrics] = field(default_factory=dict) per_mechanism: dict[str, PairCorpusMetrics] = field(default_factory=dict) - per_family: dict[str, dict[str, object]] = field(default_factory=dict) # learning-harness Req 3.3: family metrics beside the rest achievable: dict[str, PairCorpusMetrics] = field(default_factory=dict) known_limit: tuple[str, ...] = () loss: dict[str, dict[str, int]] = field(default_factory=dict) @@ -427,15 +424,11 @@ def _review_tier(item: dict[str, object], slice_name: str, name: str) -> tuple[s def evaluate_pair( case: PairCase, *, - hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None, - k_runs: int | None = None, ) -> PairOutcome: with tempfile.TemporaryDirectory(prefix="ousast-pair-") as scratch: vuln_root = _materialize_side(Path(scratch) / "vuln", case, side="vuln") fix_root = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - if hunter is not None: - return _evaluate_hunter_pair(case, vuln_root, fix_root, hunter, _k_runs(k_runs)) if case.slice in OVERLAY_SLICES: return _evaluate_overlay_pair(case, vuln_root, fix_root, ruleset) return _evaluate_inventory_pair(case, vuln_root, fix_root, ruleset=ruleset) @@ -444,19 +437,15 @@ def evaluate_pair( def evaluate_catalog( cases: Sequence[PairCase], *, - hunter: HunterScan | None = None, ruleset: tuple[PatternRule, ...] | None = None, inventory_sidecar: bool = True, pointers: bool | None = None, - k_runs: int | None = None, ) -> PairEvalResult: """Score every case. ``ruleset`` lets the improve loop score a candidate ledger; ``inventory_sidecar=False`` skips the second scan. Pointer pairs (``vendored = false``) are harvested into the cache when ``pointers`` (or OPENULTRASAST_PAIRS_NETWORK=1) allows the network, otherwise skipped with a ``pointer_pair_skipped`` degradation (Req 10.2, 10.3). """ - if hunter is not None: - _k_runs(k_runs) # refuse an impossible K before spending a single call degradations: list[dict[str, object]] = [] ready: list[PairCase] = [] for case in cases: @@ -468,18 +457,11 @@ def evaluate_catalog( cases = tuple(ready) outcomes = tuple(evaluate_pair(case, ruleset=ruleset) for case in cases) scorers: dict[str, dict[str, PairCorpusMetrics]] = {} - hunter_outcomes: list[PairOutcome] = [] # the family scorer runs on the hunter path only for slice_name in sorted({case.slice for case in cases if case.slice in OVERLAY_SLICES}): slice_cases = tuple(case for case in cases if case.slice == slice_name) scorers[slice_name] = {"overlay": _metrics(tuple(item for item in outcomes if item.slice == slice_name))} if inventory_sidecar: scorers[slice_name]["inventory"] = _metrics(tuple(_inventory_only(case, ruleset) for case in slice_cases)) - if hunter is not None: - hunted = tuple(evaluate_pair(case, hunter=hunter, k_runs=k_runs) for case in slice_cases) - hunter_outcomes.extend(hunted) - scorers[slice_name]["hunter"] = _metrics(hunted) - if hunter is None and scorers: - degradations.append({"stage": "pairs", "reason": "hunter_model_unavailable"}) achievable_outcomes = tuple(item for item in outcomes if item.known_limit is None) per_mechanism: dict[str, list[PairOutcome]] = {} for item in outcomes: @@ -503,7 +485,6 @@ def evaluate_catalog( for tier in sorted({item.review_tier for item in outcomes}) }, per_mechanism={mechanism: _metrics(tuple(items)) for mechanism, items in sorted(per_mechanism.items())}, - per_family=_family_metrics(hunter_outcomes), achievable={ slice_name: _metrics(tuple(item for item in achievable_outcomes if item.slice == slice_name)) for slice_name in sorted({item.slice for item in achievable_outcomes}) @@ -514,43 +495,6 @@ def evaluate_catalog( ) -def _family_metrics(outcomes: Sequence[PairOutcome]) -> dict[str, dict[str, object]]: - """Per-family numbers for the pairs the family scorer ran, with unscorable rows out of the denominator.""" - from .learning.families import load_families - from .learning.scoring import PairFamilyScore, aggregate - - scored = [item for item in outcomes if item.family] - if not scored: - return {} - taxonomy = load_families() - scores = [ - PairFamilyScore( - pair=item.name, - family=item.family, - runs=() - if item.unscorable - else ( - ( - ("pair_correct" if item.pair_correct else "both_flagged") - if item.detected_vuln - else ("reversed" if not item.silent_fix else "both_silent") - ), - ), - outcome="unscorable" - if item.unscorable - else ( - ("pair_correct" if item.pair_correct else "both_flagged") - if item.detected_vuln - else ("reversed" if not item.silent_fix else "both_silent") - ), - slice=item.slice, - unscorable_reason=item.unscorable, - ) - for item in scored - ] - return {name: metrics.to_dict() for name, metrics in aggregate(scores, taxonomy=taxonomy).items()} - - def build_pair_signals(outcomes: Sequence[PairOutcome], *, split: str | None = None) -> list[dict[str, object]]: """Same miss/fp vocabulary as ``build_rule_signals``, tagged with the pair name, profile, and mechanisms. @@ -737,7 +681,6 @@ def result_payload(result: PairEvalResult) -> dict[str, object]: "per_profile": {name: asdict(metrics) for name, metrics in result.per_profile.items()}, "per_tier": {name: asdict(metrics) for name, metrics in result.per_tier.items()}, "per_mechanism": {name: asdict(metrics) for name, metrics in result.per_mechanism.items()}, - "per_family": dict(result.per_family), "achievable": {name: asdict(metrics) for name, metrics in result.achievable.items()}, "known_limit": list(result.known_limit), "loss": {name: dict(counts) for name, counts in result.loss.items()}, @@ -964,53 +907,6 @@ def _with_functions(root: Path, findings: list[StaticFinding]) -> list[StaticFin return out -def make_hunter_scan(client: object, model: str, *, max_steps: int | None = None, taxonomy: object | None = None) -> HunterScan: - """Run the tool hunter over every file of an isolated pair tree (Req 1.7). - - With a ``taxonomy`` the hunter is asked to name the family of every finding, from the closed list, and the answer - becomes the finding's `family:` tag. The class-aware scorer credits a detection only to a family, so a generalist - that names none can never be scored — it would report recall zero by construction rather than by measurement. - """ - from .complexity.map import Hotspot - from .tool_hunter import _SYSTEM_PROMPT, DEFAULT_MAX_STEPS, run_tool_hunter - - prompt = _SYSTEM_PROMPT - if taxonomy is not None: - ids = ", ".join(family.id for family in taxonomy.families) # type: ignore[attr-defined] - prompt = ( - f'{_SYSTEM_PROMPT}\n\nGive every finding a "family" field, chosen from exactly this list: {ids}. ' - 'Use "unknown" when none of them fits; do not invent a name.' - ) - - def scan(root: Path) -> list[StaticFinding]: - hotspots = [ - Hotspot( - path=target.path, - function_name=None, - score=1.0, - band="high", - signals={}, - rationale="pair corpus function", - test_hint=None, - inventory_finding_ids=(), - ) - for target in _targets(root) - ] - return _with_functions( - root, - run_tool_hunter( - root, - hotspots, - client=client, # type: ignore[arg-type] - model=model, - max_steps=max_steps or DEFAULT_MAX_STEPS, - system_prompt=prompt, - ), - ) - - return scan - - def _inventory_outcome( case: PairCase, vuln_findings: list[StaticFinding], @@ -1146,98 +1042,6 @@ def _evaluate_overlay_pair(case: PairCase, vuln_root: Path, fix_root: Path, rule ) -def _k_runs(value: int | None) -> int: - """Req 3.5: three runs per pair per side is the floor, and the default, because one run misses most changes.""" - from .learning.rounds import MIN_K_RUNS - - runs = MIN_K_RUNS if value is None else int(value) - if runs < MIN_K_RUNS: - raise ValueError(f"the hunter path scores at least three runs per pair; {runs} was asked for") - return runs - - -def _evaluate_hunter_pair(case: PairCase, vuln_root: Path, fix_root: Path, hunter: HunterScan, runs: int) -> PairOutcome: - """Score the model-driven hunter by family and function (learning-harness Req 3.1-3.3, 3.5). - - Detection is a finding of the labeled family inside the labeled function; a finding of another family - or outside it is counted and never called a leak; the words in a finding are never read. A pair that - cannot be scored is reported with its reason and the detector is never run on it. - """ - from .learning.families import load_families - from .learning.scoring import score_pair_family, unscorable_reason - - taxonomy = load_families() - vuln_ranges = _overlay_scan(vuln_root).ranges - reason = unscorable_reason(case, parse_ok=bool(vuln_ranges), ranges=vuln_ranges) - family = _family_of(case, taxonomy) - if reason is not None: - return _hunter_outcome(case, family=family, unscorable=reason) - fix_ranges = _overlay_scan(fix_root).ranges - vuln_runs = [hunter(vuln_root) for _ in range(runs)] - fix_runs = [hunter(fix_root) for _ in range(runs)] - score = score_pair_family(case, family, vuln_runs, fix_runs, ranges=vuln_ranges, taxonomy=taxonomy, fixed_ranges=fix_ranges) - return _hunter_outcome(case, family=family, score=score, vuln=vuln_runs[0], fixed=fix_runs[0]) - - -def _family_of(case: PairCase, taxonomy: object) -> str: - """The family this pair is scored under: its label when it has one, else what the classifier says.""" - from .learning.classify import classify_pair - - for row in case.expected: - label = getattr(row, "family", None) - if label: - return str(label) - answer = classify_pair(case, taxonomy, client=None) # type: ignore[arg-type] - return answer.families[0] if answer.families else "unknown" - - -def _hunter_outcome( - case: PairCase, - *, - family: str, - score: object | None = None, - vuln: Sequence[StaticFinding] = (), - fixed: Sequence[StaticFinding] = (), - unscorable: str | None = None, -) -> PairOutcome: - outcome = getattr(score, "outcome", "unscorable") - detected = outcome in {"pair_correct", "both_flagged"} - silent = outcome in {"pair_correct", "both_silent"} - leaks = tuple(finding.finding_id for finding in fixed) if outcome in {"both_flagged", "reversed"} else () - misses = () if detected else tuple(f"{row.rule_id or '-'}:{row.path}:{row.cwe}" for row in case.expected) - return PairOutcome( - name=case.name, - slice=case.slice, - language=case.language, - origin=case.origin, - detected_vuln=detected, - silent_fix=silent, - pair_correct=outcome == "pair_correct", - vuln_matched=1 if detected else 0, - vuln_expected=len(case.expected), - vuln_findings=len(vuln), - fix_findings=len(fixed), - fix_leaks=len(leaks), - recall=1.0 if detected else 0.0, - misses=misses, - leaks=leaks, - commit_url=case.commit_url, - cve=case.cve, - provenance=case.provenance, - review_tier=case.review_tier, - mechanisms=_mechanisms(case), - known_limit=case.known_limit, - split=case.split, - weak_labels=0, - detection_kinds=("hunter",) if detected else (), - family=family, - unscorable=unscorable or (str(getattr(score, "unscorable_reason", "") or "") or None), - ) - - -# --- matching --------------------------------------------------------------- - - def _adjudicated(records: Sequence[OverlayRecord]) -> bool: return any(record.disposition in {"promote", "demote", "coverage"} for record in records) diff --git a/src/openultrasast/reports.py b/src/openultrasast/reports.py index 0e1b513..5f2637c 100644 --- a/src/openultrasast/reports.py +++ b/src/openultrasast/reports.py @@ -22,7 +22,6 @@ def write_markdown_report( mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligations: Mapping[str, Mapping[str, object]] | None = None, obligations_summary: Mapping[str, object] | None = None, - learning: Mapping[str, Mapping[str, object]] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications or []) @@ -73,8 +72,6 @@ def write_markdown_report( _append_mechanisms(lines, cited) if cited_obligations: _append_obligations(lines, cited_obligations, obligations_summary) - if learning: - _append_learning(lines, learning) if overlay: _append_overlay(lines, overlay) if complexity_map is not None: @@ -158,7 +155,6 @@ def write_sarif_report( overlay: Sequence[object] | None = None, mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligations: Mapping[str, Mapping[str, object]] | None = None, - learning: Mapping[str, Mapping[str, object]] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) @@ -183,7 +179,6 @@ def write_sarif_report( overlay_by_id.get(finding.finding_id), mechanisms=mechanisms, obligation=(obligations or {}).get(finding.finding_id), - learning=(learning or {}).get(finding.finding_id), ) for finding in findings ], @@ -209,7 +204,6 @@ def write_manifest( provenance: dict[str, object] | None = None, variants: dict[str, object] | None = None, obligations: dict[str, object] | None = None, - learning: dict[str, object] | None = None, ) -> None: path.parent.mkdir(parents=True, exist_ok=True) verification_by_id = _verification_by_id(verifications) @@ -250,8 +244,6 @@ def write_manifest( payload["variants"] = variants if obligations is not None: payload["obligations"] = obligations - if learning: - payload["learning"] = learning path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -295,7 +287,6 @@ def _sarif_result( *, mechanisms: Mapping[str, Mapping[str, object]] | None = None, obligation: Mapping[str, object] | None = None, - learning: Mapping[str, object] | None = None, ) -> dict[str, object]: properties: dict[str, object] = { "finding_id": finding.finding_id, @@ -327,10 +318,6 @@ def _sarif_result( properties["obligation_evidence"] = _strings_of(obligation, "evidence") properties["obligation_known_fix"] = obligation.get("known_fix") properties["obligation_intent"] = obligation.get("intent") - if learning is not None: - properties["family"] = str(learning.get("family", "")) - properties["detector"] = str(learning.get("detector", "")) - properties["verifier"] = str(learning.get("verifier", "")) return { "ruleId": _rule_id(finding), "level": _sarif_level(finding.severity), @@ -486,16 +473,6 @@ def _append_obligations(lines: list[str], cited: Mapping[str, Mapping[str, objec lines.append("") -def _append_learning(lines: list[str], cited: Mapping[str, Mapping[str, object]]) -> None: - """Which family found a claim, which detector version produced it, and what a verifier said about it.""" - lines.extend(["## Classified detectors", "", "Findings from the per-family detectors, with what confirmed them.", ""]) - for finding_id, info in sorted(cited.items()): - lines.append( - f"- `{finding_id}`: family `{info.get('family')}`, detector `{info.get('detector')}`, verifier `{info.get('verifier')}`" - ) - lines.append("") - - def _strings_of(info: Mapping[str, object], key: str) -> list[str]: value = info.get(key) return [str(item) for item in value] if isinstance(value, list | tuple) else [] diff --git a/src/openultrasast/semantic/__init__.py b/src/openultrasast/semantic/__init__.py index ae339f1..d282314 100644 --- a/src/openultrasast/semantic/__init__.py +++ b/src/openultrasast/semantic/__init__.py @@ -3,26 +3,21 @@ from .engines import joern_available, tree_sitter_available from .extra import grammar_for, has_semantic_extra from .facts import FactLoadError, SemanticFacts, load_facts -from .mechanisms import Mechanism, MechanismStore, append_mechanism, order_promotions from .overlay import OverlayError, OverlayRecord, adjudicate, finding_from_coverage, write_overlay from .prove_filter import filter_promoted_hotspots __all__ = [ "FactLoadError", - "Mechanism", - "MechanismStore", "OverlayError", "OverlayRecord", "SemanticFacts", "adjudicate", - "append_mechanism", "filter_promoted_hotspots", "finding_from_coverage", "grammar_for", "has_semantic_extra", "joern_available", "load_facts", - "order_promotions", "tree_sitter_available", "write_overlay", ] diff --git a/src/openultrasast/semantic/ir.py b/src/openultrasast/semantic/ir.py index ffe30b3..543bfc8 100644 --- a/src/openultrasast/semantic/ir.py +++ b/src/openultrasast/semantic/ir.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import re from dataclasses import dataclass @@ -230,3 +231,14 @@ def _unparse(node: ast.AST) -> str: return ast.unparse(node) except Exception: return "" + + +def trailing_name(callee: str) -> str: + """``os.path.join`` -> ``join``; ``Runtime.getRuntime().exec`` -> ``exec``; keeps the last identifier segment. + + Re-homed here from the deleted ``semantic/variants.py``: it is a pure helper over ``CallSite.name`` and has + nothing to do with the mechanism search that used to surround it. + """ + tail = callee.rsplit(".", 1)[-1].rsplit("::", 1)[-1].rsplit("->", 1)[-1] + match = re.search(r"[A-Za-z_$][\w$]*\s*$", tail) + return match.group(0).strip() if match else tail diff --git a/src/openultrasast/semantic/loo.py b/src/openultrasast/semantic/loo.py deleted file mode 100644 index 3c1c67b..0000000 --- a/src/openultrasast/semantic/loo.py +++ /dev/null @@ -1,304 +0,0 @@ -"""Leave-one-out: the pair corpus's own detection rate (corpus-seeded-mechanisms, Req 4). - -For every scorable pair, a temporary mechanism store is seeded from every *other* trusted pair, both excerpts are -materialized and searched for variants, and the pair is scored with the pair rules: detected when a hit lies inside the -labeled function and names the labeled mechanism (or sink), silent when the fixed side yields no hit. Only ``seeded`` and -``reviewed`` pairs teach; ``advisory`` and ``title`` pairs are scored as held-out targets but never seed. Reported, never -gated. - -Obligation-labeled rows (authorization-obligations, Req 7.4) run the obligation checker on both sides with the held-out -store's obligation shapes: detected when a finding tagged with the labeled obligation lies inside the labeled function *and* -carries a ``known_fix`` supplied by the store (the checker's own facts-only finding is not the corpus teaching), silent when -the fixed side carries no such finding; ``found_by`` names the shape record and the pairs that taught it. -""" - -from __future__ import annotations - -import tempfile -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..findings import StaticFinding -from ..pairs import PairCase, _materialize, _targets -from .facts import FactLoadError, SemanticFacts, load_facts -from .functions import named_function_ranges, spans_named -from .mechanisms import Mechanism, MechanismStore, append_from_pair, corpus_mechanisms -from .obligations import check_obligations, findings_to_static, load_obligation_facts, obligation_mechanisms -from .obligations.dominance import OrderDominance -from .seed import Lesson, _parse_language, pair_lessons -from .variant_search import search_tree -from .variants import Shape, trailing_name - - -@dataclass(frozen=True) -class LooOutcome: - pair: str - slice: str - provenance: str - mechanisms: tuple[str, ...] # labeled mechanism ids of the held-out pair - detected: bool - silent: bool - found_by: tuple[tuple[str, tuple[str, ...]], ...] # (mechanism record id, pairs that taught it) for detecting hits - hits_vuln: int - hits_fixed: int - teaches: bool # the pair contributed at least one shape when others were held out - leaked_by: tuple[str, ...] = () # mechanism record ids that hit the fixed side - degradations: tuple[dict[str, object], ...] = () # from variant search on either side - obligations: tuple[str, ...] = () # labeled obligation kinds of the held-out pair (Req 7.4) - - def to_dict(self) -> dict[str, object]: - return { - "pair": self.pair, - "slice": self.slice, - "provenance": self.provenance, - "mechanisms": list(self.mechanisms), - "detected": self.detected, - "silent": self.silent, - "found_by": [{"mechanism_id": mechanism_id, "pairs": list(pairs)} for mechanism_id, pairs in self.found_by], - "hits_vuln": self.hits_vuln, - "hits_fixed": self.hits_fixed, - "teaches": self.teaches, - "leaked_by": list(self.leaked_by), - "obligations": list(self.obligations), - } - - -@dataclass(frozen=True) -class LooResult: - outcomes: tuple[LooOutcome, ...] - per_slice: dict[str, dict[str, float]] - per_profile: dict[str, dict[str, float]] - per_mechanism: dict[str, dict[str, float]] - skipped: tuple[tuple[str, str], ...] = () - per_obligation_kind: dict[str, dict[str, float]] = field(default_factory=dict) - teaching_pairs: int = 0 - degradations: tuple[dict[str, object], ...] = field(default=()) - - def to_dict(self) -> dict[str, object]: - return { - "per_slice": self.per_slice, - "per_profile": self.per_profile, - "per_mechanism": self.per_mechanism, - "per_obligation_kind": self.per_obligation_kind, - "outcomes": [outcome.to_dict() for outcome in self.outcomes], - "skipped": [{"pair": pair, "reason": reason} for pair, reason in self.skipped], - "teaching_pairs": self.teaching_pairs, - "degradations": list(self.degradations), - } - - -def evaluate_loo(cases: Sequence[PairCase], *, facts: SemanticFacts | None = None) -> LooResult: - loaded = facts if facts is not None else _facts() - targets: list[PairCase] = [] - skipped: list[tuple[str, str]] = [] - for case in cases: - if case.unscorable or case.known_limit: - # `PairCase.unscorable` verbatim when the loader computed one: one spelling per corpus fact, and it - # already carries the `known_limit:` prefix when the limit is a maintainer's own word rather than one - # of the vocabulary's. A case built by hand may carry only the declaration. - skipped.append((case.name, case.unscorable or f"known_limit:{case.known_limit}")) - elif not case.vuln_file.is_file() or not case.fixed_file.is_file(): - skipped.append((case.name, "pointer_pair_not_cached" if not case.vendored else "excerpt_missing")) - else: - targets.append(case) - from ..learning.split import GATING_TIERS, holdout_names, is_teacher, refuse_if_holdout - - # learning-harness Req 5.1: every pair is still scored, but only a train-split teacher may seed a store. - lessons = {case.name: pair_lessons(case, loaded) for case in targets if is_teacher(case)} - outcomes = [_hold_out(case, targets, lessons, loaded) for case in targets] - # Req 5.2: the holdout pairs kept out of the teaching set are named, not silently dropped. Non-teachers held - # back for their tier or their unscorable reason are already in `skipped` with that reason. - excluded = [case.name for case in targets if case.name not in lessons] - refusal = refuse_if_holdout(excluded, targets) - refusals = (refusal.degradation(),) if refusal is not None else () - withheld = tuple( - (case.name, "tier" if case.review_tier not in GATING_TIERS else "unscorable") - for case in targets - if case.name in set(excluded) - set(holdout_names(targets)) - ) - skipped.extend((name, f"not_a_teacher:{why}") for name, why in withheld) - return LooResult( - outcomes=tuple(outcomes), - per_slice=_group(outcomes, lambda item: (item.slice,)), - per_profile=_group(outcomes, lambda item: (item.provenance,)), - per_mechanism=_group(outcomes, lambda item: item.mechanisms), - per_obligation_kind=_group(outcomes, lambda item: item.obligations), - skipped=tuple(skipped), - teaching_pairs=sum(1 for shapes in lessons.values() if shapes), - degradations=(*refusals, *_unique_degradations(outcomes)), - ) - - -def _unique_degradations(outcomes: Sequence[LooOutcome]) -> tuple[dict[str, object], ...]: - seen: set[str] = set() - unique: list[dict[str, object]] = [] - for outcome in outcomes: - for item in outcome.degradations: - key = repr(sorted(item.items())) - if key not in seen: - seen.add(key) - unique.append(item) - return tuple(unique) - - -def _hold_out(case: PairCase, targets: Sequence[PairCase], lessons: dict[str, list[Lesson]], facts: SemanticFacts) -> LooOutcome: - with tempfile.TemporaryDirectory(prefix="ousast-loo-") as scratch: - root = Path(scratch) - store = MechanismStore(root / "mechanisms.jsonl") - for teacher in targets: - if teacher.name == case.name: - continue - for lesson in lessons.get(teacher.name, ()): - append_from_pair( - store, - lesson.shape, - summary=lesson.summary, - cwe=lesson.cwe, - pair=teacher.name, - provenance=teacher.provenance, - tier=teacher.review_tier, - ) - scored = score_pair_with_store(case, store, facts, root=root) - return LooOutcome(**{**scored.__dict__, "teaches": bool(lessons.get(case.name))}) - - -def score_pair_with_store(case: PairCase, store: MechanismStore, facts: SemanticFacts, *, root: Path | None = None) -> LooOutcome: - """Search both sides of one pair with a fixed store and score it with the pair rules (shared by LOO and the lever).""" - labeled_mechanisms = tuple(sorted({row.mechanism for row in case.expected if row.mechanism})) - labeled_obligations = tuple(sorted({str(row.obligation) for row in case.expected if getattr(row, "obligation", None)})) - loaded = store.load() - records = {record.id: record for record in corpus_mechanisms(loaded)} - obligation_vuln: list[StaticFinding] = [] - obligation_fix: list[StaticFinding] = [] - with tempfile.TemporaryDirectory(prefix="ousast-loo-pair-") as scratch: - base = root if root is not None else Path(scratch) - vuln_root = _materialize(base / "vuln", case.vuln_file, case.relpath) - fix_root = _materialize(base / "fixed", case.fixed_file, case.relpath) - vuln_search = search_tree(vuln_root, _targets(vuln_root), store, facts, max_mechanisms=len(records) or 1) - fix_search = search_tree(fix_root, _targets(fix_root), store, facts, max_mechanisms=len(records) or 1) - vuln_hits, fix_hits = vuln_search.hits, fix_search.hits - if labeled_obligations: - shapes = obligation_mechanisms(loaded) - obligation_vuln = _obligation_findings(vuln_root, shapes) - obligation_fix = _obligation_findings(fix_root, shapes) - text = case.vuln_file.read_text(errors="ignore") - ranges = named_function_ranges(case.relpath, text, _parse_language(case)) or () - found: list[tuple[str, tuple[str, ...]]] = [] - for finding in obligation_vuln: - record = _known_fix_record(finding, records) - if record is None or finding.line is None or not _inside_label(case, finding.line, ranges): - continue - if any(f"obligation:{kind}" in finding.tags for kind in labeled_obligations): - found.append((record.id, tuple(record.pairs))) - obligation_leaks = tuple( - sorted({record.id for finding in obligation_fix if (record := _known_fix_record(finding, records)) is not None}) - ) - for hit in vuln_hits: - record = records.get(hit.mechanism_id) - if record is None or not _inside_label(case, hit.line, ranges): - continue - if _names_label(case, record, hit.sink_name): - found.append((record.id, tuple(record.pairs))) - return LooOutcome( - pair=case.name, - slice=case.slice, - provenance=case.provenance, - mechanisms=labeled_mechanisms, - detected=bool(found), - silent=not fix_hits and not obligation_leaks, - found_by=tuple(found), - hits_vuln=len(vuln_hits) + len(obligation_vuln), - hits_fixed=len(fix_hits) + len(obligation_fix), - teaches=False, - leaked_by=tuple(sorted({hit.mechanism_id for hit in fix_hits})) + obligation_leaks, - degradations=tuple(vuln_search.degradations) + tuple(fix_search.degradations), - obligations=labeled_obligations, - ) - - -def _obligation_findings(root: Path, shapes: Sequence[object]) -> list[StaticFinding]: - """Function-local obligation findings for one materialized side, with the held-out store's obligation shapes as known fixes.""" - from ..config import ObligationsConfig - from ..mapping import analyze_entry_points - from .ir import parse_file - - targets = _targets(root) - irs: dict[str, tuple[object, str]] = {} - texts: dict[str, str] = {} - for target in targets: - try: - text = (root / target.path).read_text(errors="ignore") - except OSError: - continue - texts[target.path] = text - irs[target.path] = (parse_file(target.path, text, target.language), text) - result = check_obligations( - irs=irs, # type: ignore[arg-type] - entries=analyze_entry_points(root, targets), - facts=load_obligation_facts(), - flow_facts=_facts(), - policy=None, - paths=(), - dominance=OrderDominance(texts=texts), - store_shapes=shapes, - min_siblings=ObligationsConfig().min_siblings, - ) - return findings_to_static(result) - - -def _known_fix_record(finding: StaticFinding, records: dict[str, Mechanism]) -> Mechanism | None: - for tag in finding.tags: - if tag.startswith("mechanism:"): - return records.get(tag.split(":", 1)[1]) - return None - - -def _inside_label(case: PairCase, line: int, ranges: Sequence[tuple[str, int, int]]) -> bool: - functions = {row.function for row in case.expected if row.function} - if not functions: - return True - return any(start <= line <= end for function in functions for start, end in spans_named(ranges, function)) - - -def _names_label(case: PairCase, record: Mechanism, sink_name: str) -> bool: - """The hit names the pair's labeled mechanism, or its labeled sink when the row carries one.""" - shape = Shape.from_dict(record.shape or {}) - for row in case.expected: - if row.mechanism and row.mechanism == shape.mechanism: - return True - if row.sink and trailing_name(row.sink) == sink_name: - return True - return False - - -def _group(outcomes: Sequence[LooOutcome], keys: object) -> dict[str, dict[str, float]]: - buckets: dict[str, list[LooOutcome]] = {} - for outcome in outcomes: - for key in keys(outcome): # type: ignore[operator] - buckets.setdefault(str(key), []).append(outcome) - return {name: _metrics(items) for name, items in sorted(buckets.items())} - - -def _metrics(outcomes: Sequence[LooOutcome]) -> dict[str, float]: - pairs = len(outcomes) - detected = sum(1 for item in outcomes if item.detected) - silent = sum(1 for item in outcomes if item.silent) - recall = detected / pairs if pairs else 0.0 - silence = silent / pairs if pairs else 0.0 - return { - "pairs": pairs, - "detected": detected, - "silent": silent, - "recall": recall, - "silence": silence, - "youden": recall + silence - 1.0 if pairs else 0.0, - "teaching": sum(1 for item in outcomes if item.teaches), - } - - -def _facts() -> SemanticFacts: - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) diff --git a/src/openultrasast/semantic/mechanisms.py b/src/openultrasast/semantic/mechanisms.py deleted file mode 100644 index a2ff748..0000000 --- a/src/openultrasast/semantic/mechanisms.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Append-only mechanism memory. OpenRouter embeddings rank prove budget; never demote.""" - -from __future__ import annotations - -import hashlib -import json -import math -import os -import uuid -from collections.abc import Sequence -from dataclasses import asdict, dataclass -from pathlib import Path - -from ..complexity.map import Hotspot -from ..findings import StaticFinding -from ..index import CodeChunk, EmbeddingClient, VectorIndex, VectorRecord, write_vector_index -from .obligations.shapes import ObligationShape -from .overlay import OverlayRecord -from .variants import Shape - -DEFAULT_MECHANISM_LOG = Path(".openultrasast/mechanisms.jsonl") - - -@dataclass(frozen=True) -class Mechanism: - id: str - summary: str - cwe: str - language: str - tags: tuple[str, ...] - keywords: tuple[str, ...] - what_made_it_exploitable: str - source_finding_id: str = "" - source_repo: str = "" - # corpus-seeded-mechanisms (additive; old rows load with these defaults) - origin: str = "sandbox" # "sandbox" (proven finding) | "corpus" (derived from a trusted pair) - review_tier: str = "" - pairs: tuple[str, ...] = () - shape: dict[str, object] | None = None # Shape.to_dict(); None on sandbox rows (not searchable) - guard: str = "none" - retracted: bool = False # tombstone row appended by the improve lever; load() folds the record away - - -class MechanismStore: - def __init__(self, log_path: Path, cache_path: Path | None = None) -> None: - self.log_path = log_path - self.cache_path = cache_path if cache_path is not None else log_path.with_suffix(".index.json") - - def load(self) -> tuple[Mechanism, ...]: - if not self.log_path.is_file(): - return () - by_id: dict[str, Mechanism] = {} # append-only log; the latest row for an id is its current state - for line in self.log_path.read_text().splitlines(): - if not line.strip(): - continue - payload = json.loads(line) - record = _mechanism_from_payload(payload) - if record.retracted: - by_id.pop(record.id, None) - else: - by_id[record.id] = record - return tuple(by_id.values()) - - def append(self, mechanism: Mechanism) -> None: - self.log_path.parent.mkdir(parents=True, exist_ok=True) - with self.log_path.open("a") as handle: - handle.write(json.dumps(asdict(mechanism), sort_keys=True) + "\n") - - -def append_mechanism( - store: MechanismStore, - *, - summary: str, - cwe: str, - language: str, - tags: Sequence[str], - what_made_it_exploitable: str, - source_finding_id: str = "", - source_repo: str = "", - keywords: Sequence[str] = (), -) -> Mechanism: - mechanism = Mechanism( - id=str(uuid.uuid4()), - summary=summary, - cwe=cwe, - language=language, - tags=tuple(tags), - keywords=tuple(keywords), - what_made_it_exploitable=what_made_it_exploitable, - source_finding_id=source_finding_id, - source_repo=source_repo, - ) - store.append(mechanism) - return mechanism - - -_TIER_RANK = {"seeded": 1, "reviewed": 2} - - -def corpus_mechanism_id(shape: Shape | ObligationShape) -> str: - """Deterministic id from the shape key, so the same shape gets the same record on every machine (lever-addressable).""" - return "corpus:" + hashlib.sha1(shape.key().encode()).hexdigest()[:16] - - -def append_from_pair( - store: MechanismStore, - shape: Shape | ObligationShape, - *, - summary: str, - cwe: str, - pair: str, - provenance: str, - tier: str, -) -> Mechanism: - """Second writer (Req 1.3, 1.4): one record per shape, ``origin = "corpus"``, provenance lists every pair that taught it.""" - mechanism_id = corpus_mechanism_id(shape) - existing = next((item for item in store.load() if item.id == mechanism_id), None) - pairs = tuple(existing.pairs) if existing is not None else () - if pair not in pairs: - pairs = (*pairs, pair) - if existing is not None and _TIER_RANK.get(existing.review_tier, 0) > _TIER_RANK.get(tier, 0): - tier = existing.review_tier # a shape keeps the strongest review it ever had - inherited = tuple(tag for tag in (existing.tags if existing is not None else ()) if tag.startswith(("provenance:", "tier:"))) - if isinstance(shape, Shape): - positions = ", ".join(f"arg{p} ({k})" for p, k in zip(shape.source_positions, shape.source_kinds, strict=True)) - exploitable = f"{positions} reaches {shape.sink_name}/{shape.arity}; the fix added {shape.guard}" - keywords: tuple[str, ...] = (shape.sink_name, shape.guard, *shape.source_kinds) - guard = shape.guard - else: # obligation shape: the discharger the fix carries is the 'guard' the reports show - exploitable = f"{shape.operation_kind} reached without {shape.discharger_kind}; the fix binds {shape.provenance}" - keywords = (shape.operation_kind, shape.discharger_kind, shape.provenance, shape.resource_class) - guard = shape.discharger_kind - record = Mechanism( - id=mechanism_id, - summary=summary, - cwe=cwe, - language=shape.language, - tags=( - shape.mechanism, - f"mechanism:{shape.mechanism}", - f"guard:{guard}", - *sorted({*inherited, f"provenance:{provenance}", f"tier:{tier}"}), - ), - keywords=keywords, - what_made_it_exploitable=exploitable, - source_finding_id="", - source_repo="", - origin="corpus", - review_tier=tier, - pairs=pairs, - shape=shape.to_dict(), - guard=guard, - ) - store.append(record) - return record - - -def corpus_mechanisms(records: Sequence[Mechanism]) -> tuple[Mechanism, ...]: - """Records that variant search may use: corpus origin with a shape.""" - return tuple(item for item in records if item.origin == "corpus" and item.shape is not None) - - -def order_promotions( - hotspots: Sequence[Hotspot], - records: Sequence[OverlayRecord], - findings: Sequence[StaticFinding], - *, - store: MechanismStore | None = None, - client: EmbeddingClient | None = None, - model: str | None = None, -) -> tuple[tuple[Hotspot, ...], str | None]: - """Heuristic order, then optional OpenRouter cosine boost. Never drops a promotion.""" - heuristic = tuple(sorted(hotspots, key=lambda item: (-item.score, item.path, item.function_name or ""))) - if store is None: - return heuristic, None - mechanisms = store.load() - if not mechanisms: - return heuristic, None - if client is None or not model: - return heuristic, "embeddings_unavailable" - overlay_by_id = {record.proposal_id: record for record in records} - finding_by_id = {finding.finding_id: finding for finding in findings} - try: - ranked = _rank_with_embeddings(heuristic, overlay_by_id, finding_by_id, mechanisms, store, client, model) - except Exception: - return heuristic, "embed_failed" - return ranked, None - - -def overlay_text(record: OverlayRecord) -> str: - parts = [record.cwe, record.reason, *record.sources, *record.sinks, *record.sanitizers] - return " ".join(part for part in parts if part) - - -def mechanism_text(mechanism: Mechanism) -> str: - return " ".join( - part - for part in (mechanism.summary, mechanism.what_made_it_exploitable, mechanism.cwe, *mechanism.tags, *mechanism.keywords) - if part - ) - - -def _rank_with_embeddings( - hotspots: tuple[Hotspot, ...], - overlay_by_id: dict[str, OverlayRecord], - finding_by_id: dict[str, StaticFinding], - mechanisms: tuple[Mechanism, ...], - store: MechanismStore, - client: EmbeddingClient, - model: str, -) -> tuple[Hotspot, ...]: - vectors = _mechanism_vectors(mechanisms, store, client, model) - scored: list[tuple[float, Hotspot]] = [] - for hotspot in hotspots: - record = _overlay_for_hotspot(hotspot, overlay_by_id) - finding = _finding_for_hotspot(hotspot, finding_by_id) - language = record.language if record is not None else "" - cwe = record.cwe if record is not None else "" - tags = tuple(finding.tags) if finding is not None else () - filtered = [item for item in mechanisms if _hard_filter(item, language, cwe, tags)] - boost = 0.0 - if record is not None and filtered: - query = client.embed(model=model, inputs=[overlay_text(record)])[0] - for mechanism in filtered: - vector = vectors.get(mechanism.id) - if vector is None: - continue - boost = max(boost, _cosine(query, vector)) - scored.append((hotspot.score + boost, hotspot)) - scored.sort(key=lambda item: (-item[0], item[1].path, item[1].function_name or "")) - return tuple(item[1] for item in scored) - - -def _mechanism_vectors( - mechanisms: tuple[Mechanism, ...], - store: MechanismStore, - client: EmbeddingClient, - model: str, -) -> dict[str, list[float]]: - vectors: dict[str, list[float]] = {} - missing: list[Mechanism] = [] - cached = _read_cache(store.cache_path, model) - for mechanism in mechanisms: - key = f"{mechanism.id}:{model}" - if key in cached: - vectors[mechanism.id] = cached[key] - else: - missing.append(mechanism) - if missing: - embeddings = client.embed(model=model, inputs=[mechanism_text(item) for item in missing]) - for mechanism, embedding in zip(missing, embeddings, strict=True): - vectors[mechanism.id] = embedding - records = [ - VectorRecord( - chunk=CodeChunk( - chunk_id=f"{mechanism.id}:{model}", - namespace="mechanisms", - path=mechanism.id, - language=mechanism.language or "text", - start_line=1, - end_line=1, - text=mechanism_text(mechanism), - metadata={"mechanism_id": mechanism.id, "model": model, "cwe": mechanism.cwe, "language": mechanism.language}, - ), - embedding=vectors[mechanism.id], - ) - for mechanism in mechanisms - if mechanism.id in vectors - ] - index = VectorIndex( - store="json-local", - embedding_model=model, - repo_root=str(store.log_path.parent), - repo_commit=None, - records=records, - ) - write_vector_index(index, store.cache_path) - return vectors - - -def _read_cache(path: Path, model: str) -> dict[str, list[float]]: - if not path.is_file(): - return {} - try: - payload = json.loads(path.read_text()) - except json.JSONDecodeError: - return {} - if str(payload.get("embedding_model")) != model: - return {} - cached: dict[str, list[float]] = {} - for item in payload.get("records", []): - chunk = item.get("chunk", {}) - metadata = chunk.get("metadata", {}) - mechanism_id = str(metadata.get("mechanism_id", "")) - if mechanism_id: - cached[f"{mechanism_id}:{model}"] = [float(value) for value in item.get("embedding", [])] - return cached - - -def _hard_filter(mechanism: Mechanism, language: str, cwe: str, tags: Sequence[str]) -> bool: - if mechanism.language and language and mechanism.language != language: - return False - if mechanism.cwe and cwe and mechanism.cwe != cwe: - return False - tag_ok = not mechanism.tags or not tags or bool(set(mechanism.tags) & set(tags)) - return tag_ok - - -def _overlay_for_hotspot(hotspot: Hotspot, overlay_by_id: dict[str, OverlayRecord]) -> OverlayRecord | None: - for finding_id in hotspot.inventory_finding_ids: - record = overlay_by_id.get(finding_id) - if record is not None: - return record - return None - - -def _finding_for_hotspot(hotspot: Hotspot, finding_by_id: dict[str, StaticFinding]) -> StaticFinding | None: - for finding_id in hotspot.inventory_finding_ids: - finding = finding_by_id.get(finding_id) - if finding is not None: - return finding - return None - - -def _cosine(left: list[float], right: list[float]) -> float: - if len(left) != len(right): - return 0.0 - numerator = sum(a * b for a, b in zip(left, right, strict=True)) - left_norm = math.sqrt(sum(a * a for a in left)) - right_norm = math.sqrt(sum(b * b for b in right)) - if left_norm == 0 or right_norm == 0: - return 0.0 - return numerator / (left_norm * right_norm) - - -def _mechanism_from_payload(payload: dict[str, object]) -> Mechanism: - tags = payload.get("tags") or [] - keywords = payload.get("keywords") or [] - pairs = payload.get("pairs") or [] - shape = payload.get("shape") - if not isinstance(tags, list) or not isinstance(keywords, list) or not isinstance(pairs, list): - raise ValueError("mechanism tags, keywords and pairs must be lists") - return Mechanism( - id=str(payload["id"]), - summary=str(payload.get("summary", "")), - cwe=str(payload.get("cwe", "")), - language=str(payload.get("language", "")), - tags=tuple(str(item) for item in tags), - keywords=tuple(str(item) for item in keywords), - what_made_it_exploitable=str(payload.get("what_made_it_exploitable", "")), - source_finding_id=str(payload.get("source_finding_id", "")), - source_repo=str(payload.get("source_repo", "")), - origin=str(payload.get("origin", "sandbox")), - review_tier=str(payload.get("review_tier", "")), - pairs=tuple(str(item) for item in pairs), - shape=dict(shape) if isinstance(shape, dict) else None, # validated where it is used (search_tree, the lever) - guard=str(payload.get("guard", "none")), - retracted=bool(payload.get("retracted", False)), - ) - - -def embeddings_configured(model: str | None) -> bool: - return bool(model) and bool(os.environ.get("OPENROUTER_API_KEY")) diff --git a/src/openultrasast/semantic/obligations/operations.py b/src/openultrasast/semantic/obligations/operations.py index 58770cb..c5f14a7 100644 --- a/src/openultrasast/semantic/obligations/operations.py +++ b/src/openultrasast/semantic/obligations/operations.py @@ -14,8 +14,7 @@ from collections.abc import Sequence from dataclasses import dataclass from ..facts import SemanticFacts -from ..ir import FileIR, FunctionIR -from ..variants import trailing_name +from ..ir import FileIR, FunctionIR, trailing_name from .facts import DischargerFact, ObligationFacts from .shapes import _call_matches, _constraint_provenance, _is_literal, _operations, _permissive diff --git a/src/openultrasast/semantic/obligations/shapes.py b/src/openultrasast/semantic/obligations/shapes.py index d5a8a47..ffc8c1b 100644 --- a/src/openultrasast/semantic/obligations/shapes.py +++ b/src/openultrasast/semantic/obligations/shapes.py @@ -17,8 +17,7 @@ from dataclasses import dataclass from ..facts import SemanticFacts from ..functions import named_ranges_from_ir -from ..ir import CallSite, FileIR, FunctionIR -from ..variants import trailing_name +from ..ir import CallSite, FileIR, FunctionIR, trailing_name from .facts import DISCHARGER_KINDS, OPERATION_KINDS, PROVENANCE_KINDS, DischargerFact, ObligationFacts, OperationFact FAMILY = "obligation" diff --git a/src/openultrasast/semantic/prove_budget.py b/src/openultrasast/semantic/prove_budget.py deleted file mode 100644 index b0f4e50..0000000 --- a/src/openultrasast/semantic/prove_budget.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Prove-budget ranker: heuristic order plus optional mechanism recall.""" - -from .mechanisms import MechanismStore, append_mechanism, order_promotions -from .prove_filter import filter_promoted_hotspots, promoted_findings - -__all__ = [ - "MechanismStore", - "append_mechanism", - "filter_promoted_hotspots", - "order_promotions", - "promoted_findings", -] diff --git a/src/openultrasast/semantic/seed.py b/src/openultrasast/semantic/seed.py deleted file mode 100644 index c3e039e..0000000 --- a/src/openultrasast/semantic/seed.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Export trusted pairs as mechanism records (corpus-seeded-mechanisms, Req 1). - -Only ``seeded`` and ``reviewed`` pairs seed; ``advisory`` and ``title`` pairs, ``known_limit`` pairs, pointer pairs -without a cached excerpt, and excerpts that fail to parse are skipped with a reason. Runs offline over the vendored -catalog; nothing here fetches, and no code text enters the store. -""" - -from __future__ import annotations - -from collections.abc import Sequence -from dataclasses import dataclass, field -from pathlib import Path - -from ..pairs import GATING_TIERS, PairCase -from ..preprocess import LANGUAGE_BY_EXTENSION -from .facts import FactLoadError, SemanticFacts, load_facts -from .ir import FileIR, parse_file -from .mechanisms import MechanismStore, append_from_pair -from .obligations.facts import ObligationFacts, load_obligation_facts -from .obligations.shapes import ABSENCE_MECHANISMS, ObligationShape, derive_obligation, explain_skip -from .variants import Shape, derive_shape - -LANGUAGE_UNSUPPORTED = "variants_language_unsupported" - - -@dataclass(frozen=True) -class ExportReport: - seeded: int # pairs that produced at least one record - records: int # distinct records written (deduped by shape) - skipped: tuple[tuple[str, str], ...] = () # (pair, reason) - degradations: tuple[dict[str, object], ...] = () - seeded_pairs: tuple[str, ...] = field(default=()) - - def to_dict(self) -> dict[str, object]: - return { - "seeded": self.seeded, - "records": self.records, - "skipped": [{"pair": pair, "reason": reason} for pair, reason in self.skipped], - "degradations": list(self.degradations), - "seeded_pairs": list(self.seeded_pairs), - } - - -def export_mechanisms(cases: Sequence[PairCase], store: MechanismStore, *, facts: SemanticFacts | None = None) -> ExportReport: - """Derive one shape per labeled expected row of every trusted pair and append it to ``store`` (deduped by shape).""" - loaded = facts if facts is not None else _facts() - skipped: list[tuple[str, str]] = [] - degradations: list[dict[str, object]] = [] - seeded: list[str] = [] - record_ids: set[str] = set() - unsupported_languages: set[str] = set() - for case in cases: # catalog order; the store lists pairs in the order they taught - reason = _skip_reason(case) - if reason is not None: - skipped.append((case.name, reason)) - continue - vuln_text = _read(case.vuln_file) - fixed_text = _read(case.fixed_file) - language = _parse_language(case) - vuln_ir = parse_file(case.relpath, vuln_text, language) - fixed_ir = parse_file(case.relpath, fixed_text, language) - if not vuln_ir.parse_ok or not fixed_ir.parse_ok: - bad = vuln_ir if not vuln_ir.parse_ok else fixed_ir - if bad.reason == "language_unsupported": - skipped.append((case.name, LANGUAGE_UNSUPPORTED)) - if language not in unsupported_languages: - unsupported_languages.add(language) - degradations.append({"stage": "mechanisms", "reason": LANGUAGE_UNSUPPORTED, "language": language}) - else: - skipped.append((case.name, "parse_failed")) - continue - if not any(row.function and row.mechanism for row in case.expected): - skipped.append((case.name, "no_labeled_function")) - continue - lessons = pair_lessons(case, loaded, vuln_ir=vuln_ir, fixed_ir=fixed_ir, vuln_text=vuln_text, fixed_text=fixed_text) - for lesson in lessons: - record = append_from_pair( - store, - lesson.shape, - summary=lesson.summary, - cwe=lesson.cwe, - pair=case.name, - provenance=case.provenance, - tier=case.review_tier, - ) - record_ids.add(record.id) - if lessons: - seeded.append(case.name) - else: - skipped.append((case.name, _skip_detail(case, vuln_ir, fixed_ir, vuln_text, fixed_text, loaded))) - return ExportReport( - seeded=len(seeded), - records=len(record_ids), - skipped=tuple(skipped), - degradations=tuple(degradations), - seeded_pairs=tuple(seeded), - ) - - -@dataclass(frozen=True) -class Lesson: - """One shape a trusted pair teaches, with the CWE and a text-free summary (a sink shape or an obligation shape).""" - - shape: Shape | ObligationShape - cwe: str - summary: str - - -def pair_lessons( - case: PairCase, - facts: SemanticFacts, - *, - vuln_ir: FileIR | None = None, - fixed_ir: FileIR | None = None, - vuln_text: str | None = None, - fixed_text: str | None = None, -) -> list[Lesson]: - """Shapes derived from every labeled row of a pair (shared by the exporter and leave-one-out). Empty when a side fails to parse.""" - vuln_text = vuln_text if vuln_text is not None else _read(case.vuln_file) - fixed_text = fixed_text if fixed_text is not None else _read(case.fixed_file) - language = _parse_language(case) - vuln_ir = vuln_ir if vuln_ir is not None else parse_file(case.relpath, vuln_text, language) - fixed_ir = fixed_ir if fixed_ir is not None else parse_file(case.relpath, fixed_text, language) - if not vuln_ir.parse_ok or not fixed_ir.parse_ok: - return [] - lessons: list[Lesson] = [] - for row in case.expected: - if not row.function or not row.mechanism: - continue - obligation_kind = getattr(row, "obligation", None) - if obligation_kind or row.mechanism in ABSENCE_MECHANISMS: - obligation = derive_obligation( - vuln_ir, - fixed_ir, - function=row.function, - obligation=obligation_kind, - mechanism=row.mechanism, - facts=_obligation_facts(), - flow_facts=facts, - vuln_text=vuln_text, - fixed_text=fixed_text, - ) - if obligation is not None: - summary = ( - f"{obligation.mechanism}: {obligation.operation_kind} without {obligation.discharger_kind} ({row.cwe}); " - f"fix binds {obligation.provenance}" - ) - lessons.append(Lesson(shape=obligation, cwe=row.cwe, summary=summary)) - continue - # an absence row that teaches no obligation may still teach a sink shape (a weak literal at a sink, say) - shape = derive_shape( - vuln_ir, - fixed_ir, - function=row.function, - sink=row.sink, - line=row.line, - mechanism=row.mechanism, - facts=facts, - vuln_text=vuln_text, - fixed_text=fixed_text, - cwe=row.cwe or None, - ) - if shape is not None: - summary = f"{shape.mechanism}: {'/'.join(shape.source_kinds)} into {shape.sink_name} ({row.cwe}); fix adds {shape.guard}" - lessons.append(Lesson(shape=shape, cwe=row.cwe, summary=summary)) - return lessons - - -def _skip_detail(case: PairCase, vuln_ir: FileIR, fixed_ir: FileIR, vuln_text: str, fixed_text: str, flow_facts: SemanticFacts) -> str: - """Why a parsing, labeled pair taught nothing: the obligation derivation's reason for absence rows, else the sink reason.""" - for row in case.expected: - if row.function and row.mechanism and (getattr(row, "obligation", None) or row.mechanism in ABSENCE_MECHANISMS): - return explain_skip( - vuln_ir, - fixed_ir, - function=row.function, - vuln_text=vuln_text, - fixed_text=fixed_text, - obligation=getattr(row, "obligation", None), - facts=_obligation_facts(), - flow_facts=flow_facts, - ) - return "no_labeled_sink_call_site" - - -_OBLIGATION_FACTS: list[ObligationFacts] = [] - - -def _obligation_facts() -> ObligationFacts: - if not _OBLIGATION_FACTS: - _OBLIGATION_FACTS.append(load_obligation_facts()) - return _OBLIGATION_FACTS[0] - - -def _skip_reason(case: PairCase) -> str | None: - from ..learning.split import is_teacher - - if case.review_tier not in GATING_TIERS: - return f"tier:{case.review_tier}" - if case.unscorable or case.known_limit: - # `PairCase.unscorable` verbatim when the loader computed one, the same string every other path reports: - # `identical_twin` declared in the catalog and `identical_twin` computed from the bytes are one fact, and - # calling either of them `split:train` is simply untrue. A case built by hand may carry only the - # declaration (Req 5.1, 10.3). - return case.unscorable or f"known_limit:{case.known_limit}" - if not is_teacher(case) and case.vendored: - return f"split:{case.split}" - if not case.vuln_file.is_file() or not case.fixed_file.is_file(): - return "pointer_pair_not_cached" if not case.vendored else "excerpt_missing" - return None - - -def _parse_language(case: PairCase) -> str: - by_ext = LANGUAGE_BY_EXTENSION.get(case.vuln_file.suffix.lower()) - if by_ext: - return str(by_ext) - return {"c_cpp": "c"}.get(case.language, case.language) - - -def _read(path: Path) -> str: - try: - return path.read_text(errors="ignore") - except OSError: - return "" - - -def _facts() -> SemanticFacts: - try: - return load_facts() - except FactLoadError: - return SemanticFacts(version="", sources=(), sinks=(), sanitizers=()) diff --git a/src/openultrasast/semantic/variant_search.py b/src/openultrasast/semantic/variant_search.py deleted file mode 100644 index 9c058fe..0000000 --- a/src/openultrasast/semantic/variant_search.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Variant search over a scanned tree and its merge with overlay flows (corpus-seeded-mechanisms, Req 3). - -Depends on the shapes (``variants``), the store (``mechanisms``), the overlay records and ``StaticFinding``; the shape -module itself stays free of those imports. -""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace -from pathlib import Path - -from ..findings import StaticFinding -from ..preprocess import FileTarget -from .facts import SemanticFacts -from .ir import parse_file -from .mechanisms import MechanismStore, corpus_mechanisms -from .overlay import OverlayRecord -from .variants import Shape, VariantHit, match_shapes - -LANGUAGE_UNSUPPORTED = "variants_language_unsupported" - - -@dataclass(frozen=True) -class SearchResult: - hits: tuple[VariantHit, ...] - mechanisms_searched: int - files_searched: int - degradations: tuple[dict[str, object], ...] = () - - -def search_tree( - root: Path, - targets: Sequence[FileTarget], - store: MechanismStore, - facts: SemanticFacts, - *, - max_mechanisms: int, -) -> SearchResult: - """Variant search over the parsed files of a tree with the store's corpus shapes (bounded by ``max_mechanisms``).""" - records = [record for record in corpus_mechanisms(store.load()) if (record.shape or {}).get("family", "sink") == "sink"] - records = records[: max(max_mechanisms, 0)] - shapes: list[Shape] = [] - ids: dict[str, str] = {} - degradations: list[dict[str, object]] = [] - for record in records: - assert record.shape is not None - try: - shape = Shape.from_dict(record.shape) - except (ValueError, TypeError) as exc: # one malformed operator-visible row must not abort the scan - degradations.append( - {"stage": "variants", "reason": "variants_store_row_invalid", "mechanism_id": record.id, "detail": str(exc)[:120]} - ) - continue - shapes.append(shape) - ids[shape.key()] = record.id - if not shapes: - return SearchResult(hits=(), mechanisms_searched=0, files_searched=0, degradations=tuple(degradations)) - hits: list[VariantHit] = [] - files = 0 - unsupported: set[str] = set() - for target in targets: - try: - text = (root / target.path).read_text(errors="ignore") - except OSError: - continue - ir = parse_file(target.path, text, target.language) - if not ir.parse_ok: - if ir.reason == "language_unsupported" and target.language not in unsupported: - unsupported.add(target.language) - degradations.append({"stage": "variants", "reason": "variants_language_unsupported", "language": target.language}) - continue - files += 1 - hits.extend(match_shapes(ir, shapes, facts, mechanism_ids=ids)) - return SearchResult(hits=tuple(hits), mechanisms_searched=len(shapes), files_searched=files, degradations=tuple(degradations)) - - -def hits_to_findings( - hits: Sequence[VariantHit], - records: Sequence[OverlayRecord], - summaries: Mapping[str, tuple[str, tuple[str, ...], str]], -) -> tuple[list[StaticFinding], list[OverlayRecord]]: - """Merge hits into overlay records at the same call site (Req 3.3); the rest become ``suspicion`` findings (Req 3.2).""" - by_site: dict[tuple[str, int | None], list[int]] = {} - for index, record in enumerate(records): - if record.disposition in {"promote", "coverage"} and record.sources and record.sinks: - by_site.setdefault((record.path, record.line), []).append(index) - updated = list(records) - findings: list[StaticFinding] = [] - emitted: set[str] = set() - for hit in hits: - indexes = by_site.get((hit.path, hit.line)) - if indexes: - for index in indexes: - if updated[index].mechanism_id is None: - updated[index] = replace(updated[index], mechanism_id=hit.mechanism_id) - continue - finding_id = f"variant:{hit.mechanism_id}:{hit.path}:{hit.line}" - if finding_id in emitted: - continue - emitted.add(finding_id) - summary, pairs, cwe = summaries.get(hit.mechanism_id, (hit.mechanism_id, (), "")) - taught_by = ", ".join(pairs) if pairs else "the mechanism store" - findings.append( - StaticFinding( - finding_id=finding_id, - path=hit.path, - title=f"Variant of known mechanism: {hit.sink_name}", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale=( - f"{summary}. Structural variant ({hit.source_kind} reaches {hit.sink_name}) of a mechanism learned from " - f"{taught_by}; suspicion until the overlay or the sandbox agrees." - ), - line=hit.line, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["variant", f"mechanism:{hit.mechanism_id}", *([f"cwe:{cwe}"] if cwe else [])], - ranking_priority=0.0, - ) - ) - return findings, updated diff --git a/src/openultrasast/semantic/variants.py b/src/openultrasast/semantic/variants.py deleted file mode 100644 index 56a6a2c..0000000 --- a/src/openultrasast/semantic/variants.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Structural mechanism shapes derived from labeled pairs (corpus-seeded-mechanisms, Req 2). - -A shape is the closed, text-free description of how a known vulnerable call looked: the -sink's trailing callee name, its arity, which argument positions carried source-derived -values and of what kind, and the kind of guard the fix added. Shapes are derived from -``FileIR`` (never from regex over code) and compared structurally, so a variant in another -project matches even when every identifier differs. Stdlib plus ``.ir``, ``.facts`` and -``.functions`` only; never imports the overlay, the scorer, or the store. -""" - -from __future__ import annotations - -import re -from collections.abc import Mapping, Sequence -from dataclasses import dataclass - -from .facts import SemanticFacts, SinkFact -from .functions import named_ranges_from_ir -from .ir import Bind, CallSite, FileIR, FunctionIR - -GUARD_KINDS = ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") -SOURCE_KINDS = ("fact_source", "parameter", "container_read") -_MAX_BIND_DEPTH = 8 - -# Closed guard classification over the statements the fix added inside the labeled function. -# Order matters: the first matching kind wins. These classify the *fix*, they never detect. -_GUARD_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( - ("parameterized_call", re.compile(r"execute\w*\([^,]+,\s*[\(\[]|prepareStatement\(|\?\s*['\"]?\s*[,)]|\$\d\b|%s['\"]\s*,\s*[\(\[]")), - ( - "auth_check", - re.compile( - r"\b(request\.user|req\.user|current_user|request\.state|session\[|is_authenticated|authoriz|require_auth" - r"|login_required|verify_token|check_permission|owner_id\s*[!=]=)" - ), - ), - ( - "allowlist_test", - re.compile( - r"\bnot in\b|\bin\s*[\(\{\[]|\.includes\(|\.has\(|\bstartswith\(|\bstartsWith\(|\bendswith\(|\bendsWith\(" - r"|re\.(?:match|fullmatch)\(|\.test\(|allow_?list|whitelist|\bALLOWED\b|\bsafe_join\(|secure_filename\(" - r"|\brealpath\(|\bnormpath\(|\bresolve\(" - ), - ), - ("bounds_test", re.compile(r"\blen\(|\.length\b|\bsizeof\(|\bstrn?len\(|\bsnprintf\(|\bstrlcpy\(|\bmin\(|\bmax\(|\s(?:<|>|<=|>=)\s")), - ( - "null_test", - re.compile( - r"\bis None\b|\bis not None\b|[!=]= *NULL\b|[!=]==? *null\b|\bnullptr\b|\bif\s*\(\s*!\s*\w|\bif not \w+\s*:" - r"|\bif\s+\w+\s+is\s+None|\?\?|\?\." - ), - ), - ( - "type_change", - re.compile(r"\bint\(|\bparseInt\(|\bNumber\(|\(int\)|\(size_t\)|\(unsigned\b|\bstr\(|\.toString\(|\bstatic_cast<|\bBigInt\("), - ), -) -# Design §Shape: a container read is a subscript or a `.get(` read; source names themselves live in facts, never here. -_CONTAINER_READ = re.compile(r"\[[^\]]+\]|\.get\(") - - -@dataclass(frozen=True) -class Shape: - language: str - sink_name: str # trailing callee segment - arity: int - source_positions: tuple[int, ...] - source_kinds: tuple[str, ...] - guard: str # GUARD_KINDS - mechanism: str # closed vocabulary id from the pair label - - def key(self) -> str: - """Stable identity of the shape; carries no path, line, or literal.""" - positions = ",".join(str(p) for p in self.source_positions) - kinds = ",".join(self.source_kinds) - return f"{self.language}|{self.sink_name}/{self.arity}|pos={positions}|kinds={kinds}|guard={self.guard}|{self.mechanism}" - - def to_dict(self) -> dict[str, object]: - return { - "language": self.language, - "sink_name": self.sink_name, - "arity": self.arity, - "source_positions": list(self.source_positions), - "source_kinds": list(self.source_kinds), - "guard": self.guard, - "mechanism": self.mechanism, - } - - @classmethod - def from_dict(cls, payload: dict[str, object]) -> Shape: - positions = payload.get("source_positions") or [] - kinds = payload.get("source_kinds") or [] - if not isinstance(positions, list) or not isinstance(kinds, list): - raise ValueError("shape positions and kinds must be lists") - guard = str(payload.get("guard", "none")) - if guard not in GUARD_KINDS: - raise ValueError(f"unknown guard kind {guard!r}") - return cls( - language=str(payload.get("language", "")), - sink_name=str(payload.get("sink_name", "")), - arity=int(str(payload.get("arity", 0))), - source_positions=tuple(int(p) for p in positions), - source_kinds=tuple(str(k) for k in kinds), - guard=guard, - mechanism=str(payload.get("mechanism", "other")), - ) - - -def trailing_name(callee: str) -> str: - """``os.path.join`` -> ``join``; ``Runtime.getRuntime().exec`` -> ``exec``; keeps the last identifier segment.""" - tail = callee.rsplit(".", 1)[-1].rsplit("::", 1)[-1].rsplit("->", 1)[-1] - match = re.search(r"[A-Za-z_$][\w$]*\s*$", tail) - return match.group(0).strip() if match else tail - - -def derive_shape( - vuln: FileIR, - fixed: FileIR, - *, - function: str, - sink: str | None, - line: int | None, - mechanism: str, - facts: SemanticFacts, - vuln_text: str = "", - fixed_text: str = "", - cwe: str | None = None, -) -> Shape | None: - """The shape of the labeled sink call in ``function`` on the vulnerable side, with the guard the fixed side added. - - None when either side failed to parse, the function or the labeled call site is not found, or no argument of the - call carries a source (a constant-only call teaches nothing). - """ - if not vuln.parse_ok or not fixed.parse_ok: - return None - target = _function_named(vuln, function, vuln_text) - if target is None: - return None - scoped = facts.for_language(vuln.language) - call = _labeled_call(target, sink, line, scoped, cwe) - if call is None: - return None - kinds = [_source_kind_of_argument(target, call, index, scoped) for index in range(len(call.arg_texts))] - positions = tuple(index for index, kind in enumerate(kinds) if kind is not None) - if not positions: - return None - guard = classify_guard(vuln, fixed, function=function, vuln_text=vuln_text, fixed_text=fixed_text) - return Shape( - language=vuln.language, - sink_name=trailing_name(call.name), - arity=len(call.arg_texts), - source_positions=positions, - source_kinds=tuple(kind for kind in kinds if kind is not None), - guard=guard, - mechanism=mechanism, - ) - - -def classify_guard(vuln: FileIR, fixed: FileIR, *, function: str, vuln_text: str, fixed_text: str) -> str: - """Closed guard kind from the statements inside ``function`` that exist only on the fixed side (``none`` when unknown).""" - before = _function_lines(vuln, function, vuln_text) - after = _function_lines(fixed, function, fixed_text) - added = [line for line in after if line not in set(before)] - for kind, pattern in _GUARD_PATTERNS: - if any(pattern.search(line) for line in added): - return kind - return "none" - - -def source_kind_of_name(function: FunctionIR, name: str, facts: SemanticFacts, *, before_line: int, depth: int = 0) -> str | None: - """How ``name`` came to carry attacker data inside ``function``: a fact source, a parameter, or a container read.""" - if depth > _MAX_BIND_DEPTH: - return None - bind = _last_bind(function, name, before_line) - if bind is not None: - if _source_in(bind.value_text, facts): - return "fact_source" - if _CONTAINER_READ.search(bind.value_text): - return "container_read" - for inner in bind.names: - if inner == name: - continue - kind = source_kind_of_name(function, inner, facts, before_line=bind.line, depth=depth + 1) - if kind is not None: - return kind - if name in function.params: - return "parameter" - return None - - -def _source_kind_of_argument(function: FunctionIR, call: CallSite, index: int, facts: SemanticFacts) -> str | None: - text = call.arg_texts[index] - if _source_in(text, facts): - return "fact_source" - if call.arg_is_constant[index] and not call.arg_names[index]: - return None - if _CONTAINER_READ.search(text) and any(name in function.params for name in call.arg_names[index]): - return "container_read" - found: list[str] = [] - for name in call.arg_names[index]: - kind = source_kind_of_name(function, name, facts, before_line=call.line + 1) - if kind is not None: - found.append(kind) - for preferred in SOURCE_KINDS: - if preferred in found: - return preferred - return None - - -def _source_in(text: str, facts: SemanticFacts) -> bool: - return any(pattern and pattern in text for source in facts.sources for pattern in source.patterns) - - -def _last_bind(function: FunctionIR, name: str, before_line: int) -> Bind | None: - candidates = [bind for bind in function.binds if bind.name == name and bind.line < before_line] - return max(candidates, key=lambda bind: bind.line) if candidates else None - - -def _labeled_call(function: FunctionIR, sink: str | None, line: int | None, facts: SemanticFacts, cwe: str | None) -> CallSite | None: - """The call the label names: line and sink intersected when both exist, else either; else the fact sink in the function. - - Labels that carry only a function and a mechanism (Real-Vuln rows) fall back to the fact-sink call inside the labeled - function, preferring the sink whose CWE matches the row and a call with a non-constant argument. - """ - wanted = trailing_name(sink) if sink else None - on_line = [call for call in function.calls if line is not None and call.line == line] - if on_line and wanted: - exact = [call for call in on_line if trailing_name(call.name) == wanted] - if exact: - return exact[0] - if on_line: - return on_line[0] - if wanted: - for call in function.calls: - if trailing_name(call.name) == wanted: - return call - return None - ranked: list[tuple[int, int, CallSite]] = [] - for order, call in enumerate(function.calls): - fact = _sink_fact_for(call.name, facts) - if fact is None: - continue - cwe_match = 0 if (cwe and fact.cwe == cwe) else 1 - non_constant = 0 if any(not const or names for const, names in zip(call.arg_is_constant, call.arg_names, strict=True)) else 1 - ranked.append((cwe_match + non_constant, order, call)) - ranked.sort(key=lambda item: (item[0], item[1])) - return ranked[0][2] if ranked else None - - -def _sink_fact_for(name: str, facts: SemanticFacts) -> SinkFact | None: - for fact in facts.sinks: - if any(_call_matches(name, call) for call in fact.calls): - return fact - return None - - -def _call_matches(name: str, fact_call: str) -> bool: - """Same rule as the taint walker: exact, dotted suffix, or a bare name equal to the fact's last segment.""" - if name == fact_call or name.endswith("." + fact_call): - return True - return "." not in name and name == fact_call.split(".")[-1] - - -def _function_named(ir: FileIR, function: str, text: str) -> FunctionIR | None: - for item in ir.functions: - if item.name == function: - return item - if text: # labels use declarator names; the parser may name the function differently (or ) - for name, start, _end in named_ranges_from_ir(ir, text): - if name == function: - for item in ir.functions: - if item.start_line == start: - return item - return None - - -def _function_lines(ir: FileIR, function: str, text: str) -> list[str]: - target = _function_named(ir, function, text) - if target is None or not text: - return [] - lines = text.splitlines() - body = lines[target.start_line - 1 : target.end_line] - return [line.strip() for line in body if line.strip()] - - -def shapes_by_sink(shapes: Sequence[Shape]) -> dict[tuple[str, str], list[Shape]]: - """Index for matching: (language, sink_name) -> shapes.""" - index: dict[tuple[str, str], list[Shape]] = {} - for shape in shapes: - index.setdefault((shape.language, shape.sink_name), []).append(shape) - return index - - -@dataclass(frozen=True) -class VariantHit: - path: str - line: int - mechanism_id: str - sink_name: str - source_kind: str - - -def match_shapes( - ir: FileIR, - shapes: Sequence[Shape], - facts: SemanticFacts, - *, - mechanism_ids: Mapping[str, str], -) -> list[VariantHit]: - """Call sites of ``ir`` whose trailing callee name, arity and source-carrying positions match a shape (Req 3.1). - - Any source kind at the shape's positions matches: a variant learned from ``request.args`` still fires when the same - sink takes a parameter. Constant-only arguments never match. ``mechanism_ids`` maps ``Shape.key()`` to the store id. - """ - if not ir.parse_ok: - return [] - index = shapes_by_sink([shape for shape in shapes if shape.language == ir.language]) - if not index: - return [] - scoped = facts.for_language(ir.language) if hasattr(facts, "for_language") else facts - hits: list[VariantHit] = [] - seen: set[tuple[int, str]] = set() - for function in ir.functions: - for call in function.calls: - candidates = index.get((ir.language, trailing_name(call.name))) - if not candidates: - continue - for shape in candidates: - if shape.arity != len(call.arg_texts) or (call.line, shape.key()) in seen: - continue - kinds = [ - _source_kind_of_argument(function, call, position, scoped) - for position in shape.source_positions - if position < len(call.arg_texts) - ] - if len(kinds) != len(shape.source_positions) or any(kind is None for kind in kinds): - continue - seen.add((call.line, shape.key())) - hits.append( - VariantHit( - path=ir.path, - line=call.line, - mechanism_id=mechanism_ids.get(shape.key(), shape.key()), - sink_name=shape.sink_name, - source_kind=str(kinds[0]), - ) - ) - return hits diff --git a/src/openultrasast/vectorstore.py b/src/openultrasast/vectorstore.py deleted file mode 100644 index f5d081e..0000000 --- a/src/openultrasast/vectorstore.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict, dataclass -from pathlib import Path - -from .index import VectorIndex, index_reuse_key, query_vector_index, read_vector_index, write_vector_index -from .preprocess import RepoSnapshot - -SELECTED_LOCAL_VECTOR_STORE = "json-local" - - -@dataclass(frozen=True) -class VectorStoreBakeoffResult: - store: str - metadata_filtering: bool - local_persistence: bool - incremental_keying: bool - export_import: bool - retrieval_quality: bool - selected_default: bool - - @property - def passed(self) -> bool: - return all( - ( - self.metadata_filtering, - self.local_persistence, - self.incremental_keying, - self.export_import, - self.retrieval_quality, - self.selected_default, - ) - ) - - -def bakeoff_json_local(index: VectorIndex, output_path: Path, *, query_embedding: list[float]) -> VectorStoreBakeoffResult: - write_vector_index(index, output_path) - loaded = read_vector_index(output_path) - metadata_matches = query_vector_index(loaded, query_embedding, metadata_filter={"language": "python"}, limit=1) - unfiltered_matches = query_vector_index(loaded, query_embedding, limit=1) - snapshot = RepoSnapshot( - root=loaded.repo_root, - commit=loaded.repo_commit, - file_count=len({record.chunk.path for record in loaded.records}), - languages=_language_counts(loaded), - ) - result = VectorStoreBakeoffResult( - store=SELECTED_LOCAL_VECTOR_STORE, - metadata_filtering=bool(metadata_matches) and metadata_matches[0][1].metadata.get("language") == "python", - local_persistence=output_path.exists() and output_path.stat().st_size > 0, - incremental_keying=index_reuse_key(snapshot, loaded.embedding_model) == index_reuse_key(snapshot, loaded.embedding_model), - export_import=len(loaded.records) == len(index.records) and loaded.embedding_model == index.embedding_model, - retrieval_quality=bool(unfiltered_matches) and unfiltered_matches[0][0] > 0, - selected_default=SELECTED_LOCAL_VECTOR_STORE == "json-local", - ) - return result - - -def bakeoff_payload(result: VectorStoreBakeoffResult) -> dict[str, object]: - payload = asdict(result) - payload["passed"] = result.passed - return payload - - -def _language_counts(index: VectorIndex) -> dict[str, int]: - counts: dict[str, int] = {} - for record in index.records: - counts[record.chunk.language] = counts.get(record.chunk.language, 0) + 1 - return counts diff --git a/tests/test_candidate_enumeration.py b/tests/test_candidate_enumeration.py index ca5277b..7ad2474 100644 --- a/tests/test_candidate_enumeration.py +++ b/tests/test_candidate_enumeration.py @@ -16,7 +16,7 @@ import pytest sys.path.insert(0, str(Path(__file__).parent)) from test_obligation_operations import APP # noqa: E402 -from openultrasast.learning.families import load_families # noqa: E402 +from openultrasast.model.taxonomy import load_families # noqa: E402 from openultrasast.semantic.extra import has_semantic_extra # noqa: E402 INJECTION = ( @@ -33,7 +33,7 @@ INJECTION = ( def _region(path: str = "app.py", function: str | None = "run"): # type: ignore[no-untyped-def] - from openultrasast.learning.detectors import Region + from openultrasast.model.candidates import Region return Region(path=path, function=function) @@ -44,7 +44,7 @@ def _write(tmp_path: Path, name: str, text: str) -> Path: def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", INJECTION) result = enumerate_candidates(root, _region(), "injection", taxonomy=load_families()) @@ -62,7 +62,7 @@ def test_a_call_site_candidate_carries_what_the_model_needs_to_judge_it(tmp_path def test_the_family_decides_which_kinds_are_candidates() -> None: - from openultrasast.learning.candidates import FAMILY_SHAPE + from openultrasast.model.candidates import FAMILY_SHAPE taxonomy = load_families() assert set(FAMILY_SHAPE) == {family.id for family in taxonomy.families}, "every family says what carries its bug" @@ -76,7 +76,7 @@ def test_the_family_decides_which_kinds_are_candidates() -> None: @pytest.mark.skipif(not has_semantic_extra(), reason="the guards come from the parsed IR") def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_path: Path) -> None: """An absence bug has no sink. Its candidate is the operation, and what the model needs is what guards it.""" - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) guarded = enumerate_candidates(root, _region(function="guarded"), "access_control", taxonomy=load_families()) @@ -95,7 +95,7 @@ def test_access_control_candidates_are_operations_with_the_guards_in_scope(tmp_p def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", "def broken(:\n pass\n") result = enumerate_candidates(root, _region(function="broken"), "injection", taxonomy=load_families()) @@ -103,7 +103,7 @@ def test_a_side_the_parser_rejects_yields_no_candidates_and_says_why(tmp_path: P def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", "def empty():\n return 1\n") result = enumerate_candidates(root, _region(function="empty"), "injection", taxonomy=load_families()) @@ -111,7 +111,7 @@ def test_a_labeled_function_with_nothing_to_judge_says_no_candidate(tmp_path: Pa def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> None: - from openultrasast.learning.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates + from openultrasast.model.candidates import MAX_CANDIDATES_PER_CALL, enumerate_candidates # one call per line, so the arithmetic is the bound and not the fixture body = "\n".join(f" sink_{index}(value)" for index in range(MAX_CANDIDATES_PER_CALL + 5)) @@ -123,7 +123,7 @@ def test_the_bound_is_a_constant_and_the_overflow_is_counted(tmp_path: Path) -> def test_enumerating_twice_returns_an_identical_sequence(tmp_path: Path) -> None: """The batches, and therefore the sequence of model calls, are determined by the input alone (Req 2.4).""" - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) taxonomy = load_families() @@ -139,7 +139,7 @@ def _order(result): # type: ignore[no-untyped-def] def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", APP) whole = enumerate_candidates(root, _region(function=None), "injection", taxonomy=load_families()) @@ -150,7 +150,7 @@ def test_a_region_without_a_function_enumerates_the_whole_file(tmp_path: Path) - @pytest.mark.parametrize("family", ["injection", "path", "deserialization", "untrusted_destination", "config_secrets"]) def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: Path, family: str) -> None: - from openultrasast.learning.candidates import enumerate_candidates + from openultrasast.model.candidates import enumerate_candidates root = _write(tmp_path, "app.py", INJECTION) result = enumerate_candidates(root, _region(), family, taxonomy=load_families()) @@ -164,7 +164,7 @@ def test_every_family_shape_enumerates_something_on_a_file_that_has_it(tmp_path: def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_path: Path) -> None: from openultrasast.benchmark import ExpectedFinding - from openultrasast.learning.candidates import ceiling + from openultrasast.model.candidates import ceiling from openultrasast.pairs import PairCase def case(name: str, body: str, *, slice_name: str, family: str) -> PairCase: @@ -210,7 +210,7 @@ def test_the_ceiling_counts_labeled_functions_with_at_least_one_candidate(tmp_pa def test_the_ruleset_generator_is_measured_too_so_the_comparison_is_reproducible(tmp_path: Path) -> None: """12.1% against 96.6% is why the ruleset is not the generator. A remembered number is not a measurement.""" - from openultrasast.learning.candidates import ceiling + from openultrasast.model.candidates import ceiling from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_slice, select_vendored cases = [c for c in select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py")) if not c.unscorable][:6] @@ -225,12 +225,12 @@ def test_the_command_runs_offline_and_writes_a_stable_artifact(tmp_path: Path, m for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): monkeypatch.delenv(name, raising=False) - out = tmp_path / "learning" - assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 + out = tmp_path / "model" + assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 artifact = out / "candidate-ceiling.json" assert artifact.is_file() first = artifact.read_bytes() - assert main(["learning", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 + assert main(["model", "candidates", "--slice", "vibe-py", "--out", str(out)]) == 0 assert artifact.read_bytes() == first, "the ceiling is a property of the corpus, not of when it was measured" import json diff --git a/tests/test_degradation_matrix.py b/tests/test_degradation_matrix.py index 48767e6..ad94422 100644 --- a/tests/test_degradation_matrix.py +++ b/tests/test_degradation_matrix.py @@ -44,71 +44,6 @@ def test_the_baseline_records_where_it_came_from() -> None: # --- one optional dependency at a time --------------------------------------- -def test_without_harnessx_the_meta_agent_proposer_proposes_nothing_and_says_why(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning import proposer as proposer_module - from openultrasast.learning.detectors import FamilyConfig - from openultrasast.learning.proposer import FailureFacts, HarnessXProposer - - monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False) - (tmp_path / "injection").mkdir() - facts = FailureFacts( - family="injection", - config=FamilyConfig( - family="injection", version="0", prompt="p", checklist="c", tools=("read_file",), max_steps=1, max_cost_usd=0.1, max_chars=100 - ), - ) - proposer = HarnessXProposer(configs_dir=tmp_path, model="m") - assert proposer.propose(facts) is None - assert proposer.reason == "harnessx_unavailable" - - -def test_without_a_chat_endpoint_the_learning_run_records_it_and_still_writes_its_artifact( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from openultrasast.cli import main - - for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): - monkeypatch.delenv(name, raising=False) - (tmp_path / "a-v.py").write_text("def run():\n return 1\n") - (tmp_path / "a-f.py").write_text("def run():\n return 1\n") - (tmp_path / "pairs.toml").write_text( - '[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n' - 'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n' - ) - printed: list[str] = [] - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: printed.append(" ".join(str(item) for item in args))) - assert main(["learning", "baseline", "--catalog", str(tmp_path / "pairs.toml"), "--out", str(tmp_path / "l"), "--model", "none"]) == 0 - assert any("learning_endpoint_unavailable" in line for line in printed) - assert (tmp_path / "l" / "baseline" / "none" / "report.json").is_file() - - -def test_without_family_configurations_a_scan_records_the_reason_rather_than_the_traceback( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from openultrasast.cli import _run_family_detectors - - class _Settings: - configs_dir = "does/not/exist" - families_path = None - - class _Runtime: - def __init__(self) -> None: - self.state: dict[str, list[dict[str, object]]] = {"degradations": []} - - runtime = _Runtime() - findings, payload, cited = _run_family_detectors( - root=tmp_path, - hotspots=[], - cases=None, - settings=_Settings(), - config=None, - runtime=runtime, # type: ignore[arg-type] - ) - assert (findings, payload, cited) == ([], {}, {}) - assert [item["reason"] for item in runtime.state["degradations"]] == ["learning_configs_unavailable"] - - def test_without_the_network_a_pointer_pair_is_skipped_by_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: from openultrasast.pairs import evaluate_catalog, load_pair_catalog, select_slice diff --git a/tests/test_fixture_difficulty.py b/tests/test_fixture_difficulty.py deleted file mode 100644 index e5b2012..0000000 --- a/tests/test_fixture_difficulty.py +++ /dev/null @@ -1,117 +0,0 @@ -"""A unit fixture may not be easier than the corpus it claims to cover (learning-harness 4.2, Req 10.3, 10.4). - -Two things a corpus number cannot survive: a byte-identical pair counted as a detector result, and a fixture that -passes because the world it describes is simpler than the one the detector meets. The first is excluded by name; -the second is measured on three axes the corpus actually varies on, and the comparison is recorded. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).parent)) -from test_obligation_operations import APP # noqa: E402 - -from openultrasast.learning.difficulty import DifficultyVector, difficulty_of, hardest, strictly_easier # noqa: E402 -from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored # noqa: E402 - -TWINS = { - "dsvw-dsvw-do-get", - "python-insecure-app-main-try-hack-me", - "pythonssti-main-read-root", - "vampi-books-get-by-title", - "vulnpy-deserialization-do-pickle-load", -} -# The rows the shared obligations fixture claims to cover: the vibe-py absence pairs. -COVERED = ("vampi-books-get-by-title", "vampi-users-update-password", "threatbyte-api-v1-get", "threatbyte-api-v1-delete") -FIXTURE_HANDLERS = ("guarded", "constrained", "constrained_too", "leaky", "from_spec") -REPORT = Path("benchmarks/measurements/2026-09-06-fixture-difficulty.json") - - -# --- identical twins --------------------------------------------------------- - - -def test_every_identical_twin_declares_the_limit_rather_than_only_hashing_to_it() -> None: - """A content hash is evidence; the catalog row is the maintainer standing behind it. - - A re-harvest that changes one byte would otherwise turn a known-unscorable row back into a scored one with no - review, and the number would move for a reason nobody chose.""" - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - for name in sorted(TWINS): - assert cases[name].known_limit == "identical_twin", f"{name} is a twin by content but says nothing in the catalog" - assert cases[name].unscorable == "identical_twin", "the declared limit and the computed one must be one reason, not two" - - -def test_the_twins_leave_every_denominator_with_their_reason_listed() -> None: - from openultrasast.learning.families import load_families - from openultrasast.learning.scoring import PairFamilyScore, aggregate - - cases = [case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)) if case.name in TWINS] - scores = [ - PairFamilyScore( - pair=case.name, family="injection", slice=case.slice, runs=(), outcome="unscorable", unscorable_reason=case.unscorable - ) - for case in cases - ] - metrics = aggregate(scores, taxonomy=load_families())["injection"] - assert metrics.scorable == 0 - assert metrics.unscorable == {"identical_twin": len(TWINS)} - - -# --- the difficulty check ---------------------------------------------------- - - -def _corpus_vectors() -> list[DifficultyVector]: - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - vectors = [] - for name in COVERED: - case = cases[name] - function = next(row.function for row in case.expected if row.function) - context = [vuln.read_text() for _relpath, vuln, _fixed in case.context_files] - vectors.append(difficulty_of(case.vuln_file.read_text(), function, context=context)) - return vectors - - -def test_the_shared_obligations_fixture_is_not_easier_than_the_rows_it_covers() -> None: - """The check Req 10.4 asks for, on the fixture every obligations test shares.""" - fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS] - corpus = _corpus_vectors() - assert not strictly_easier(fixture, corpus), ( - f"the fixture reaches {hardest(fixture)} where the corpus reaches {hardest(corpus)} " - "(binding hops, registration distance, guard distance)" - ) - - -def test_the_check_catches_a_fixture_that_is_easier_on_one_axis(tmp_path: Path) -> None: - """Mutation of the check itself: drop the corpus-shaped handler and it has to fail.""" - without = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS if name != "from_spec"] - assert strictly_easier(without, _corpus_vectors()) - assert hardest(without)[1] == 0, "every remaining handler carries its route decorator; nothing is registered elsewhere" - - -def test_the_verdict_is_recorded_beside_the_numbers_it_explains() -> None: - assert REPORT.is_file(), "the fixture-difficulty verdict is committed, not recomputed from memory" - payload = json.loads(REPORT.read_text()) - fixture = [difficulty_of(APP, name) for name in FIXTURE_HANDLERS] - corpus = _corpus_vectors() - assert payload["fixture"] == [row.to_dict() for row in fixture] - assert payload["corpus"] == [row.to_dict() for row in corpus] - assert payload["strictly_easier"] is False and payload["verdict"] == "fixture_matches_corpus" - assert payload["axes"] == ["binding_hops", "registration_distance", "guard_distance"] - - -@pytest.mark.parametrize( - ("source", "expected"), - [ - ("@app.route('/x')\ndef h():\n return Book.query.filter_by(user_id=request.user.id).first()\n", (0, 0, 2)), - ("@app.route('/x')\n@login_required\ndef h():\n return 1\n", (0, 0, 0)), - ("def h():\n u = current_user.id\n v = u\n return v\n", (2, 2, 1)), - ("app.add_url_rule('/x', view_func=h)\n\n\ndef h():\n return 1\n", (0, 1, 2)), - ], -) -def test_each_axis_is_read_off_the_code(source: str, expected: tuple[int, int, int]) -> None: - assert difficulty_of(source, "h").axes() == expected diff --git a/tests/test_index.py b/tests/test_index.py index 8b7ed2e..59a9442 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -15,7 +15,6 @@ from openultrasast.index import ( write_vector_index, ) from openultrasast.preprocess import RepoSnapshot, preprocess_repository -from openultrasast.vectorstore import bakeoff_json_local, bakeoff_payload class FakeEmbeddingClient: @@ -103,15 +102,3 @@ def test_retrieval_package_is_bounded_by_character_budget() -> None: assert package.truncated is True -def test_json_local_bakeoff_selects_default_store(tmp_path: Path) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def login(user):\n return user\n") - snapshot, targets = preprocess_repository(repo) - chunks = build_code_chunks(repo, targets) - index = build_vector_index(snapshot=snapshot, chunks=chunks, embedding_model="model", client=FakeEmbeddingClient()) - - result = bakeoff_json_local(index, tmp_path / "vector-index.json", query_embedding=[20.0, 1.0]) - - assert result.passed is True - assert bakeoff_payload(result)["selected_default"] is True diff --git a/tests/test_learning_acceptance.py b/tests/test_learning_acceptance.py deleted file mode 100644 index b0cfdf7..0000000 --- a/tests/test_learning_acceptance.py +++ /dev/null @@ -1,115 +0,0 @@ -"""learning-harness task 3.6: the acceptance rule and the directory revert (offline).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.learning.rounds import NoiseFloor - - -def _floor(family: str, budget: int = 0, *, gates: bool = True) -> NoiseFloor: - return NoiseFloor(family=family, k_runs=5, pairs=10, flaky_pairs=budget, negative_flip_rate=0.0, budget_flips=budget, gates=gates) - - -def _decide(train: float, holdout: float, sweep: dict[str, int], floors: dict[str, NoiseFloor]): # type: ignore[no-untyped-def] - from openultrasast.learning.acceptance import decide - - return decide(target="access_control", train_delta=train, holdout_delta=holdout, sweep=sweep, floors=floors) - - -def test_a_change_that_helps_and_hurts_nobody_is_accepted() -> None: - verdict = _decide(0.1, 0.2, {"injection": 0}, {"injection": _floor("injection")}) - assert verdict.accepted and verdict.reason == "improved" - - -def test_one_side_may_stand_still_but_the_other_must_move() -> None: - assert _decide(0.0, 0.2, {}, {}).accepted - assert _decide(0.2, 0.0, {}, {}).accepted - tie = _decide(0.0, 0.0, {}, {}) - assert not tie.accepted and tie.reason == "tie" # a tie is rejected, or the loop drifts on noise - - -def test_a_decrease_on_either_side_is_rejected() -> None: - train_down = _decide(-0.1, 0.5, {}, {}) - assert not train_down.accepted and train_down.reason == "train_regressed" - holdout_down = _decide(0.5, -0.1, {}, {}) - assert not holdout_down.accepted and holdout_down.reason == "holdout_regressed" - - -def test_another_family_beyond_its_measured_budget_rejects_the_round() -> None: - within = _decide(0.2, 0.2, {"injection": 2}, {"injection": _floor("injection", 2)}) - assert within.accepted # inside the noise this family shows against itself - beyond = _decide(0.2, 0.2, {"injection": 3}, {"injection": _floor("injection", 2)}) - assert not beyond.accepted and beyond.reason == "collateral_regression" - assert beyond.offenders == ("injection",) - - -def test_a_family_that_never_gates_cannot_reject_a_round() -> None: - verdict = _decide(0.2, 0.2, {"memory": 99}, {"memory": _floor("memory", 0, gates=False)}) - assert verdict.accepted and verdict.offenders == () - - -def test_a_family_with_no_measured_floor_gets_no_budget() -> None: - verdict = _decide(0.2, 0.2, {"path": 1}, {}) - assert not verdict.accepted and verdict.offenders == ("path",) # unmeasured is not the same as unlimited - - -def test_a_snapshot_restores_a_family_directory_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.learning.acceptance import DirectorySnapshot - - directory = tmp_path / "access_control" - directory.mkdir() - (directory / "prompt.md").write_text("original\n") - (directory / "family.toml").write_text('version = "0"\n') - snapshot = DirectorySnapshot.of(directory) - (directory / "prompt.md").write_text("changed\n") - (directory / "extra.md").write_text("added by a round\n") - (directory / "family.toml").unlink() - snapshot.restore() - assert (directory / "prompt.md").read_text() == "original\n" - assert (directory / "family.toml").read_text() == 'version = "0"\n' - assert not (directory / "extra.md").exists() # a file the round added is not left behind - assert sorted(path.name for path in directory.iterdir()) == ["family.toml", "prompt.md"] - - -def test_every_other_family_directory_is_untouched_across_a_round(tmp_path: Path) -> None: - """Req 6.2: a round can only write its target, so the freeze is structural rather than promised.""" - from openultrasast.learning.acceptance import DirectorySnapshot, digest_configs - from openultrasast.learning.detectors import write_default_configs - from openultrasast.learning.families import load_families - from openultrasast.learning.proposer import Proposal, apply_proposal - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - before = digest_configs(configs) - snapshot = DirectorySnapshot.of(configs / "access_control") - apply_proposal(Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}), configs / "access_control") - after = digest_configs(configs) - assert after["access_control"] != before["access_control"] - assert {name: value for name, value in after.items() if name != "access_control"} == { - name: value for name, value in before.items() if name != "access_control" - } - snapshot.restore() - assert digest_configs(configs) == before # a rejected round leaves the whole tree as it found it - - -def test_a_bumped_version_is_written_only_on_acceptance(tmp_path: Path) -> None: - from openultrasast.learning.acceptance import bump_version - from openultrasast.learning.detectors import load_family_configs, write_default_configs - from openultrasast.learning.families import load_families - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - bump_version(configs / "access_control", "1") - taxonomy = load_families() - assert load_family_configs(configs, taxonomy)["access_control"].version == "1" - assert load_family_configs(configs, taxonomy)["injection"].version == "0" - - -@pytest.mark.parametrize(("spent", "cap", "expected"), [(0.5, 10.0, True), (10.0, 10.0, True), (10.01, 10.0, False)]) -def test_a_round_over_its_cost_cap_is_reverted(spent: float, cap: float, expected: bool) -> None: - from openultrasast.learning.acceptance import within_budget - - assert within_budget(spent, cap) is expected diff --git a/tests/test_learning_aggregation.py b/tests/test_learning_aggregation.py deleted file mode 100644 index 8561a79..0000000 --- a/tests/test_learning_aggregation.py +++ /dev/null @@ -1,116 +0,0 @@ -"""learning-harness task 2.4: aggregation, noise handling and honest denominators (offline).""" - -from __future__ import annotations - -import json - -from openultrasast.learning.families import load_families -from openultrasast.learning.scoring import FamilyOutcome, PairFamilyScore - - -def _score( - pair: str, outcome: FamilyOutcome, *, family: str = "access_control", slice_name: str = "vibe-py", **extra: object -) -> PairFamilyScore: - runs = () if outcome == "unscorable" else (outcome,) - return PairFamilyScore(pair=pair, family=family, slice=slice_name, runs=runs, outcome=outcome, **extra) # type: ignore[arg-type] - - -def test_rates_use_only_the_scorable_rows_and_the_reasons_are_kept() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - scores = [ - _score("a", "pair_correct"), - _score("b", "both_silent"), - _score("c", "unscorable", unscorable_reason="identical_twin"), - _score("d", "unscorable", unscorable_reason="identical_twin"), - _score("e", "unscorable", unscorable_reason="unsupported_language"), - ] - metrics = aggregate(scores, taxonomy=taxonomy)["access_control"] - assert metrics.scorable == 2 and metrics.unscorable == {"identical_twin": 2, "unsupported_language": 1} - assert metrics.recall == 0.5 and metrics.silence == 1.0 and metrics.youden == 0.5 - assert metrics.outcomes == {"pair_correct": 1, "both_silent": 1} - assert metrics.taxonomy_version == taxonomy.version - json.dumps(metrics.to_dict()) - - -def test_a_family_with_nothing_scorable_reports_no_rate_rather_than_a_perfect_one() -> None: - from openultrasast.learning.scoring import aggregate - - metrics = aggregate([_score("a", "unscorable", unscorable_reason="identical_twin")], taxonomy=load_families())["access_control"] - assert metrics.scorable == 0 and metrics.recall == 0.0 and metrics.silence == 0.0 and metrics.youden == 0.0 - - -def test_directional_bias_separates_flagging_everything_from_saying_everything_is_safe() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - loud = aggregate([_score("a", "both_flagged"), _score("b", "both_flagged")], taxonomy=taxonomy)["access_control"] - quiet = aggregate([_score("a", "both_silent"), _score("b", "both_silent")], taxonomy=taxonomy)["access_control"] - assert loud.directional_bias == 1.0 and quiet.directional_bias == -1.0 - assert loud.youden == quiet.youden == 0.0 # the same Youden, opposite failures: the index is what tells them apart - - -def test_recall_at_a_fixed_false_positive_ceiling_is_forfeited_when_the_family_leaks() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - clean = [_score(str(index), "pair_correct") for index in range(9)] + [_score("x", "both_silent")] - assert aggregate(clean, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"].fixed_fpr_recall == 0.9 - leaky = [_score(str(index), "pair_correct") for index in range(8)] + [_score("y", "both_flagged"), _score("z", "reversed")] - metrics = aggregate(leaky, taxonomy=taxonomy, fpr_ceiling=0.1)["access_control"] - assert metrics.recall == 0.9 and metrics.fixed_fpr_recall == 0.0 # a 20% leak rate buys no credit at a 10% ceiling - - -def test_negative_flips_and_the_reliable_change_test_compare_against_a_baseline() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - baseline = [_score(str(index), "pair_correct") for index in range(10)] - same = aggregate(baseline, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert same.negative_flips == 0 and same.reliable_change is False - broke_one = [_score("0", "both_silent")] + [_score(str(index), "pair_correct") for index in range(1, 10)] - one = aggregate(broke_one, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert one.negative_flips == 1 and one.reliable_change is False # one flip in ten is inside the noise - broke_six = [_score(str(index), "both_silent") for index in range(6)] + [_score(str(index), "pair_correct") for index in range(6, 10)] - six = aggregate(broke_six, taxonomy=taxonomy, baseline=baseline)["access_control"] - assert six.negative_flips == 6 and six.reliable_change is True - assert six.p_value is not None and six.p_value < 0.05 - - -def test_metrics_are_reported_per_slice_when_asked() -> None: - from openultrasast.learning.scoring import aggregate - - taxonomy = load_families() - scores = [_score("a", "pair_correct", slice_name="vibe-py"), _score("b", "both_silent", slice_name="agent-vfc")] - overall = aggregate(scores, taxonomy=taxonomy) - assert set(overall) == {"access_control"} and overall["access_control"].scorable == 2 - per_slice = aggregate(scores, taxonomy=taxonomy, per_slice=True) - assert set(per_slice) == {"vibe-py/access_control", "agent-vfc/access_control"} - assert per_slice["vibe-py/access_control"].recall == 1.0 and per_slice["agent-vfc/access_control"].recall == 0.0 - - -def test_the_sign_test_is_two_sided_and_symmetric() -> None: - from openultrasast.learning.scoring import sign_test - - assert sign_test(0, 0) == 1.0 - assert sign_test(6, 0) == sign_test(0, 6) < 0.05 - assert sign_test(1, 0) > 0.05 and sign_test(3, 3) == 1.0 - - -def test_every_family_figure_says_when_hierarchical_credit_could_not_apply() -> None: - """Req 3.7: with a flat taxonomy no answer can earn partial credit, and the number must carry that.""" - from openultrasast.learning.families import Family, FamilyTaxonomy - from openultrasast.learning.scoring import aggregate - - flat = load_families() - metrics = aggregate([_score("a", "pair_correct")], taxonomy=flat)["access_control"] - assert metrics.hierarchical_credit is False and metrics.to_dict()["hierarchical_credit"] is False - nested = FamilyTaxonomy( - version="test", - families=( - Family(id="injection", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none"), - Family(id="access_control", description="d", cwes=frozenset(), mechanisms=frozenset(), verifier="none", parent="injection"), - ), - ) - assert aggregate([_score("a", "pair_correct")], taxonomy=nested)["access_control"].hierarchical_credit is True diff --git a/tests/test_learning_canaries.py b/tests/test_learning_canaries.py deleted file mode 100644 index d7cdfb4..0000000 --- a/tests/test_learning_canaries.py +++ /dev/null @@ -1,197 +0,0 @@ -"""learning-harness task 2.7: canary templates and the second judge (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.config import SandboxConfig -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.regress.safety import check_snippet_safety -from openultrasast.sandbox import SandboxResult -from openultrasast.sandbox.runner import FakeSandboxRunner -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient - -CANARY_FAMILIES = ("injection", "path", "deserialization") - - -def _claim(family: str, function: str = "lookup", line: int = 3) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=function, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}"], - ranking_priority=0.0, - ) - - -def test_every_canary_template_renders_and_passes_the_snippet_safety_check() -> None: - from openultrasast.learning.canaries import render_canary - - for family in CANARY_FAMILIES: - snippet, token = render_canary(family, language="python", path="app.py", function="lookup") - assert token and token in snippet - check_snippet_safety(snippet) # raises when the template would be refused before it ever runs - assert "/workspace" in snippet # the candidate is imported from the read-only source mount - assert render_canary("memory", language="python", path="app.py", function="f") is None - assert render_canary("injection", language="rust", path="a.rs", function="f") is None - - -def test_a_canary_token_is_unique_to_the_experiment_it_names() -> None: - from openultrasast.learning.canaries import render_canary - - first = render_canary("injection", language="python", path="app.py", function="lookup") - second = render_canary("injection", language="python", path="app.py", function="other") - third = render_canary("path", language="python", path="app.py", function="lookup") - assert first and second and third - assert first[1] != second[1] != third[1] and first[1] != third[1] - # the same family, file and function is the same experiment, so it is deliberately the same token - assert render_canary("injection", language="python", path="app.py", function="lookup")[1] == first[1] # type: ignore[index] - - -def test_the_canary_verifier_proves_only_when_the_oracle_fires(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - - (tmp_path / "app.py").write_text("def lookup(value):\n return value\n") - fired = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, f"CANARY {token}", "", False)), limits=SandboxConfig()) - proven = fired.verify(_claim("injection"), root=tmp_path, language="python") - assert proven.rung == "proven" and proven.reason == "oracle_fired" - quiet = CanaryVerifier(runner=_runner(lambda _t: SandboxResult(1, "boom", "Traceback", False)), limits=SandboxConfig()) - unproven = quiet.verify(_claim("injection"), root=tmp_path, language="python") - assert unproven.rung == "suspicion" and unproven.reason == "oracle_quiet" # a crash is not this family's evidence - - -def test_a_family_or_language_with_no_template_stays_at_suspicion(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - - (tmp_path / "app.py").write_text("def lookup(value):\n return value\n") - verifier = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig()) - assert verifier.verify(_claim("memory"), root=tmp_path, language="python").reason == "no_template" - assert verifier.verify(_claim("injection"), root=tmp_path, language="rust").reason == "no_template" - - -def test_the_registry_uses_the_canary_verifier_when_one_is_supplied(tmp_path: Path) -> None: - from openultrasast.learning.canaries import CanaryVerifier - from openultrasast.learning.verifiers import NoVerifier, verifier_for - - taxonomy = load_families() - canary = CanaryVerifier(runner=_runner(lambda token: SandboxResult(0, token, "", False)), limits=SandboxConfig()) - assert verifier_for(taxonomy.by_id("injection"), canary=canary) is canary - assert isinstance(verifier_for(taxonomy.by_id("access_control"), canary=canary), type(verifier_for(taxonomy.by_id("access_control")))) - assert isinstance(verifier_for(taxonomy.by_id("memory"), canary=canary), NoVerifier) - - -def test_nothing_is_published_as_proven_until_a_second_judge_confirms() -> None: - from openultrasast.learning.canaries import confirm_proven - from openultrasast.learning.verifiers import Verification - - proven = Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc") - agrees = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "the row came back"}))]) - kept = confirm_proven(proven, _claim("injection"), client=agrees, model="judge", snippet="print('x')") - assert kept.rung == "proven" - declines = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": False, "reason": "unrelated output"}))]) - dropped = confirm_proven(proven, _claim("injection"), client=declines, model="judge", snippet="print('x')") - assert dropped.rung == "static_corroboration" and dropped.reason == "judge_declined" - assert confirm_proven(proven, _claim("injection"), client=None, model="", snippet="x").rung == "static_corroboration" - lower = Verification(rung="suspicion", reason="oracle_quiet") - assert confirm_proven(lower, _claim("injection"), client=agrees, model="judge", snippet="x") is lower # nothing to confirm - - -def test_the_judge_sees_the_snippet_and_the_oracle_output_and_never_the_label() -> None: - from openultrasast.learning.canaries import confirm_proven - from openultrasast.learning.verifiers import Verification - - client = ScriptedChatClient([ChatResponse(content=json.dumps({"confirms": True, "reason": "ok"}))]) - confirm_proven( - Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY abc"), - _claim("injection"), - client=client, - model="judge", - snippet="print('CANARY abc')", - ) - prompt = json.dumps(client.calls[0]["messages"]) - assert "CANARY abc" in prompt and "print('CANARY abc')" in prompt - assert client.calls[0].get("json_object") is True and client.calls[0]["tools"] == [] - assert "family:injection" not in prompt # the judge is told the claim, not the answer key - - -def test_templates_and_oracle_code_live_outside_every_proposer_write_root(tmp_path: Path) -> None: - from openultrasast.learning import canaries - from openultrasast.learning.detectors import write_default_configs - - write_roots = set(write_default_configs(tmp_path, load_families(), prompt="p", version="0").values()) - module = Path(canaries.__file__).resolve() - assert not any(module.is_relative_to(root.resolve()) for root in write_roots) - assert "site-packages" in str(module) or "src/openultrasast" in str(module) - - -def _runner(reply: object) -> FakeSandboxRunner: - """A sandbox that answers from whatever canary token the snippet actually planted.""" - - class _Runner(FakeSandboxRunner): # type: ignore[misc] - def run(self, spec: object) -> SandboxResult: - body = "\n".join(str(value) for value in (getattr(spec, "scratch_files", {}) or {}).values()) - marker = next((word.strip("'\"") for word in body.split() if "ousast-canary-" in word), "ousast-canary-none") - return reply(marker) # type: ignore[operator] - - return _Runner() - - -VULNERABLE = { - "injection": "import os\n\n\ndef lookup(value):\n os.system(value)\n return 'done'\n", - "path": "def lookup(value):\n with open(value, encoding='utf-8') as handle:\n return handle.read()\n", - "deserialization": "import pickle\n\n\ndef lookup(value):\n return pickle.loads(value)\n", -} -FIXED = { - "injection": ( - "import shlex, subprocess\n\n\ndef lookup(value):\n" - " subprocess.run(['echo', shlex.quote(value)], check=False)\n return 'done'\n" - ), - "path": ( - "from pathlib import Path\n\n\ndef lookup(value):\n" - " name = Path(value).name\n return (Path('/tmp') / name).read_text(encoding='utf-8') if False else ''\n" - ), - "deserialization": "import json\n\n\ndef lookup(value):\n return json.loads(value.decode('utf-8', 'ignore') or '{}')\n", -} - - -@pytest.mark.parametrize("family", CANARY_FAMILIES) -def test_each_canary_fires_on_a_real_bug_and_stays_quiet_on_its_fixed_twin(tmp_path: Path, family: str) -> None: - """The oracle must be a property of the code, not of the harness: run the snippet for real, both sides. - - The injection fixture's fixed twin deliberately echoes its input, which is why the payload carries a - value unrelated to the oracle token: an echoing handler must not be able to satisfy the oracle. - """ - import subprocess - import sys - - from openultrasast.learning.canaries import render_canary - - for side, sources in (("vuln", VULNERABLE), ("fixed", FIXED)): - workspace = tmp_path / side / "workspace" - scratch = tmp_path / side / "scratch" - workspace.mkdir(parents=True) - scratch.mkdir(parents=True) - (workspace / "app.py").write_text(sources[family]) - rendered = render_canary( - family, language="python", path="app.py", function="lookup", workspace=str(workspace), scratch=str(scratch) - ) - assert rendered is not None - snippet, token = rendered - script = scratch / "case.py" - script.write_text(snippet) - completed = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, timeout=60, check=False) - fired = token in completed.stdout or token in completed.stderr - assert fired is (side == "vuln"), f"{family} {side}: {completed.stdout!r} {completed.stderr!r}" diff --git a/tests/test_learning_classifier_report.py b/tests/test_learning_classifier_report.py deleted file mode 100644 index e423435..0000000 --- a/tests/test_learning_classifier_report.py +++ /dev/null @@ -1,115 +0,0 @@ -"""learning-harness task 2.2: the classifier reports on itself and queues disagreements (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored - - -def _case(tmp_path: Path, name: str, *, cwe: str, mechanism: str = "other", family: str | None = None, tier: str = "reviewed") -> PairCase: - (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n") - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}.py", - fixed_file=tmp_path / f"{name}.py", - relpath="app.py", - expected=( - ExpectedFinding(cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function="f", mechanism=mechanism, family=family), - ), - min_recall=1.0, - fix_policy="silent", - review_tier=tier, - reviewer="m" if tier == "reviewed" else "", - ) - - -def test_the_report_covers_the_whole_corpus_and_names_the_unknowns(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG)) - report = measure_classifier(cases, taxonomy, client=None) - assert report.taxonomy_version == taxonomy.version - assert report.classified == len(cases) - assert sum(report.per_tier.values()) == len(cases) and set(report.per_tier) <= {"static", "model", "none"} - assert report.per_family["memory"] >= 150 # the vfc rows land in one family instead of reading `other` - assert report.per_family["access_control"] >= 10 and report.per_family["injection"] >= 40 - assert report.unknown == 2 and report.per_tier["static"] == len(cases) - 2 # tier one places all but two rows - payload = report.to_dict() - json.dumps(payload) - # Task 4.1 declared `family` on the ten absence rows; only the four at a reference tier (`seeded`) count as labels - # a maintainer stands behind, and the six `title`-tier agent rows do not. Agreement over four rows of one family is - # not yet a measurement of the classifier, so the number is recorded, not celebrated. - assert payload["labeled"] == 4 and payload["agreement"] == 1.0 - assert set(report.confusion) == {"access_control"} - - -def test_agreement_uses_the_first_answer_and_the_oracle_uses_any_of_them(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = [ - _case(tmp_path, "hit", cwe="CWE-89", family="injection"), # tier one answers injection: agreement and oracle - _case(tmp_path, "miss", cwe="CWE-89", family="access_control"), # answers injection: neither - ] - report = measure_classifier(cases, taxonomy, client=None) - assert report.labeled == 2 - assert report.agreement == 0.5 and report.oracle_router == 0.5 - assert abs(report.random_router - 1 / 9) < 1e-9 # a uniform pick over the nine real families - assert report.confusion["injection"]["injection"] == 1 and report.confusion["access_control"]["injection"] == 1 - - -def test_every_disagreement_is_queued_once_with_both_answers_and_the_tier(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier, write_review_queue - - taxonomy = load_families() - cases = [_case(tmp_path, "miss", cwe="CWE-89", family="access_control"), _case(tmp_path, "hit", cwe="CWE-89", family="injection")] - report = measure_classifier(cases, taxonomy, client=None) - assert [entry.pair for entry in report.queue] == ["miss"] - entry = report.queue[0] - assert entry.label == "access_control" and entry.classified == ("injection",) and entry.tier == "static" - queue_path = tmp_path / "review-queue.jsonl" - write_review_queue(queue_path, report.queue) - write_review_queue(queue_path, report.queue) # a second measurement must not duplicate the row - rows = [json.loads(line) for line in queue_path.read_text().splitlines() if line.strip()] - assert [row["pair"] for row in rows] == ["miss"] - assert rows[0]["label"] == "access_control" and rows[0]["classified"] == ["injection"] and rows[0]["tier"] == "static" - - -def test_measuring_never_writes_a_label(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - case = _case(tmp_path, "miss", cwe="CWE-89", family="access_control") - measure_classifier([case], taxonomy, client=None) - assert case.expected[0].family == "access_control" # the classifier proposes; the maintainer decides - - -def test_only_reviewed_and_seeded_rows_count_towards_agreement(tmp_path: Path) -> None: - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - cases = [ - _case(tmp_path, "trusted", cwe="CWE-89", family="access_control", tier="reviewed"), - _case(tmp_path, "untrusted", cwe="CWE-89", family="access_control", tier="title"), - ] - report = measure_classifier(cases, taxonomy, client=None) - assert report.labeled == 1 and report.classified == 2 # an unreviewed label is not a reference answer - assert [entry.pair for entry in report.queue] == ["trusted"] - - -def test_the_report_says_when_hierarchical_credit_could_not_apply(tmp_path: Path) -> None: - """Req 2.6: the shipped taxonomy is flat, so partial credit is inert and every figure must say so.""" - from openultrasast.learning.classify import measure_classifier - - taxonomy = load_families() - report = measure_classifier([_case(tmp_path, "a", cwe="CWE-89", family="injection")], taxonomy, client=None) - assert report.hierarchical_credit is False - assert report.to_dict()["hierarchical_credit"] is False diff --git a/tests/test_learning_classify.py b/tests/test_learning_classify.py deleted file mode 100644 index 25d44fa..0000000 --- a/tests/test_learning_classify.py +++ /dev/null @@ -1,163 +0,0 @@ -"""learning-harness task 2.1: the three-tier auto-classifier (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) - - -def _case(tmp_path: Path, **row: object) -> PairCase: - (tmp_path / "v.py").write_text(APP) - (tmp_path / "f.py").write_text(APP + "# fixed\n") - expected = ExpectedFinding( - cwe=str(row.get("cwe", "")), - vulnerability_class="x", - path="app.py", - evidence="", - function="leaky", - mechanism=str(row.get("mechanism", "other")), - family=row.get("family"), # type: ignore[arg-type] - ) - return PairCase( - name="t", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=(expected,), - min_recall=1.0, - fix_policy="silent", - ) - - -def test_tier_one_decides_from_a_declared_family_a_mechanism_or_a_weakness(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - declared = classify_pair(_case(tmp_path, family="injection", cwe="CWE-639"), taxonomy, client=None) - assert declared.families == ("injection",) and declared.tier == "static" # a declared label wins - mechanism = classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=None) - assert mechanism.families == ("access_control",) and mechanism.tier == "static" - weakness = classify_pair(_case(tmp_path, cwe="CWE-89", mechanism="source_reaches_sink"), taxonomy, client=None) - assert weakness.families == ("injection",) and weakness.tier == "static" # the generic mechanism defers to the weakness - nothing = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=None) - assert nothing.families == () and nothing.tier == "none" # unknown, and no client to ask - - -def test_tier_one_reads_a_finding_from_its_tags_and_its_rule(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_finding - - taxonomy = load_families() - - def finding(finding_id: str, tags: list[str]) -> StaticFinding: - return StaticFinding( - finding_id=finding_id, - path="app.py", - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=12, - function_name="leaky", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=tags, - ranking_priority=0.0, - ) - - tagged = classify_finding(finding("x:app.py:1", ["family:access_control"]), taxonomy, client=None) - assert tagged.families == ("access_control",) and tagged.tier == "static" - obligated = classify_finding(finding("obligation:protected_read:app.py:12:x", ["obligation:protected_read"]), taxonomy, client=None) - assert obligated.families == ("access_control",) - ruled = classify_finding(finding("python-os-command:app.py:3", ["tool-hunter"]), taxonomy, client=None) - assert ruled.families == ("injection",) and ruled.tier == "static" # the rule's weakness names the family - assert classify_finding(finding("tool-hunter:app.py:3", ["tool-hunter"]), taxonomy, client=None).families == () - - -def test_tier_one_classifies_the_shared_handler_without_any_client(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_region - - (tmp_path / "app.py").write_text(APP) - taxonomy = load_families() - leaky = classify_region(tmp_path, "app.py", function="leaky", taxonomy=taxonomy, client=None) - assert leaky.families == ("access_control",) and leaky.tier == "static" - (tmp_path / "flow.py").write_text("from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n") - injected = classify_region(tmp_path, "flow.py", function="run", taxonomy=taxonomy, client=None) - assert "injection" in injected.families - - -def test_tier_two_asks_the_model_only_when_static_signals_say_nothing(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["output_encoding"]}), mean_logprob=-0.05)]) - answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - assert answered.families == ("output_encoding",) and answered.tier == "model" and answered.confidence == -0.05 - quiet = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}))]) - classify_pair(_case(tmp_path, mechanism="missing_auth_guard"), taxonomy, client=quiet) - assert quiet.calls == [] # a static answer never spends a model call - - -def test_an_unsure_or_off_vocabulary_answer_becomes_unknown(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - invented = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["sql_injection"]}), mean_logprob=-0.01)]) - assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=invented).families == () - unsure = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["injection"]}), mean_logprob=-4.0)]) - hedged = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=unsure) - assert hedged.families == () and hedged.tier == "none" and hedged.confidence == -4.0 - broken = ScriptedChatClient([ChatResponse(content="not json at all")]) - assert classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=broken).families == () - - -def test_the_model_is_asked_in_json_mode_with_the_taxonomy_and_no_thinking(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["path"]}), mean_logprob=-0.1)]) - classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - call = client.calls[0] - prompt = json.dumps(call["messages"]) - assert "json" in prompt.lower() and "access_control" in prompt and "untrusted_destination" in prompt - assert call.get("json_object") is True and call.get("logprobs") is True and call["tools"] == [] - - -def test_multi_label_answers_are_kept_in_taxonomy_order(tmp_path: Path) -> None: - from openultrasast.learning.classify import classify_pair - - taxonomy = load_families() - client = ScriptedChatClient([ChatResponse(content=json.dumps({"families": ["access_control", "injection"]}), mean_logprob=-0.2)]) - answered = classify_pair(_case(tmp_path, mechanism="other"), taxonomy, client=client) - assert answered.families == ("injection", "access_control") # taxonomy order, so two runs compare equal - - -def test_a_fabricated_family_label_in_a_catalog_is_rejected_by_name(tmp_path: Path) -> None: - from openultrasast.learning.classify import FamilyLabelError, validate_family_labels - - taxonomy = load_families() - good = _case(tmp_path, family="access_control") - validate_family_labels([good], taxonomy) - with pytest.raises(FamilyLabelError, match="not_a_family"): - validate_family_labels([_case(tmp_path, family="not_a_family")], taxonomy) diff --git a/tests/test_learning_cli.py b/tests/test_learning_cli.py deleted file mode 100644 index b33e45a..0000000 --- a/tests/test_learning_cli.py +++ /dev/null @@ -1,302 +0,0 @@ -"""learning-harness task 3.9: the learning subcommands (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" -CATALOG = ( - '[[pair]]\nname = "{name}"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "{name}-v.py"\nfixed = "{name}-f.py"\n' - 'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "command injection"\npath = "app.py"\nfunction = "run"\n' - 'sink = "system"\nmechanism = "source_reaches_sink"\nfamily = "injection"\n' -) - - -def _detector(monkeypatch: pytest.MonkeyPatch, line: int = 6) -> None: - """A deterministic detector that reads the file it was given and answers differently on the two sides. - - The shipped `scripted` stub answers twice and then goes quiet, and its finding sits at line 3, outside `run`. - Both make every family number zero, and a scoreboard assertion against zeros passes with the detector missing - entirely — which is what these tests used to assert.""" - from openultrasast.learning.endpoint import ChatEndpoint - from openultrasast.tool_hunter import ChatResponse, ToolCall - - class Detector: - def __init__(self) -> None: - self.turn = 0 - - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - del model, tools, options - self.turn += 1 - if self.turn % 2 == 1: - return ChatResponse(tool_calls=(ToolCall(id=f"c{self.turn}", name="grep_repo", arguments={"pattern": "system"}),)) - body = "\n".join(str(message.get("content") or "") for message in messages) - if "os.system" not in body: - return ChatResponse(content="[]") # the fixed side really is different, and the detector sees it - return ChatResponse(content=json.dumps([{"path": "app.py", "line": line, "title": "t", "rationale": "r"}])) - - endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False) - monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (Detector(), endpoint)) - - -@pytest.fixture -def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - for name in ("train-a", "hold-a"): - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - (tmp_path / "pairs.toml").write_text( - CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout") - ) - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - return tmp_path # no chdir: the mechanism vocabulary is found relative to the repository root - - -def test_classify_reports_the_corpus_and_writes_the_queue(workspace: Path, capsys: pytest.CaptureFixture[str]) -> None: - assert main(["learning", "classify", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning")]) == 0 - payload = json.loads((workspace / "learning" / "classifier.json").read_text()) - assert payload["classified"] == 2 and payload["per_family"]["injection"] == 2 - assert payload["hierarchical_credit"] is False - assert "classified 2" in capsys.readouterr().out - - -def test_score_reports_family_numbers_for_a_catalog( - workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch -) -> None: - """The scripted client detects the vulnerable side and is silent on the fix, so this is a real 2/2, not a zero. - - An all-zero scoreboard would satisfy "the family key exists"; it is what a generalist detector scoring every - family produced before round zero ran each family's own configuration.""" - _detector(monkeypatch) - assert main(["learning", "score", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--json"]) == 0 - payload = json.loads(capsys.readouterr().out) - block = payload["families"]["injection"] - assert block["taxonomy_version"] and block["scorable"] == 2 - assert block["outcomes"] == {"pair_correct": 2} and block["recall"] == 1.0 and block["youden"] == 1.0 - - -def test_baseline_writes_the_noise_floor_and_publish_reads_it( - workspace: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch -) -> None: - _detector(monkeypatch) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "scripted", - ] - ) - == 0 - ) - floors = json.loads((workspace / "learning" / "baseline" / "scripted" / "noise-floors.json").read_text()) - assert floors["k_runs"] == 3 and floors["floors"]["injection"]["pairs"] == 2 - report = json.loads((workspace / "learning" / "baseline" / "scripted" / "report.json").read_text()) - # Measured with the injection configuration, not the generalist: `family:unknown` earns no credit against any - # real family, so a baseline built from one shared detector reports a structural zero here. - assert report["metrics"]["injection"]["recall"] == 1.0 - assert report["per_slice"]["vibe-py/injection"]["scorable"] == 2 - capsys.readouterr() - roadmap = workspace / "roadmap.md" - roadmap.write_text("# Roadmap\n\n## Overview\n\ntext\n") - assert ( - main( - ["learning", "publish", "--out", str(workspace / "learning"), "--measurements", str(workspace / "m"), "--roadmap", str(roadmap)] - ) - == 0 - ) - assert "learning-harness:begin" in roadmap.read_text() - assert "| injection |" in roadmap.read_text() - - -def test_a_round_runs_from_the_command_line_and_journals_its_outcome(workspace: Path) -> None: - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "scripted", - ] - ) - == 0 - ) - assert ( - main( - [ - "learning", - "round", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--family", - "injection", - "--k-runs", - "3", - ] - ) - == 0 - ) - rounds = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()] - assert len(rounds) == 1 and rounds[0]["family"] == "injection" - # Without the extra there is no meta-agent, so the honest outcome is exactly this one. Accepting any member of - # the enum would have hidden that the round could never propose anything at all. - # The reason now carries *why* the proposer had nothing: "no proposal" and "could not start" are different - # worlds, and only one of them is a fixable configuration error. - assert rounds[0]["outcome"] == "rejected" and rounds[0]["reason"].startswith("no_proposal") - assert (workspace / "learning" / "rounds" / "1" / "attribution.json").is_file() - - -def test_a_recorded_proposal_makes_the_round_do_something(workspace: Path) -> None: - """`--proposals` is what makes a round runnable at all without the optional extra.""" - for command in ( - ["learning", "baseline", "--catalog", str(workspace / "pairs.toml"), "--out", str(workspace / "learning"), "--k-runs", "3"], - ): - assert main([*command, "--model", "scripted"]) == 0 - proposals = workspace / "proposals.json" - proposals.write_text( - json.dumps([{"hypothesis": "ask about quoting", "lever": "checklist", "change": {"checklist.md": "- ask about quoting\n"}}]) - ) - assert ( - main( - [ - "learning", - "round", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--family", - "injection", - "--k-runs", - "3", - "--proposals", - str(proposals), - ] - ) - == 0 - ) - (record,) = [json.loads(line) for line in (workspace / "learning" / "journal.jsonl").read_text().splitlines() if line.strip()] - assert record["reason"] != "no_proposal", "a recorded proposal reached the round" - assert json.loads((workspace / "learning" / "rounds" / "1" / "proposal.json").read_text())["lever"] == "checklist" - - -def test_a_missing_endpoint_is_recorded_and_the_rest_still_runs(workspace: Path, monkeypatch: pytest.MonkeyPatch, capsys) -> None: # type: ignore[no-untyped-def] - for name in ("OPENULTRASAST_HUNTER_CLIENT", "DEEPSEEK_API_KEY", "OPENROUTER_API_KEY"): - monkeypatch.delenv(name, raising=False) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "none", - ] - ) - == 0 - ) - out = capsys.readouterr().out - assert "learning_endpoint_unavailable" in out - assert (workspace / "learning" / "baseline" / "none" / "report.json").is_file() # the run still recorded what it could - - -def test_every_subcommand_is_listed_in_help(capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["learning", "--help"]) - out = capsys.readouterr().out - for name in ("classify", "score", "baseline", "round", "publish"): - assert name in out - - -def test_the_pairs_command_scores_k_runs_per_side(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Grepping `--help` for the flag passes with the capability entirely absent, which is what it used to be.""" - from openultrasast import pairs as pairs_module - - calls: list[str] = [] - real = pairs_module.make_hunter_scan - - def counting(client: object, model: str, **kwargs: object): # type: ignore[no-untyped-def] - inner = real(client, model, **kwargs) # type: ignore[arg-type] - - def scan(root: Path): # type: ignore[no-untyped-def] - calls.append(root.name) - return inner(root) - - return scan - - monkeypatch.setattr(pairs_module, "make_hunter_scan", counting) - monkeypatch.setattr("openultrasast.cli.make_hunter_scan", counting) - monkeypatch.setenv("OPENULTRASAST_HUNTER_MODEL", "scripted") - assert main(["pairs", "--catalog", str(workspace / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "4"]) == 0 - assert calls.count("vuln") == 8 and calls.count("fixed") == 8 # two pairs, four runs per side - - -def test_a_baseline_records_what_the_endpoint_actually_spent(workspace: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Req 8.3: cost per pair is published beside every number, so the run has to read it off the client. - - The client meters itself; nothing passed the meter to the baseline, so every published cost would have been - 0.0 — and the round's cost cap, which reads the same meter, could never have fired either (Req 9.8).""" - from openultrasast.learning.endpoint import ChatEndpoint - from openultrasast.tool_hunter import ChatResponse - - class Metered: - def __init__(self) -> None: - self.calls = 0 - - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - del model, messages, tools, options - self.calls += 1 - return ChatResponse(content="[]") - - def cost_usd(self) -> float: - return 0.25 * self.calls - - client = Metered() - endpoint = ChatEndpoint(provider="scripted", base_url="", thinking=False) - monkeypatch.setattr("openultrasast.learning.endpoint.resolve_chat_endpoint", lambda *a, **k: (client, endpoint)) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(workspace / "pairs.toml"), - "--out", - str(workspace / "learning"), - "--k-runs", - "3", - "--model", - "metered", - ] - ) - == 0 - ) - report = json.loads((workspace / "learning" / "baseline" / "metered" / "report.json").read_text()) - assert client.calls > 0 - assert report["cost_usd"] == pytest.approx(0.25 * client.calls) diff --git a/tests/test_learning_cutoff.py b/tests/test_learning_cutoff.py deleted file mode 100644 index f22fd3d..0000000 --- a/tests/test_learning_cutoff.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Rows fixed after the detector's training cutoff are reported apart (learning-harness 4.3, Req 10.5). - -A model that may have read the fix cannot be measured on it the same way as one that cannot have. The corpus -carries the date at the precision the recipe recorded — a year for every row harvested so far — so a row counts as -post-cutoff only when its *earliest possible* date is after the cutoff. Rows with no date at all are counted -separately rather than quietly folded into either side. -""" - -from __future__ import annotations - -import json -import tomllib -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.learning.families import load_families -from openultrasast.learning.scoring import PairFamilyScore, aggregate -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored - - -def _score(pair: str, outcome: str, *, fix_date: str = "") -> PairFamilyScore: - return PairFamilyScore(pair=pair, family="injection", slice="vibe-py", runs=(outcome,), outcome=outcome, fix_date=fix_date) # type: ignore[arg-type] - - -def test_every_vendored_row_carries_the_date_its_recipe_recorded() -> None: - cases = select_vendored(load_pair_catalog(DEFAULT_CATALOG)) - dated = [case for case in cases if case.fix_date] - assert len(dated) >= 250, "the generator fills the date from the recipe; almost every row has a year" - assert all(len(case.fix_date) in {4, 10} for case in dated), "a date is a year or a full ISO day, never anything else" - by_name = {case.name: case for case in cases} - for slice_name in ("vibe-py", "agent-vfc"): - for recipe in tomllib.loads((Path("benchmarks/pairs") / slice_name / "recipes.toml").read_text())["recipe"]: - case = by_name.get(str(recipe["name"])) - if case is not None and recipe.get("year"): - assert case.fix_date.startswith(str(recipe["year"])) - - -@pytest.mark.parametrize( - ("fix_date", "cutoff", "expected"), - [ - ("2026", "2025-07-01", True), # every day of 2026 is after the cutoff - ("2026", "2026-03-01", False), # a year-precision row that might be before it is not claimed - ("2026-04-02", "2026-03-01", True), - ("2024", "2025-07-01", False), - ("2026-05", "2026-03-01", True), # month precision: the earliest day of May is after 1 March - ("2026-05", "2026-06-01", False), # and before 1 June, so the row is not claimed - ("2025", "2025", False), # a cutoff coarser than a day is refused rather than compared - ("", "2025-07-01", False), - ], -) -def test_a_row_is_post_cutoff_only_when_its_earliest_possible_date_is_after_it(fix_date: str, cutoff: str, expected: bool) -> None: - from openultrasast.learning.scoring import is_post_cutoff - - assert is_post_cutoff(fix_date, cutoff) is expected - - -def test_the_scoreboard_reports_the_post_cutoff_slice_the_cutoff_and_the_undated_count() -> None: - scores = [ - _score("new-correct", "pair_correct", fix_date="2026"), - _score("new-missed", "both_silent", fix_date="2026"), - _score("old-correct", "pair_correct", fix_date="2021"), - _score("undated", "pair_correct"), - ] - block = aggregate(scores, taxonomy=load_families(), cutoff="2025-07-01")["injection"] - assert block.cutoff == "2025-07-01" and block.undated == 1 - assert block.post_cutoff["scorable"] == 2 and block.post_cutoff["recall"] == 0.5 - assert block.scorable == 4, "the slice is reported beside the whole number, never instead of it" - payload = block.to_dict() - json.dumps(payload) - assert payload["cutoff"] == "2025-07-01" and payload["post_cutoff"]["scorable"] == 2 - - -def test_without_a_configured_cutoff_the_scoreboard_says_so_rather_than_inventing_one() -> None: - block = aggregate([_score("a", "pair_correct", fix_date="2026")], taxonomy=load_families())["injection"] - assert block.cutoff == "" and block.post_cutoff == {} and block.undated == 0 - - -def test_the_score_carries_the_date_from_the_pair(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text("def run():\n return 1\n") - case = PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "v.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - fix_date="2026", - ) - score = score_pair_family(case, "injection", [[]], [[]], ranges={"app.py": (("run", 1, 2),)}, taxonomy=load_families()) - assert score.fix_date == "2026" - - -def test_the_published_table_names_the_post_cutoff_slice(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - directory = out / "baseline" / "m" - directory.mkdir(parents=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": "m", - "k_runs": 5, - "cost_usd": 1.0, - "taxonomy_version": "1", - "metrics": { - "injection": { - "family": "injection", - "scorable": 4, - "unscorable": {}, - "recall": 0.5, - "youden": 0.25, - "hierarchical_credit": False, - "cutoff": "2025-07-01", - "undated": 1, - "post_cutoff": {"scorable": 2, "recall": 0.5, "youden": 0.5}, - } - }, - "floors": {}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text("# Roadmap\n") - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "2025-07-01" in report.table - assert "post-cutoff" in report.table.lower() - assert "undated" in report.table.lower() - - -def test_the_command_line_reports_the_slice_the_cutoff_and_the_undated_count(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Req 10.5 says every scoreboard, which includes the one an operator actually reads.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415 - - from openultrasast.cli import main - - for name in ("train-a", "hold-a"): - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - catalog = CATALOG.format(name="train-a", split="train") + "\n" + CATALOG.format(name="hold-a", split="holdout") - (tmp_path / "pairs.toml").write_text(catalog.replace('review_tier = "seeded"', 'review_tier = "seeded"\nfix_date = "2026"')) - (tmp_path / "openultrasast.toml").write_text('[learning]\nenabled = true\ncutoff_date = "2025-07-01"\n') - _detector(monkeypatch) - captured: list[str] = [] - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: captured.append(" ".join(str(item) for item in args))) - assert ( - main( - [ - "learning", - "score", - "--catalog", - str(tmp_path / "pairs.toml"), - "--out", - str(tmp_path / "learning"), - "--config", - str(tmp_path / "openultrasast.toml"), - ] - ) - == 0 - ) - line = next(item for item in captured if item.startswith("learning score injection")) - assert "post-cutoff(2025-07-01)=2" in line and "undated=0" in line - - -def test_a_baseline_over_a_named_slice_measures_that_slice(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`--slice vfc` selected the vfc rows and then `run_baseline` filtered them all out again. - - Its `slices` default is the two web slices, so a baseline over the memory-safety slice — the run Req 8.4 asks - for as a measured, non-gating number — produced an empty report and said nothing about it.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_cli import CATALOG, FIXED, VULN, _detector # noqa: PLC0415 - - from openultrasast.cli import main - - (tmp_path / "a-v.py").write_text(VULN) - (tmp_path / "a-f.py").write_text(FIXED) - (tmp_path / "pairs.toml").write_text(CATALOG.format(name="a", split="train").replace('slice = "vibe-py"', 'slice = "vfc"')) - _detector(monkeypatch) - assert ( - main( - [ - "learning", - "baseline", - "--catalog", - str(tmp_path / "pairs.toml"), - "--slice", - "vfc", - "--out", - str(tmp_path / "l"), - "--k-runs", - "3", - "--model", - "m", - ] - ) - == 0 - ) - # A run over a named slice is its own baseline: keyed by model *and* slice, so it cannot overwrite the run - # over the web families. - report = json.loads((tmp_path / "l" / "baseline" / "m-vfc" / "report.json").read_text()) - assert report["slices"] == ["vfc"] - assert report["metrics"]["injection"]["scorable"] == 1 diff --git a/tests/test_learning_detectors.py b/tests/test_learning_detectors.py deleted file mode 100644 index 6043611..0000000 --- a/tests/test_learning_detectors.py +++ /dev/null @@ -1,167 +0,0 @@ -"""learning-harness task 2.5: per-family detector configurations and the family runner (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.learning.families import load_families -from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient, ToolCall - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) -REPLY = '[{"path": "app.py", "line": 12, "title": "unconstrained read", "rationale": "no owner check"}]' - - -@pytest.fixture -def repo(tmp_path: Path) -> Path: - (tmp_path / "app.py").write_text(APP) - return tmp_path - - -def _write_config(root: Path, family: str, **overrides: object) -> Path: - directory = root / family - directory.mkdir(parents=True, exist_ok=True) - (directory / "prompt.md").write_text(str(overrides.get("prompt", f"You hunt {family} only."))) - (directory / "checklist.md").write_text(str(overrides.get("checklist", "- did the handler check ownership?"))) - tools = overrides.get("tools", ["read_definition", "obligations"]) - (directory / "family.toml").write_text( - f'version = "{overrides.get("version", "1")}"\ntools = {json.dumps(tools)}\n' - f"max_steps = {overrides.get('max_steps', 3)}\nmax_cost_usd = {overrides.get('max_cost_usd', 0.5)}\n" - f"max_chars = {overrides.get('max_chars', 4000)}\n" - ) - (directory / "hard_negatives.jsonl").write_text(json.dumps({"note": "a guarded handler is not a finding"}) + "\n") - (directory / "counterexamples.jsonl").write_text("") - return directory - - -def test_a_configuration_loads_every_part_it_needs(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs - - _write_config(tmp_path, "access_control") - configs = load_family_configs(tmp_path, load_families()) - config = configs["access_control"] - assert config.family == "access_control" and config.version == "1" - assert config.prompt.startswith("You hunt access_control") and "ownership" in config.checklist - assert config.tools == ("read_definition", "obligations") - assert config.max_steps == 3 and config.max_cost_usd == 0.5 and config.max_chars == 4000 - assert config.hard_negatives[0]["note"].startswith("a guarded handler") - assert config.counterexamples == () - assert load_family_configs(tmp_path, load_families())["access_control"] == config # loading twice is the same config - - -@pytest.mark.parametrize( - ("overrides", "needle"), - [ - ({"prompt": "x" * 9000}, "length"), - ({"tools": ["read_file", "run_shell"]}, "run_shell"), - ({"version": ""}, "version"), - ], - ids=["over_cap_prompt", "unknown_tool", "no_version"], -) -def test_the_loader_refuses_a_broken_configuration_by_name(tmp_path: Path, overrides: dict[str, object], needle: str) -> None: - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "access_control", **overrides) - with pytest.raises(DetectorConfigError, match=needle): - load_family_configs(tmp_path, load_families()) - - -def test_a_directory_that_is_not_a_family_is_refused_by_name(tmp_path: Path) -> None: - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "access_control") - (tmp_path / "sql_injection").mkdir() - (tmp_path / "sql_injection" / "prompt.md").write_text("x") - with pytest.raises(DetectorConfigError, match="sql_injection"): - load_family_configs(tmp_path, load_families()) - - -def test_the_runner_uses_the_family_prompt_tools_and_tags(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector - - configs_dir = tmp_path / "configs" - _write_config(configs_dir, "access_control") - config = load_family_configs(configs_dir, load_families())["access_control"] - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="1", name="obligations", arguments={"path": "app.py", "function": "leaky"}),)), - ChatResponse(content=REPLY), - ] - ) - findings = run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m") - assert [finding.tags for finding in findings] == [["family:access_control", "detector:access_control@1"]] - assert findings[0].evidence_level == "suspicion" - call = client.calls[0] - assert call["messages"][0]["content"] == "You hunt access_control only." - assert [tool["function"]["name"] for tool in call["tools"]] == ["read_definition", "obligations"] - user = str(call["messages"][1]["content"]) - assert "def leaky" in user and "def guarded" in user # the whole file, not an excerpt around a hotspot - assert "ownership" in user and "a guarded handler is not a finding" in user - - -def test_a_region_runs_every_admitted_family_plus_the_generalist(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_region_detectors - - configs_dir = tmp_path / "configs" - for family in ("access_control", "injection", "unknown"): - _write_config(configs_dir, family, tools=["read_file"]) - configs = load_family_configs(configs_dir, load_families()) - client = ScriptedChatClient( - [ - ChatResponse(tool_calls=(ToolCall(id="1", name="read_file", arguments={"path": "app.py"}),)), - ChatResponse(content=REPLY), - ] - * 4 - ) - region = Region(path="app.py", function="leaky", families=("access_control",)) - findings = run_region_detectors(repo, region, configs, client=client, model="m") - families = sorted({tag for finding in findings for tag in finding.tags if tag.startswith("family:")}) - assert families == ["family:access_control", "family:unknown"] # the admitted family and the generalist, not injection - assert len(client.calls) == 4 # two turns each, and nothing else was run - - -def test_the_runner_never_offers_a_tool_outside_the_curated_set(repo: Path, tmp_path: Path) -> None: - from openultrasast.learning.detectors import Region, load_family_configs, run_family_detector - from openultrasast.tool_hunter import HUNTER_TOOLS - - configs_dir = tmp_path / "configs" - _write_config( - configs_dir, "injection", tools=["read_file", "grep_repo", "find_refs", "read_definition", "entry_points", "flows", "obligations"] - ) - config = load_family_configs(configs_dir, load_families())["injection"] - client = ScriptedChatClient([ChatResponse(content="")]) - run_family_detector(repo, Region(path="app.py", function="leaky"), config, client=client, model="m") - offered = {tool["function"]["name"] for tool in client.calls[0]["tools"]} - assert offered == {str(tool["function"]["name"]) for tool in HUNTER_TOOLS} - assert "bash" not in offered and "run_shell" not in offered - - -def test_round_zero_can_write_a_configuration_for_every_family(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs, write_default_configs - - taxonomy = load_families() - written = write_default_configs(tmp_path, taxonomy, prompt="You are a security hunter.", version="0") - assert set(written) == {family.id for family in taxonomy.families} - configs = load_family_configs(tmp_path, taxonomy) - assert set(configs) == {family.id for family in taxonomy.families} - assert all(config.version == "0" for config in configs.values()) - assert all(config.prompt.startswith("You are a security hunter.") for config in configs.values()) - assert configs["access_control"].prompt != configs["injection"].prompt # each carries its own family's definition - - -def test_a_configuration_with_no_tools_is_refused_rather_than_reporting_nothing(tmp_path: Path) -> None: - """A detector offered no tools never calls one, and the hunter loop then returns nothing at all.""" - from openultrasast.learning.detectors import DetectorConfigError, load_family_configs - - _write_config(tmp_path, "injection", tools=[]) - with pytest.raises(DetectorConfigError, match="tools"): - load_family_configs(tmp_path, load_families()) diff --git a/tests/test_learning_journal.py b/tests/test_learning_journal.py deleted file mode 100644 index 6a936c8..0000000 --- a/tests/test_learning_journal.py +++ /dev/null @@ -1,112 +0,0 @@ -"""learning-harness task 2.8: the round journal, the archive and the rejected buffer (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.learning.scoring import PairFamilyScore - - -def _record(round_number: int, family: str, outcome: str, **extra: object) -> object: - from openultrasast.learning.journal import Attribution, RoundRecord - - defaults: dict[str, object] = { - "round": round_number, - "family": family, - "hypothesis": "the checklist never asks about ownership", - "levers": ("checklist",), - "predicted_affected": (family,), - "predicted_at_risk": ("injection",), - "outcome": outcome, - "reason": "", - "target_train_delta": 0.1, - "target_holdout_delta": 0.2, - "sweep": {"injection": 0}, - "attribution": Attribution(flipped_predicted=2, flipped_unpredicted=0, precision=1.0), - "cost_usd": 0.42, - "taxonomy_version": "1", - "config_version": "2", - } - return RoundRecord(**{**defaults, **extra}) # type: ignore[arg-type] - - -def test_a_record_round_trips_through_the_file(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - assert journal.rounds() == () - record = _record(1, "access_control", "accepted") - journal.append(record) - journal.append(_record(2, "injection", "rejected", reason="tie")) - rounds = journal.rounds() - assert [item.round for item in rounds] == [1, 2] and rounds[0] == record - assert rounds[0].attribution.precision == 1.0 and rounds[0].sweep == {"injection": 0} - assert rounds[1].outcome == "rejected" and rounds[1].reason == "tie" - rows = [json.loads(line) for line in (tmp_path / "journal.jsonl").read_text().splitlines() if line.strip()] - assert len(rows) == 2 and rows[0]["hypothesis"].startswith("the checklist") - - -def test_the_journal_is_append_only_and_a_round_number_is_never_reused(tmp_path: Path) -> None: - from openultrasast.learning.journal import JournalError, LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "accepted")) - before = (tmp_path / "journal.jsonl").read_text() - try: - journal.append(_record(1, "injection", "accepted")) - raise AssertionError("a repeated round number must be refused") - except JournalError as error: - assert "1" in str(error) - assert (tmp_path / "journal.jsonl").read_text() == before - assert journal.next_round() == 2 - - -def test_the_rejected_buffer_returns_only_the_target_family(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "rejected", hypothesis="ask about ownership", reason="tie")) - journal.append(_record(2, "injection", "rejected", hypothesis="ask about quoting", reason="tie")) - journal.append(_record(3, "access_control", "accepted", hypothesis="give it the obligations tool")) - assert journal.rejected_buffer("access_control") == ("ask about ownership",) - assert journal.rejected_buffer("injection") == ("ask about quoting",) - assert journal.rejected_buffer("memory") == () - - -def test_the_archive_names_one_winning_configuration_per_pair_and_family(tmp_path: Path) -> None: - from openultrasast.learning.journal import Archive - - archive = Archive(tmp_path / "archive.jsonl") - correct = PairFamilyScore(pair="a", family="access_control", runs=("pair_correct",), outcome="pair_correct") - missed = PairFamilyScore(pair="a", family="access_control", runs=("both_silent",), outcome="both_silent") - archive.record("access_control", "1", correct) - archive.record("access_control", "2", missed) # a later version that loses this pair does not take it - archive.record( - "access_control", "2", PairFamilyScore(pair="b", family="access_control", runs=("pair_correct",), outcome="pair_correct") - ) - assert archive.winners("access_control") == {"a": "1", "b": "2"} - assert archive.winners("injection") == {} - reopened = Archive(tmp_path / "archive.jsonl") - assert reopened.winners("access_control") == {"a": "1", "b": "2"} # survives a reload, like the store it copies - - -def test_a_later_version_takes_a_pair_only_by_winning_it(tmp_path: Path) -> None: - from openultrasast.learning.journal import Archive - - archive = Archive(tmp_path / "archive.jsonl") - archive.record("injection", "1", PairFamilyScore(pair="a", family="injection", runs=("both_silent",), outcome="both_silent")) - assert archive.winners("injection") == {} - archive.record("injection", "2", PairFamilyScore(pair="a", family="injection", runs=("pair_correct",), outcome="pair_correct")) - assert archive.winners("injection") == {"a": "2"} - - -def test_everything_persisted_passes_redaction(tmp_path: Path) -> None: - from openultrasast.learning.journal import LearningJournal - - leaked = "api_key = " + "'" + "A" * 24 + "'" # assembled here so no key-shaped literal sits in the file - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append(_record(1, "access_control", "rejected", hypothesis=f"the prompt should show {leaked}", reason="tie")) - text = (tmp_path / "journal.jsonl").read_text() - assert "A" * 24 not in text and "REDACTED" in text - assert journal.rounds()[0].hypothesis.endswith("'") diff --git a/tests/test_learning_pairs_evidence.py b/tests/test_learning_pairs_evidence.py deleted file mode 100644 index ef7b777..0000000 --- a/tests/test_learning_pairs_evidence.py +++ /dev/null @@ -1,151 +0,0 @@ -"""What the pairs hunter path must prove (learning-harness review round 3, Req 3.2, 3.5). - -Three defects, one test each: K was declared and never threaded, so every stage scored one run; the fixed -side was judged against the *vulnerable* side's function ranges, which differ on most of the corpus; and -the generalist hunter emitted findings with no family tag, which the class-aware scorer can never credit, -so `ousast pairs --hunter` reported recall zero by construction. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase, evaluate_catalog - -# The fix inserts two guard lines above the handler, so `run` starts on a different line on each side. -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = ( - "from flask import request\nimport shlex\nimport os\n\n\n" - "def _safe(value):\n return shlex.quote(value or '')\n\n\n" - "def run():\n os.system(_safe(request.args.get('cmd')))\n" -) - - -def _case(tmp_path: Path, name: str = "p") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", - vulnerability_class="x", - path="app.py", - evidence="", - function="run", - mechanism="other", - family="injection", - ), - ), - min_recall=1.0, - fix_policy="silent", - review_tier="seeded", - ) - - -def _finding(line: int, *, family: str | None = "injection") -> StaticFinding: - return StaticFinding( - finding_id=f"tool-hunter:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["tool-hunter", *([f"family:{family}"] if family else [])], - ranking_priority=0.0, - ) - - -def test_the_k_runs_flag_reaches_the_detector(tmp_path: Path) -> None: - """Req 3.5: three runs per side, per pair. A parameter no caller can set is not a K-run floor.""" - calls: list[str] = [] - - def scan(root: Path) -> list[StaticFinding]: - calls.append(root.name) - return [_finding(6)] if root.name == "vuln" else [] - - evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - assert calls.count("vuln") == 3 and calls.count("fixed") == 3 - - -def test_fewer_than_three_runs_is_refused(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="three runs"): - evaluate_catalog([_case(tmp_path)], hunter=lambda root: [], k_runs=1) - - -def test_the_fixed_side_is_judged_against_the_fixed_side_line_numbers(tmp_path: Path) -> None: - """`run` sits at line 5 on the vulnerable side and line 10 on the fixed side. - - Scoring the fixed side against the vulnerable side's span reads a finding inside the fixed handler as - outside it, so a detector that flags both sides is recorded as silent on the fix — a leak counted as a - clean pass, and always in the direction that flatters the number.""" - - def scan(root: Path) -> list[StaticFinding]: - return [_finding(6)] if root.name == "vuln" else [_finding(11)] - - result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - hunter = result.scorers["vibe-py"]["hunter"] - assert hunter.detected_vuln == 1 - assert hunter.silent_fix == 0, "the finding is inside `run` on the fixed side; that is a leak" - assert hunter.pair_correct == 0 - - -def test_a_generalist_finding_without_a_family_is_never_a_detection(tmp_path: Path) -> None: - """The other half of the same defect: `pairs --hunter` used to feed the scorer untagged findings.""" - - def scan(root: Path) -> list[StaticFinding]: - return [_finding(6, family=None)] if root.name == "vuln" else [] - - result = evaluate_catalog([_case(tmp_path)], hunter=scan, k_runs=3) - assert result.scorers["vibe-py"]["hunter"].detected_vuln == 0 - - -def test_the_hunter_reports_the_family_it_believes_and_that_is_what_is_credited(tmp_path: Path) -> None: - """A generalist that must name the family is answerable; one that names nothing cannot be scored at all.""" - from openultrasast.tool_hunter import _findings_from_content - - findings = _findings_from_content( - tmp_path, - [], - '[{"path": "app.py", "line": 6, "family": "injection", "title": "t", "rationale": "r"}]', - ) - assert findings and "family:injection" in findings[0].tags - untagged = _findings_from_content(tmp_path, [], '[{"path": "app.py", "line": 6, "title": "t"}]') - assert untagged and not [tag for tag in untagged[0].tags if tag.startswith("family:")] - - -def test_the_pair_hunter_asks_for_a_family_from_the_closed_list(tmp_path: Path) -> None: - """The prompt has to name the taxonomy, or the model invents family names and every finding is noise.""" - from openultrasast.pairs import make_hunter_scan - - seen: list[str] = [] - - class Client: - def complete(self, *, model: str, messages, tools=None, **options): # type: ignore[no-untyped-def] - seen.append("\n".join(str(message.get("content") or "") for message in messages)) - from openultrasast.tool_hunter import ChatResponse - - return ChatResponse(content="[]", tool_calls=()) - - (tmp_path / "app.py").write_text(VULN) - make_hunter_scan(Client(), "m", taxonomy=load_families())(tmp_path) - prompt = "\n".join(seen) - assert "injection" in prompt and "access_control" in prompt and "untrusted_destination" in prompt - assert '"family"' in prompt diff --git a/tests/test_learning_pairs_path.py b/tests/test_learning_pairs_path.py deleted file mode 100644 index cab5abe..0000000 --- a/tests/test_learning_pairs_path.py +++ /dev/null @@ -1,110 +0,0 @@ -"""learning-harness task 3.2: family scoring on the pairs hunter path (offline).""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.pairs import PairCase, evaluate_pair - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', request.args.get('cmd')], check=False)\n" - - -def _case(tmp_path: Path, *, family: str | None = "injection", unscorable: str | None = None) -> PairCase: - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED) - return PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - unscorable=unscorable, - ) - - -def _finding(family: str, line: int) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:injection@1"], - ranking_priority=0.0, - ) - - -def _hunter(vuln: list[StaticFinding], fixed: list[StaticFinding]): # type: ignore[no-untyped-def] - seen: list[Path] = [] - - def scan(root: Path) -> list[StaticFinding]: - seen.append(root) - return vuln if root.name == "vuln" else fixed - - scan.seen = seen # type: ignore[attr-defined] - return scan - - -def test_the_hunter_path_scores_by_family_and_function_not_by_text(tmp_path: Path) -> None: - right = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [])) - assert right.detected_vuln and right.silent_fix and right.pair_correct - assert right.detection_kinds == ("hunter",) and right.family == "injection" - wrong_family = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("access_control", 6)], [])) - assert not wrong_family.detected_vuln # the right place, the wrong family - loud = StaticFinding(**{**_finding("access_control", 6).__dict__, "title": "CWE-78 command injection in run"}) - assert not evaluate_pair(_case(tmp_path), hunter=_hunter([loud], [])).detected_vuln # the words never count - - -def test_a_finding_of_another_family_on_the_fixed_side_is_not_a_leak(tmp_path: Path) -> None: - outcome = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("access_control", 6)])) - assert outcome.pair_correct and outcome.silent_fix - assert outcome.fix_findings == 1 and outcome.fix_leaks == 0 # counted, never charged - leaked = evaluate_pair(_case(tmp_path), hunter=_hunter([_finding("injection", 6)], [_finding("injection", 6)])) - assert leaked.detected_vuln and not leaked.silent_fix and leaked.fix_leaks == 1 - - -def test_an_unscorable_pair_is_reported_as_such_and_never_run(tmp_path: Path) -> None: - hunter = _hunter([_finding("injection", 6)], []) - outcome = evaluate_pair(_case(tmp_path, unscorable="identical_twin"), hunter=hunter) - assert outcome.unscorable == "identical_twin" and not outcome.detected_vuln - assert hunter.seen == [] # type: ignore[attr-defined] - - -def test_a_pair_with_no_family_label_falls_back_to_the_classifier(tmp_path: Path) -> None: - outcome = evaluate_pair(_case(tmp_path, family=None), hunter=_hunter([_finding("injection", 6)], [])) - assert outcome.family == "injection" and outcome.pair_correct # CWE-78 classifies as injection - - -def test_the_overlay_path_is_untouched_by_the_family_scorer(tmp_path: Path) -> None: - overlay = evaluate_pair(_case(tmp_path)) - assert overlay.detection_kinds != ("hunter",) and overlay.family == "" - - -def test_the_scoreboard_carries_the_family_block(tmp_path: Path) -> None: - from openultrasast.pairs import evaluate_catalog, result_payload - - result = evaluate_catalog([_case(tmp_path)], hunter=_hunter([_finding("injection", 6)], []), inventory_sidecar=False) - payload = result_payload(result) - assert "per_family" in payload and payload["per_family"]["injection"]["scorable"] == 1 - assert payload["per_family"]["injection"]["recall"] == 1.0 - assert payload["per_family"]["injection"]["hierarchical_credit"] is False - assert payload["per_family"]["injection"]["taxonomy_version"] diff --git a/tests/test_learning_proposer.py b/tests/test_learning_proposer.py deleted file mode 100644 index 2a0ca2f..0000000 --- a/tests/test_learning_proposer.py +++ /dev/null @@ -1,222 +0,0 @@ -"""learning-harness task 3.5: failure facts, the scripted proposer and the HarnessX proposer (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.learning.detectors import FamilyConfig -from openultrasast.learning.journal import LearningJournal, RoundRecord -from openultrasast.learning.scoring import PairFamilyScore - - -def _config(family: str = "access_control") -> FamilyConfig: - return FamilyConfig( - family=family, - version="0", - prompt="You hunt access control.", - checklist="- did the handler check ownership?", - tools=("read_definition", "obligations"), - max_steps=3, - max_cost_usd=0.5, - max_chars=4000, - ) - - -def _score(pair: str, outcome: str, *, family: str = "access_control") -> PairFamilyScore: - return PairFamilyScore(pair=pair, family=family, runs=(outcome,), outcome=outcome, slice="vibe-py") # type: ignore[arg-type] - - -def test_failure_facts_carry_the_misses_the_leaks_and_the_rejected_buffer(tmp_path: Path) -> None: - from openultrasast.learning.proposer import build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - journal.append( - RoundRecord( - round=1, - family="access_control", - hypothesis="ask about ownership", - levers=("checklist",), - predicted_affected=("access_control",), - predicted_at_risk=(), - outcome="rejected", - reason="tie", - ) - ) - scores = [_score("miss-one", "both_silent"), _score("leak-one", "both_flagged"), _score("good", "pair_correct")] - facts = build_failure_facts( - scores, family="access_control", config=_config(), journal=journal, train_pairs={"miss-one", "leak-one", "good"} - ) - assert [item.pair for item in facts.misses] == ["miss-one"] - assert [item.pair for item in facts.leaks] == ["leak-one"] - assert facts.rejected == ("ask about ownership",) - assert facts.config.family == "access_control" and facts.family == "access_control" - - -def test_the_proposer_is_never_handed_a_holdout_pair(tmp_path: Path) -> None: - from openultrasast.learning.proposer import build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - scores = [_score("train-miss", "both_silent"), _score("holdout-miss", "both_silent")] - facts = build_failure_facts(scores, family="access_control", config=_config(), journal=journal, train_pairs={"train-miss"}) - assert [item.pair for item in facts.misses] == ["train-miss"] - assert "holdout-miss" not in json.dumps(facts.to_dict()) - - -def test_the_scripted_proposer_returns_one_change_for_one_family(tmp_path: Path) -> None: - from openultrasast.learning.proposer import Proposal, ScriptedProposer, build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"}) - scripted = ScriptedProposer( - [ - Proposal( - hypothesis="h", - lever="checklist", - change={"checklist.md": "- ask about ownership\n"}, - predicted_affected=("access_control",), - ) - ] - ) - proposal = scripted.propose(facts) - assert proposal is not None and proposal.lever == "checklist" and set(proposal.change) == {"checklist.md"} - assert scripted.propose(facts) is None # a scripted proposer runs out rather than repeating itself - - -@pytest.mark.parametrize( - ("change", "lever", "needle"), - [ - ({"../../etc/passwd": "x"}, "checklist", "outside"), - ({"family.toml": "x", "prompt.md": "y"}, "prompt", "one file"), - ({"secrets.md": "x"}, "checklist", "secrets.md"), - ({"prompt.md": "x" * 9000}, "prompt", "cap"), - ], - ids=["escapes_the_family_directory", "two_files", "unknown_file", "over_the_cap"], -) -def test_a_proposal_that_reaches_beyond_its_family_is_refused_by_name(change: dict[str, str], lever: str, needle: str) -> None: - from openultrasast.learning.proposer import Proposal, refuse_proposal - - refusal = refuse_proposal(Proposal(hypothesis="h", lever=lever, change=change), _config()) # type: ignore[arg-type] - assert refusal is not None and needle in refusal - - -def test_a_well_formed_proposal_is_accepted_and_applies_inside_its_own_directory(tmp_path: Path) -> None: - from openultrasast.learning.proposer import Proposal, apply_proposal, refuse_proposal - - directory = tmp_path / "access_control" - directory.mkdir() - (directory / "checklist.md").write_text("- old\n") - proposal = Proposal(hypothesis="h", lever="checklist", change={"checklist.md": "- new\n"}, predicted_affected=("access_control",)) - assert refuse_proposal(proposal, _config()) is None - apply_proposal(proposal, directory) - assert (directory / "checklist.md").read_text() == "- new\n" - - -def test_the_write_roots_are_one_family_directory_and_exclude_the_checker_and_the_corpus(tmp_path: Path) -> None: - from openultrasast.learning import canaries - from openultrasast.learning.proposer import write_roots - - roots = write_roots(tmp_path / "configs", "access_control") - assert roots == (tmp_path / "configs" / "access_control",) - verifier_module = Path(canaries.__file__).resolve() - assert not any(verifier_module.is_relative_to(root.resolve()) for root in roots) - corpus = Path("benchmarks").resolve() - assert not any(str(corpus).startswith(str(root.resolve())) for root in roots) - - -def test_the_harnessx_proposer_degrades_with_a_reason_when_the_extra_is_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning import proposer as proposer_module - from openultrasast.learning.proposer import HarnessXProposer, build_failure_facts - - journal = LearningJournal(tmp_path / "journal.jsonl") - facts = build_failure_facts([_score("a", "both_silent")], family="access_control", config=_config(), journal=journal, train_pairs={"a"}) - monkeypatch.setattr(proposer_module, "has_harnessx", lambda: False) - agent = HarnessXProposer(configs_dir=tmp_path / "configs", model="m") - assert agent.propose(facts) is None - assert agent.reason == "harnessx_unavailable" - assert agent.write_roots("access_control") == (tmp_path / "configs" / "access_control",) - - -# --- the meta-agent proposer (review round 3, task 3.5 finding 1) ------------ - - -def test_the_meta_agent_proposer_reads_its_change_out_of_a_scratch_copy(tmp_path: Path) -> None: - """The meta-agent never writes into the live family directory. - - A proposer that edited the tree directly would have changed the configuration before the round snapshotted - it, so a rejection could not put it back. It gets a copy, and the proposal is the diff of that copy.""" - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - (configs / "injection" / "family.toml").write_text('family = "injection"\nversion = "0"\n') - seen: dict[str, object] = {} - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - seen["workspace"] = workspace - seen["family"] = facts.family - (workspace / "checklist.md").write_text("- ask about quoting\n") - return "ask about quoting" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - proposal = proposer.propose(_facts(tmp_path, "injection")) - assert proposal is not None - assert proposal.lever == "checklist" and proposal.change == {"checklist.md": "- ask about quoting\n"} - assert proposal.hypothesis == "ask about quoting" - assert seen["workspace"] != configs / "injection" - assert (configs / "injection" / "checklist.md").read_text() == "- old\n", "the live tree was edited" - - -def test_the_meta_agent_proposer_refuses_more_than_one_changed_file(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - (configs / "injection" / "prompt.md").write_text("p\n") - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - (workspace / "checklist.md").write_text("- a\n") - (workspace / "prompt.md").write_text("- b\n") - return "two at once" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert "one file" in proposer.reason - - -def test_the_meta_agent_proposer_refuses_a_file_no_lever_owns(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - - def agent(*, workspace: Path, facts, model: str): # type: ignore[no-untyped-def] - (workspace / "notes.txt").write_text("x\n") - return "sideways" - - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=agent) - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert "notes.txt" in proposer.reason - - -def test_the_meta_agent_proposer_says_so_when_it_changed_nothing(tmp_path: Path) -> None: - from openultrasast.learning.proposer import HarnessXProposer - - configs = tmp_path / "configs" - (configs / "injection").mkdir(parents=True) - (configs / "injection" / "checklist.md").write_text("- old\n") - proposer = HarnessXProposer(configs_dir=configs, model="m", agent=lambda **_kwargs: "nothing to do") - assert proposer.propose(_facts(tmp_path, "injection")) is None - assert proposer.reason == "no_change_proposed" - - -def _facts(tmp_path: Path, family: str): # type: ignore[no-untyped-def] - from openultrasast.learning.proposer import FailureFacts - - del tmp_path - return FailureFacts(family=family, config=_config(family)) diff --git a/tests/test_learning_publish.py b/tests/test_learning_publish.py deleted file mode 100644 index 07601e2..0000000 --- a/tests/test_learning_publish.py +++ /dev/null @@ -1,209 +0,0 @@ -"""learning-harness task 3.8: publishing every number from its committed artifacts (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -ROADMAP = "# Roadmap\n\n## Overview\n\nSomething about the project.\n\n## Constraints\n\nCore dependencies stay empty.\n" - - -def _baseline(out: Path, model: str, *, recall: float = 1.0, cost: float = 0.5) -> None: - directory = out / "baseline" / model - directory.mkdir(parents=True, exist_ok=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": model, - "k_runs": 5, - "taxonomy_version": "1", - "slices": ["vibe-py"], - "cost_usd": cost, - "floors": {"injection": {"family": "injection", "budget_flips": 1, "negative_flip_rate": 0.1, "k_runs": 5, "gates": True}}, - "metrics": { - "injection": { - "family": "injection", - "taxonomy_version": "1", - "scorable": 10, - "unscorable": {"identical_twin": 2}, - "outcomes": {"pair_correct": round(recall * 10)}, # keep the fixture internally consistent - "recall": recall, - "silence": 1.0, - "youden": recall, - "directional_bias": 0.0, - "leak_rate": 0.0, - "fixed_fpr_recall": recall, - "hierarchical_credit": False, - } - }, - }, - indent=2, - sort_keys=True, - ) - + "\n" - ) - - -def test_publishing_twice_without_new_artifacts_is_byte_identical(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - measurements = tmp_path / "measurements" - first = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - text = roadmap.read_text() - second = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - assert roadmap.read_text() == text - assert first.table == second.table and first.families == second.families - - -def test_every_number_carries_its_denominator_floor_and_cost(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.9, cost=1.8) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - table = report.table - assert "| injection |" in table - assert "10" in table and "identical_twin: 2" in table # scorable and why the rest are not - assert "taxonomy 1" in table and "budget 1" in table - assert "0.20" in table # cost per pair-correct: 1.8 over 9 - assert "hierarchical partial credit could not apply" in table - assert report.command and report.command in roadmap.read_text() - - -def test_a_second_model_appears_beside_the_first(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.9) - _baseline(out, "deepseek-v4-pro", recall=1.0) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "deepseek-v4-flash" in report.table and "deepseek-v4-pro" in report.table - assert set(report.families) == {"injection"} - - -def test_the_corrected_lever_number_replaces_the_earlier_one(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - measurements = tmp_path / "m" - measurements.mkdir() - (measurements / "2026-09-06-mechanism-lever-split.json").write_text( - json.dumps( - { - "slice": "vibe-py", - "holdout_pairs": 17, - "before": {"detected": 7, "youden": 0.0588}, - "after": {"detected": 5, "youden": -0.0588}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - stale = "human holdout profile pair_correct 0 -> 6 of 17, Youden 0 -> +0.353, round 2 `no_proposals`." - roadmap.write_text(f"{ROADMAP}\nMeasured 2026-09-05 with the vibe-py candidates: {stale}\n") - report = publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - text = roadmap.read_text() - assert "-0.0588" in report.table or "-0.059" in report.table - assert "5 of 17" in report.table or "detected 5" in report.table - assert "holdout pairs stopped teaching" in report.table.lower() or "train split only" in report.table.lower() - assert text.count("learning-harness:begin") == 1 - # Req 5.3 says *in place of* the earlier one. Leaving the old claim above the new section is how a document - # ends up asserting two different numbers for the same measurement. - assert stale not in text - assert "superseded" in text and "learning-harness" in text - assert "Measured 2026-09-05 with the vibe-py candidates:" in text, "the sentence is corrected, not deleted" - before = roadmap.read_text() - publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - assert roadmap.read_text() == before, "a second publish must be a no-op once the claim is corrected" - - -def test_the_roadmap_section_is_replaced_not_appended_again(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash", recall=0.5) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - _baseline(out, "deepseek-v4-flash", recall=0.9) - publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - text = roadmap.read_text() - assert text.count("learning-harness:begin") == 1 and text.count("learning-harness:end") == 1 - assert "## Overview" in text and "## Constraints" in text # nothing else in the roadmap is disturbed - assert "0.5" not in text.split("learning-harness:begin")[1] # the stale number is gone - - -def test_the_raw_artifacts_are_written_where_the_numbers_are_read_from(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - _baseline(out, "deepseek-v4-flash") - measurements = tmp_path / "m" - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - publish(learning_dir=out, measurements_dir=measurements, roadmap=roadmap) - written = sorted(path.name for path in measurements.iterdir()) - assert any(name.endswith("-learning-families.json") for name in written) - payload = json.loads((measurements / next(name for name in written if name.endswith("-learning-families.json"))).read_text()) - # keyed by what identifies the run, so two slices of one model cannot collapse into one entry - assert payload["models"]["deepseek-v4-flash (vibe-py)"]["injection"]["scorable"] == 10 - assert payload["command"] - - -def test_publishing_with_no_baseline_says_so_rather_than_inventing_a_table(tmp_path: Path) -> None: - from openultrasast.learning.publish import publish - - roadmap = tmp_path / "roadmap.md" - roadmap.write_text(ROADMAP) - report = publish(learning_dir=tmp_path / "empty", measurements_dir=tmp_path / "m", roadmap=roadmap) - assert report.families == () and "no baseline" in report.table.lower() - assert "learning-harness:begin" in roadmap.read_text() - - -def test_two_slices_of_one_model_are_two_rows_not_one_overwriting_the_other(tmp_path: Path) -> None: - """Keyed by model alone, the vibe-py run silently replaced the agent-vfc run and the published table showed - hand-written teaching code as if it were the corpus — the exact failure Req 3 exists to prevent.""" - from openultrasast.learning.publish import publish - - out = tmp_path / "learning" - for key, slice_name, recall in (("deepseek-v4-flash-vibe-py", "vibe-py", 0.833), ("deepseek-v4-flash-agent-vfc", "agent-vfc", 0.444)): - directory = out / "baseline" / key - directory.mkdir(parents=True) - (directory / "report.json").write_text( - json.dumps( - { - "model": "deepseek-v4-flash", - "slices": [slice_name], - "k_runs": 5, - "cost_usd": 1.0, - "taxonomy_version": "1", - "metrics": { - "injection": { - "family": "injection", - "scorable": 10, - "unscorable": {}, - "recall": recall, - "silence": 1.0, - "youden": recall, - "hierarchical_credit": False, - "outcomes": {"pair_correct": 5}, - } - }, - "floors": {}, - } - ) - ) - roadmap = tmp_path / "roadmap.md" - roadmap.write_text("# Roadmap\n") - report = publish(learning_dir=out, measurements_dir=tmp_path / "m", roadmap=roadmap) - assert "vibe-py" in report.table and "agent-vfc" in report.table - assert report.table.count("| injection |") == 2, "one row per slice, not one row that survived" - assert "0.833" in report.table and "0.444" in report.table diff --git a/tests/test_learning_refusals.py b/tests/test_learning_refusals.py deleted file mode 100644 index 16ca7fe..0000000 --- a/tests/test_learning_refusals.py +++ /dev/null @@ -1,94 +0,0 @@ -"""A refusal is data, not a `continue` (learning-harness review round 3, Req 5.1, 5.2). - -`Refusal.degradation()` existed and nothing ever called it, so every path that declined to learn from a pair -declined silently: the mechanism lever skipped a holdout-taught shape without a word, leave-one-out dropped -non-teachers with no record, and the seeder gave a train-split identical twin the reason `split:train`, which -is not true. A leak the run does not mention is a leak nobody reads. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase - -VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str = "train", unscorable: str | None = None, tier: str = "seeded") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier=tier, - unscorable=unscorable, - ) - - -def test_a_train_split_twin_is_refused_for_being_a_twin_not_for_its_split(tmp_path: Path) -> None: - """`split:train` on a train-split row is a lie, and it hides a corpus fact behind a split fact. - - Two vibe-py rows are exactly this: `python-insecure-app-main-try-hack-me` and - `vulnpy-deserialization-do-pickle-load` are train-split identical twins.""" - from openultrasast.semantic.seed import _skip_reason - - assert _skip_reason(_case(tmp_path, "twin", split="train", unscorable="identical_twin")) == "identical_twin" - assert _skip_reason(_case(tmp_path, "held", split="holdout")) == "split:holdout" - assert _skip_reason(_case(tmp_path, "good", split="train")) is None - - -def test_leave_one_out_records_the_pairs_it_would_not_learn_from(tmp_path: Path) -> None: - """Req 5.2: the refusal is recorded with the pair names, on a channel a caller can read.""" - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo([_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")]) - refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"] - assert refusals, "leave-one-out excluded a holdout pair from teaching and said nothing" - assert refusals[0]["pairs"] == ["hold-a"] - assert result.teaching_pairs <= 1 - - -def test_the_mechanism_lever_records_every_shape_a_holdout_pair_taught(tmp_path: Path) -> None: - """The lever used to `continue` past a refused candidate; the run then showed a shorter list with no reason.""" - from openultrasast.improve.evolve import propose_mechanism_edits - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variants import Shape - - cases = [_case(tmp_path, "train-a"), _case(tmp_path, "hold-a", split="holdout")] - candidates = MechanismStore(tmp_path / "candidates.jsonl") - append_from_pair( - candidates, - Shape( - language="python", - sink_name="system", - arity=1, - source_positions=(0,), - source_kinds=("request",), - guard="none", - mechanism="source_reaches_sink", - ), - summary="s", - cwe="CWE-78", - pair="hold-a", - provenance="human", - tier="seeded", - ) - refusals: list[dict[str, object]] = [] - edits = propose_mechanism_edits(cases, candidates, tmp_path / "scan.jsonl", degradations=refusals) - assert [edit.mechanism_id for edit in edits if edit.action == "admit"] == [] - assert [item["reason"] for item in refusals] == ["holdout_pair_refused"] - assert refusals[0]["pairs"] == ["hold-a"] - json.dumps(refusals) # a degradation has to survive the artifact it is written into diff --git a/tests/test_learning_reliability.py b/tests/test_learning_reliability.py deleted file mode 100644 index abda25d..0000000 --- a/tests/test_learning_reliability.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5). - -The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw -deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and -a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up. - -Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision, -the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether -any of those decisions could be told from a coin flip. -""" - -from __future__ import annotations - -from openultrasast.learning.acceptance import reliability - - -def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None: - result = reliability(better=8, worse=0) - assert result.better == 8 and result.worse == 0 - assert result.p_value < 0.05 and result.reliable is True - - -def test_two_pairs_moving_the_right_way_is_not() -> None: - """n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing.""" - result = reliability(better=2, worse=0) - assert result.p_value == 0.5 and result.reliable is False - - -def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None: - result = reliability(better=0, worse=0) - assert result.p_value == 1.0 and result.reliable is False - - -def test_an_even_split_is_never_reliable_however_large() -> None: - assert reliability(better=20, worse=20).reliable is False - - -def test_the_round_journals_the_reliability_of_its_own_decision() -> None: - from openultrasast.learning.journal import RoundRecord - - record = RoundRecord( - round=1, - family="injection", - hypothesis="h", - levers=("checklist",), - predicted_affected=(), - predicted_at_risk=(), - outcome="accepted", - train_better=8, - train_worse=0, - holdout_better=1, - holdout_worse=0, - ) - payload = record.to_dict() - assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05 - assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so" diff --git a/tests/test_learning_round4_evidence.py b/tests/test_learning_round4_evidence.py deleted file mode 100644 index 3d09877..0000000 --- a/tests/test_learning_round4_evidence.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Three defects the round-4 review found, one test each (learning-harness, Req 3.2, 4.1, 9.1). - -Two of them were the same shape as defects already rejected once: a proposer that cannot propose because the call -is wrong and the error is swallowed, and a K-run floor that turned a documented command into a traceback. The -third is the fixed-side half of the range defect — 28 rows resolve a span on the vulnerable side and none on the -fixed side, so every finding there reads as outside the labeled function and the pair scores as silence. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.harness_ext import has_harnessx -from openultrasast.learning.families import load_families -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_vendored -from openultrasast.semantic.extra import has_semantic_extra - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED_OTHER = "from flask import request\n\n\ndef unrelated():\n return 'ok'\n" - - -def _finding(line: int) -> StaticFinding: - return StaticFinding( - finding_id=f"tool-hunter:app.py:{line}", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["tool-hunter", "family:injection"], - ranking_priority=0.0, - ) - - -# --- the meta-agent call ----------------------------------------------------- - - -@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable") -def test_the_meta_agent_is_built_with_a_model_config_not_a_model_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`MetaAgent(inner_model=...)` wants a `ModelConfig`; a string reaches `self.inner_model.agentic(...)` and dies. - - Every other HarnessX composition site in this repository builds the config properly. Here the mistake was - invisible because a `# type: ignore` hid it from mypy and a bare `except Exception` turned it into the - degradation reason `meta_agent_failed`, which reads exactly like the model declining.""" - from openultrasast.learning import proposer as proposer_module - - seen: dict[str, object] = {} - - class _Agent: - def __init__(self, **kwargs: object) -> None: - seen.update(kwargs) - - async def evolve(self, **kwargs: object) -> Path: - seen["evolve"] = kwargs - workspace = kwargs["current_config"] - assert isinstance(workspace, Path) - (workspace / "checklist.md").write_text("- from the meta agent\n") - return workspace - - monkeypatch.setattr(proposer_module, "_meta_agent_class", lambda: _Agent) - monkeypatch.setattr(proposer_module, "_model_config", lambda model, provider="anthropic": f"ModelConfig({model}@{provider})") - (tmp_path / "injection").mkdir() - (tmp_path / "injection" / "checklist.md").write_text("- old\n") - proposal = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m").propose(_facts()) - assert proposal is not None and proposal.change == {"checklist.md": "- from the meta agent\n"} - assert seen["inner_model"] == "ModelConfig(m@anthropic)", "the agent was handed a model name where it wanted a config" - assert seen["allowed_write_roots"] != (tmp_path / "injection",), "the live family directory is never a write root" - - -@pytest.mark.skipif(not has_harnessx(), reason="the composition seam only runs when the extra is importable") -def test_a_meta_agent_built_wrongly_is_not_reported_as_a_declined_proposal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """ "We called the agent wrong" and "the agent had nothing to propose" are different facts.""" - from openultrasast.learning import proposer as proposer_module - - def _boom() -> object: - raise TypeError("inner_model must be a ModelConfig") - - monkeypatch.setattr(proposer_module, "_meta_agent_class", _boom) - (tmp_path / "injection").mkdir() - proposer = proposer_module.HarnessXProposer(configs_dir=tmp_path, model="m") - with pytest.raises(TypeError, match="ModelConfig"): - proposer.propose(_facts()) - - -def _facts(): # type: ignore[no-untyped-def] - from openultrasast.learning.detectors import FamilyConfig - from openultrasast.learning.proposer import FailureFacts - - return FailureFacts( - family="injection", - config=FamilyConfig( - family="injection", - version="0", - prompt="p", - checklist="- old", - tools=("read_file",), - max_steps=1, - max_cost_usd=0.1, - max_chars=100, - ), - ) - - -# --- the pairs command in its documented form -------------------------------- - - -def test_the_pairs_hunter_path_runs_without_a_k_runs_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """The floor is a default, not a trap: `ousast pairs --hunter` must work as documented.""" - from openultrasast.cli import main - - (tmp_path / "a-v.py").write_text(VULN) - (tmp_path / "a-f.py").write_text(FIXED_OTHER) - (tmp_path / "pairs.toml").write_text( - '[[pair]]\nname = "a"\nslice = "vibe-py"\nlanguage = "python"\nvuln = "a-v.py"\nfixed = "a-f.py"\n' - 'relpath = "app.py"\nsplit = "train"\nreview_tier = "seeded"\n\n' - '[[pair.expected]]\ncwe = "CWE-78"\nclass = "x"\npath = "app.py"\nfunction = "run"\nmechanism = "other"\nfamily = "injection"\n' - ) - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - assert main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted"]) == 0 - - -def test_a_k_below_the_floor_names_the_flag_instead_of_raising(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.cli import main - - (tmp_path / "pairs.toml").write_text("") - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - with pytest.raises(SystemExit, match="k-runs"): - main(["pairs", "--catalog", str(tmp_path / "pairs.toml"), "--hunter", "--hunter-model", "scripted", "--k-runs", "1"]) - - -# --- the fixed side of a trap pair ------------------------------------------- - - -def test_a_trap_pair_is_scored_against_the_function_its_fixed_side_really_holds(tmp_path: Path) -> None: - """A Real-Vuln-Benchmark trap takes its fixed side from a different, correctly guarded handler. - - With no span for it, every fixed-side finding counts as outside the labeled function and the pair scores as - silence — the same flattering direction as the range defect, for the 28 rows that resolve one side only.""" - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED_OTHER) - case = PairCase( - name="trap", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection" - ), - ), - min_recall=1.0, - fix_policy="silent", - fix_function="unrelated", - ) - score = score_pair_family( - case, - "injection", - [[_finding(6)]], - [[_finding(5)]], - ranges={"app.py": (("run", 5, 6),)}, - fixed_ranges={"app.py": (("unrelated", 4, 5),)}, - taxonomy=load_families(), - ) - assert score.outcome == "both_flagged", "the finding is inside the fixed side's own handler; that is a leak" - - -def test_a_row_whose_fixed_side_resolves_no_span_says_so_instead_of_scoring_silent(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - (tmp_path / "v.py").write_text(VULN) - (tmp_path / "f.py").write_text(FIXED_OTHER) - case = PairCase( - name="orphan", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family="injection" - ), - ), - min_recall=1.0, - fix_policy="silent", - ) - score = score_pair_family( - case, - "injection", - [[_finding(6)]], - [[]], - ranges={"app.py": (("run", 5, 6),)}, - fixed_ranges={"app.py": (("unrelated", 4, 5),)}, - taxonomy=load_families(), - ) - assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label:fixed" - - -def test_the_trap_rows_carry_their_fixed_side_function_in_the_catalog() -> None: - """`fix_function` lived only in the recipes; the scorer could not see which handler the fixed side holds.""" - cases = {case.name: case for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG))} - assert cases["threatbyte-api-v1-get"].fix_function == "post" - assert cases["threatbyte-api-v1-delete"].fix_function == "post" - assert cases["vampi-users-update-password"].fix_function == "register_user" - assert cases["vampi-books-get-by-title"].fix_function == "", "a row whose two sides are one function names none" - - -@pytest.mark.semantic -@pytest.mark.skipif(not has_semantic_extra(), reason="resolving a span on both sides needs the tree-sitter grammars") -def test_no_vendored_row_resolves_one_side_and_not_the_other() -> None: - """The corpus-wide form of the same defect: 28 rows resolved a vulnerable span and no fixed-side span. - - Every one of them therefore scored the fixed side as silent by construction, whatever the detector said. The - catalog generator now derives `fix_function` from the fixed excerpt itself, so the span exists and the leak is - visible.""" - import tempfile - - from openultrasast.learning.scoring import _spans - from openultrasast.pairs import _materialize_side, _overlay_scan - - orphans = [] - for case in select_vendored(load_pair_catalog(DEFAULT_CATALOG)): - if case.unscorable: - continue - with tempfile.TemporaryDirectory() as scratch: - vuln = _materialize_side(Path(scratch) / "vuln", case, side="vuln") - fixed = _materialize_side(Path(scratch) / "fixed", case, side="fixed") - vuln_ranges, fixed_ranges = _overlay_scan(vuln).ranges, _overlay_scan(fixed).ranges - if _spans(case, vuln_ranges) and not _spans(case, fixed_ranges, function=case.fix_function): - orphans.append(case.name) - assert orphans == [], f"rows whose fixed side can only ever score as silence: {orphans}" diff --git a/tests/test_learning_round_evidence.py b/tests/test_learning_round_evidence.py deleted file mode 100644 index 5775f25..0000000 --- a/tests/test_learning_round_evidence.py +++ /dev/null @@ -1,428 +0,0 @@ -"""What a round must prove before it is kept (learning-harness review round 3, Req 4.1, 5.2, 9.5, 9.6, 9.8). - -The round loop was accepting and rejecting on numbers that did not mean what their names said: the -cross-family sweep counted every failing pair rather than the pairs this change broke, unscorable rows -voted against a round they cannot judge, cost was metered once after everything had been paid for, and a -stage that raised left no record at all. Each test here fails on exactly one of those. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.detectors import FamilyConfig, write_default_configs -from openultrasast.learning.families import load_families -from openultrasast.learning.journal import Archive, LearningJournal -from openultrasast.learning.proposer import Proposal, ScriptedProposer -from openultrasast.learning.rounds import NoiseFloor, run_learning_round -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - unscorable=unscorable, - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _proposal(text: str = "- ask about quoting\n") -> Proposal: - return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("train-a",)) - - -@pytest.fixture -def world(tmp_path: Path): # type: ignore[no-untyped-def] - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - (configs / "injection" / "checklist.md").write_text("- old\n") - return { - "configs": configs, - "cases": [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")], - "journal": LearningJournal(tmp_path / "journal.jsonl"), - "archive": Archive(tmp_path / "archive.jsonl"), - "out": tmp_path / "out", - "floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)}, - "root": tmp_path, - } - - -def _run(world, proposal, build, **kwargs): # type: ignore[no-untyped-def] - return run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([proposal] if proposal else []), - scan_factory=build, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - **kwargs, - ) - - -def _injection_improves(config: FamilyConfig): # type: ignore[no-untyped-def] - """Injection detects only once the proposed checklist lands; every other family finds nothing, ever.""" - - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln" or config.family != "injection": - return [] - return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else [] - - return scan - - -# --- the sweep --------------------------------------------------------------- - - -def test_a_family_that_was_already_failing_does_not_reject_the_round(world) -> None: # type: ignore[no-untyped-def] - """Req 9.5 counts *negative flips*, not absolute failures. - - On the real corpus most pairs fail at round zero, so counting failures rejects every round for damage it - did not do — and the harness then reports that it cannot improve.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 0, "access control failed before and after; this change flipped nothing" - assert record.outcome == "accepted" and record.reason == "improved" - - -def test_a_family_this_change_really_broke_still_rejects_the_round(world) -> None: # type: ignore[no-untyped-def] - """The other arm: a pair that was correct before and is not after is a flip, and one is over the budget.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True - return [_finding("injection")] - return [] - - return scan - - record = _run(world, _proposal(), build, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 1 - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_an_unscorable_row_never_votes_in_the_sweep(world) -> None: # type: ignore[no-untyped-def] - """Req 4.1: an unscorable row leaves every denominator, and it certainly does not reject a round.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control", unscorable="identical_twin")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=0, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - record = _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - assert record.sweep["access_control"] == 0 and record.outcome == "accepted" - - -def test_attribution_names_the_pairs_this_change_flipped_and_no_others(world) -> None: # type: ignore[no-untyped-def] - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - _run(world, _proposal(), _injection_improves, sweep_families=("access_control",)) - attribution = json.loads((world["out"] / "rounds" / "1" / "attribution.json").read_text()) - assert attribution["flipped_predicted"] == ["hold-a", "train-a"] - assert attribution["flipped_unpredicted"] == [], "a pair that was already failing was never flipped by this round" - - -# --- cost, exceptions, the archive ------------------------------------------ - - -def test_the_cap_stops_a_round_before_it_pays_for_the_next_stage(world) -> None: # type: ignore[no-untyped-def] - """Req 9.8 says stop, revert and record. Metering once, after every stage, can only ever record.""" - stages: list[str] = [] - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - stages.append(f"{config.family}:{root.name}") - return [] - - return scan - - spend = {"usd": 0.0} - - def spent() -> float: - spend["usd"] += 1.0 - return spend["usd"] - - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - record = _run(world, _proposal(), build, cost_cap_usd=0.5, spent_usd=spent, sweep_families=("access_control",)) - assert record.outcome == "reverted_cost" and record.reason == "cost_cap_exceeded" - assert "access_control:vuln" not in stages, "the sweep ran after the cap was already crossed" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_stage_that_raises_is_reverted_journalled_and_re_raised(world) -> None: # type: ignore[no-untyped-def] - """An endpoint failure mid-round is an outcome. Losing the round record loses the money it already spent.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if config.checklist.strip() == "- ask about quoting": - raise RuntimeError("endpoint said no") - return [] - - return scan - - with pytest.raises(RuntimeError, match="endpoint said no"): - _run(world, _proposal(), build) - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()] - assert outcomes == [("reverted", "stage_failed: RuntimeError")] # the exception type is part of the record - assert (world["out"] / "rounds" / "1" / "scores.json").is_file() - - -def test_a_rejected_version_is_archived_so_a_pair_it_alone_wins_is_not_lost(world) -> None: # type: ignore[no-untyped-def] - """Req 9.7: per-pair winners, or the loop collapses onto one lineage and forgets what a rejected try could do. - - This round wins both of its own pairs and is still rejected, because it broke another family. Version 1 is the - only configuration that ever got those two pairs right; dropping it because the round as a whole lost is how a - search collapses onto one lineage.""" - world["cases"].append(_case(world["root"], "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True - return [_finding("injection")] - return [] - - return scan - - record = _run(world, _proposal(), build, sweep_families=("access_control",)) - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert world["archive"].winners("injection") == {"train-a": "1", "hold-a": "1"} - assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text(), "a rejected round is not versioned" - - -# --- the holdout refusal ----------------------------------------------------- - - -def test_a_proposal_that_names_a_holdout_pair_is_refused_by_name(world) -> None: # type: ignore[no-untyped-def] - """Req 5.2: a candidate that touches a holdout pair is refused and the refusal names the pairs.""" - record = _run( - world, - Proposal( - hypothesis="memorize the answer", - lever="memory", - change={"hard_negatives.jsonl": json.dumps({"pair": "hold-a", "note": "always flag"}) + "\n"}, - predicted_affected=("hold-a",), - ), - _injection_improves, - ) - assert record.outcome == "refused_holdout" - assert "hold-a" in record.reason - assert (world["configs"] / "injection" / "hard_negatives.jsonl").read_text() == "" - - -def test_a_proposal_that_only_names_train_pairs_is_not_refused(world) -> None: # type: ignore[no-untyped-def] - record = _run( - world, - Proposal( - hypothesis="remember this shape", - lever="memory", - change={"hard_negatives.jsonl": json.dumps({"pair": "train-a", "note": "shape"}) + "\n"}, - predicted_affected=("train-a",), - ), - _injection_improves, - ) - assert record.outcome != "refused_holdout" - - -# --- round zero -------------------------------------------------------------- - - -def test_round_zero_measures_each_family_with_its_own_detector(tmp_path: Path) -> None: - """Req 8.2. One detector for every family means the generalist scores them all, and abstention earns no credit - (`related(X, "unknown")` is lateral), so every family but `unknown` baselines at a structural zero.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - seen: list[str] = [] - - def factory(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - seen.append(config.family) - return [_finding(config.family)] if root.name == "vuln" else [] - - return scan - - cases = [_case(tmp_path, "inj", split="train"), _case(tmp_path, "acl", split="train", family="access_control")] - report = run_baseline( - cases, - taxonomy=load_families(), - configs_dir=configs, - scan_factory=factory, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert set(seen) == {"injection", "access_control"}, "each family must be measured with its own configuration" - assert report.metrics["injection"]["recall"] == 1.0 and report.metrics["access_control"]["recall"] == 1.0 - - -def test_round_zero_reports_each_slice_separately(tmp_path: Path) -> None: - """Req 8.4: vibe-py and agent-vfc are different worlds; one pooled number hides which one moved.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - web = _case(tmp_path, "web", split="train") - agent = _case(tmp_path, "agent", split="train") - object.__setattr__(agent, "slice", "agent-vfc") - - def factory(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - return [_finding(config.family)] if root.name == "vuln" else [] - - return scan - - report = run_baseline( - [web, agent], - taxonomy=load_families(), - configs_dir=configs, - scan_factory=factory, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert set(report.per_slice) == {"vibe-py/injection", "agent-vfc/injection"} - payload = json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text()) - assert set(payload["per_slice"]) == {"vibe-py/injection", "agent-vfc/injection"} - - -def test_round_zero_refuses_fewer_than_three_runs(tmp_path: Path) -> None: - """The floor exists because a single run misses most real changes; a baseline measured at K=1 is a noise floor - of zero and every later round then inherits a budget of nothing.""" - from openultrasast.learning.rounds import run_baseline - - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - with pytest.raises(ValueError, match="three runs"): - run_baseline( - [_case(tmp_path, "inj", split="train")], - taxonomy=load_families(), - configs_dir=configs, - scan_factory=lambda config: lambda root: [], - model="m", - out_dir=tmp_path / "out", - k_runs=1, - ) - - -def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: # type: ignore[no-untyped-def] - """`no_proposal` covers two different worlds: a proposer with nothing to add, and one that could not start. - - Measured: the meta-agent round journalled `no_proposal` while the real cause was an authentication failure in a - provider seam pointing at the wrong vendor. A round that failed for a fixable configuration reason must not - look identical to a model that considered the facts and declined.""" - - class Mute: - reason = "meta_agent_failed: RuntimeError" - - def propose(self, facts): # type: ignore[no-untyped-def] - del facts - return None - - from openultrasast.learning.rounds import run_learning_round - - record = run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=Mute(), - scan_factory=_injection_improves, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - ) - assert record.outcome == "rejected" - assert record.reason == "no_proposal: meta_agent_failed: RuntimeError" - - -def test_a_round_that_dies_before_it_applies_anything_is_still_journalled(world) -> None: # type: ignore[no-untyped-def] - """Measured: round 5 died on a network read during its *first* stage and left no record at all. - - The exception path was wrapped around the stages after the proposal is applied, because those are the ones that - have something to revert. The ones before it had already spent money.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - raise ConnectionError("the body stopped arriving") - - return scan - - with pytest.raises(ConnectionError): - _run(world, _proposal(), build) - outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()] - assert outcomes == [("reverted", "stage_failed: ConnectionError")] - assert (world["out"] / "rounds" / "1").is_dir() - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" diff --git a/tests/test_learning_round_loop.py b/tests/test_learning_round_loop.py deleted file mode 100644 index c3c8892..0000000 --- a/tests/test_learning_round_loop.py +++ /dev/null @@ -1,230 +0,0 @@ -"""learning-harness task 3.7: one evolve round, end to end (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.detectors import FamilyConfig, write_default_configs -from openultrasast.learning.families import load_families -from openultrasast.learning.journal import Archive, LearningJournal -from openultrasast.learning.proposer import Proposal, ScriptedProposer -from openultrasast.learning.rounds import NoiseFloor -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, split: str, family: str = "injection") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _factory(finds: dict[str, bool]): # type: ignore[no-untyped-def] - """A detector whose behaviour depends on the configuration version it was built from.""" - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - return [_finding(config.family)] if finds.get(config.checklist.strip(), False) else [] - - return scan - - return build - - -def _proposal(text: str) -> Proposal: - return Proposal(hypothesis="ask about quoting", lever="checklist", change={"checklist.md": text}, predicted_affected=("injection",)) - - -@pytest.fixture -def world(tmp_path: Path): # type: ignore[no-untyped-def] - configs = tmp_path / "configs" - write_default_configs(configs, load_families(), prompt="p", version="0") - (configs / "injection" / "checklist.md").write_text("- old\n") - cases = [_case(tmp_path, "train-a", split="train"), _case(tmp_path, "hold-a", split="holdout")] - return { - "configs": configs, - "cases": cases, - "journal": LearningJournal(tmp_path / "journal.jsonl"), - "archive": Archive(tmp_path / "archive.jsonl"), - "out": tmp_path / "out", - "floors": {"injection": NoiseFloor(family="injection", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0)}, - } - - -def _run(world, proposal: Proposal | None, finds: dict[str, bool], **kwargs): # type: ignore[no-untyped-def] - from openultrasast.learning.rounds import run_learning_round - - return run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([proposal] if proposal else []), - scan_factory=_factory(finds), - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - **kwargs, - ) - - -def test_a_change_that_helps_is_accepted_versioned_and_journalled(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}) - assert record.outcome == "accepted" and record.family == "injection" - assert record.target_train_delta > 0 and record.target_holdout_delta > 0 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- ask about quoting\n" - assert 'version = "1"' in (world["configs"] / "injection" / "family.toml").read_text() - assert [item.outcome for item in world["journal"].rounds()] == ["accepted"] - assert world["archive"].winners("injection") - - -def test_a_change_that_helps_nothing_is_rejected_and_reverted(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- useless\n"), {}) - assert record.outcome == "rejected" and record.reason == "tie" - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - assert 'version = "0"' in (world["configs"] / "injection" / "family.toml").read_text() - assert world["journal"].rejected_buffer("injection") == ("ask about quoting",) - - -def test_a_round_is_rejected_when_the_sweep_finds_another_family_below_its_floor(world) -> None: # type: ignore[no-untyped-def] - """The sweep counts pairs this change turned from right to wrong in another family (Req 9.5). - - It used to count every failing pair, which rejected a round for damage it did not do; the pair below is right - before the change and wrong after it, which is the only thing that may reject a round on another family's - behalf. The other arm — a family that was already failing must not reject — lives in - `tests/test_learning_round_evidence.py`. - """ - world["cases"].append(_case(world["configs"].parent, "other-hold", split="holdout", family="access_control")) - world["floors"]["access_control"] = NoiseFloor( - family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0 - ) - landed = {"yes": False} - - def build(config: FamilyConfig): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if root.name != "vuln": - return [] - if config.family == "access_control": - return [] if landed["yes"] else [_finding("access_control")] - if config.checklist.strip() == "- ask about quoting": - landed["yes"] = True # the injection change lands, and access control stops finding its pair - return [_finding("injection")] - return [] - - return scan - - from openultrasast.learning.rounds import run_learning_round - - record = run_learning_round( - world["cases"], - family="injection", - taxonomy=load_families(), - configs_dir=world["configs"], - proposer=ScriptedProposer([_proposal("- ask about quoting\n")]), - scan_factory=build, - model="m", - journal=world["journal"], - archive=world["archive"], - floors=world["floors"], - out_dir=world["out"], - sweep_families=("access_control",), - ) - assert record.outcome == "rejected" and record.reason == "collateral_regression" - assert record.sweep["access_control"] == 1 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_round_over_its_cost_cap_is_reverted_and_says_so(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}, cost_cap_usd=0.0, spent_usd=lambda: 1.0) - assert record.outcome == "reverted_cost" and record.cost_usd == 1.0 - assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n" - - -def test_a_proposal_that_reaches_beyond_its_family_never_runs(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, Proposal(hypothesis="h", lever="checklist", change={"../escape.md": "x"}), {}) - assert record.outcome == "rejected" and "outside" in record.reason # the holdout refusal has its own test - assert not (world["configs"].parent / "escape.md").exists() - - -def test_a_round_with_no_proposal_is_recorded_rather_than_skipped(world) -> None: # type: ignore[no-untyped-def] - record = _run(world, None, {}) - assert record.outcome == "rejected" and record.reason == "no_proposal" - assert [item.round for item in world["journal"].rounds()] == [1] - - -def test_the_round_directory_holds_what_a_replay_needs(world) -> None: # type: ignore[no-untyped-def] - record = _run( - world, - _proposal("- ask about quoting\n"), - {"- ask about quoting": True}, - trajectories=lambda: [{"pair": "train-a", "prompt": "api_key = " + "'" + "B" * 24 + "'"}], - ) - directory = world["out"] / "rounds" / "1" - assert (directory / "proposal.json").is_file() and (directory / "scores.json").is_file() - attribution = json.loads((directory / "attribution.json").read_text()) - assert attribution["flipped_predicted"] == ["hold-a", "train-a"] # the round is deterministic; one exact answer - assert attribution["flipped_unpredicted"] == [] - trajectories = (directory / "trajectories.jsonl").read_text() - assert "REDACTED" in trajectories and "B" * 24 not in trajectories - assert record.attribution.precision == 1.0 - - -def test_a_round_number_is_never_reused(world) -> None: # type: ignore[no-untyped-def] - from openultrasast.learning.journal import JournalError - - _run(world, _proposal("- ask about quoting\n"), {"- ask about quoting": True}) - (world["out"] / "rounds" / "2").mkdir(parents=True) - with pytest.raises(JournalError, match="2"): - _run(world, _proposal("- another\n"), {}) - - -def test_every_stage_scores_at_least_three_runs_per_pair(world) -> None: # type: ignore[no-untyped-def] - with pytest.raises(ValueError, match="three"): - _run(world, _proposal("- x\n"), {}, k_runs=2) diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py deleted file mode 100644 index 04e554a..0000000 --- a/tests/test_learning_rounds.py +++ /dev/null @@ -1,377 +0,0 @@ -"""learning-harness task 3.3: round zero and the per-family noise floor (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase - -VULN = "from flask import request\nimport os\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "from flask import request\nimport subprocess\n\n\ndef run():\n subprocess.run(['echo', 'x'], check=False)\n" - - -def _case(tmp_path: Path, name: str, *, family: str = "injection", slice_name: str = "vibe-py", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice=slice_name, - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - review_tier="seeded", - ) - - -def _finding(family: str) -> StaticFinding: - return StaticFinding( - finding_id="detector:app.py:6", - path="app.py", - title="t", - severity="high", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=6, - function_name="run", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@0"], - ranking_priority=0.0, - ) - - -def _is_vulnerable_side(root: Path) -> bool: - """The materialized side, by directory name only. - - An earlier version also matched "-v" anywhere in the path, which the random temporary directory name - sometimes contained, so the detector "found" the bug on the fixed side too and the pair flipped. A - flaky harness is indistinguishable from a flaky detector, which is precisely what a noise floor must - not measure. - """ - return root.name == "vuln" - - -def _steady(root: Path) -> list[StaticFinding]: - return [_finding("injection")] if _is_vulnerable_side(root) else [] - - -def _always(scan): # type: ignore[no-untyped-def] - """Round zero asks for a detector per family; these tests deliberately give every family the same one so the - noise-floor arithmetic is the only thing under test.""" - - def build(_config): # type: ignore[no-untyped-def] - return scan - - return build - - -def _flaky(counter: list[int]): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if not _is_vulnerable_side(root): - return [] - counter.append(1) - return [_finding("injection")] if len(counter) % 2 else [] - - return scan - - -def test_round_zero_writes_a_configuration_for_every_family(tmp_path: Path) -> None: - from openultrasast.learning.detectors import load_family_configs - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - report = run_baseline( - [_case(tmp_path, "a")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - ) - configs = load_family_configs(tmp_path / "configs", taxonomy) - assert set(configs) == {family.id for family in taxonomy.families} - assert all(config.version == "0" for config in configs.values()) - assert report.model == "m" and report.k_runs == 5 - - -def test_a_steady_detector_has_a_zero_noise_floor_and_a_flaky_one_does_not(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - steady = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "c1", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "o1", - ) - floor = steady.floors["injection"] - assert floor.k_runs == 5 and floor.negative_flip_rate == 0.0 and floor.budget_flips == 0 and floor.gates is True - flaky = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "c2", - scan_factory=_always(_flaky([])), - model="m", - out_dir=tmp_path / "o2", - ) - assert flaky.floors["injection"].negative_flip_rate > 0.0 and flaky.floors["injection"].budget_flips >= 1 - - -def test_the_memory_family_is_measured_but_never_gates(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - report = run_baseline( - [_case(tmp_path, "m", family="memory", slice_name="vfc")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - slices=("vfc",), - ) - assert "memory" in report.floors and report.floors["memory"].gates is False - assert report.metrics["memory"]["scorable"] == 1 - - -def test_the_default_selection_is_the_web_slices(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - cases = [_case(tmp_path, "web", slice_name="vibe-py"), _case(tmp_path, "c", family="memory", slice_name="vfc")] - report = run_baseline( - cases, taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(_steady), model="m", out_dir=tmp_path / "out" - ) - assert set(report.metrics) == {"injection"} and "memory" not in report.floors - - -def test_artifacts_are_keyed_by_model_so_another_model_can_be_compared(tmp_path: Path) -> None: - from openultrasast.learning.rounds import compare_baselines, run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - for index, model in enumerate(("deepseek-v4-flash", "deepseek-v4-pro")): - run_baseline( - [_case(tmp_path, f"a{index}")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model=model, - out_dir=out, - ) - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-pro"] - floors = json.loads((out / "baseline" / "deepseek-v4-flash" / "noise-floors.json").read_text()) - assert floors["model"] == "deepseek-v4-flash" and floors["floors"]["injection"]["k_runs"] == 5 - report = json.loads((out / "baseline" / "deepseek-v4-flash" / "report.json").read_text()) - assert report["metrics"]["injection"]["recall"] == 1.0 and report["taxonomy_version"] - table = compare_baselines(out) - assert set(table) == {"deepseek-v4-flash", "deepseek-v4-pro"} - assert table["deepseek-v4-pro"]["injection"]["youden"] == 1.0 - # a second baseline never rewrites a configuration - assert (tmp_path / "configs" / "injection" / "family.toml").read_text().count("version") == 1 - - -def test_an_unscorable_pair_is_listed_and_never_run(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - calls: list[Path] = [] - - def counting(root: Path) -> list[StaticFinding]: - calls.append(root) - return _steady(root) - - taxonomy = load_families() - case = _case(tmp_path, "twin") - twin = PairCase(**{**case.__dict__, "unscorable": "identical_twin"}) - report = run_baseline( - [twin], taxonomy=taxonomy, configs_dir=tmp_path / "configs", scan_factory=_always(counting), model="m", out_dir=tmp_path / "out" - ) - assert report.metrics["injection"]["scorable"] == 0 - assert report.metrics["injection"]["unscorable"] == {"identical_twin": 1} - assert calls == [] - - -def test_round_zero_records_what_it_spent(tmp_path: Path) -> None: - """Publishing puts a cost beside every number, so the baseline has to know what it cost.""" - from openultrasast.learning.rounds import run_baseline - - report = run_baseline( - [_case(tmp_path, "a")], - taxonomy=load_families(), - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=tmp_path / "out", - spent_usd=lambda: 1.25, - ) - assert report.cost_usd == 1.25 - assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25 - - -def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_run(tmp_path: Path) -> None: - """One transient error must not cost a run that has already paid for hundreds of calls. - - Round zero is the most expensive thing this harness does. A provider hiccup two thirds of the way through used - to raise straight out of `run_baseline`, and everything measured so far went with it — so the honest response - to a flaky endpoint was to run the whole thing again and hope.""" - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - seen: list[str] = [] - - def build(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - seen.append(root.name) - if len(seen) == 3: - raise TimeoutError("the endpoint hung up") - return _steady(root) - - return scan - - report = run_baseline( - [_case(tmp_path, "a"), _case(tmp_path, "b")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=build, - model="m", - out_dir=tmp_path / "out", - k_runs=3, - ) - assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured" - assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1} - assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file() - - -def test_two_slices_on_one_model_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: - """Artifacts are keyed by what identifies the run, and the slice selection is part of that. - - Req 8.4 asks for the web families and, separately, the memory-safety family over the vfc slice. Keyed by model - alone, the second run silently replaces the first and the published table shows one of them as if it were - everything.""" - from openultrasast.learning.rounds import compare_baselines, run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - web = run_baseline( - [_case(tmp_path, "w", slice_name="vibe-py")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - ) - memory = run_baseline( - [_case(tmp_path, "m", family="memory", slice_name="vfc")], - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - slices=("vfc",), - ) - assert set(web.metrics) == {"injection"} and set(memory.metrics) == {"memory"} - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-flash-vfc"], written - table = compare_baselines(out) - assert set(table) == {"deepseek-v4-flash", "deepseek-v4-flash (vfc)"} - assert "memory" in table["deepseek-v4-flash (vfc)"] and "injection" in table["deepseek-v4-flash"] - - -def test_round_zero_keeps_every_pair_it_has_already_paid_for(tmp_path: Path) -> None: - """A four-hour run that is interrupted must not lose four hours of paid calls. - - Measured the hard way: the 176-pair vfc baseline hit its timeout and wrote nothing, because scores were - accumulated in memory and persisted once at the end. An exception was already handled; a timeout, a kill and a - machine going away were not, and they cost the same money.""" - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - scored: list[str] = [] - - def build(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - scored.append(root.name) - # K=3 means three vulnerable-side scans per pair, so this gives up part way through the third pair - if len([item for item in scored if item == "vuln"]) > 6: - raise KeyboardInterrupt("the operator gave up") - return _steady(root) - - return scan - - cases = [_case(tmp_path, name) for name in ("a", "b", "c", "d")] - with pytest.raises(KeyboardInterrupt): - run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=build, - model="m", - out_dir=out, - k_runs=3, - ) - checkpoint = out / "baseline" / "m" / "scores.jsonl" - assert checkpoint.is_file(), "the pairs already measured were not written down" - rows = [json.loads(line) for line in checkpoint.read_text().splitlines() if line.strip()] - assert [row["pair"] for row in rows] == ["a", "b"] - assert all(row["outcome"] == "pair_correct" for row in rows) - - -def test_a_resumed_baseline_does_not_pay_twice(tmp_path: Path) -> None: - from openultrasast.learning.rounds import run_baseline - - taxonomy = load_families() - out = tmp_path / "out" - cases = [_case(tmp_path, name) for name in ("a", "b")] - run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="m", - out_dir=out, - k_runs=3, - ) - calls: list[str] = [] - - def counting(_config): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - calls.append(root.name) - return _steady(root) - - return scan - - report = run_baseline( - cases, - taxonomy=taxonomy, - configs_dir=tmp_path / "configs", - scan_factory=counting, - model="m", - out_dir=out, - k_runs=3, - resume=True, - ) - assert calls == [], "every pair was already measured and paid for" - assert report.metrics["injection"]["scorable"] == 2 diff --git a/tests/test_learning_scan.py b/tests/test_learning_scan.py deleted file mode 100644 index fe7b065..0000000 --- a/tests/test_learning_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -"""learning-harness task 3.4: family detectors, verifiers and tags in a scan (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n" -) - - -def _repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, toml: str, *, configs: bool = True) -> Path: - from openultrasast.learning.detectors import write_default_configs - from openultrasast.learning.families import load_families - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(APP) - (repo / "openultrasast.toml").write_text(toml) - if configs: # round zero writes these; a scan reads them and never creates them - write_default_configs(repo / ".openultrasast/learning/configs", load_families(), prompt="You hunt.", version="0") - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - monkeypatch.setenv("OPENULTRASAST_HUNTER_CLIENT", "scripted") - return repo - - -def _run_dir(repo: Path) -> Path: - return sorted((repo / ".runs").iterdir())[-1] - - -def _scan(repo: Path) -> tuple[list[dict[str, object]], dict[str, object]]: - assert main(["scan", str(repo), "--mode", "standard", "--config", str(repo / "openultrasast.toml")]) == 0 - run_dir = _run_dir(repo) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - manifest = json.loads((run_dir / "manifest.json").read_text()) - return findings, manifest - - -def test_a_standard_scan_tags_findings_with_family_detector_and_verifier(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - findings, manifest = _scan(repo) - detected = [item for item in findings if any(str(tag).startswith("family:") for tag in item["tags"])] - assert detected, "the family detectors ran and their findings are in the report" - tags = set(detected[0]["tags"]) - assert any(tag.startswith("family:") for tag in tags) - assert any(tag.startswith("detector:") for tag in tags) - assert any(tag.startswith("verifier:") for tag in tags) - assert detected[0]["evidence_level"] == "suspicion" # nothing rises without a verifier saying so - block = manifest["learning"] - assert block["taxonomy_version"] and block["round"] is None - assert block["families"] and isinstance(block["classifier"], dict) - assert block["classifier"]["hierarchical_credit"] is False - - -def test_a_missing_family_directory_is_a_recorded_reason_not_a_crash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo( - tmp_path, - monkeypatch, - '[learning]\nenabled = true\nconfigs_dir = "nowhere/configs"\n\n[models]\nhunter = "scripted-model"\n', - configs=False, - ) - findings, manifest = _scan(repo) - assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) - reasons = {str(item.get("reason")) for item in manifest["degradations"]} - assert "learning_configs_unavailable" in reasons - assert "learning" not in manifest or manifest["learning"]["families"] == {} - - -def test_the_feature_is_off_by_configuration_and_leaves_no_trace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, "[learning]\nenabled = false\n", configs=False) - findings, manifest = _scan(repo) - assert not any(str(tag).startswith("family:") for item in findings for tag in item["tags"]) - assert "learning" not in manifest - - -def test_family_findings_never_become_sandbox_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_RUNNER", "fake") - assert main(["scan", str(repo), "--mode", "deep", "--config", str(repo / "openultrasast.toml")]) == 0 - run_dir = _run_dir(repo) - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - payload = json.loads((run_dir / "findings.json").read_text())["findings"] - family_findings = {item["finding_id"] for item in payload if any(str(tag).startswith("family:") for tag in item["tags"])} - assert family_findings, "the fixture must actually produce family findings, or this proves nothing" - # A family detector's claim is a suspicion until a verifier speaks; handing it to the sandbox would let an - # unverified model claim drive execution (Req 7.1). - candidates = {str(item.get("finding_id") or item.get("id") or "") for item in verdicts} - assert candidates.isdisjoint(family_findings) - - -def test_the_sarif_report_carries_the_family_properties(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = _repo(tmp_path, monkeypatch, '[learning]\nenabled = true\n\n[models]\nhunter = "scripted-model"\n') - _scan(repo) - sarif = json.loads((_run_dir(repo) / "report.sarif").read_text()) - props = [result["properties"] for result in sarif["runs"][0]["results"]] - family_props = [item for item in props if item.get("family")] - assert family_props and all(item.get("detector") and item.get("verifier") for item in family_props) diff --git a/tests/test_learning_scoring.py b/tests/test_learning_scoring.py deleted file mode 100644 index 3c679e6..0000000 --- a/tests/test_learning_scoring.py +++ /dev/null @@ -1,185 +0,0 @@ -"""learning-harness task 2.3: per-pair family scoring (offline).""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.pairs import PairCase - -RANGES = {"app.py": (("leaky", 10, 13), ("guarded", 4, 7))} - - -def _finding(family: str, line: int, path: str = "app.py") -> StaticFinding: - return StaticFinding( - finding_id=f"detector:{path}:{line}", - path=path, - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}", "detector:x@1"], - ranking_priority=0.0, - ) - - -def _case(tmp_path: Path, *, family: str = "access_control", unscorable: str | None = None) -> PairCase: - (tmp_path / "v.py").write_text("x = 1\n") - (tmp_path / "f.py").write_text("x = 2\n") - return PairCase( - name="p", - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / "v.py", - fixed_file=tmp_path / "f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function="leaky", mechanism="other", family=family - ), - ), - min_recall=1.0, - fix_policy="silent", - unscorable=unscorable, - ) - - -def test_detection_needs_the_labeled_family_inside_the_labeled_function(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - hit = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert hit.outcome == "pair_correct" and hit.runs == ("pair_correct",) - outside = score_pair_family(case, "access_control", [[_finding("access_control", 5)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert outside.outcome == "both_silent" # the right family in the wrong function is not a detection - other_file = score_pair_family( - case, "access_control", [[_finding("access_control", 12, "other.py")]], [[]], ranges=RANGES, taxonomy=taxonomy - ) - assert other_file.outcome == "both_silent" - wrong_family = score_pair_family(case, "access_control", [[_finding("injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert wrong_family.outcome == "both_silent" and wrong_family.other_findings_vuln == 1 - - -def test_a_fabricated_family_never_counts_and_is_recorded(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[_finding("sql_injection", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "both_silent" and score.fabricated_vuln == 1 - assert score.other_findings_vuln == 0 # a fabricated id is not another family, it is noise - - -def test_silence_ignores_findings_of_other_families_and_never_calls_them_leaks(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - quiet = score_pair_family( - case, - "access_control", - [[_finding("access_control", 12)]], - [[_finding("injection", 12), _finding("access_control", 5)]], - ranges=RANGES, - taxonomy=taxonomy, - ) - assert quiet.outcome == "pair_correct" and quiet.other_findings_fixed == 2 - leaked = score_pair_family( - case, "access_control", [[_finding("access_control", 12)]], [[_finding("access_control", 13)]], ranges=RANGES, taxonomy=taxonomy - ) - assert leaked.outcome == "both_flagged" - reversed_pair = score_pair_family(case, "access_control", [[]], [[_finding("access_control", 12)]], ranges=RANGES, taxonomy=taxonomy) - assert reversed_pair.outcome == "reversed" - - -def test_the_outcome_of_several_runs_is_their_majority(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path) - hit = [_finding("access_control", 12)] - score = score_pair_family(case, "access_control", [hit, [], hit], [[], [], []], ranges=RANGES, taxonomy=taxonomy) - assert score.runs == ("pair_correct", "both_silent", "pair_correct") and score.outcome == "pair_correct" - flaky = score_pair_family(case, "access_control", [hit, [], []], [[], [], []], ranges=RANGES, taxonomy=taxonomy) - assert flaky.outcome == "both_silent" and flaky.flips == 1 # a run that disagrees with the majority is a flip - - -def test_an_unscorable_pair_is_never_run_and_keeps_its_reason(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - case = _case(tmp_path, unscorable="identical_twin") - score = score_pair_family(case, "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "unscorable" and score.unscorable_reason == "identical_twin" and score.runs == () - - -def test_a_pair_with_no_parsed_function_range_is_unscorable_not_a_miss(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy) - assert score.outcome == "unscorable" and score.unscorable_reason == "unresolved_label" - - -def test_scoring_never_reads_the_text_of_a_finding(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - shouting = _finding("injection", 12) - loud = StaticFinding(**{**shouting.__dict__, "title": "CWE-639 broken object level authorization", "rationale": "IDOR in leaky"}) - score = score_pair_family(_case(tmp_path), "access_control", [[loud]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "both_silent" # the words say access control; the family tag says injection, and only that counts - - -def test_the_rung_is_the_best_verifier_outcome_among_the_detections(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - corroborated = _finding("access_control", 12) - corroborated.tags.append("verifier:static_corroboration") - score = score_pair_family(_case(tmp_path), "access_control", [[corroborated]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert score.outcome == "pair_correct" and score.rung == "static_corroboration" - plain = score_pair_family(_case(tmp_path), "access_control", [[_finding("access_control", 12)]], [[]], ranges=RANGES, taxonomy=taxonomy) - assert plain.rung == "suspicion" - - -def test_the_unscorable_reason_tells_a_tooling_gap_from_a_corpus_defect(tmp_path: Path) -> None: - """Req 4.1: an unparsable row is a tooling gap; a row whose labeled function no range names is a corpus defect.""" - from openultrasast.learning.scoring import unscorable_reason - - case = _case(tmp_path) - assert unscorable_reason(case, parse_ok=True, ranges=RANGES) is None - assert unscorable_reason(case, parse_ok=False, ranges={}) == "unsupported_language" - assert unscorable_reason(case, parse_ok=False, ranges=RANGES) == "unsupported_language" # the parse decides, not the ranges - assert unscorable_reason(case, parse_ok=True, ranges={}) == "unresolved_label" - assert unscorable_reason(_case(tmp_path, unscorable="identical_twin"), parse_ok=True, ranges=RANGES) == "identical_twin" - twin = _case(tmp_path, unscorable="known_limit:java hash") - assert unscorable_reason(twin, parse_ok=False, ranges={}) == "known_limit:java hash" # a declared limit wins - - -def test_a_row_whose_registration_context_is_missing_is_named_as_such(tmp_path: Path) -> None: - """Req 4.1: an access-control row whose handler arrived without its registration cannot be scored for that reason.""" - from openultrasast.learning.scoring import unscorable_reason - - case = _case(tmp_path, family="access_control") - assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=("leaky",)) is None - assert unscorable_reason(case, parse_ok=True, ranges=RANGES, entry_points=()) == "missing_context" - # only families whose bug shape needs the registration are judged on it - assert unscorable_reason(_case(tmp_path, family="injection"), parse_ok=True, ranges=RANGES, entry_points=()) is None - - -def test_an_unparsable_side_is_scored_unscorable_rather_than_missed(tmp_path: Path) -> None: - from openultrasast.learning.scoring import score_pair_family - - taxonomy = load_families() - score = score_pair_family(_case(tmp_path), "access_control", [[]], [[]], ranges={}, taxonomy=taxonomy, parse_ok=False) - assert score.outcome == "unscorable" and score.unscorable_reason == "unsupported_language" diff --git a/tests/test_learning_split.py b/tests/test_learning_split.py deleted file mode 100644 index c365340..0000000 --- a/tests/test_learning_split.py +++ /dev/null @@ -1,148 +0,0 @@ -"""learning-harness task 3.1: one teacher rule across every learning path (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import DEFAULT_CATALOG, PairCase, load_pair_catalog, select_slice, select_vendored - -VULN = "import os\nfrom flask import request\n\n\ndef run():\n os.system(request.args.get('cmd'))\n" -FIXED = "import subprocess\nfrom flask import request\n\n\ndef run():\n subprocess.run(['echo'], check=False)\n" - - -def _teaching_case(tmp_path: Path, name: str, *, split: str = "train") -> PairCase: - """A pair that really carries a mechanism shape. - - The trivial `x = 1` fixture below teaches nothing whatever the split says, so a teacher-rule assertion made - against it passes with the rule deleted. This one is the difference between measuring the rule and measuring - an empty corpus.""" - (tmp_path / f"{name}-v.py").write_text(VULN) - (tmp_path / f"{name}-f.py").write_text(FIXED) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="run", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier="seeded", - ) - - -def _case(tmp_path: Path, name: str, *, split: str = "train", tier: str = "seeded", unscorable: str | None = None) -> PairCase: - (tmp_path / f"{name}.py").write_text(f"# {name}\nx = 1\n") - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}.py", - fixed_file=tmp_path / f"{name}.py", - relpath="app.py", - expected=(ExpectedFinding(cwe="CWE-89", vulnerability_class="x", path="app.py", evidence="", function="f", mechanism="other"),), - min_recall=1.0, - fix_policy="silent", - split=split, - review_tier=tier, - reviewer="m" if tier == "reviewed" else "", - unscorable=unscorable, - ) - - -def test_only_vendored_trusted_scorable_train_pairs_may_teach(tmp_path: Path) -> None: - from openultrasast.learning.split import teachers - - cases = [ - _case(tmp_path, "train-seeded"), - _case(tmp_path, "train-reviewed", tier="reviewed"), - _case(tmp_path, "holdout-seeded", split="holdout"), - _case(tmp_path, "train-untrusted", tier="title"), - _case(tmp_path, "train-twin", unscorable="identical_twin"), - ] - assert [case.name for case in teachers(cases)] == ["train-seeded", "train-reviewed"] - - -def test_a_candidate_taught_by_a_holdout_pair_is_refused_by_name(tmp_path: Path) -> None: - from openultrasast.learning.split import refuse_if_holdout - - cases = [_case(tmp_path, "train-one"), _case(tmp_path, "holdout-one", split="holdout")] - assert refuse_if_holdout(["train-one"], cases) is None - refusal = refuse_if_holdout(["train-one", "holdout-one"], cases) - assert refusal is not None and refusal.pairs == ("holdout-one",) - assert refusal.reason == "holdout_pair_refused" - degradation = refusal.degradation() - assert degradation["stage"] == "learning" and degradation["pairs"] == ["holdout-one"] - assert "holdout-one" in str(degradation["detail"]) - - -def test_the_exporter_seeds_from_train_pairs_only(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import MechanismStore, corpus_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - cases = select_vendored(select_slice(load_pair_catalog(DEFAULT_CATALOG), "vibe-py")) - holdout = {case.name for case in cases if case.split == "holdout"} - store = MechanismStore(tmp_path / "candidates.jsonl") - report = export_mechanisms(cases, store) - taught_by_holdout = [record.id for record in corpus_mechanisms(store.load()) if set(record.pairs) & holdout] - assert taught_by_holdout == [] - assert report.records > 0 and any(reason.startswith("split:") for _pair, reason in report.skipped) - - -def test_leave_one_out_teaches_from_train_pairs_only(tmp_path: Path) -> None: - """Both arms, because `teaching_pairs <= 1` on a two-pair corpus is satisfied by a rule that does nothing. - - The same two excerpts teach twice when both are on the train split and once when one is held out, and the run - names the pair it refused (Req 5.1, 5.2).""" - from openultrasast.semantic.loo import evaluate_loo - - both_train = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "train-two")]) - assert both_train.teaching_pairs == 2 and both_train.degradations == () - result = evaluate_loo([_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")]) - assert {outcome.pair for outcome in result.outcomes} == {"train-one", "holdout-one"} # every pair is still scored - assert result.teaching_pairs == 1 - refusals = [item for item in result.degradations if item.get("reason") == "holdout_pair_refused"] - assert [item["pairs"] for item in refusals] == [["holdout-one"]] - - -def test_the_mechanism_lever_refuses_a_candidate_a_holdout_pair_taught(tmp_path: Path) -> None: - """Both arms. The shape is the one the corpus really teaches, taken from the pair itself. - - Asserting only the refusal would pass against a lever that admits nothing at all, and a hand-written shape that - matches no excerpt can never be admitted either, so neither would measure the split rule (Req 5.2).""" - from openultrasast.improve.evolve import propose_mechanism_edits - from openultrasast.semantic.loo import _facts - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.seed import pair_lessons - - cases = [_teaching_case(tmp_path, "train-one"), _teaching_case(tmp_path, "holdout-one", split="holdout")] - (lesson,) = pair_lessons(cases[0], _facts()) - - def store(name: str, pair: str) -> MechanismStore: - path = MechanismStore(tmp_path / name) - append_from_pair(path, lesson.shape, summary=lesson.summary, cwe=lesson.cwe, pair=pair, provenance="human", tier="seeded") - return path - - reasons: list[dict[str, object]] = [] - refused = propose_mechanism_edits(cases, store("refused.jsonl", "holdout-one"), tmp_path / "scan.jsonl", degradations=reasons) - assert [edit.mechanism_id for edit in refused if edit.action == "admit"] == [] - assert [item["pairs"] for item in reasons] == [["holdout-one"]] - admitted = propose_mechanism_edits(cases, store("allowed.jsonl", "train-one"), tmp_path / "scan2.jsonl") - assert [edit.mechanism_id for edit in admitted if edit.action == "admit"], "a train-taught shape is still admissible" - - -def test_the_corrected_lever_number_is_measured_and_committed() -> None: - report = Path("benchmarks/measurements/2026-09-06-mechanism-lever-split.json") - assert report.is_file() - payload = json.loads(report.read_text()) - assert payload["slice"] == "vibe-py" and payload["holdout_pairs"] > 0 - assert set(payload["before"]) >= {"detected", "silent", "youden", "shapes"} - assert set(payload["after"]) >= {"detected", "silent", "youden", "shapes"} - assert payload["after"]["shapes"] < payload["before"]["shapes"] # holdout pairs no longer teach - assert payload["after"]["youden"] <= payload["before"]["youden"] diff --git a/tests/test_learning_thinking.py b/tests/test_learning_thinking.py deleted file mode 100644 index cf2fcfa..0000000 --- a/tests/test_learning_thinking.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3). - -Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys -determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did -not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is -built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.config import load_config - - -def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None: - """The default preserves what round zero was measured under; changing it silently would make the two runs - incomparable while looking like one number.""" - assert load_config(None).learning.thinking is False - path = tmp_path / "openultrasast.toml" - path.write_text("[learning]\nthinking = true\n") - assert load_config(path).learning.thinking is True - - -def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint - - monkeypatch.setenv("DEEPSEEK_API_KEY", "k") - monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False) - path = tmp_path / "openultrasast.toml" - path.write_text("[learning]\nthinking = true\n") - client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] - assert endpoint.thinking is True - assert client._disable_thinking is False # type: ignore[attr-defined] - path.write_text("[learning]\nthinking = false\n") - client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] - assert endpoint.thinking is False and client._disable_thinking is True # type: ignore[attr-defined] - - -def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: - """The condition that produced a number is part of what identifies it.""" - import sys - - sys.path.insert(0, str(Path(__file__).parent)) - from test_learning_rounds import _always, _case, _steady # noqa: PLC0415 - - from openultrasast.learning.families import load_families - from openultrasast.learning.rounds import compare_baselines, run_baseline - - out = tmp_path / "out" - for thinking in (False, True): - run_baseline( - [_case(tmp_path, "a")], - taxonomy=load_families(), - configs_dir=tmp_path / "configs", - scan_factory=_always(_steady), - model="deepseek-v4-flash", - out_dir=out, - k_runs=3, - thinking=thinking, - ) - written = sorted(path.name for path in (out / "baseline").iterdir()) - assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written - assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"} - report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text()) - assert report["thinking"] is True diff --git a/tests/test_learning_verifiers.py b/tests/test_learning_verifiers.py deleted file mode 100644 index ce198fc..0000000 --- a/tests/test_learning_verifiers.py +++ /dev/null @@ -1,145 +0,0 @@ -"""learning-harness task 2.6: the verifier registry and the oracle callback (offline).""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.findings import StaticFinding -from openultrasast.learning.families import load_families -from openultrasast.regress.verdict import NOT_TRIGGERABLE, TRIGGERABLE, verdict_from_result -from openultrasast.sandbox import SandboxResult - -APP = ( - "from flask import request\n\n\n" - "@app.route('/books/<title>')\n@login_required\ndef guarded(title):\n" - " return Book.query.filter_by(book_title=title).first()\n\n\n" - "@app.route('/books/any/<title>')\ndef leaky(title):\n" - " owner = request.args.get('owner')\n" - " return Book.query.filter_by(user_id=owner, book_title=title).first()\n\n\n" - "@app.route('/health')\ndef health():\n return {'ok': True}\n" -) - - -def _claim(function: str, line: int, family: str) -> StaticFinding: - return StaticFinding( - finding_id=f"detector:app.py:{line}", - path="app.py", - title="t", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="r", - line=line, - function_name=function, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=[f"family:{family}"], - ranking_priority=0.0, - ) - - -@pytest.fixture -def repo(tmp_path: Path) -> Path: - (tmp_path / "app.py").write_text(APP) - return tmp_path - - -def test_an_oracle_decides_the_verdict_and_the_exit_code_no_longer_does() -> None: - from openultrasast.learning.verifiers import token_oracle - - oracle = token_oracle("CANARY:9f2a") - fired = SandboxResult(exit_code=0, stdout="ok\nCANARY:9f2a\n", stderr="", timed_out=False) - quiet = SandboxResult(exit_code=1, stdout="boom", stderr="Traceback", timed_out=False) - assert verdict_from_result(fired, oracle=oracle).verdict == TRIGGERABLE - assert verdict_from_result(quiet, oracle=oracle).verdict == NOT_TRIGGERABLE # a crash is not this family's oracle - assert verdict_from_result(quiet).verdict == TRIGGERABLE # without an oracle the exit code still decides - assert verdict_from_result(fired, oracle=oracle).reason == "oracle_fired" - - -def test_an_oracle_never_overrides_an_inconclusive_run() -> None: - from openultrasast.learning.verifiers import token_oracle - - oracle = token_oracle("CANARY:9f2a") - timed_out = SandboxResult(exit_code=0, stdout="CANARY:9f2a", stderr="", timed_out=True) - assert verdict_from_result(timed_out, oracle=oracle).verdict == "inconclusive" - assert verdict_from_result(None, oracle=oracle).verdict == "inconclusive" - assert verdict_from_result(sandbox_missing=True, oracle=oracle).reason == "sandbox_unavailable" - - -def test_the_registry_answers_by_the_family_s_declared_verifier_kind() -> None: - from openultrasast.learning.verifiers import NoVerifier, StaticVerifier, verifier_for - - taxonomy = load_families() - assert isinstance(verifier_for(taxonomy.by_id("access_control")), StaticVerifier) - assert isinstance(verifier_for(taxonomy.by_id("memory")), NoVerifier) - assert isinstance(verifier_for(taxonomy.by_id("output_encoding")), NoVerifier) - # a canary family with no canary implementation registered yet reports at suspicion rather than pretending - injection = verifier_for(taxonomy.by_id("injection")) - assert isinstance(injection, NoVerifier) and injection.kind == "canary" - - -def test_a_family_with_no_verifier_leaves_the_claim_at_suspicion(repo: Path) -> None: - from openultrasast.learning.verifiers import verifier_for - - taxonomy = load_families() - verification = verifier_for(taxonomy.by_id("memory")).verify(_claim("leaky", 12, "memory"), root=repo, language="python") - assert verification.rung == "suspicion" and verification.reason == "no_verifier" - - -def test_the_static_verifier_raises_an_access_control_claim_the_checker_agrees_with(repo: Path) -> None: - from openultrasast.learning.verifiers import verifier_for - - taxonomy = load_families() - verifier = verifier_for(taxonomy.by_id("access_control")) - agreed = verifier.verify(_claim("leaky", 12, "access_control"), root=repo, language="python") - assert agreed.rung == "static_corroboration" and "identity_constraint" in agreed.oracle_output - # the checker owes nothing in a handler that touches no protected resource, so it refuses to corroborate - healthy = verifier.verify(_claim("health", 16, "access_control"), root=repo, language="python") - assert healthy.rung == "suspicion" and healthy.reason == "checker_disagrees" - elsewhere = verifier.verify(_claim("missing", 99, "access_control"), root=repo, language="python") - assert elsewhere.rung == "suspicion" - - -def test_a_verification_is_written_onto_the_claim_as_a_tag_and_nothing_else(repo: Path) -> None: - """The evidence ladder belongs to another boundary; a verifier says what it saw with a tag.""" - from openultrasast.learning.verifiers import apply_verification, verifier_for - - taxonomy = load_families() - claim = _claim("leaky", 12, "access_control") - verified = apply_verification(claim, verifier_for(taxonomy.by_id("access_control")).verify(claim, root=repo, language="python")) - assert "verifier:static_corroboration" in verified.tags - assert verified.evidence_level == "suspicion" and claim.evidence_level == "suspicion" # the ladder is untouched - plain = apply_verification(claim, verifier_for(taxonomy.by_id("memory")).verify(claim, root=repo, language="python")) - assert "verifier:suspicion" in plain.tags and plain.tags.count("verifier:suspicion") == 1 - - -def test_a_rung_only_reaches_the_ladder_through_an_explicit_mapping() -> None: - """Every rung must land on a level the project's ladder actually defines, or the report raises later.""" - from openultrasast.learning.verifiers import RUNGS, evidence_level_for - from openultrasast.verification import EvidenceLevel - - assert {rung: evidence_level_for(rung) for rung in RUNGS} == { - "suspicion": EvidenceLevel.SUSPICION, - "static_corroboration": EvidenceLevel.STATIC_CORROBORATION, - "proven": EvidenceLevel.EXPLOIT_DEMONSTRATED, - } - assert all(EvidenceLevel(evidence_level_for(rung).value) for rung in RUNGS) # never a value outside the ladder - - -def test_a_verified_claim_still_passes_the_projects_own_verification_helpers(repo: Path) -> None: - """The bug this pins: stamping a learning rung onto evidence_level raised ValueError in three call sites.""" - from openultrasast.learning.verifiers import Verification, apply_verification, raise_to - from openultrasast.verification import VerificationStatus, is_report_verified, verify_finding - - claim = _claim("lookup", 3, "injection") - proven = apply_verification(claim, Verification(rung="proven", reason="oracle_fired", oracle_output="CANARY x")) - assert "verifier:proven" in proven.tags - assert verify_finding(proven).evidence_level == "suspicion" # would raise ValueError if the rung were stamped on - assert is_report_verified(proven.evidence_level, VerificationStatus.ACCEPTED) is False - raised = raise_to(proven, "proven") - assert raised.evidence_level == "exploit_demonstrated" - assert verify_finding(raised).evidence_level == "exploit_demonstrated" - assert is_report_verified(raised.evidence_level, VerificationStatus.ACCEPTED) is True diff --git a/tests/test_loo.py b/tests/test_loo.py deleted file mode 100644 index 8975adf..0000000 --- a/tests/test_loo.py +++ /dev/null @@ -1,235 +0,0 @@ -"""corpus-seeded-mechanisms task 3.1: leave-one-out is the corpus's own detection rate (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase - -EXEC_A = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute(\"SELECT * FROM t WHERE u = '%s'\" % user)\n" -) -EXEC_A_FIX = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n c.execute('SELECT * FROM t WHERE u = ?', (user,))\n" -) -EXEC_B = ( - "import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n cur.execute('DELETE FROM t WHERE n = ' + name)\n" -) -EXEC_B_FIX = ( - "import sqlite3\n\n\ndef delete(name):\n cur = sqlite3.connect('y').cursor()\n" - + " cur.execute('DELETE FROM t WHERE n = ?', (name,))\n" -) -SYS_C = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n" -SYS_C_FIX = "import os\nimport subprocess\n\n\ndef ping(host):\n subprocess.run(['ping', host])\n" - - -def _case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="toy", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink" - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance=provenance, - review_tier="seeded", - ) - - -def _toy(tmp_path: Path) -> list[PairCase]: - return [ - _case(tmp_path, "a", EXEC_A, EXEC_A_FIX, function="lookup", cwe="CWE-89"), - _case(tmp_path, "b", EXEC_B, EXEC_B_FIX, function="delete", cwe="CWE-89", provenance="agent"), - _case(tmp_path, "c", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78"), - ] - - -def test_leave_one_out_reports_which_pair_taught_which(tmp_path: Path) -> None: - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo(_toy(tmp_path)) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert set(by_name) == {"a", "b", "c"} - assert by_name["a"].detected and by_name["a"].silent and by_name["a"].found_by # taught by b's execute shape - assert by_name["b"].detected and by_name["b"].silent and any("a" in pairs for _, pairs in by_name["b"].found_by) - assert not by_name["c"].detected and by_name["c"].found_by == () # nobody else teaches os.system: c only tests - slice_metrics = result.per_slice["toy"] - assert slice_metrics["pairs"] == 3 and slice_metrics["detected"] == 2 and slice_metrics["silent"] == 3 - assert abs(slice_metrics["recall"] - 2 / 3) < 1e-9 and slice_metrics["silence"] == 1.0 and abs(slice_metrics["youden"] - 2 / 3) < 1e-9 - assert result.per_profile["agent"]["pairs"] == 1 and result.per_profile["human"]["pairs"] == 2 - assert "source_reaches_sink" in result.per_mechanism - payload = result.to_dict() - assert set(payload) >= {"per_slice", "per_profile", "per_mechanism", "outcomes"} - json.dumps(payload) # artifact-ready - - -def test_leave_one_out_seeds_only_trusted_pairs_and_skips_known_limit(tmp_path: Path) -> None: - from openultrasast.semantic.loo import evaluate_loo - - cases = _toy(tmp_path) - cases[1] = PairCase(**{**cases[1].__dict__, "review_tier": "advisory"}) # b may be tested, never teaches - cases[2] = PairCase(**{**cases[2].__dict__, "known_limit": "engine"}) - result = evaluate_loo(cases) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert set(by_name) == {"a", "b"} # known_limit excluded entirely - assert not by_name["a"].detected # b is advisory: it cannot teach a - assert by_name["b"].detected # a (seeded) teaches b, which is scored as a held-out target - # `b` is excluded from teaching for its tier and says so; without that line a pair that silently stopped - # teaching would look identical to one that never could (Req 5.2). - assert result.skipped == (("c", "known_limit:engine"), ("b", "not_a_teacher:tier")) - - -def test_cli_pairs_loo_writes_the_artifact_offline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - monkeypatch.chdir(Path.cwd()) - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["pairs", "--slice", "vibe-py", "--loo", "--json", "--loo-out", str(tmp_path / "loo.json")]) == 0 - payload = json.loads(buf.getvalue()) - # 35 vendored rows less the five byte-identical twins, which task 4.2 declared unscorable in the catalog: a - # pair whose two sides are the same code cannot be held out against itself. - assert payload["per_slice"]["vibe-py"]["pairs"] == 30 and "found_by" in payload["outcomes"][0] - assert json.loads((tmp_path / "loo.json").read_text()) == payload - - -HELPER_SINK = "import os\n\n\ndef run(cmd):\n os.system(cmd)\n\n\ndef ping(host):\n return run('ping ' + host)\n" -HELPER_SINK_FIX = ( - "import subprocess\n\n\ndef run(cmd):\n subprocess.run(cmd.split())\n\n\ndef ping(host):\n return run('ping ' + host)\n" -) - - -def test_hits_outside_the_labeled_function_or_off_label_never_count(tmp_path: Path) -> None: - """Round-3 mutants: a hit must sit inside the labeled function and name the labeled mechanism or sink.""" - from openultrasast.semantic.loo import evaluate_loo - - teacher = _case(tmp_path, "t", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78") - outside = _case( - tmp_path, "outside", HELPER_SINK, HELPER_SINK_FIX, function="ping", cwe="CWE-78" - ) # sink call lives in `run`, not `ping` - off_label = PairCase( - **{ - **_case(tmp_path, "offlabel", SYS_C, SYS_C_FIX, function="ping", cwe="CWE-78").__dict__, - "expected": ( - ExpectedFinding( - cwe="CWE-78", vulnerability_class="x", path="app.py", evidence="", function="ping", mechanism="missing_auth_guard" - ), - ), - } - ) - result = evaluate_loo([teacher, outside, off_label]) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - assert by_name["outside"].hits_vuln >= 1 and not by_name["outside"].detected - assert by_name["offlabel"].hits_vuln >= 1 and not by_name["offlabel"].detected - - -def test_leave_one_out_forwards_search_degradations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.loo import evaluate_loo - - monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0") - js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n" - (tmp_path / "j-v.js").write_text(js) - (tmp_path / "j-f.js").write_text(js) - case = PairCase( - **{ - **_toy(tmp_path)[0].__dict__, - "name": "j", - "language": "javascript", - "vuln_file": tmp_path / "j-v.js", - "fixed_file": tmp_path / "j-f.js", - "relpath": "app.js", - } - ) - result = evaluate_loo([*_toy(tmp_path), case]) - assert any(item["reason"] == "variants_language_unsupported" for item in result.degradations) - - -# --- authorization-obligations 4.2: leave-one-out per obligation kind ------------------------------------------------- - -_HANDLER = "from flask import request\n\n\n@app.route('/{route}/<key>')\ndef {fn}(key):\n{body}" -OBL_A = _HANDLER.format(route="books", fn="get_book", body=" return Book.query.filter_by(book_title=key).first()\n") -OBL_A_FIX = _HANDLER.format( - route="books", - fn="get_book", - body=" user_id = request.user.id\n return Book.query.filter_by(user_id=user_id, book_title=key).first()\n", -) -OBL_B = _HANDLER.format(route="orders", fn="get_order", body=" return Order.query.filter_by(ref=key).first()\n") -OBL_B_FIX = _HANDLER.format( - route="orders", fn="get_order", body=" owner = current_user.id\n return Order.query.filter_by(owner=owner, ref=key).first()\n" -) -OBL_C = _HANDLER.format( - route="login", fn="login", body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='None')\n return resp\n" -) -OBL_C_FIX = _HANDLER.format( - route="login", - fn="login", - body=" resp = make_response('ok')\n resp.set_cookie('sid', key, samesite='Strict', httponly=1)\n return resp\n", -) - - -def _obligation_case(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, obligation: str, mechanism: str) -> PairCase: - case = _case(tmp_path, name, vuln, fixed, function=function, cwe="CWE-639") - row = case.expected[0] - return PairCase( - **{ - **case.__dict__, - "slice": "vibe-py", - "expected": (ExpectedFinding(**{**row.__dict__, "mechanism": mechanism, "obligation": obligation}),), - } - ) - - -def _obligation_toy(tmp_path: Path) -> list[PairCase]: - read = {"obligation": "protected_read", "mechanism": "unconstrained_protected_read"} - return [ - _obligation_case(tmp_path, "a", OBL_A, OBL_A_FIX, function="get_book", **read), - _obligation_case(tmp_path, "b", OBL_B, OBL_B_FIX, function="get_order", **read), - _obligation_case(tmp_path, "c", OBL_C, OBL_C_FIX, function="login", obligation="security_setting", mechanism="permissive_default"), - ] - - -def test_leave_one_out_scores_obligation_rows_per_kind_and_names_the_teaching_pair(tmp_path: Path) -> None: - """Req 7.4: recall, silence and Youden per obligation kind; found_by names the shape record that supplied known_fix.""" - from openultrasast.semantic.loo import evaluate_loo - - result = evaluate_loo(_obligation_toy(tmp_path)) - by_name = {outcome.pair: outcome for outcome in result.outcomes} - a, b, c = by_name["a"], by_name["b"], by_name["c"] - assert a.obligations == ("protected_read",) and c.obligations == ("security_setting",) - assert a.detected and a.silent and [pairs for _, pairs in a.found_by] == [("b",)] # b taught the identity discharger - assert b.detected and b.silent and [pairs for _, pairs in b.found_by] == [("a",)] - assert all(record_id.startswith("corpus:") for record_id, _ in a.found_by) - assert c.teaches and not c.detected and c.found_by == () and c.silent # reached and undischarged, but nobody else teaches the kind - kinds = result.per_obligation_kind - assert kinds["protected_read"]["pairs"] == 2 and kinds["protected_read"]["detected"] == 2 and kinds["protected_read"]["youden"] == 1.0 - assert kinds["security_setting"]["pairs"] == 1 and kinds["security_setting"]["detected"] == 0 - assert "unconstrained_protected_read" in result.per_mechanism # flow mechanisms and obligation kinds sit side by side - payload = result.to_dict() - assert "per_obligation_kind" in payload and payload["outcomes"][0]["obligations"] == ["protected_read"] - json.dumps(payload) - - -def test_obligation_rows_are_not_found_by_the_facts_alone(tmp_path: Path) -> None: - """A finding without a known fix from the held-out store is the checker's own work, not the corpus teaching.""" - from openultrasast.semantic.loo import evaluate_loo - - (only,) = [outcome for outcome in evaluate_loo(_obligation_toy(tmp_path)[:1]).outcomes] - assert only.hits_vuln >= 1 and not only.detected and only.found_by == () # the leaky read is flagged, but no pair taught it diff --git a/tests/test_mechanism_export.py b/tests/test_mechanism_export.py deleted file mode 100644 index 227247b..0000000 --- a/tests/test_mechanism_export.py +++ /dev/null @@ -1,74 +0,0 @@ -"""corpus-seeded-mechanisms: additive store fields and the corpus writer (task 1.2, Req 1.3/1.4).""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.semantic.mechanisms import MechanismStore, append_mechanism - - -def _shape(**overrides: object): - from openultrasast.semantic.variants import Shape - - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("fact_source",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return Shape(**base) # type: ignore[arg-type] - - -def test_old_sandbox_rows_load_with_corpus_defaults(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism(store, summary="s", cwe="CWE-78", language="python", tags=("t",), what_made_it_exploitable="x") - # a row written before this spec has no origin/tier/pairs/shape/guard fields at all - raw = json.loads((tmp_path / "mechanisms.jsonl").read_text().splitlines()[0]) - for key in ("origin", "review_tier", "pairs", "shape", "guard"): - raw.pop(key, None) - (tmp_path / "mechanisms.jsonl").write_text(json.dumps(raw) + "\n") - (record,) = store.load() - assert ( - record.origin == "sandbox" and record.review_tier == "" and record.pairs == () and record.shape is None and record.guard == "none" - ) - - -def test_append_from_pair_dedupes_by_shape_and_extends_provenance(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import append_from_pair, corpus_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - first = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") - second = append_from_pair(store, _shape(), summary="request into os.system", cwe="CWE-78", pair="p2", provenance="agent", tier="seeded") - assert first.id == second.id - assert ( - second.pairs == ("p1", "p2") and second.origin == "corpus" and second.review_tier == "seeded" and second.guard == "allowlist_test" - ) - third = append_from_pair( - store, _shape(guard="none"), summary="request into os.system", cwe="CWE-78", pair="p3", provenance="human", tier="reviewed" - ) - assert third.id != first.id - loaded = store.load() - assert len(loaded) == 2 and {r.id for r in loaded} == {first.id, third.id} # the store is append-only, load folds by id - corpus = corpus_mechanisms(loaded) - assert {r.id for r in corpus} == {first.id, third.id} - assert all(r.shape is not None and r.shape["sink_name"] == "system" for r in corpus) - assert "source_reaches_sink" in first.tags and "mechanism:source_reaches_sink" in first.tags - - -def test_corpus_rows_never_carry_source_text_and_ids_are_deterministic(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import append_from_pair - - a = append_from_pair( - MechanismStore(tmp_path / "a.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="p", provenance="human", tier="seeded" - ) - b = append_from_pair( - MechanismStore(tmp_path / "b.jsonl"), _shape(), summary="s", cwe="CWE-78", pair="q", provenance="human", tier="seeded" - ) - assert a.id == b.id # identity is the shape key, not a random uuid: the lever can name records across machines - text = (tmp_path / "a.jsonl").read_text() - assert "app.py" not in text and "cmd" not in text diff --git a/tests/test_mechanism_lever.py b/tests/test_mechanism_lever.py deleted file mode 100644 index 103ed7b..0000000 --- a/tests/test_mechanism_lever.py +++ /dev/null @@ -1,112 +0,0 @@ -"""corpus-seeded-mechanisms task 4.1: the `mechanisms` lever admits or retracts exporter records, nothing else.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair -from openultrasast.semantic.variants import Shape - - -def _record(store: MechanismStore, **overrides: object): - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("parameter",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return append_from_pair(store, Shape(**base), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") # type: ignore[arg-type] - - -def test_mechanism_edit_is_a_bounded_lever_with_a_stable_key(tmp_path: Path) -> None: - from openultrasast.improve.validator import VALID_LEVERS, MechanismEdit - - assert "mechanisms" in VALID_LEVERS - edit = MechanismEdit(action="admit", mechanism_id="corpus:abc", source="loo", rationale="recovers holdout pair x") - assert edit.lever == "mechanisms" and edit.key() == "mechanisms:admit:corpus:abc" - - -def test_validator_accepts_only_exporter_records_with_closed_guards(tmp_path: Path) -> None: - from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - good = _record(candidates) - validator = EvolveValidator() - by_id = {record.id: record for record in candidates.load()} - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=good.id, source="loo"), by_id) - validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=good.id, source="loo"), by_id) - with pytest.raises(StrictValidationError, match="unknown mechanism"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id="corpus:not-exported", source="loo"), by_id) - with pytest.raises(StrictValidationError, match="action"): - validator.validate_mechanism(MechanismEdit(action="rewrite", mechanism_id=good.id, source="loo"), by_id) - with pytest.raises(StrictValidationError, match="origin"): - from openultrasast.semantic.mechanisms import append_mechanism - - sandbox = append_mechanism(candidates, summary="s", cwe="CWE-78", language="python", tags=(), what_made_it_exploitable="x") - validator.validate_mechanism( - MechanismEdit(action="admit", mechanism_id=sandbox.id, source="loo"), {r.id: r for r in candidates.load()} - ) - # free-form shapes never pass: a record whose shape text is not a closed structure is rejected before it can be admitted - from dataclasses import replace - - forged = replace(good, shape={**(good.shape or {}), "guard": "if user not in ALLOWED"}) - with pytest.raises(StrictValidationError, match="guard"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=forged.id, source="export"), {forged.id: forged}) - forged_text = replace(good, shape={**(good.shape or {}), "sink_name": "os.system(cmd + ' ')"}) - with pytest.raises(StrictValidationError, match="identifier"): - validator.validate_mechanism( - MechanismEdit(action="admit", mechanism_id=forged_text.id, source="export"), {forged_text.id: forged_text} - ) - - -def test_apply_mechanism_edits_admits_into_the_scan_store_and_reverts_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - a = _record(candidates) - b = _record(candidates, sink_name="popen") - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - scan_store.parent.mkdir() - scan_store.write_text("") - before = scan_store.read_bytes() - snapshot = apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=a.id, source="loo")], candidates, scan_store) - admitted = MechanismStore(scan_store).load() - assert [r.id for r in admitted] == [a.id] and admitted[0].origin == "corpus" - apply_mechanism_edits([MechanismEdit(action="retract", mechanism_id=a.id, source="loo")], candidates, scan_store) - assert MechanismStore(scan_store).load() == () # retraction is a tombstone row; load() folds it away - snapshot.restore() - assert scan_store.read_bytes() == before # byte-for-byte revert of the whole round - assert b.id != a.id - - -def test_retract_is_allowed_for_an_admitted_record_that_left_the_candidate_set(tmp_path: Path) -> None: - from openultrasast.improve.validator import EvolveValidator, MechanismEdit, StrictValidationError - - scan = MechanismStore(tmp_path / "scan.jsonl") - stale = _record(scan, sink_name="popen") - validator = EvolveValidator() - with pytest.raises(StrictValidationError, match="unknown mechanism"): - validator.validate_mechanism(MechanismEdit(action="admit", mechanism_id=stale.id), {}, admitted={stale.id: stale}) - validator.validate_mechanism(MechanismEdit(action="retract", mechanism_id=stale.id), {}, admitted={stale.id: stale}) - - -def test_export_defaults_to_a_candidates_file_and_never_writes_the_scan_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - monkeypatch.chdir(tmp_path) - (tmp_path / "benchmarks").symlink_to(Path(__file__).resolve().parents[1] / "benchmarks") - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["mechanisms", "export", "--slice", "vibe-py", "--json"]) == 0 - payload = __import__("json").loads(buf.getvalue()) - assert payload["store"].endswith("mechanism-candidates.jsonl") - assert not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() diff --git a/tests/test_mechanism_loop.py b/tests/test_mechanism_loop.py deleted file mode 100644 index 558b735..0000000 --- a/tests/test_mechanism_loop.py +++ /dev/null @@ -1,330 +0,0 @@ -"""corpus-seeded-mechanisms task 4.2: the improve loop admits mechanisms that recover holdout pairs and retracts leakers.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding, load_benchmark_manifest -from openultrasast.pairs import PairCase -from openultrasast.ruleset import PatternRule, write_ruleset -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair -from openultrasast.semantic.variants import Shape - -EXEC_VULN = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute('SELECT * FROM t WHERE u = ' + user)\n" -) -EXEC_FIX = ( - "import sqlite3\n\n\ndef lookup(user):\n c = sqlite3.connect('x').cursor()\n" - + " c.execute('SELECT * FROM t WHERE u = ?', (user,))\n" -) -SYS_VULN = "import os\n\n\ndef ping(host):\n os.system('ping ' + host)\n" -SYS_LEAKY_FIX = ( - "import os\n\n\ndef ping(host):\n log(host)\n os.system('ping ' + host)\n" # the 'fix' still calls the sink: a leaking shape -) - - -def _shape(sink: str, guard: str = "none") -> Shape: - return Shape( - language="python", - sink_name=sink, - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard=guard, - mechanism="source_reaches_sink", - ) - - -def _pair(tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, cwe: str, provenance: str = "human") -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="github", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe=cwe, vulnerability_class="x", path="app.py", evidence="", function=function, mechanism="source_reaches_sink" - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance=provenance, - split="holdout", - review_tier="reviewed", - reviewer="t", - ) - - -def _round_env(tmp_path: Path) -> dict[str, object]: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("def f(x):\n return eval(x)\n") - rd = tmp_path / "ruleset" / "python" - write_ruleset( - rd / "rules.toml", - [PatternRule(rule_id="good-eval", title="e", languages=("python",), cwe="CWE-95", tags=("injection",), pattern=r"\beval\s*\(")], - ) - manifest_path = tmp_path / "bench.toml" - manifest_path.write_text( - 'name = "t"\nlanguage = "python_web"\n\n[source]\npath = "repo"\n\n' - '[[expected]]\ncwe = "CWE-95"\nclass = "code injection"\npath = "app.py"\n' - 'line = 2\nrule_id = "good-eval"\nsink = "eval"\nevidence = "eval executes input"\n' - ) - return {"target": repo, "manifest": load_benchmark_manifest(manifest_path), "ruleset_dir": tmp_path / "ruleset"} - - -def test_proposer_admits_recovering_records_and_retracts_leakers(tmp_path: Path) -> None: - from openultrasast.improve.evolve import evaluate_mechanism_profiles, propose_mechanism_edits - - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - sys_record = append_from_pair( - candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" - ) - holdout = [ - _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), - _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78", provenance="agent"), - ] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - edits = propose_mechanism_edits(holdout, candidates, scan_store) - assert [(e.action, e.mechanism_id) for e in edits] == [("admit", exec_record.id)] # sys leaks on y's fixed side: never proposed - assert "x" in edits[0].rationale - # once a leaking record is admitted (by a human or an earlier round), the proposer asks to retract it - from openultrasast.improve.validator import MechanismEdit, apply_mechanism_edits - - apply_mechanism_edits([MechanismEdit(action="admit", mechanism_id=sys_record.id)], candidates, scan_store) - edits = propose_mechanism_edits(holdout, candidates, scan_store) - assert ("retract", sys_record.id) in [(e.action, e.mechanism_id) for e in edits] - profiles = evaluate_mechanism_profiles(holdout, scan_store) - assert profiles["agent"]["pairs"] == 1.0 and profiles["agent"]["pair_correct"] == 0.0 # y: detected but leaked - - -def test_round_accepts_a_recovering_admission_and_rejects_a_leaking_one_byte_for_byte(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_round - from openultrasast.improve.validator import MechanismEdit - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - sys_record = append_from_pair( - candidates, _shape("system"), summary="s", cwe="CWE-78", pair="teacher-b", provenance="human", tier="seeded" - ) - holdout = [ - _pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89"), - _pair(tmp_path, "y", SYS_VULN, SYS_LEAKY_FIX, function="ping", cwe="CWE-78"), - ] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - ledger = tmp_path / "rule_policy.json" - journal = tmp_path / "journal.json" - outcome = run_round( - env["target"], - env["manifest"], - ledger_path=ledger, - journal_path=journal, - ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] - pair_cases=holdout, - min_holdout_pairs=1, - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - ) - assert outcome.accepted and outcome.reason == "accepted" - assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] - entry = json.loads(journal.read_text())[0] - mech_edits = [e for e in entry["edits"] if e["lever"] == "mechanisms"] - assert mech_edits and mech_edits[0]["mechanism_id"] == exec_record.id and mech_edits[0]["pairs"] == ["teacher-a"] - assert entry["mechanisms_before"]["human"]["pair_correct"] == 0.0 and entry["mechanisms_after"]["human"]["pair_correct"] == 1.0 - before = scan_store.read_bytes() - forced = run_round( - env["target"], - env["manifest"], - ledger_path=ledger, - journal_path=journal, - ruleset_dir=env["ruleset_dir"], # type: ignore[arg-type] - pair_cases=holdout, - min_holdout_pairs=1, - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - scripted_mechanism_edits=[MechanismEdit(action="admit", mechanism_id=sys_record.id, source="export", rationale="scripted")], - ) - assert not forced.accepted and (forced.reason.startswith("mechanism_leak") or forced.reason.startswith("profile_regression")) - assert scan_store.read_bytes() == before # byte-for-byte revert - - -def test_no_gain_admission_is_rejected_and_reverted_mechanism_edits_are_not_reproposed(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_round - from openultrasast.improve.journal import reverted_edit_keys - from openultrasast.improve.validator import MechanismEdit - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - unrelated = append_from_pair( - candidates, _shape("popen"), summary="s", cwe="CWE-78", pair="teacher-c", provenance="human", tier="seeded" - ) - holdout = [_pair(tmp_path, "x", EXEC_VULN, EXEC_FIX, function="lookup", cwe="CWE-89")] - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - ledger = tmp_path / "rule_policy.json" - journal = tmp_path / "journal.json" - kwargs: dict[str, object] = { - "ledger_path": ledger, - "journal_path": journal, - "ruleset_dir": env["ruleset_dir"], - "pair_cases": holdout, - "min_holdout_pairs": 1, - "mechanism_candidates": candidates.log_path, - "mechanism_store": scan_store, - } - edit = MechanismEdit(action="admit", mechanism_id=unrelated.id, source="export", rationale="scripted: recovers nothing") - outcome = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] - assert not outcome.accepted and outcome.reason == "mechanism_no_gain" - assert not scan_store.exists() or scan_store.read_bytes() == b"" - assert edit.key() in reverted_edit_keys(__import__("json").loads(journal.read_text())) - # the same reverted edit, offered again with a rationale, is filtered before validation (novelty gate) - again = run_round(env["target"], env["manifest"], scripted_mechanism_edits=[edit], **kwargs) # type: ignore[arg-type] - assert again.reason == "no_proposals" and again.mechanism_edits == [] - - -def _holdout_catalog(tmp_path: Path) -> Path: - """A train pair to teach and a holdout pair to recover. - - The catalog used to hold the holdout pair alone, so the only shape available was the one that pair - taught about itself; learning-harness Req 5.1 refuses that, and rightly. - """ - (tmp_path / "x-v.py").write_text(EXEC_VULN) - (tmp_path / "x-f.py").write_text(EXEC_FIX) - (tmp_path / "t-v.py").write_text(EXEC_VULN.replace("lookup", "search").replace("u = ", "name = ")) - (tmp_path / "t-f.py").write_text(EXEC_FIX.replace("lookup", "search").replace("u = ", "name = ")) - row = ( - '[[pair]]\nname = "{name}"\nslice = "github"\nlanguage = "python"\nvuln = "{name[0]}-v.py"\nfixed = "{name[0]}-f.py"\n' - 'relpath = "app.py"\nsplit = "{split}"\nreview_tier = "reviewed"\nreviewer = "t"\n\n' - '[[pair.expected]]\ncwe = "CWE-89"\nclass = "sql injection"\npath = "app.py"\nfunction = "{function}"\n' - 'sink = "execute"\nmechanism = "source_reaches_sink"\n' - ) - catalog = tmp_path / "pairs.toml" - catalog.write_text( - row.replace("{name}", "x").replace("{name[0]}", "x").replace("{split}", "holdout").replace("{function}", "lookup") - + "\n" - + row.replace("{name}", "t").replace("{name[0]}", "t").replace("{split}", "train").replace("{function}", "search") - ) - return catalog - - -def test_ousast_improve_admits_candidates_into_the_scan_store_by_default(tmp_path: Path) -> None: - """Req 5: the lever is reachable from the operator command; export -> candidates -> improve -> mechanisms.jsonl the scan reads.""" - from openultrasast.cli import main - - env = _round_env(tmp_path) - repo = Path(str(env["target"])) - calibration = repo / ".openultrasast" / "calibration" - candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - catalog = _holdout_catalog(tmp_path) - manifest = tmp_path / "bench.toml" - assert ( - main( - ["improve", str(manifest), "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] - ) - == 0 - ) - admitted = MechanismStore(calibration / "mechanisms.jsonl").load() - assert [r.id for r in admitted] == [exec_record.id] - journal = json.loads((calibration / "improve_journal.json").read_text()) - assert any(e["lever"] == "mechanisms" and e["mechanism_id"] == exec_record.id for entry in journal for e in entry["edits"]) - - -def test_run_improvement_threads_mechanism_paths(tmp_path: Path) -> None: - from openultrasast.improve.evolve import run_improvement - from openultrasast.pairs import load_pair_catalog - - env = _round_env(tmp_path) - candidates = MechanismStore(tmp_path / "candidates.jsonl") - exec_record = append_from_pair( - candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded" - ) - cases = load_pair_catalog(_holdout_catalog(tmp_path)) - scan_store = tmp_path / "scan" / "mechanisms.jsonl" - outcomes = run_improvement( - env["target"], - env["manifest"], - ledger_path=tmp_path / "ledger.json", - journal_path=tmp_path / "journal.json", # type: ignore[arg-type] - ruleset_dir=env["ruleset_dir"], - pair_cases=cases, - min_holdout_pairs=1, # type: ignore[arg-type] - mechanism_candidates=candidates.log_path, - mechanism_store=scan_store, - max_rounds=3, - ) - assert any(o.accepted and o.mechanism_edits for o in outcomes) - assert [r.id for r in MechanismStore(scan_store).load()] == [exec_record.id] - assert outcomes[-1].reason == "no_proposals" # converges once the candidate is admitted - - -def _cli_env(tmp_path: Path) -> tuple[dict[str, object], Path, Path]: - env = _round_env(tmp_path) - calibration = Path(str(env["target"])) / ".openultrasast" / "calibration" - candidates = MechanismStore(calibration / "mechanism-candidates.jsonl") - append_from_pair(candidates, _shape("execute"), summary="s", cwe="CWE-89", pair="teacher-a", provenance="human", tier="seeded") - return env, calibration, _holdout_catalog(tmp_path) - - -def test_dry_run_and_no_mechanisms_never_write_the_scan_store(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] - from openultrasast.cli import main - - env, calibration, catalog = _cli_env(tmp_path) - base = [ - "improve", - str(tmp_path / "bench.toml"), - "--ruleset-dir", - str(env["ruleset_dir"]), - "--pair-catalog", - str(catalog), - "--min-holdout-pairs", - "1", - ] - assert main([*base, "--dry-run"]) == 0 - assert not (calibration / "mechanisms.jsonl").exists() # a dry run mutates nothing under the target - assert main([*base, "--no-mechanisms"]) == 0 - assert not (calibration / "mechanisms.jsonl").exists() - (calibration / "mechanism-candidates.jsonl").unlink() - capsys.readouterr() # the dry run legitimately reports its scratch admission; judge only the candidate-less round below - assert main(base) == 0 - assert not (calibration / "mechanisms.jsonl").exists() # no candidates file: the lever is inert, the round is a plain rule round - out = capsys.readouterr().out - assert "admit " not in out and "retract " not in out - - -def test_documented_default_flow_connects_export_to_improve(tmp_path: Path, monkeypatch, capsys) -> None: # type: ignore[no-untyped-def] - """README flow: `ousast mechanisms export` (cwd default) then `ousast improve --pair-catalog` admits into the target's store.""" - from openultrasast.cli import main - - env = _round_env(tmp_path) - catalog = _holdout_catalog(tmp_path) - monkeypatch.chdir(tmp_path) - # export a candidates file exactly as documented (cwd default), seeded from a tiny trusted catalog of our own - assert main(["mechanisms", "export", "--catalog", str(catalog), "--slice", "all"]) == 0 - exported = tmp_path / ".openultrasast" / "calibration" / "mechanism-candidates.jsonl" - assert exported.is_file() and not (tmp_path / ".openultrasast" / "calibration" / "mechanisms.jsonl").exists() - # the target is a different directory (the benchmark fixture); improve must still find the cwd candidates by default - assert ( - main( - ["improve", "bench.toml", "--ruleset-dir", str(env["ruleset_dir"]), "--pair-catalog", str(catalog), "--min-holdout-pairs", "1"] - ) - == 0 - ) - target_store = Path(str(env["target"])) / ".openultrasast" / "calibration" / "mechanisms.jsonl" - assert MechanismStore(target_store).load() # admitted where the scan of that target reads - out = capsys.readouterr().out - assert "admit corpus:" in out # the accepted round names its mechanism edits diff --git a/tests/test_mechanism_reports.py b/tests/test_mechanism_reports.py deleted file mode 100644 index c19b702..0000000 --- a/tests/test_mechanism_reports.py +++ /dev/null @@ -1,141 +0,0 @@ -"""corpus-seeded-mechanisms task 5.1: reports name the mechanism, the known fix guard, and the pair provenance.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from openultrasast.findings import StaticFinding -from openultrasast.reports import write_markdown_report, write_sarif_report -from openultrasast.semantic.overlay import OverlayRecord - - -def _variant() -> StaticFinding: - return StaticFinding( - finding_id="variant:corpus:abc:app.py:5", - path="app.py", - title="Variant of known mechanism: run_shell", - severity="medium", - confidence="low", - evidence_level="suspicion", - rationale="parameter into run_shell; fix adds allowlist_test. Structural variant of a mechanism learned from p1, p2.", - line=5, - function_name="handler", - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=["variant", "mechanism:corpus:abc"], - ranking_priority=0.0, - ) - - -MECHANISMS = { - "corpus:abc": { - "summary": "source_reaches_sink: parameter into run_shell (CWE-78); fix adds allowlist_test", - "guard": "allowlist_test", - "pairs": ["p1", "p2"], - "cwe": "CWE-78", - } -} - - -def test_markdown_names_mechanism_guard_and_provenance_and_keeps_the_suspicion_label(tmp_path: Path) -> None: - path = tmp_path / "report.md" - write_markdown_report([_variant()], path, mechanisms=MECHANISMS) - text = path.read_text() - assert "- Mechanism: `corpus:abc`" in text - assert "source_reaches_sink: parameter into run_shell" in text - assert "- Known fix guard: `allowlist_test`" in text - assert "- Learned from pairs: `p1, p2`" in text - assert "- Evidence level: `suspicion`" in text - assert "## Mechanisms" in text # a section summarising every mechanism the report cites - - -def test_sarif_carries_mechanism_properties_for_variant_and_merged_overlay_findings(tmp_path: Path) -> None: - merged = OverlayRecord( - proposal_id="python-os-command:app.py:9", - path="app.py", - line=9, - disposition="promote", - reason="request reaches os.system", - cwe="CWE-78", - sources=("request",), - sinks=("os.system",), - sanitizers=(), - evidence_level="static_corroboration", - origin="inventory", - engine="python-ast", - language="python", - mechanism_id="corpus:abc", - ) - inventory = StaticFinding( - **{ - **_variant().__dict__, - "finding_id": "python-os-command:app.py:9", - "line": 9, - "tags": ["injection"], - "evidence_level": "static_corroboration", - } - ) - path = tmp_path / "report.sarif" - write_sarif_report([_variant(), inventory], [], path, overlay=[merged], mechanisms=MECHANISMS) - results = json.loads(path.read_text())["runs"][0]["results"] - by_id = {item["properties"]["finding_id"]: item["properties"] for item in results} - variant = by_id["variant:corpus:abc:app.py:5"] - assert ( - variant["mechanism_id"] == "corpus:abc" - and variant["mechanism_guard"] == "allowlist_test" - and variant["mechanism_pairs"] == ["p1", "p2"] - ) - assert variant["evidence_level"] == "suspicion" - merged_props = by_id["python-os-command:app.py:9"] - assert merged_props["mechanism_id"] == "corpus:abc" and merged_props["mechanism_guard"] == "allowlist_test" # from the overlay record - - -def test_reports_without_mechanisms_are_unchanged(tmp_path: Path) -> None: - plain = StaticFinding(**{**_variant().__dict__, "finding_id": "python-os-command:app.py:9", "tags": ["injection"]}) - path = tmp_path / "report.md" - write_markdown_report([plain], path) - text = path.read_text() - assert "Mechanism:" not in text and "## Mechanisms" not in text - - -def test_standard_scan_report_names_the_mechanism_behind_a_variant_finding(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] - from openultrasast.cli import main - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variants import Shape - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text("from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n") - shape = Shape( - language="python", - sink_name="run_shell", - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard="allowlist_test", - mechanism="source_reaches_sink", - ) - record = append_from_pair( - MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"), - shape, - summary="parameter into run_shell", - cwe="CWE-78", - pair="p1", - provenance="human", - tier="seeded", - ) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = sorted((repo / ".runs").iterdir())[-1] - report = (run_dir / "report.md").read_text() - assert ( - f"- Mechanism: `{record.id}`" in report - and "- Known fix guard: `allowlist_test`" in report - and "- Learned from pairs: `p1`" in report - ) - sarif = json.loads((run_dir / "report.sarif").read_text()) - props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"]["finding_id"].startswith("variant:")] - assert props and props[0]["mechanism_guard"] == "allowlist_test" and props[0]["evidence_level"] == "suspicion" diff --git a/tests/test_mechanism_seed.py b/tests/test_mechanism_seed.py deleted file mode 100644 index 9bedb2d..0000000 --- a/tests/test_mechanism_seed.py +++ /dev/null @@ -1,112 +0,0 @@ -"""corpus-seeded-mechanisms task 2.1: export trusted pairs as mechanism records (offline).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase, load_pair_catalog, select_slice -from openultrasast.semantic.mechanisms import MechanismStore - -VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FIXED = ( - "from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _case(tmp_path: Path, name: str, *, tier: str, known_limit: str | None = None, vuln: str = VULN, language: str = "python") -> PairCase: - ext = {"python": "py", "javascript": "js"}[language] - (tmp_path / f"{name}-v.{ext}").write_text(vuln) - (tmp_path / f"{name}-f.{ext}").write_text(FIXED if language == "python" else vuln) - return PairCase( - name=name, - slice="vibe-py", - language=language, - origin="test", - vuln_file=tmp_path / f"{name}-v.{ext}", - fixed_file=tmp_path / f"{name}-f.{ext}", - relpath=f"app.{ext}", - expected=( - ExpectedFinding( - cwe="CWE-78", - vulnerability_class="command injection", - path=f"app.{ext}", - evidence="", - sink="os.system", - function="run", - mechanism="source_reaches_sink", - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance="human", - review_tier=tier, - reviewer="t" if tier == "reviewed" else "", - known_limit=known_limit, - ) - - -def test_export_seeds_only_trusted_parsing_pairs_and_reports_every_skip(tmp_path: Path) -> None: - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - cases = [ - _case(tmp_path, "seeded", tier="seeded"), - _case(tmp_path, "reviewed-twin", tier="reviewed"), - _case(tmp_path, "advisory", tier="advisory"), - _case(tmp_path, "title", tier="title"), - _case(tmp_path, "limit", tier="seeded", known_limit="engine cannot see it"), - _case(tmp_path, "broken", tier="seeded", vuln="def run(:\n"), - ] - report = export_mechanisms(cases, store) - records = store.load() - assert len(records) == 1 # seeded and reviewed-twin share one shape: one record, two pairs - (record,) = records - assert record.origin == "corpus" and record.pairs == ("seeded", "reviewed-twin") and record.shape is not None - assert report.seeded == 2 and report.records == 1 - reasons = dict(report.skipped) - assert reasons["advisory"] == "tier:advisory" and reasons["title"] == "tier:title" - assert reasons["limit"].startswith("known_limit") and reasons["broken"] == "parse_failed" - text = (tmp_path / "mechanisms.jsonl").read_text() - assert "request.args" not in text and "app.py" not in text # Req 2.4: no code text in the store - - -def test_export_skips_languages_without_a_grammar_with_a_degradation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.seed import export_mechanisms - - monkeypatch.setenv("OPENULTRASAST_TREE_SITTER_PROBE", "0") - store = MechanismStore(tmp_path / "mechanisms.jsonl") - js = "function run(req) {\n const cmd = req.query.cmd;\n exec(cmd);\n}\n" - report = export_mechanisms([_case(tmp_path, "js", tier="seeded", vuln=js, language="javascript")], store) - assert store.load() == () and dict(report.skipped)["js"] == "variants_language_unsupported" - assert {"stage": "mechanisms", "reason": "variants_language_unsupported", "language": "javascript"} in report.degradations - - -def test_export_over_the_vendored_vibe_py_slice_is_offline_and_deterministic(tmp_path: Path) -> None: - from openultrasast.pairs import select_vendored - from openultrasast.semantic.seed import export_mechanisms - - cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py")) - first = export_mechanisms(cases, MechanismStore(tmp_path / "a.jsonl")) - second = export_mechanisms(cases, MechanismStore(tmp_path / "b.jsonl")) - assert first.records >= 1 and first.records == second.records - assert (tmp_path / "a.jsonl").read_text() == (tmp_path / "b.jsonl").read_text() - assert first.seeded + len(first.skipped) == len(cases) - - -def test_cli_mechanisms_export_writes_the_store_and_prints_the_report(tmp_path: Path) -> None: - import io - from contextlib import redirect_stdout - - from openultrasast.cli import main - - buf = io.StringIO() - with redirect_stdout(buf): - assert main(["mechanisms", "export", "--slice", "vibe-py", "--store", str(tmp_path / "m.jsonl"), "--json"]) == 0 - payload = json.loads(buf.getvalue()) - assert payload["records"] >= 1 and payload["slice"] == "vibe-py" and "skipped" in payload - assert (tmp_path / "m.jsonl").is_file() diff --git a/tests/test_learning_endpoint.py b/tests/test_model_endpoint.py similarity index 85% rename from tests/test_learning_endpoint.py rename to tests/test_model_endpoint.py index fd2c5d6..074c260 100644 --- a/tests/test_learning_endpoint.py +++ b/tests/test_model_endpoint.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import urllib.request +from pathlib import Path from typing import Any import pytest @@ -33,7 +34,7 @@ def _endpoint_env(monkeypatch: pytest.MonkeyPatch, **values: str | None) -> None def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint + from openultrasast.model.endpoint import DEEPSEEK_BASE_URL, resolve_chat_endpoint _endpoint_env(monkeypatch, DEEPSEEK_API_KEY="ds-key", OPENROUTER_API_KEY="or-key", OPENROUTER_BASE_URL="https://openrouter.ai/api/v1") resolved = resolve_chat_endpoint(ResolvedConfig()) @@ -47,7 +48,7 @@ def test_deepseek_wins_over_openrouter_and_never_touches_the_embedding_endpoint( def test_resolution_order_override_then_scripted_then_openrouter_then_none(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint + from openultrasast.model.endpoint import resolve_chat_endpoint from openultrasast.tool_hunter import ChatResponse, ScriptedChatClient injected = ScriptedChatClient([ChatResponse(content="{}")]) @@ -68,7 +69,7 @@ def test_resolution_order_override_then_scripted_then_openrouter_then_none(monke def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.learning.endpoint import resolve_chat_endpoint + from openultrasast.model.endpoint import resolve_chat_endpoint _endpoint_env(monkeypatch, OPENROUTER_API_KEY="or-key") @@ -81,16 +82,14 @@ def test_a_provider_that_refuses_to_build_degrades_to_no_endpoint(monkeypatch: p def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: from openultrasast.config import load_config - from openultrasast.learning.endpoint import resolve_chat_endpoint, resolve_models + from openultrasast.model.endpoint import resolve_chat_endpoint, resolve_models _endpoint_env(monkeypatch, MY_KEY="local-key") monkeypatch.setenv("MY_KEY", "local-key") (tmp_path / "ousast.toml").write_text( - '[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n\n' - "[learning]\nk_runs = 3\nround_cost_cap_usd = 2.5\nminibatch = 4\n" + '[models]\njudge = "deepseek-v4-pro"\nchat_base_url = "http://localhost:11434/v1"\nchat_api_key_env = "MY_KEY"\n' ) config = load_config(tmp_path / "ousast.toml") - assert config.learning.k_runs == 3 and config.learning.round_cost_cap_usd == 2.5 and config.learning.minibatch == 4 resolved = resolve_chat_endpoint(config) assert resolved is not None and resolved[1].base_url == "http://localhost:11434/v1" detector, judge = resolve_models(config) @@ -98,7 +97,7 @@ def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pyte def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_field() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": '{"intent": "public"}', "reasoning_content": "step one"}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -110,7 +109,7 @@ def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_fie def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": ""}, {"content": '{"ok": true}'}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -123,7 +122,7 @@ def test_the_adapter_retries_once_on_empty_json_content_then_gives_up() -> None: def test_log_probabilities_are_requested_only_when_asked_for() -> None: - from openultrasast.learning.endpoint import DeepSeekChatClient + from openultrasast.model.endpoint import DeepSeekChatClient recorder = _Recorder([{"content": "{}", "logprobs": {"content": [{"logprob": -0.1}, {"logprob": -0.3}]}}]) client = DeepSeekChatClient(recorder) # type: ignore[arg-type] @@ -133,7 +132,7 @@ def test_log_probabilities_are_requested_only_when_asked_for() -> None: def test_cost_comes_from_the_providers_own_usage_fields() -> None: - from openultrasast.learning.endpoint import ChatEndpoint, price_of + from openultrasast.model.endpoint import ChatEndpoint, price_of flash = ChatEndpoint(provider="deepseek", base_url="https://api.deepseek.com", thinking=False, prices=price_of("deepseek-v4-flash")) cost = flash.cost({"prompt_cache_hit_tokens": 1_000_000, "prompt_cache_miss_tokens": 1_000_000, "completion_tokens": 1_000_000}) @@ -179,7 +178,7 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None `_message_of` returned only `choices[0].message`, so the confidence signal the classifier is supposed to abstain on was always `None` — a knob that reads as "never unsure" whatever the model said.""" - from openultrasast.learning.endpoint import _message_of + from openultrasast.model.endpoint import _message_of payload = { "choices": [ @@ -197,8 +196,29 @@ def test_logprobs_are_read_off_the_choice_where_the_provider_puts_them() -> None def test_a_reply_without_logprobs_reports_none_rather_than_a_number() -> None: - from openultrasast.learning.endpoint import _message_of + from openultrasast.model.endpoint import _message_of from openultrasast.tool_hunter import chat_response_from_message message = _message_of({"choices": [{"message": {"content": "yes"}}]}) assert chat_response_from_message(message).mean_logprob is None + + +def test_thinking_is_a_measured_condition_on_the_models_section() -> None: + """The knob moved from `[learning]` to `[models]` with the noise architecture's removal. + + Thinking is off by default because that is what the committed baseline was measured under — the provider + silently ignores `temperature` while thinking — not because it is known to be better. + """ + from openultrasast.config import load_config + + assert load_config(None).models.thinking is False + + +def test_the_config_can_turn_thinking_on(tmp_path: Path) -> None: + from openultrasast.config import load_config + + path = tmp_path / "ousast.toml" + path.write_text("[models]\nthinking = true\n") + assert load_config(path).models.thinking is True + path.write_text("[models]\nthinking = false\n") + assert load_config(path).models.thinking is False diff --git a/tests/test_model_specs.py b/tests/test_model_specs.py new file mode 100644 index 0000000..93e212d --- /dev/null +++ b/tests/test_model_specs.py @@ -0,0 +1,101 @@ +"""model-grounded-detection task 1.1: the security vocabularies survive the modules that carried them. + +The guard vocabulary lived in `semantic/variants.py`, whose optimisation loop this feature deletes. The +source/sink/sanitizer facts and the obligation facts live in `ruleset/` data and are retained, so the specs +*derive* from them rather than copying them — a copy would be a second place to update. What must be ported +verbatim is the guard vocabulary, because nothing else holds it. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SPECS = Path("src/openultrasast/model/specs.py") + + +def test_the_guard_vocabulary_is_preserved_verbatim() -> None: + from openultrasast.model.specs import GUARD_KINDS + + assert GUARD_KINDS == ("null_test", "bounds_test", "allowlist_test", "auth_check", "parameterized_call", "type_change", "none") + + +def test_the_guard_classifier_still_recognises_each_kind() -> None: + """The patterns classify the *fix*, first match wins — the order is part of the vocabulary.""" + from openultrasast.model.specs import classify_guard_text + + assert classify_guard_text('cur.execute("SELECT * FROM t WHERE id = %s", (uid,))') == "parameterized_call" + assert classify_guard_text("if not current_user.is_authenticated: abort(403)") == "auth_check" + assert classify_guard_text('if name not in ALLOWED: abort(400)') == "allowlist_test" + assert classify_guard_text("if len(buf) > 64: return") == "bounds_test" + assert classify_guard_text("if value is None: return") == "null_test" + assert classify_guard_text("page = int(raw)") == "type_change" + assert classify_guard_text("total = total + 1") == "none" + + +def test_the_seeded_families_and_the_named_gaps_are_both_exact() -> None: + """What the retained fact data can seed, and what it provably cannot — Req 9.2, a gap is *named*. + + `path`, `output_encoding`, `untrusted_destination` and `prototype` have no sink in `ruleset/semantic` + whose CWE routes to them, which is the same hole the entailment ceiling diagnosed ("closed literal sink + table: misses per-family sinks"). Task 4.2 authors those sinks and must update this test when it does — + the set is asserted exactly so a gap can neither appear nor close silently. + """ + from openultrasast.model.specs import dominance_specs, taint_specs + from openultrasast.model.taxonomy import load_families + + taxonomy = load_families() + seeded = set() + for language in ("python", "javascript", "java", "c"): + seeded |= {spec.family for spec in taint_specs(language=language).values()} + seeded |= {spec.family for spec in dominance_specs(language=language).values()} + deferred = {"unknown", "memory"} # the declined bucket and the deferred execution tier + assert seeded == {"injection", "deserialization", "config_secrets", "access_control", "memory"} + gaps = {family.id for family in taxonomy.families} - seeded - deferred + assert gaps == {"path", "output_encoding", "untrusted_destination", "prototype"}, ( + "a family lost or gained a seed without this record being updated" + ) + + +def test_the_injection_taint_spec_carries_sources_sinks_and_sanitizers() -> None: + from openultrasast.model.specs import taint_specs + + spec = taint_specs(language="python")["injection"] + assert any("request.args" in source for source in spec.sources) + assert any(sink in ("eval", "exec", "os.system") for sink in spec.sinks) + assert spec.sanitizers, "a family with no sanitizer model can never distinguish a fixed twin" + + +def test_the_access_control_dominance_spec_carries_operations_and_dischargers() -> None: + from openultrasast.model.specs import dominance_specs + + spec = dominance_specs(language="python")["access_control"] + assert any("filter_by" in operation for operation in spec.operations) + assert spec.dischargers, "an absence family with no discharger model can only ever report suspicion" + + +def test_specs_import_nothing_from_the_removed_packages() -> None: + """Req 1.2: the vocabularies must not depend on the modules this feature deletes.""" + tree = ast.parse(SPECS.read_text()) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + elif isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + forbidden = ("learning", "improve", "variants", "mechanisms", "evolve") + offending = [name for name in imported if any(token in name for token in forbidden)] + assert not offending, f"model/specs.py imports removed code: {offending}" + + +def test_the_specs_module_exposes_no_writer() -> None: + """Req 7.4, the verifier boundary: the models are read-only data no LLM or optimiser can rewrite.""" + tree = ast.parse(SPECS.read_text()) + writers = [ + node.name + for node in tree.body + if isinstance(node, ast.FunctionDef) + and any(node.name.startswith(verb) for verb in ("write", "save", "dump", "store", "append", "update", "set_")) + ] + assert not writers, f"specs must be read-only, found writers: {writers}" + assert "open(" not in SPECS.read_text().replace("# ", ""), "specs must not open a file for writing" diff --git a/tests/test_learning_families.py b/tests/test_model_taxonomy.py similarity index 93% rename from tests/test_learning_families.py rename to tests/test_model_taxonomy.py index 24a9e66..c3510a8 100644 --- a/tests/test_learning_families.py +++ b/tests/test_model_taxonomy.py @@ -14,7 +14,7 @@ GOOD = ( def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None: - from openultrasast.learning.families import FAMILY_IDS, VERIFIER_KINDS, load_families + from openultrasast.model.taxonomy import FAMILY_IDS, VERIFIER_KINDS, load_families taxonomy = load_families() assert len(FAMILY_IDS) == 10 and FAMILY_IDS[-1] == "unknown" @@ -27,7 +27,7 @@ def test_taxonomy_loads_ten_families_each_with_a_verifier() -> None: def test_every_mechanism_id_resolves_to_exactly_one_family() -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families from openultrasast.pairs import load_mechanisms taxonomy = load_families() @@ -43,7 +43,7 @@ def test_every_mechanism_id_resolves_to_exactly_one_family() -> None: def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families taxonomy = load_families() seen: dict[str, str] = {} @@ -63,7 +63,7 @@ def test_every_labeled_corpus_cwe_resolves_and_no_cwe_is_claimed_twice() -> None def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared(tmp_path: Path) -> None: - from openultrasast.learning.families import load_families + from openultrasast.model.taxonomy import load_families taxonomy = load_families() assert taxonomy.related("injection", "injection") == "same" @@ -109,7 +109,7 @@ def test_relation_answers_same_lateral_fabricated_and_parent_child_when_declared ], ) def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str) -> None: - from openultrasast.learning.families import FamiliesError, load_families + from openultrasast.model.taxonomy import FamiliesError, load_families (tmp_path / "families.toml").write_text(text) with pytest.raises(FamiliesError, match=needle): @@ -117,7 +117,7 @@ def test_loader_rejects_data_bugs_by_name(tmp_path: Path, text: str, needle: str def test_the_shipped_file_must_declare_the_whole_closed_set_in_order(tmp_path: Path) -> None: - from openultrasast.learning.families import FamiliesError, load_families + from openultrasast.model.taxonomy import FamiliesError, load_families (tmp_path / "families.toml").write_text(GOOD) with pytest.raises(FamiliesError, match="in that order"): diff --git a/tests/test_module_audit.py b/tests/test_module_audit.py new file mode 100644 index 0000000..b6fc5f8 --- /dev/null +++ b/tests/test_module_audit.py @@ -0,0 +1,45 @@ +"""model-grounded-detection Req 3: outgrowth is visible and checkable, not noticed two specs later.""" + +from __future__ import annotations + +import json +from pathlib import Path + +SRC = Path("src/openultrasast") +MANIFEST = Path("benchmarks/measurements/2026-09-08-module-audit.json") + + +def test_no_module_in_the_tree_is_orphaned() -> None: + from openultrasast.model.audit import audit + + orphans = [row.module for row in audit(SRC) if row.classification == "orphaned"] + assert not orphans, f"orphaned modules must be removed, not carried: {orphans}" + + +def test_every_module_is_classified() -> None: + from openultrasast.model.audit import audit, module_name + + classified = {row.module for row in audit(SRC)} + on_disk = {module_name(path, SRC) for path in SRC.rglob("*.py") if "__pycache__" not in path.parts} + assert on_disk - classified <= {""}, f"unclassified: {sorted(on_disk - classified - {''})}" + + +def test_the_named_subsystems_each_carry_an_explicit_decision() -> None: + """Req 3.3: the five older-phase subsystems get a decision, not silence.""" + from openultrasast.model.audit import audit + + rows = {row.module: row for row in audit(SRC)} + for name in ("fusion", "mcp", "skills", "harness_ext", "hunter_harness"): + assert name in rows, f"{name} vanished without a recorded decision" + assert rows[name].reason, f"{name} has no stated reason" + + +def test_the_manifest_matches_the_tree() -> None: + """A later reader can check the tree against the committed manifest (Req 3.4).""" + from openultrasast.model.audit import audit, to_dict + + assert MANIFEST.is_file(), "the audit manifest is committed evidence, not a local artifact" + committed = json.loads(MANIFEST.read_text()) + current = to_dict(audit(SRC)) + assert committed["counts"] == current["counts"], "the tree drifted from its committed audit" + assert committed["orphaned"] == current["orphaned"] == [] diff --git a/tests/test_obligation_check.py b/tests/test_obligation_check.py index 3344034..a890589 100644 --- a/tests/test_obligation_check.py +++ b/tests/test_obligation_check.py @@ -115,37 +115,24 @@ def test_path_records_gate_reachability_and_a_dominating_hop_discharges() -> Non assert covered.findings # without the hop witness the finding stands -def test_findings_become_suspicion_static_findings_with_known_fix_and_labels(tmp_path: Path) -> None: - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.obligations.check import findings_to_static - from openultrasast.semantic.obligations.shapes import ObligationShape, obligation_mechanisms +def test_findings_become_suspicion_static_findings_with_labels() -> None: + """The finding-id contract, the suspicion rung and the tags, without the deleted mechanism store. - store = MechanismStore(tmp_path / "m.jsonl") - shape = ObligationShape( - language="python", - operation_kind="protected_read", - discharger_kind="identity_constraint", - provenance="authenticated_context", - resource_class="owned", - mechanism="missing_auth_guard", - ) - record = append_from_pair(store, shape, summary="s", cwe="CWE-639", pair="vampi", provenance="human", tier="seeded") - result = _check(store_shapes=obligation_mechanisms(store.load())) + The original also asserted `known_fix` — a hint the store supplied. The store is gone, so the hint is gone + with it; everything the checker establishes on its own is still pinned here. + """ + from openultrasast.semantic.obligations.check import findings_to_static + + result = _check() leaky = next(f for f in result.findings if f.operation.function == "leaky" and f.missing == "identity_constraint") - assert leaky.known_fix == record.id + guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint") statics = findings_to_static(result) ids = {s.finding_id for s in statics} - guarded = next(f for f in result.findings if f.operation.function == "guarded" and f.missing == "identity_constraint") assert f"obligation:protected_read:app.py:{leaky.operation.line}:identity_constraint" in ids # exact id contract assert f"obligation:protected_read:app.py:{guarded.operation.line}:identity_constraint" in ids static = next(s for s in statics if s.line == leaky.operation.line and "discharger:identity_constraint" in s.tags) assert static.evidence_level == "suspicion" - assert { - "obligation:protected_read", - "discharger:identity_constraint", - "obligation_evidence:consistency_violation", - f"mechanism:{record.id}", - } <= set(static.tags) + assert {"obligation:protected_read", "discharger:identity_constraint", "obligation_evidence:consistency_violation"} <= set(static.tags) assert "app.py::constrained" in static.rationale diff --git a/tests/test_obligation_export.py b/tests/test_obligation_export.py deleted file mode 100644 index 7846569..0000000 --- a/tests/test_obligation_export.py +++ /dev/null @@ -1,160 +0,0 @@ -"""authorization-obligations task 1.3: obligation shapes travel through the existing exporter and store.""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.benchmark import ExpectedFinding -from openultrasast.pairs import PairCase -from openultrasast.semantic.mechanisms import MechanismStore - -VULN = ( - "from flask import request\n\n\ndef get_book(book_title):\n" - " resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n" - " book = Book.query.filter_by(book_title=book_title).first()\n return book\n" -) -FIXED = ( - "from flask import request\n\n\ndef get_book(book_title):\n" - " resp = token_validator(request.headers.get('Authorization'))\n if 'error' in resp:\n return None\n" - " user = User.query.filter_by(username=resp['sub']).first()\n" - " book = Book.query.filter_by(user=user, book_title=book_title).first()\n return book\n" -) -FLOW_VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FLOW_FIXED = ( - "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - + " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _case( - tmp_path: Path, name: str, vuln: str, fixed: str, *, function: str, mechanism: str, tier: str = "seeded", sink: str | None = None -) -> PairCase: - (tmp_path / f"{name}-v.py").write_text(vuln) - (tmp_path / f"{name}-f.py").write_text(fixed) - return PairCase( - name=name, - slice="vibe-py", - language="python", - origin="test", - vuln_file=tmp_path / f"{name}-v.py", - fixed_file=tmp_path / f"{name}-f.py", - relpath="app.py", - expected=( - ExpectedFinding( - cwe="CWE-639", vulnerability_class="x", path="app.py", evidence="", function=function, mechanism=mechanism, sink=sink - ), - ), - min_recall=1.0, - fix_policy="silent", - provenance="human", - review_tier=tier, - reviewer="t" if tier == "reviewed" else "", - ) - - -def test_exporter_writes_obligation_rows_deduped_with_provenance_and_skips_untrusted(tmp_path: Path) -> None: - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "mechanisms.jsonl") - cases = [ - _case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard"), - _case( - tmp_path, - "b", - VULN.replace("book", "item"), - FIXED.replace("book", "item"), - function="get_item", - mechanism="missing_auth_guard", - tier="reviewed", - ), - _case(tmp_path, "advisory", VULN, FIXED, function="get_book", mechanism="missing_auth_guard", tier="advisory"), - _case(tmp_path, "flow", FLOW_VULN, FLOW_FIXED, function="run", mechanism="source_reaches_sink", sink="os.system"), - ] - report = export_mechanisms(cases, store) - rows = store.load() - obligations = obligation_mechanisms(rows) - assert len(obligations) == 1 and obligations[0].pairs == ("a", "b") and obligations[0].origin == "corpus" - assert ( - obligations[0].shape is not None - and obligations[0].shape["family"] == "obligation" - and obligations[0].guard == "identity_constraint" - ) - sinks = [r for r in rows if r.shape and r.shape.get("family", "sink") != "obligation"] - assert len(sinks) == 1 and sinks[0].shape is not None and sinks[0].shape["sink_name"] == "system" # flow rows unaffected - assert report.seeded == 3 and dict(report.skipped)["advisory"] == "tier:advisory" - text = (tmp_path / "mechanisms.jsonl").read_text() - assert "book_title" not in text and "resp" not in text and "app.py" not in text - - -def test_absence_pair_without_a_lesson_is_skipped_with_the_derivation_reason(tmp_path: Path) -> None: - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "m.jsonl") - report = export_mechanisms([_case(tmp_path, "same", VULN, VULN, function="get_book", mechanism="missing_auth_guard")], store) - assert store.load() == () and dict(report.skipped)["same"].startswith("no_discharger_added:identity_constraint") - - -def test_sink_search_ignores_obligation_rows(tmp_path: Path) -> None: - from openultrasast.preprocess import preprocess_repository - from openultrasast.semantic.facts import load_facts - from openultrasast.semantic.seed import export_mechanisms - from openultrasast.semantic.variant_search import search_tree - - store = MechanismStore(tmp_path / "m.jsonl") - export_mechanisms([_case(tmp_path, "a", VULN, FIXED, function="get_book", mechanism="missing_auth_guard")], store) - repo = tmp_path / "repo" - repo.mkdir() - (repo / "views.py").write_text(VULN) - _, targets = preprocess_repository(repo) - result = search_tree(repo, targets, store, load_facts(), max_mechanisms=500) - assert result.mechanisms_searched == 0 and result.hits == () and result.degradations == () - - -def test_the_train_split_of_vibe_py_teaches_no_obligation_shape_today(tmp_path: Path) -> None: - """learning-harness Req 5.1 turned this number honest, and the honest number is zero. - - Every vibe-py absence pair that yields an obligation lesson sits on the holdout side, so once only - train-split pairs may teach, the corpus teaches no obligation shape at all. The earlier result - ("at least one obligation shape from vibe-py") was true only because holdout pairs were teaching. - The derivation itself stays covered by the fixture tests above; this pins the corpus gap, which the - absence re-harvest and a maintainer pass over the splits are what close. - """ - from openultrasast.pairs import load_pair_catalog, select_slice, select_vendored - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - store = MechanismStore(tmp_path / "m.jsonl") - cases = select_vendored(select_slice(load_pair_catalog(), "vibe-py")) - report = export_mechanisms(cases, store) - assert obligation_mechanisms(store.load()) == () - refused = {pair: reason for pair, reason in report.skipped} - assert refused["vampi-users-update-password"] == "split:holdout" - assert refused["threatbyte-api-v1-get"] == "split:holdout" - # Its own reason, not its split: this row is a byte-identical twin that happens to sit on the holdout side, - # and reporting `split:holdout` would hide a corpus fact behind a split fact (Req 5.2). - assert refused["vampi-books-get-by-title"] == "identical_twin" - flow_rows = [row for row in store.load() if row.shape is not None and row.shape.get("family", "sink") == "sink"] - assert report.records == len(flow_rows) and len(flow_rows) >= 1 # the flow family still teaches from the train split - - -def test_absence_row_without_an_obligation_lesson_still_teaches_its_sink_shape(tmp_path: Path) -> None: - """The fallback that keeps the flow store unchanged: a `missing_auth_guard` row whose twin adds no discharger but whose - labeled sink call carries a source still seeds a sink shape (the threatbyte pairs do exactly this).""" - from openultrasast.semantic.obligations.shapes import obligation_mechanisms - from openultrasast.semantic.seed import export_mechanisms - - vuln = ( - "import sqlite3\n\n\ndef get(user_id):\n cur = sqlite3.connect('x').cursor()\n" - " cur.execute('SELECT * FROM users WHERE id = ' + user_id)\n return cur.fetchone()\n" - ) - fixed = ( - "import sqlite3\n\n\ndef login(username):\n cur = sqlite3.connect('x').cursor()\n" - " cur.execute('SELECT * FROM users WHERE name = ?', (username,))\n return cur.fetchone()\n" - ) - store = MechanismStore(tmp_path / "m.jsonl") - report = export_mechanisms([_case(tmp_path, "tb", vuln, fixed, function="get", mechanism="missing_auth_guard", sink="execute")], store) - rows = store.load() - assert report.seeded == 1 and obligation_mechanisms(rows) == () - assert len(rows) == 1 and rows[0].shape is not None and rows[0].shape.get("family", "sink") == "sink" - assert rows[0].shape["sink_name"] == "execute" diff --git a/tests/test_pair_corpus.py b/tests/test_pair_corpus.py index cf947f5..cf932ec 100644 --- a/tests/test_pair_corpus.py +++ b/tests/test_pair_corpus.py @@ -370,55 +370,6 @@ def test_known_limit_pairs_are_evaluated_but_excluded_from_achievable(tmp_path: assert set(result.per_mechanism) == {"cross_artifact", "source_reaches_sink"} assert result.loss["sast"]["known_limit"] == 1 assert "overlay" in result.scorers["sast"] and "inventory" in result.scorers["sast"] - assert result.degradations and result.degradations[0]["reason"] == "hunter_model_unavailable" - - -def test_hunter_scorer_credits_a_family_tag_and_never_the_words(tmp_path: Path) -> None: - """learning-harness Req 3.1/3.6 replaced this path's rule: the family tag decides, the rationale never does.""" - from openultrasast.findings import StaticFinding - - src = "from flask import request\n\ndef f():\n return eval(request.args.get('x'))\n" - row = ExpectedFinding( - cwe="CWE-95", - vulnerability_class="code injection", - path="app.py", - evidence="", - sink="eval", - function="f", - mechanism="source_reaches_sink", - ) - - def scripted(tags: list[str]): # type: ignore[no-untyped-def] - def scan(root: Path) -> list[StaticFinding]: - if "eval(" not in (root / "app.py").read_text(): - return [] - return [ - StaticFinding( - finding_id="tool-hunter:app.py:4", - path="app.py", - title="eval of request input", - severity="high", - confidence="medium", - evidence_level="suspicion", - rationale="CWE-95 eval executes attacker input", - line=4, - function_name=None, - reachability_status="unknown", - reachability_evidence=[], - reachability_conditions=[], - tags=tags, - ranking_priority=1.0, - ) - ] - - return scan - - case = _case(tmp_path, "hunt", src, "def f():\n return 1\n", row) - tagged = evaluate_catalog([case], hunter=scripted(["family:injection"])).scorers["sast"]["hunter"] - assert tagged.pair_correct == 1 and tagged.detected_vuln == 1 and tagged.silent_fix == 1 - # the same finding, the same words, no family tag: not a detection, because the rationale is not evidence - untagged = evaluate_catalog([case], hunter=scripted([])).scorers["sast"]["hunter"] - assert untagged.detected_vuln == 0 and untagged.silent_fix == 1 def test_juliet_strcpy_pair_bodies_differ_and_java_hash_is_known_limit() -> None: diff --git a/tests/test_pair_registration_context.py b/tests/test_pair_registration_context.py index e1fbdac..2e6dfc2 100644 --- a/tests/test_pair_registration_context.py +++ b/tests/test_pair_registration_context.py @@ -17,8 +17,8 @@ from pathlib import Path import pytest -from openultrasast.learning.families import load_families from openultrasast.mapping import analyze_entry_points +from openultrasast.model.taxonomy import load_families from openultrasast.pairs import DEFAULT_CATALOG, load_pair_catalog, select_vendored from openultrasast.preprocess import preprocess_repository from openultrasast.semantic.extra import has_semantic_extra diff --git a/tests/test_semantic_pipeline.py b/tests/test_semantic_pipeline.py index 493e0e0..b85d3ba 100644 --- a/tests/test_semantic_pipeline.py +++ b/tests/test_semantic_pipeline.py @@ -1,4 +1,4 @@ -"""MAP overlay, prove filter, reports, pair scoring, mechanism memory.""" +"""MAP overlay, prove filter, reports, pair scoring.""" from __future__ import annotations @@ -15,7 +15,7 @@ from openultrasast.gate import LANGUAGE_MANIFESTS from openultrasast.pairs import PairCase, evaluate_pair, load_pair_catalog from openultrasast.preprocess import preprocess_repository from openultrasast.rank import rank_targets -from openultrasast.semantic import MechanismStore, OverlayRecord, append_mechanism, filter_promoted_hotspots, order_promotions +from openultrasast.semantic import OverlayRecord, filter_promoted_hotspots from openultrasast.semantic.prove_filter import promoted_findings @@ -265,79 +265,3 @@ def test_markdown_report_includes_four_overlay_labels(tmp_path: Path) -> None: assert "coverage" in text -def test_mechanism_order_boosts_paraphrased_match(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism( - store, - summary="request parameter reaches eval", - cwe="CWE-95", - language="python", - tags=["syscall_entry"], - what_made_it_exploitable="dynamic execution of attacker input", - ) - eval_id = "python-unsafe-eval:app.py:1" - hash_id = "python-weak-hash:hash.py:1" - hotspots = (_hotspot(hash_id, score=5.0, path="hash.py"), _hotspot(eval_id, score=5.0, path="app.py")) - records = ( - _record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",), language="python"), - _record(eval_id, "promote", sources=("request",), cwe="CWE-95", sinks=("eval",), language="python"), - ) - findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)] - - class _Client: - def embed(self, *, model: str, inputs: list[str], timeout_seconds: int = 60) -> list[list[float]]: - del model, timeout_seconds - vectors: list[list[float]] = [] - for text in inputs: - if "eval" in text.lower() or "CWE-95" in text: - vectors.append([1.0, 0.0]) - else: - vectors.append([0.0, 1.0]) - return vectors - - ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=_Client(), model="test-embed") - assert degradation is None - assert ordered[0].inventory_finding_ids == (eval_id,) - assert {item.inventory_finding_ids[0] for item in ordered} == {eval_id, hash_id} - - -def test_mechanism_degrades_to_heuristic_without_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) - store = MechanismStore(tmp_path / "mechanisms.jsonl") - append_mechanism( - store, - summary="request parameter reaches eval", - cwe="CWE-95", - language="python", - tags=["syscall_entry"], - what_made_it_exploitable="dynamic execution of attacker input", - ) - eval_id = "python-unsafe-eval:app.py:1" - hash_id = "python-weak-hash:hash.py:1" - hotspots = (_hotspot(hash_id, score=6.0, path="hash.py"), _hotspot(eval_id, score=3.0, path="app.py")) - records = ( - _record(hash_id, "promote", path="hash.py", cwe="CWE-327", sinks=("hashlib",)), - _record(eval_id, "promote", sources=("request",)), - ) - findings = [_finding(hash_id, path="hash.py"), _finding(eval_id)] - ordered, degradation = order_promotions(hotspots, records, findings, store=store, client=None, model=None) - assert degradation == "embeddings_unavailable" - assert [item.inventory_finding_ids[0] for item in ordered] == [hash_id, eval_id] - - -def test_mechanisms_are_not_written_from_unadjudicated(tmp_path: Path) -> None: - store = MechanismStore(tmp_path / "mechanisms.jsonl") - record = OverlayRecord( - proposal_id="python-unsafe-eval:app.py:1", - path="app.py", - line=1, - disposition="unadjudicated", - reason="parse_failed", - cwe="CWE-95", - sources=(), - sinks=(), - sanitizers=(), - evidence_level="static_corroboration", - ) - assert record.disposition != "promote" - assert store.load() == () diff --git a/tests/test_subtraction_complete.py b/tests/test_subtraction_complete.py new file mode 100644 index 0000000..4ea55cd --- /dev/null +++ b/tests/test_subtraction_complete.py @@ -0,0 +1,81 @@ +"""model-grounded-detection Req 1.3: nothing survives that imports what this feature removed. + +The noise architecture and the mechanism/evolve loop are deleted, not deprecated. This walks the AST of every +surviving module and fails on any import that names one of them — the check that turns a five-thousand-line +deletion from crash-driven discovery into one assertion. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +SRC = Path("src/openultrasast") + +REMOVED_MODULES = frozenset( + { + # the noise architecture (the whole learning/ package; taxonomy, candidates and endpoint re-homed to model/) + "learning", + # the mechanism/evolve optimisation loop + "semantic.mechanisms", + "semantic.variants", + "semantic.variant_search", + "semantic.prove_budget", + "semantic.loo", + "semantic.seed", + } +) + + +def _module_names(tree: ast.AST) -> set[str]: + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + elif isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + return names + + +def _names_a_removed_module(imported: str) -> bool: + trimmed = imported.lstrip(".") + return any(trimmed == removed or trimmed.startswith(f"{removed}.") for removed in REMOVED_MODULES) + + +def test_no_surviving_module_imports_a_removed_one() -> None: + offenders: dict[str, list[str]] = {} + for path in sorted(SRC.rglob("*.py")): + named = sorted(name for name in _module_names(ast.parse(path.read_text())) if _names_a_removed_module(name)) + if named: + offenders[str(path)] = named + assert not offenders, f"dangling imports of removed modules: {offenders}" + + +def test_the_removed_modules_are_gone_from_the_tree() -> None: + assert not (SRC / "learning").exists(), "the learning package is removed; taxonomy/candidates/endpoint live under model/" + for relative in ("semantic/mechanisms.py", "semantic/variants.py", "semantic/variant_search.py", + "semantic/prove_budget.py", "semantic/loo.py", "semantic/seed.py"): + assert not (SRC / relative).exists(), f"{relative} still exists" + + +def test_the_mechanism_lever_is_gone_from_the_retained_improve_loop() -> None: + """`improve/` survives with its lever removed, not deleted. + + The design's Removed list named `improve/evolve.py` whole, but the file is a mix: the mechanism lever + (deleted here) and the deterministic rule-status loop that `ousast improve` still runs from benchmark + evidence, which has no LLM in it and is not what this feature supersedes. Deleting the file would have + taken a retained capability with it, so the lever is stripped and the loop stays — asserted by symbol so + the lever cannot creep back. + """ + evolve = (SRC / "improve" / "evolve.py").read_text() + validator = (SRC / "improve" / "validator.py").read_text() + for symbol in ("MechanismEdit", "MechanismStore", "apply_mechanism_edits", "evaluate_mechanism_profiles", + "propose_mechanism_edits", "validate_mechanism", "mechanism_store", "mechanism_candidates"): + assert symbol not in evolve, f"{symbol} survives in improve/evolve.py" + assert symbol not in validator, f"{symbol} survives in improve/validator.py" + assert "def run_improvement(" in evolve and "def propose_status_edits(" in evolve, "the rule-status loop must survive" + + +def test_the_re_homed_modules_are_where_the_model_layer_expects_them() -> None: + for relative in ("model/taxonomy.py", "model/candidates.py", "model/endpoint.py", "model/specs.py"): + assert (SRC / relative).is_file(), f"{relative} is missing" diff --git a/tests/test_variant_scan.py b/tests/test_variant_scan.py deleted file mode 100644 index 76f7963..0000000 --- a/tests/test_variant_scan.py +++ /dev/null @@ -1,130 +0,0 @@ -"""corpus-seeded-mechanisms task 2.3: variant findings in a standard scan, sandbox-eligible, counted in the manifest.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from openultrasast.cli import main -from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - -VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n" -UNSEEN_SINK = ( - "from shelltools import run_shell\n\n\ndef handler(q):\n run_shell(q)\n" # a project helper: no inventory rule, no fact sink -) - - -def _seed(repo: Path) -> str: - from openultrasast.semantic.variants import Shape - - shape = Shape( - language="python", - sink_name="run_shell", - arity=1, - source_positions=(0,), - source_kinds=("parameter",), - guard="allowlist_test", - mechanism="source_reaches_sink", - ) - store = MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl") - return append_from_pair( - store, shape, summary="parameter into run_shell; fix adds allowlist", cwe="CWE-78", pair="p1", provenance="human", tier="seeded" - ).id - - -def _latest_run(repo: Path) -> Path: - runs = sorted((repo / ".runs").iterdir()) - return runs[-1] - - -def test_standard_scan_emits_variant_findings_and_manifest_counters(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) # only the learned shape can find a project-specific sink - mechanism_id = _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = _latest_run(repo) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - variant = [f for f in findings if f["finding_id"].startswith("variant:")] - assert len(variant) == 1 and variant[0]["finding_id"] == f"variant:{mechanism_id}:app.py:5" - assert variant[0]["evidence_level"] == "suspicion" and f"mechanism:{mechanism_id}" in variant[0]["tags"] - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert manifest["variants"] == {"mechanisms_searched": 1, "files_searched": 1, "findings": 1, "merged_into_overlay": 0} - - -def test_variant_hit_merges_into_an_overlay_flow_at_the_same_call_site(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast.semantic.variants import Shape - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(VARIANT) # os.system is both an inventory rule and a fact sink: the overlay records the flow - shape = Shape( - language="python", - sink_name="system", - arity=1, - source_positions=(0,), - source_kinds=("fact_source",), - guard="none", - mechanism="source_reaches_sink", - ) - record = append_from_pair( - MechanismStore(repo / ".openultrasast" / "calibration" / "mechanisms.jsonl"), - shape, - summary="s", - cwe="CWE-78", - pair="p", - provenance="human", - tier="seeded", - ) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "0") - assert main(["scan", str(repo), "--mode", "standard"]) == 0 - run_dir = _latest_run(repo) - overlay = json.loads((run_dir / "overlay.json").read_text())["records"] - assert any(item["mechanism_id"] == record.id and item["line"] == 7 for item in overlay) - findings = json.loads((run_dir / "findings.json").read_text())["findings"] - assert not any(f["finding_id"].startswith("variant:") for f in findings) # merged, not duplicated - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert manifest["variants"]["merged_into_overlay"] == 1 and manifest["variants"]["findings"] == 0 - # Req 6.1 end to end: the merged inventory finding names the mechanism in both reports and keeps its own evidence level - report = (run_dir / "report.md").read_text() - assert f"- Mechanism: `{record.id}`" in report and "- Learned from pairs: `p`" in report - sarif = json.loads((run_dir / "report.sarif").read_text()) - props = [item["properties"] for item in sarif["runs"][0]["results"] if item["properties"].get("mechanism_id") == record.id] - assert props and props[0]["mechanism_guard"] == "none" and props[0]["evidence_level"] != "suspicion" - - -def test_quick_scan_never_searches_variants(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) - _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - assert main(["scan", str(repo), "--mode", "quick"]) == 0 - run_dir = _latest_run(repo) - manifest = json.loads((run_dir / "manifest.json").read_text()) - assert "variants" not in manifest - assert not any(f["finding_id"].startswith("variant:") for f in json.loads((run_dir / "findings.json").read_text())["findings"]) - - -def test_deep_scan_offers_variant_findings_to_the_sandbox_after_promotions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from openultrasast import cli - from openultrasast.sandbox import FakeSandboxRunner, SandboxResult - - repo = tmp_path / "repo" - repo.mkdir() - (repo / "app.py").write_text(UNSEEN_SINK) - mechanism_id = _seed(repo) - monkeypatch.setenv("OPENULTRASAST_RUNS_DIR", ".runs") - monkeypatch.setenv("OPENULTRASAST_SANDBOX_PROBE", "1") - fake = FakeSandboxRunner(SandboxResult(exit_code=0, stdout="", stderr="", timed_out=False)) - monkeypatch.setattr(cli, "resolve_sandbox_runner", lambda: fake) - assert main(["scan", str(repo), "--mode", "deep"]) == 0 - run_dir = _latest_run(repo) - verdicts = json.loads((run_dir / "verdicts.json").read_text())["verdicts"] - offered = {finding_id for item in verdicts for finding_id in item["inventory_finding_ids"]} - assert f"variant:{mechanism_id}:app.py:5" in offered diff --git a/tests/test_variant_search.py b/tests/test_variant_search.py deleted file mode 100644 index 066871e..0000000 --- a/tests/test_variant_search.py +++ /dev/null @@ -1,124 +0,0 @@ -"""corpus-seeded-mechanisms task 2.2: structural variant search over a scanned tree, merged with overlay flows.""" - -from __future__ import annotations - -from pathlib import Path - -from openultrasast.semantic.facts import load_facts -from openultrasast.semantic.ir import parse_file -from openultrasast.semantic.overlay import OverlayRecord - -PY_FACTS = load_facts() - - -def _shape(**overrides: object): - from openultrasast.semantic.variants import Shape - - base: dict[str, object] = { - "language": "python", - "sink_name": "system", - "arity": 1, - "source_positions": (0,), - "source_kinds": ("fact_source",), - "guard": "allowlist_test", - "mechanism": "source_reaches_sink", - } - base.update(overrides) - return Shape(**base) # type: ignore[arg-type] - - -VARIANT = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q)\n" -PARAM_VARIANT = "import os\n\n\ndef handler(q):\n os.system(q)\n" -CONSTANT = "import os\n\n\ndef handler():\n os.system('ls -la')\n" -ARITY = "import os\nfrom flask import request\n\n\ndef handler():\n q = request.args.get('q')\n os.system(q, 1)\n" - - -def test_match_shapes_finds_structural_variants_and_ignores_constants() -> None: - from openultrasast.semantic.variants import match_shapes - - shape = _shape() - hits = match_shapes(parse_file("a.py", VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) - assert [(h.path, h.line, h.mechanism_id, h.sink_name, h.source_kind) for h in hits] == [ - ("a.py", 7, "corpus:abc", "system", "fact_source") - ] - param_hits = match_shapes(parse_file("b.py", PARAM_VARIANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) - assert [(h.line, h.source_kind) for h in param_hits] == [(5, "parameter")] # any source kind at the shape's positions matches - assert match_shapes(parse_file("c.py", CONSTANT, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == [] - assert match_shapes(parse_file("d.py", ARITY, "python"), [shape], PY_FACTS, mechanism_ids={shape.key(): "corpus:abc"}) == [] - js_shape = _shape(language="javascript", sink_name="exec") - assert match_shapes(parse_file("a.py", VARIANT, "python"), [js_shape], PY_FACTS, mechanism_ids={js_shape.key(): "x"}) == [] - - -def test_hits_become_suspicion_findings_or_merge_into_an_overlay_flow() -> None: - from openultrasast.semantic.variant_search import hits_to_findings - from openultrasast.semantic.variants import VariantHit - - hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="fact_source") - other = VariantHit(path="z.py", line=3, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter") - coverage = OverlayRecord( - proposal_id="overlay-coverage:a.py:7:os.system", - path="a.py", - line=7, - disposition="coverage", - reason="uninventoried sink os.system reached from request", - cwe="CWE-78", - sources=("request",), - sinks=("os.system",), - sanitizers=(), - evidence_level="static_corroboration", - origin="overlay", - engine="python-ast", - language="python", - ) - summaries = {"corpus:abc": ("request into os.system; fix adds allowlist_test", ("p1", "p2"), "CWE-78")} - findings, records = hits_to_findings([hit, other], [coverage], summaries) - assert len(records) == 1 and records[0].mechanism_id == "corpus:abc" and records[0].disposition == "coverage" - assert [f.finding_id for f in findings] == ["variant:corpus:abc:z.py:3"] # the a.py hit merged, no duplicate - (finding,) = findings - assert finding.evidence_level == "suspicion" and "variant" in finding.tags and "mechanism:corpus:abc" in finding.tags - assert "p1" in finding.rationale and finding.line == 3 and finding.path == "z.py" - - -def test_search_tree_uses_the_store_and_respects_the_mechanism_cap(tmp_path: Path) -> None: - from openultrasast.preprocess import preprocess_repository - from openultrasast.semantic.mechanisms import MechanismStore, append_from_pair - from openultrasast.semantic.variant_search import search_tree - - root = tmp_path / "repo" - root.mkdir() - (root / "views.py").write_text(VARIANT) - (root / "safe.py").write_text(CONSTANT) - store = MechanismStore(tmp_path / "mechanisms.jsonl") - record = append_from_pair(store, _shape(), summary="s", cwe="CWE-78", pair="p1", provenance="human", tier="seeded") - _, targets = preprocess_repository(root) - result = search_tree(root, targets, store, PY_FACTS, max_mechanisms=500) - assert [(h.path, h.line, h.mechanism_id) for h in result.hits] == [("views.py", 7, record.id)] - assert result.mechanisms_searched == 1 and result.files_searched == 2 - capped = search_tree(root, targets, store, PY_FACTS, max_mechanisms=0) - assert capped.hits == () and capped.mechanisms_searched == 0 - - -def test_hits_never_merge_into_demoted_or_flowless_records() -> None: - from openultrasast.semantic.variant_search import hits_to_findings - from openultrasast.semantic.variants import VariantHit - - hit = VariantHit(path="a.py", line=7, mechanism_id="corpus:abc", sink_name="system", source_kind="parameter") - demoted = OverlayRecord( - proposal_id="python-os-command:a.py:7", - path="a.py", - line=7, - disposition="demote", - reason="constant argument", - cwe="CWE-78", - sources=(), - sinks=("os.system",), - sanitizers=(), - evidence_level="suspicion", - dominating_fact="constant", - origin="inventory", - engine="python-ast", - language="python", - ) - findings, records = hits_to_findings([hit], [demoted], {}) - assert records[0].mechanism_id is None and records[0].disposition == "demote" # Req 3.6: never demote, never re-label a demotion - assert [f.finding_id for f in findings] == ["variant:corpus:abc:a.py:7"] diff --git a/tests/test_variants.py b/tests/test_variants.py deleted file mode 100644 index bc72042..0000000 --- a/tests/test_variants.py +++ /dev/null @@ -1,76 +0,0 @@ -"""corpus-seeded-mechanisms: structural shapes derived from pairs and matched on trees (offline, stdlib ast).""" - -from __future__ import annotations - -from openultrasast.semantic.facts import load_facts -from openultrasast.semantic.ir import parse_file - -VULN = "from flask import request\nimport os\n\n\ndef run():\n cmd = request.args.get('cmd')\n os.system(cmd)\n" -FIXED = ( - "from flask import request\nimport os\n\nALLOWED = {'ls', 'date'}\n\n\ndef run():\n cmd = request.args.get('cmd')\n" - " if cmd not in ALLOWED:\n return None\n os.system(cmd)\n" -) - - -def _derive(vuln: str = VULN, fixed: str = FIXED, **overrides: object): - from openultrasast.semantic.variants import derive_shape - - kwargs: dict[str, object] = {"function": "run", "sink": "os.system", "line": None, "mechanism": "source_reaches_sink"} - kwargs.update(overrides) - return derive_shape( - parse_file("app.py", vuln, "python"), - parse_file("app.py", fixed, "python"), - vuln_text=vuln, - fixed_text=fixed, - facts=load_facts(), - **kwargs, - ) - - -def test_derive_shape_from_a_request_to_os_system_pair_with_an_allowlist_fix() -> None: - from openultrasast.semantic.variants import GUARD_KINDS, SOURCE_KINDS - - shape = _derive() - assert shape is not None - assert shape.language == "python" and shape.sink_name == "system" and shape.arity == 1 - assert shape.source_positions == (0,) and shape.source_kinds == ("fact_source",) - assert shape.guard == "allowlist_test" and shape.guard in GUARD_KINDS and shape.mechanism == "source_reaches_sink" - assert set(shape.source_kinds) <= set(SOURCE_KINDS) - - -def test_shape_key_is_stable_and_carries_no_path_line_or_literal() -> None: - a, b = _derive(), _derive() - assert a is not None and b is not None and a.key() == b.key() - key = a.key() - for forbidden in ("app.py", "cmd", "'ls'", "request.args", ":6", ":7"): - assert forbidden not in key - # the same code in another file with different variable names is the same shape - other_vuln = VULN.replace("cmd", "command") - other = _derive(other_vuln, FIXED.replace("cmd", "command")) - assert other is not None and other.key() == key - - -def test_parameter_and_container_read_sources_are_classified() -> None: - vuln = "import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n os.system(cmd + target)\n" - fixed = ( - "import os\n\n\ndef run(payload, cmd):\n target = payload['t']\n if len(target) > 32:\n return None\n" - " os.system(cmd + target)\n" - ) - shape = _derive(vuln, fixed) - assert shape is not None - assert shape.source_positions == (0,) - assert shape.source_kinds in {("parameter",), ("container_read",)} # one argument carrying both; the kind is the first found - assert shape.guard == "bounds_test" - - -def test_unlabeled_or_missing_sink_returns_none_and_guard_defaults_to_none() -> None: - assert _derive(sink="subprocess.run") is None # the label names a sink the function never calls - assert _derive(function="other") is None # no such function - same = _derive(VULN, VULN) - assert same is not None and same.guard == "none" # no statement differs on the fixed side - constant = "import os\n\n\ndef run():\n os.system('ls')\n" - assert _derive(constant, constant) is None # a constant argument carries no source: nothing to learn - - -def test_parse_failed_side_yields_no_shape() -> None: - assert _derive("def run(:\n", FIXED) is None