From 88ada8e48c53222a657d1abc73f29ea316773c3a Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:22:18 +0200 Subject: [PATCH] feat(ranking): phase 0 -- the baseline, the evidence instrument, and a 7x taint speedup found on the way flow-aware-ranking phase 0: measure before changing anything. THE BASELINE, committed as benchmarks/ranking/baselines/2026-09-10-access-rank.json from benchmarks/ranking/position.py, which sorts regions exactly as the scan does: pmpro 4,463 regions 83.8% at 0.30 CVE at 390 budget_at_recall 391 mwwpform 983 regions 81.7% at 0.30 CVE at 512 budget_at_recall 513 wpstatistics 1,673 regions 75.3% at 0.30 CVE at 503 budget_at_recall 504 vampi 25 regions BOLA at 15, 10; SQLI reached transitively That corrects the brief's own cross-assessment: "concentration, not inclusion" was measured on pmpro alone. On the other two plugins the CVE sits OUTSIDE a 500-region budget and is never examined; the slices found those CVEs only because a slice has too few regions to exclude anything. THE INSTRUMENT: `evidenceOnly` mode on the taint query -- what is in scope, which sink calls, what shape, whether a sanitizer wraps THEIR argument, whether a source is local or reachable -- returning before reachableByFlows is ever called. model/evidence.py turns its rows into a vector per (region, family) and a tier; scan.py records every pair's tier on the result and acts on none of it. Every later phase is measured against a record made by scans that did not know they were being measured. 2,500 pairs in 98.9s incl. JVM load, ~40 ms per pair tier 0 -- no sink of that family in reach: 2,161 of 2,500 = 86% Two earlier attempts ran past an hour each with no payload, and both were my fault: repository-wide walks repeated per request -- the sink scan, the reachable-method BFS, familyInRepo. Memoised across the batch. AND THE SAME FIX REACHES THE DATAFLOW MODE. sinkCalls and reachableMethods do not depend on the family, and were recomputed for all 2,500 requests. The identical 250-request set with both joins on: 2,026.3s -> 287.0s 8,105 -> 1,148 ms per request same 5,779 rows Most of what 5.13 measured as "the base taint query" was never dataflow. A 7x speedup with nothing about the analysis changed; both corrections recorded in the specs. Combined with 86% tier 0, the whole-repository taint query that timed out at 40 minutes is ~339 real requests at ~1.1s: about six minutes. Also: the module audit manifest is regenerated (91 load-bearing, 7 standalone, 0 orphaned) because evidence.py is now imported by the scan, and the failed-query test is order-independent since the evidence pass records its own failure before taint's. 869 tests pass; query compile test 4 passed; gates byte-identical. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG --- .kiro/specs/contributor-scan/tasks.md | 7 + .kiro/specs/flow-aware-ranking/brief.md | 35 + .../measurements/2026-09-08-module-audit.json | 709 +++++++++--------- .../baselines/2026-09-10-access-rank.json | 96 +++ benchmarks/ranking/position.py | 121 +++ src/openultrasast/cpg/queries/taint.sc | 77 +- src/openultrasast/model/evidence.py | 150 ++++ src/openultrasast/model/scan.py | 47 +- tests/test_model_evidence.py | 99 +++ tests/test_model_scan.py | 4 +- 10 files changed, 1000 insertions(+), 345 deletions(-) create mode 100644 benchmarks/ranking/baselines/2026-09-10-access-rank.json create mode 100644 benchmarks/ranking/position.py create mode 100644 src/openultrasast/model/evidence.py create mode 100644 tests/test_model_evidence.py diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 8d0e4f8..612c38c 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -821,6 +821,13 @@ half is free (inside noise, and measured slightly faster), the hook half adds 7.5%, both add 22%. Removing them buys a fifth of the time and costs two of the three CVEs, which is not a trade worth making. + - **Corrected 2026-09-10, same day: most of that 6.63 s was never dataflow.** `sinkCalls` scanned every + call in the graph per request, and `reachableMethods` ran a call-graph BFS per request — both + independent of the region's family, both repeated 2,500 times. Memoised across the batch (found while + building flow-aware-ranking's evidence mode, which asks no dataflow and still took an hour), the same + 250 requests with both joins on: **2,026.3 s → 287.0 s, 8,105 → 1,148 ms per request, identical 5,779 + rows.** A 7× speedup with nothing about the analysis changed. The base cost of a taint request is about + 1.1 s, and the levers above still apply to it. - **The estimate this replaces was wrong and worth recording as wrong.** ">0.94s per request" came from assuming taint COMPLETED within its 2,400s ceiling. It timed out, so that was a floor presented as a figure. The measured cost is seven times higher. diff --git a/.kiro/specs/flow-aware-ranking/brief.md b/.kiro/specs/flow-aware-ranking/brief.md index 1e46900..1c9f7b6 100644 --- a/.kiro/specs/flow-aware-ranking/brief.md +++ b/.kiro/specs/flow-aware-ranking/brief.md @@ -55,6 +55,41 @@ A phase that cannot show that is not an improvement, it is a story. - **Phase 0 — the oracle and the baseline.** The evidence extractor (no model) and a scoring harness over the pinned CVEs, reporting each one's rank position. Gate: reproduces today's numbers — CVE-2023-23488 at position 390 of 4,463, 83.8% of regions in one tier. Without a baseline nothing later is measurable. + + **Done 2026-09-10.** `benchmarks/ranking/position.py`, committed as + `benchmarks/ranking/baselines/2026-09-10-access-rank.json`: + + | repository | regions | largest tier | CVE position | `budget_at_recall` | + |---|---|---|---|---| + | `pmpro` | 4,463 | 83.8% at 0.30 | 390 | 391 | + | `mwwpform` | 983 | 81.7% at 0.30 | **512** | **513** | + | `wpstatistics` | 1,673 | 75.3% at 0.30 | **503** | **504** | + | `vampi` | 25 | 36.0% at 1.00 | 15, 10 (SQLI reached transitively) | 16 | + + **And this corrects the cross-assessment above.** "Ranking's job is concentration, not inclusion" was + measured on `pmpro` alone, where the CVE is inside the budget. On the other two plugins the CVE sits at + 512 and 503 — **outside a 500-region budget** — so the current ranker never examines it at all, and the + slices found those CVEs only because a slice has too few regions to exclude anything. Both jobs are + real: inclusion on two repositories, concentration on all three. The single metric `budget_at_recall` + covers both, which is why it is the one that matters. + + The instrument is `evidenceOnly` mode on the taint query, recorded on every scan as `ModelScanResult.tiers` + and `tier_counts`, and acted on by nothing — so every later phase is measured against a record made by + scans that did not know they were being measured. + + **Measured cost of the instrument**, 637-file graph, the scan's own 500-region request set: + + 2,500 pairs in 98.9 s including JVM load — about 40 ms per pair + tier 0 (no sink of that family in reach): 2,161 of 2,500 = 86% + + Two attempts before that ran past an hour each without producing a payload, and the cause was mine both + times: repository-wide walks — the sink scan, the reachable-method BFS, `familyInRepo` — repeated per + request. Memoising them across the batch is what made "milliseconds per pair" true rather than claimed. + + **And the same memoisation reaches the dataflow mode.** The identical 250-request set that cost 2,026.3 s + (8,105 ms per request) with both joins on now costs **287.0 s — 1,148 ms per request — for the same 5,779 + rows.** Most of what contributor-scan 5.13 measured as "the base taint query" was the per-request scans, + not the dataflow. That correction is recorded there. - **Phase 1 — tier 0 only, no model.** Exact pruning. Gate: **53% fewer taint requests issued, measured** — and nothing more. 1,177 requests at 6.63 s is still 2.2 hours, so this phase does NOT make the scan finish; it halves the problem. Claiming completion here would send someone looking for a bug where there is none. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index a1bd487..4d1e5a1 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,28 +1,24 @@ { + "modules": 98, "counts": { - "load_bearing": 90, + "load_bearing": 91, "standalone_capability": 7 }, - "generated": "2026-09-09 (regenerated after contributor-scan 2.10 added model/shipped.py)", - "kept_with_a_lever_stripped": { - "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." - }, - "modules": 97, - "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], - "removed_in_this_pass": { - "learning/ (16 modules)": "the noise architecture: rounds, acceptance, proposer, journal, canaries, verifiers, difficulty, scoring, classify, detectors, split, publish. Taxonomy, candidates and endpoint were re-homed to model/ first.", - "semantic/loo.py": "leave-one-out scoring of the mechanism store", - "semantic/mechanisms.py": "the mechanism store and the shape search that fed the optimisation loop", - "semantic/prove_budget.py": "embedding-ordered promotion budget, driven by the mechanism store", - "semantic/seed.py": "the mechanism exporter", - "semantic/variant_search.py": "variant search over the mechanism store", - "semantic/variants.py": "the shape-derivation optimisation surface; its guard vocabulary was ported verbatim to model/specs.py first, and trailing_name re-homed to semantic/ir.py", - "vectorstore.py": "a vector-store bakeoff harness with no production caller and no CLI entry, kept alive only by its own test -- the one genuine orphan the audit found" + "standalone": { + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "slot_contract": "the slot/contract protocol stage_processors implements", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" }, "rows": [ { + "module": "benchmark", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "gate", @@ -30,28 +26,28 @@ "map_gate", "mcp", "pairs" - ], - "module": "benchmark", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "calibration", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "calibration", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "cli", "classification": "load_bearing", + "reason": "entry point", "importers": [ "mcp" - ], - "module": "cli", - "reason": "entry point" + ] }, { + "module": "complexity", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity.hints", @@ -61,32 +57,32 @@ "regress.candidate", "semantic.prove_filter", "tool_hunter" - ], - "module": "complexity", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.hints", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.map" - ], - "module": "complexity.hints", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.ledger", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity", "complexity.map", "regress.candidate" - ], - "module": "complexity.ledger", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.map", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.hints", @@ -95,22 +91,22 @@ "regress.candidate", "semantic.prove_filter", "tool_hunter" - ], - "module": "complexity.map", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.signals", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.hints", "complexity.map" - ], - "module": "complexity.signals", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "config", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "harness", @@ -120,45 +116,49 @@ "regress.candidate", "regress.recipes", "run" - ], - "module": "config", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "cpg", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4: the CPG seam package", "importers": [ + "cli", "cpg.backend", "model.config_value", "model.dominance", "model.judge", "model.pipeline", + "model.scan", "model.taint" - ], - "module": "cpg", - "reason": "model-grounded-detection Req 4: the CPG seam package" + ] }, { + "module": "cpg.backend", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "importers": [ + "cli", "model.config_value", "model.dominance", "model.judge", "model.pipeline", + "model.scan", "model.taint" - ], - "module": "cpg.backend", - "reason": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary" + ] }, { + "module": "cpg.capability", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", "importers": [ "cpg.backend" - ], - "module": "cpg.capability", - "reason": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion" + ] }, { + "module": "findings", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "benchmark", "calibration", @@ -187,147 +187,148 @@ "tool_hunter", "verification", "verify_judge" - ], - "module": "findings", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "standalone_capability", - "importers": [ - "cli" - ], "module": "fusion", - "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone" - }, - { - "classification": "load_bearing", + "classification": "standalone_capability", + "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone", "importers": [ "cli" - ], - "module": "gate", - "reason": "entry point" + ] }, { + "module": "gate", "classification": "load_bearing", + "reason": "entry point", + "importers": [ + "cli" + ] + }, + { + "module": "harness", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "slot_contract" - ], - "module": "harness", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "harness_ext", "classification": "standalone_capability", + "reason": "HarnessX extension surface; loaded by name when the optional harness is present", "importers": [ "cli", "fusion", "hunter_harness", "stage_processors", "verify_judge" - ], - "module": "harness_ext", - "reason": "HarnessX extension surface; loaded by name when the optional harness is present" + ] }, { + "module": "hunter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_harness" - ], - "module": "hunter", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "hunter_harness", "classification": "standalone_capability", + "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp", "importers": [ "cli" - ], - "module": "hunter_harness", - "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp" + ] }, { + "module": "hunter_tools", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "tool_hunter" - ], - "module": "hunter_tools", - "reason": "imported by tool_hunter" + ] }, { + "module": "improve", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve.evolve" - ], - "module": "improve", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "improve.evolve", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve" - ], - "module": "improve.evolve", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "improve", - "improve.evolve" - ], "module": "improve.journal", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve", "improve.evolve" - ], - "module": "improve.validator", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "improve.validator", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "improve", + "improve.evolve" + ] + }, + { + "module": "index", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "skills" - ], - "module": "index", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "map_gate", - "reason": "entry point" + "classification": "load_bearing", + "reason": "entry point", + "importers": [] }, { + "module": "mapping", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "gate", "hunter_tools", "improve.evolve", "map_gate", + "model.scan", "pairs", "preprocess", "stage_processors" - ], - "module": "mapping", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "mcp", "classification": "standalone_capability", + "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", "importers": [ "cli" - ], - "module": "mcp", - "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration" + ] }, { + "module": "model", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.candidates", @@ -337,70 +338,83 @@ "model.judge", "model.pipeline", "model.regions", + "model.report", "model.scan", "model.specs", "model.taint" - ], - "module": "model", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.audit", - "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan", + "importers": [] }, { - "classification": "load_bearing", - "importers": [], "module": "model.calibrate", - "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set", + "importers": [] }, { + "module": "model.candidates", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.scan" - ], - "module": "model.candidates", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.pipeline" - ], "module": "model.config_value", - "reason": "model-grounded-detection Req 6.3: constant abstraction for the configuration families" - }, - { "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", "importers": [ - "model.pipeline" - ], + "model.pipeline", + "model.scan" + ] + }, + { "module": "model.dominance", - "reason": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash", + "importers": [ + "model.pipeline", + "model.scan" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.endpoint", - "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3", + "importers": [ + "cli" + ] }, { + "module": "model.evidence", "classification": "load_bearing", - "importers": [], + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "model.scan" + ] + }, + { "module": "model.execution", - "reason": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built", + "importers": [] }, { - "classification": "load_bearing", - "importers": [], "module": "model.judge", - "reason": "model-grounded-detection Req 8: one bounded question, checked against the model" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: one bounded question, checked against the model", + "importers": [] }, { + "module": "model.ladder", "classification": "load_bearing", + "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries", "importers": [ "model.config_value", "model.dominance", @@ -409,94 +423,110 @@ "model.pipeline", "model.scan", "model.taint" - ], - "module": "model.ladder", - "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.scan" - ], "module": "model.pipeline", - "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes" - }, - { "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes", "importers": [ "model.scan" - ], + ] + }, + { "module": "model.regions", - "reason": "contributor-scan Req 2: regions from entry points, with the families each admits" + "classification": "load_bearing", + "reason": "contributor-scan Req 2: regions from entry points, with the families each admits", + "importers": [ + "cli", + "model.scan" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.report", - "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", + "importers": [ + "cli" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.scan", - "reason": "contributor-scan Req 3: the repository driver -- one CPG, one budget, ranked spend" + "classification": "load_bearing", + "reason": "contributor-scan Req 3: the repository driver -- one CPG, one budget, ranked spend", + "importers": [ + "cli" + ] }, { + "module": "model.shipped", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "cli" + ] + }, + { + "module": "model.specs", + "classification": "load_bearing", + "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by", "importers": [ "model.config_value", "model.dominance", "model.judge", "model.pipeline", "model.regions", + "model.report", "model.scan", "model.taint" - ], - "module": "model.specs", - "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by" + ] }, { + "module": "model.taint", "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter", "importers": [ "model.judge", - "model.pipeline" - ], - "module": "model.taint", - "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter" + "model.pipeline", + "model.scan" + ] }, { + "module": "model.taxonomy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.candidates", + "model.report", "model.scan", "model.specs" - ], - "module": "model.taxonomy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "pair_gate", "classification": "load_bearing", + "reason": "entry point", "importers": [ "cli" - ], - "module": "pair_gate", - "reason": "entry point" + ] }, { + "module": "pairs", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve", "improve.evolve", "model.candidates", "pair_gate" - ], - "module": "pairs", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "policy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "findings", @@ -507,20 +537,20 @@ "scoring.project_score", "semantic.overlay", "stage_processors" - ], - "module": "policy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "policy.verycode", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "policy" - ], - "module": "policy.verycode", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "preprocess", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity.hints", @@ -536,44 +566,45 @@ "map_gate", "mapping", "model.candidates", + "model.scan", "pairs", "rank", "regress.candidate", "semantic.overlay", "skills", "stage_processors" - ], - "module": "preprocess", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "provenance", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "provenance", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.endpoint", - "tool_hunter" - ], "module": "provider", - "reason": "imported by model.endpoint, tool_hunter" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.endpoint", "tool_hunter" - ], - "module": "provider.openrouter", - "reason": "imported by model.endpoint, tool_hunter" + ] }, { + "module": "provider.openrouter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "model.endpoint", + "tool_hunter" + ] + }, + { + "module": "rank", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "calibration", "cli", @@ -586,76 +617,84 @@ "map_gate", "pairs", "stage_processors" - ], - "module": "rank", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "redaction", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "harness", "model.judge", "model.pipeline", "reports", "semantic.obligations.intent" - ], - "module": "redaction", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "regress.candidate" - ], - "module": "regress", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress.candidate", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress" - ], - "module": "regress.candidate", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "regress", - "regress.candidate" - ], "module": "regress.recipes", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress", "regress.candidate" - ], + ] + }, + { "module": "regress.safety", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress", "regress.candidate" - ], - "module": "regress.verdict", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress.verdict", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "regress", + "regress.candidate" + ] + }, + { + "module": "reports", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "reports", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "repos", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "cli" + ] + }, + { + "module": "ruleset", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "findings", @@ -665,83 +704,84 @@ "improve.validator", "pairs", "stage_processors" - ], - "module": "ruleset", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "ruleset.store", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve.validator", "ruleset" - ], - "module": "ruleset.store", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "run", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "reports" - ], - "module": "run", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "regress.candidate", "regress.recipes", "regress.safety", "regress.verdict" - ], - "module": "sandbox", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox.probe", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "sandbox" - ], - "module": "sandbox.probe", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox.runner", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress.candidate", "regress.recipes", "regress.safety", "sandbox" - ], - "module": "sandbox.runner", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "scoring", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve.evolve", "stage_processors" - ], - "module": "scoring", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "scoring.project_score", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "scoring" - ], - "module": "scoring.project_score", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "cpg.capability", "hunter_tools", + "mapping", "model.candidates", "model.specs", "pairs", @@ -755,43 +795,44 @@ "semantic.overlay", "semantic.prove_filter", "semantic.taint" - ], - "module": "semantic", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.cst", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.ir" - ], - "module": "semantic.cst", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.engines", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cpg.capability", "semantic", "semantic.overlay" - ], - "module": "semantic.engines", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.extra", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic", "semantic.cst", "semantic.engines" - ], - "module": "semantic.extra", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.facts", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", + "mapping", "model.candidates", "model.specs", "pairs", @@ -801,23 +842,23 @@ "semantic.obligations.shapes", "semantic.overlay", "semantic.taint" - ], - "module": "semantic.facts", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.functions", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "hunter_tools", "pairs", "semantic.obligations.shapes", "semantic.overlay" - ], - "module": "semantic.functions", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.ir", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", @@ -830,12 +871,12 @@ "semantic.obligations.shapes", "semantic.overlay", "semantic.taint" - ], - "module": "semantic.ir", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", @@ -849,33 +890,33 @@ "semantic.obligations.policy", "semantic.obligations.shapes", "semantic.obligations.siblings" - ], - "module": "semantic.obligations", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.check", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "semantic.obligations", "semantic.obligations.intent" - ], - "module": "semantic.obligations.check", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.dominance", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", "pairs", "semantic.obligations.check" - ], - "module": "semantic.obligations.dominance", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.facts", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.specs", "pairs", @@ -885,148 +926,136 @@ "semantic.obligations.policy", "semantic.obligations.shapes", "semantic.obligations.siblings" - ], - "module": "semantic.obligations.facts", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.intent", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "semantic.obligations.intent", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.operations", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.candidates", "semantic.obligations.check", "semantic.obligations.dominance", "semantic.obligations.siblings" - ], - "module": "semantic.obligations.operations", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.policy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "semantic.obligations.check" - ], - "module": "semantic.obligations.policy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.shapes", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.obligations", "semantic.obligations.operations" - ], - "module": "semantic.obligations.shapes", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.siblings", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.obligations.check" - ], - "module": "semantic.obligations.siblings", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.overlay", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic", "semantic.prove_filter" - ], - "module": "semantic.overlay", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.prove_filter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic" - ], - "module": "semantic.prove_filter", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.taint", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.overlay" - ], - "module": "semantic.taint", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "skills", "classification": "standalone_capability", + "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", "importers": [ "hunter", "hunter_harness" - ], - "module": "skills", - "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path" + ] }, { + "module": "slot_contract", "classification": "standalone_capability", + "reason": "the slot/contract protocol stage_processors implements", "importers": [ "stage_processors" - ], - "module": "slot_contract", - "reason": "the slot/contract protocol stage_processors implements" + ] }, { - "classification": "standalone_capability", - "importers": [], "module": "stage_processors", - "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + "classification": "standalone_capability", + "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate", + "importers": [] }, { + "module": "stages", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "stages", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "tool_hunter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.endpoint" - ], - "module": "tool_hunter", - "reason": "imported by model.endpoint" + ] }, { + "module": "verification", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "calibration", "cli", "fusion", "reports", "verify_judge" - ], - "module": "verification", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "verify_judge", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "verify_judge", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] } - ], - "spec": "model-grounded-detection Req 3 (the outgrowth audit)", - "standalone": { - "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", - "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", - "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", - "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", - "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", - "slot_contract": "the slot/contract protocol stage_processors implements", - "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" - } + ] } diff --git a/benchmarks/ranking/baselines/2026-09-10-access-rank.json b/benchmarks/ranking/baselines/2026-09-10-access-rank.json new file mode 100644 index 0000000..8bbd6d8 --- /dev/null +++ b/benchmarks/ranking/baselines/2026-09-10-access-rank.json @@ -0,0 +1,96 @@ +[ + { + "repo": "mwwpform", + "regions": 983, + "distinct_ranks": 5, + "largest_tier_share": 0.817, + "largest_tier_rank": 0.3, + "budget_at_recall": 513, + "known": [ + { + "id": "CVE-2023-6559", + "family": "path", + "site": "classes/services/class.mail.php:259:_delete_files", + "position": 512, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "pmpro", + "regions": 4463, + "distinct_ranks": 5, + "largest_tier_share": 0.838, + "largest_tier_rank": 0.3, + "budget_at_recall": 391, + "known": [ + { + "id": "CVE-2023-23488", + "family": "injection", + "site": "classes/class.memberorder.php:936:getMemberOrderByCode", + "position": 390, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "vampi", + "regions": 25, + "distinct_ranks": 5, + "largest_tier_share": 0.36, + "largest_tier_rank": 1.0, + "budget_at_recall": 16, + "known": [ + { + "id": "VAmPI-SQLI", + "family": "injection", + "site": "models/user_model.py:73:get_user", + "position": null, + "rank": null, + "in_scope": true, + "reached_transitively": true + }, + { + "id": "VAmPI-BOLA", + "family": "access_control", + "site": "api_views/users.py:187:update_password", + "position": 15, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + }, + { + "id": "VAmPI-BOLA-BOOKS", + "family": "access_control", + "site": "api_views/books.py:51:get_by_title", + "position": 10, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "wpstatistics", + "regions": 1673, + "distinct_ranks": 6, + "largest_tier_share": 0.753, + "largest_tier_rank": 0.3, + "budget_at_recall": 504, + "known": [ + { + "id": "CVE-2022-25148", + "family": "injection", + "site": "includes/class-wp-statistics-pages.php:225:record", + "position": 503, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + } +] diff --git a/benchmarks/ranking/position.py b/benchmarks/ranking/position.py new file mode 100644 index 0000000..5b9c15b --- /dev/null +++ b/benchmarks/ranking/position.py @@ -0,0 +1,121 @@ +"""Phase 0 of flow-aware-ranking: where does each pinned CVE sit in the order the scan actually examines? + +The ranker's success metric is not "is the CVE in the budget" -- on Paid Memberships Pro it already is, at +position 390 of 4,463 -- but **how small the budget can be while every CVE stays in it**. That number is +`budget_at_recall`, and until it is written down nothing later can claim to have improved it. + +This is the baseline. It runs no engine and asks no model: it is the region pipeline exactly as the scan +uses it, sorted exactly as the scan sorts it, with each known vulnerability's position read off. Offline, +against pinned checkouts, so two runs of one repository agree. + + ousast-venv/bin/python benchmarks/ranking/position.py # every pinned recipe with a checkout + ousast-venv/bin/python benchmarks/ranking/position.py --json # the same, as an artifact +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +from openultrasast.mapping import analyze_entry_points # noqa: E402 +from openultrasast.model.regions import ScanRegion, regions_for # noqa: E402 +from openultrasast.model.shipped import declared_sources # noqa: E402 +from openultrasast.preprocess import preprocess_repository # noqa: E402 +from openultrasast.repos import checkout_path, load_repo_recipes # noqa: E402 + + +def ordered_regions(root: Path) -> list[ScanRegion]: + """The regions in the order `scan_repository` examines them. One definition, imported, never copied.""" + _, targets = preprocess_repository(root) + regions = regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root)) + # The scan's own total order. If this drifts from scan.py the baseline measures the wrong thing, which + # is why the sort key is the one thing here worth a test against the driver. + return sorted(regions, key=lambda r: (not r.shipped, -r.rank, r.path, r.function or "")) + + +def measure(name: str, root: Path, known: list[dict[str, object]]) -> dict[str, object]: + regions = ordered_regions(root) + tiers = Counter(r.rank for r in regions) + largest = max(tiers.values()) if tiers else 0 + rows: list[dict[str, object]] = [] + for item in known: + # A known vulnerability names a file and a function. Match on both, path by suffix because a recipe + # writes it relative to the repository and a region may carry it the same way or deeper. + hit = next( + (i for i, r in enumerate(regions) if r.path.endswith(str(item["file"])) and r.function == item["function"]), + None, + ) + rows.append( + { + "id": item["id"], + "family": item["family"], + "site": f"{item['file']}:{item['line']}:{item['function']}", + # Zero-based. None means no region is named after this function: the site is a SINK that an + # entry-point region elsewhere reaches through `callDepth` -- VAmPI's SQL injection lives in + # `models/user_model.py:get_user` and is found from the handler that calls it. Its true + # position is that handler's, which only the flow can say, so it is reported and left out of + # `budget_at_recall` rather than counted as a miss it is not. + "position": hit, + "rank": regions[hit].rank if hit is not None else None, + "in_scope": item["in_scope"], + "reached_transitively": hit is None, + } + ) + positioned = [r["position"] for r in rows if r["position"] is not None and r["in_scope"]] + return { + "repo": name, + "regions": len(regions), + "distinct_ranks": len(tiers), + "largest_tier_share": round(largest / len(regions), 3) if regions else 0.0, + "largest_tier_rank": max(tiers, key=lambda k: tiers[k]) if tiers else None, + # The smallest examined-region budget that still holds every in-scope known vulnerability. + "budget_at_recall": (max(positioned) + 1) if positioned else None, + "known": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--json", action="store_true", help="emit one JSON object per repository") + parser.add_argument("--repo", help="only this recipe") + args = parser.parse_args() + + results: list[dict[str, object]] = [] + for recipe in load_repo_recipes(): + if args.repo and recipe.name != args.repo: + continue + if "detection" not in recipe.measures: + continue + root = checkout_path(recipe) + if not root.is_dir(): + print(f"{recipe.name}: not checked out, skipping", file=sys.stderr) + continue + known = [ + {"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope} + for k in recipe.known + ] + results.append(measure(recipe.name, root, known)) + + if args.json: + print(json.dumps(results, indent=2)) + return 0 + for r in results: + print( + f"{r['repo']}: {r['regions']} regions, {r['distinct_ranks']} distinct ranks, " + f"{100 * float(r['largest_tier_share']):.1f}% share rank {r['largest_tier_rank']}" + ) + for k in r["known"]: + where = "reached via another region" if k["position"] is None else f"position {k['position']} (rank {k['rank']})" + scope = "" if k["in_scope"] else " [out of scope]" + print(f" {k['id']:16} {where:28} {k['site']}{scope}") + print(f" budget_at_recall = {r['budget_at_recall']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index 855875e..772ef4b 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -49,6 +49,7 @@ boundedSinks: String = "", parameterSources: String = "false", fieldSources: String = "true", + evidenceOnly: String = "false", hookCallbacks: String = "", dispatchApply: String = "", requests: String = "", @@ -65,6 +66,12 @@ def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList + // Per-family answers that do not depend on the region, kept across the whole batch. + val familyInRepoMemo = scala.collection.mutable.Map.empty[String, Boolean] + val sinkCandidatesMemo = scala.collection.mutable.Map.empty[String, List[io.shiftleft.codepropertygraph.generated.nodes.Call]] + val reachableMemo = scala.collection.mutable.Map.empty[(String, String, Int), Set[String]] + val methodSourceMemo = scala.collection.mutable.Map.empty[String, Boolean] + def rowsFor( sourcesS: String, sinksS: String, @@ -74,6 +81,7 @@ functionS: String, paramSrc: String, fieldSrc: String, + evidenceS: String, fileS: String, depthS: String, boundedS: String @@ -132,7 +140,9 @@ // Bounded by `callDepth`, because "every method transitively reachable from an entry point" is most of a // repository, and an unbounded question is how this becomes slow again. val depth = scala.util.Try(depthS.toInt).getOrElse(0) - lazy val reachableMethods: Set[String] = + lazy val reachableMethods: Set[String] = reachableMemo.getOrElseUpdate((function, fileS, depth), computeReachable) + + def computeReachable: Set[String] = if (depth <= 0 || labeledMethods.isEmpty) Set.empty else { var frontier = labeledMethods.toSet @@ -502,7 +512,66 @@ byName || sanitizerNames.exists(n => code.startsWith(n + "(") || code.contains("'" + n + "'") || code.contains("\"" + n + "\"")) } - def sinkCalls = cpg.call.filter(c => sinkNames.exists(n => sinkMatches(c, n))).filter(c => inScope(c.method)) + // The candidate sink calls for a FAMILY do not depend on the region, and scanning every call in the graph + // for each of 2,487 requests was a large share of what made a request cost seconds -- in evidence mode, + // which asks for no dataflow at all, it alone took a query past an hour. Scanned once per sink list and + // kept for the batch; the per-request work is the scope filter over that list. + def sinkCalls = + sinkCandidatesMemo + .getOrElseUpdate(sinksS, cpg.call.filter(c => sinkNames.exists(n => sinkMatches(c, n))).l) + .filter(c => inScope(c.method)) + + // ---- EVIDENCE MODE: the same question without the dataflow (flow-aware-ranking, phase 0) -------------- + // + // Tiering is the CPG query WITHOUT dataflow; arbitration is the query WITH it. Same graph, same facts, two + // costs. This branch answers "what is in scope" -- which sink calls, what shape they take, whether a + // cleansing call wraps THEIR argument, whether a source is local or reachable -- and returns before + // `reachableByFlows` is ever called. Milliseconds per call, where a flow costs 6.63 seconds. + // + // Every field is stated in fact-table terms, so the vector means the same thing in every language the + // graph can hold. A `summary` row is always emitted, even when there are no sinks, because "no sinks" is + // tier 0 and must be distinguishable from "the query failed". + if (evidenceS == "true") { + // Whether a method holds a modelled source, memoised by full name across the batch: the same helper + // is reachable from hundreds of regions, and re-walking its AST for each of them is what made a + // no-dataflow query take minutes. `cpg.method.filter(...)` over every method per request was the + // other half of that, replaced by a direct lookup of the reachable names. + def methodHasSource(m: io.shiftleft.codepropertygraph.generated.nodes.Method): Boolean = + methodSourceMemo.getOrElseUpdate(m.fullName, m.ast.isCall.exists(c => sourcePatterns.exists(p => c.code.contains(p)))) + val sinkList = sinkCalls.l + val sourceLocal = labeledMethods.exists(methodHasSource) + val sourceNear = depth > 0 && cpg.method.fullNameExact(reachableMethods.toSeq: _*).exists(methodHasSource) + // Anywhere in the repository at all. Exact at every callDepth, and the cheapest cut there is: a family + // with no sink anywhere cannot produce a flow for any region. Memoised on the sink list, because it is + // the same answer for every request of one family and a repository-wide walk per request turned a + // milliseconds-per-pair query into minutes. + val familyInRepo = familyInRepoMemo.getOrElseUpdate(sinksS, cpg.call.exists(c => sinkNames.exists(n => sinkMatches(c, n)))) + val summary = ujson.Obj( + "kind" -> "summary", + "sinks" -> sinkList.size, + "sourceLocal" -> sourceLocal, + "sourceNear" -> sourceNear, + "familyInRepo" -> familyInRepo + ) + val perSink = sinkList.map { sink => + val realArgs = sink.argument.argumentIndexGt(0).l + ujson.Obj( + "kind" -> "sink", + "sink" -> sink.code.take(200), + "sinkLine" -> sink.lineNumber.getOrElse(-1).toString, + "sinkMethod" -> sink.method.name, + "sinkFile" -> sink.method.filename.split("/").last, + "sinkArity" -> realArgs.size, + "sinkArg0Literal" -> realArgs.headOption.map(a => a.isLiteral).getOrElse(false), + // A sanitizer of this family is an ANCESTOR of an argument in the AST -- it wraps what flows in -- + // as opposed to merely appearing somewhere in the same function. This is the field a text scan + // gets wrong, and the one that would have misranked CVE-2023-23488: `esc_sql` is in that function, + // on a different statement. + "cleansedOnCall" -> sink.argument.ast.l.exists(node => sanitizesHere(node)) + ) + } + return summary :: perSink + } val rows = sinkCalls.l.flatMap { sink => // Data flows into the arguments; asking the call node itself finds nothing. @@ -576,6 +645,7 @@ def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") val paramSrc = req.obj.get("parameterSources").map(_.str).getOrElse("false") val fieldSrc = req.obj.get("fieldSources").map(_.str).getOrElse("true") + val evidence = req.obj.get("evidenceOnly").map(_.str).getOrElse("false") id -> ujson.Arr( rowsFor( field("sources"), @@ -589,6 +659,7 @@ field("function"), paramSrc, fieldSrc, + evidence, field("file"), field("callDepth"), field("boundedSinks") @@ -604,7 +675,7 @@ println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census)))) } else { println( - ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, hookCallbacks, dispatchApply, function, parameterSources, fieldSources, file, callDepth, boundedSinks): _*)) + ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, hookCallbacks, dispatchApply, function, parameterSources, fieldSources, evidenceOnly, file, callDepth, boundedSinks): _*)) ) } println("---OUSAST-CPG-END---") diff --git a/src/openultrasast/model/evidence.py b/src/openultrasast/model/evidence.py new file mode 100644 index 0000000..d553154 --- /dev/null +++ b/src/openultrasast/model/evidence.py @@ -0,0 +1,150 @@ +"""The evidence vector: what a (region, family) pair carries BEFORE any dataflow is asked (flow-aware-ranking). + +Tiering is the CPG query without dataflow; arbitration is the query with it. Same graph, same facts, two costs. +This module turns the evidence rows that `taint.sc` emits in `evidenceOnly` mode into one vector per pair, +and assigns the tier. Every field is stated in fact-table terms -- source, sink, sanitizer -- so a heuristic +that mentions a language's own syntax has, by construction, left this module. + +Only tier 0 excludes. Its predicate has a proof -- a family with no sink in reach cannot produce a flow, in any +language, under any dataflow model -- and it is the only place exactness is required. Everything above orders, +where being wrong costs position, and `regions_truncated` reports what position cost. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field + +TIER_EXCLUDE = 0 # no sink of this family in reach: never ask +TIER_NO_SOURCE = 1 # sinks, but nothing untrusted can arrive: ask last +TIER_CLEANSED = 2 # a source exists, but every sink call is bound or cleansed on the call: ask late +TIER_OPEN = 3 # a source exists and some sink call is computed and uncleansed: ask first +TIER_OPEN_PUBLIC = 4 # tier 3 and declared public, or a source on the same statement: ask very first + + +@dataclass(frozen=True) +class SinkEvidence: + """One sink call of the family, as the graph saw it, with no flow asked.""" + + code: str + line: int + method: str + arity: int + arg0_literal: bool + cleansed_on_call: bool # a sanitizer of this family wraps an ARGUMENT -- not merely present in the function + + @property + def shape(self) -> str: + """``literal`` cannot carry taint; ``computed`` is the residue that can.""" + return "literal" if self.arg0_literal else "computed" + + +@dataclass(frozen=True) +class Evidence: + """The vector for one (region, family) pair.""" + + sinks: tuple[SinkEvidence, ...] = () + family_in_repo: bool = True # False is the repository-wide tier 0, exact at every callDepth + source_local: bool = False # a modelled source in the region's own body + source_near: bool = False # a modelled source within the arbiter's callDepth + entry: bool = False # parameters untrusted by contract + access_declared_public: bool = False # declared, not inferred -- "no decorator found" is not "public" + carried: bool | None = None # stage one of the two-stage join; None where it was not computed + bound_names: tuple[str, ...] = field(default=()) # sink names the shape test treats as bound at arity >= 2 + + @property + def any_source(self) -> bool: + return self.source_local or self.source_near or self.entry or bool(self.carried) + + def is_bound(self, sink: SinkEvidence) -> bool: + """The safe-shape rule the arbiter already applies: a named sink with two or more arguments binds.""" + return sink.arity >= 2 and any(name in sink.code for name in self.bound_names) + + @property + def open_sinks(self) -> tuple[SinkEvidence, ...]: + """Sink calls that could carry an uncleansed, unbound, computed value.""" + return tuple(s for s in self.sinks if s.shape == "computed" and not s.cleansed_on_call and not self.is_bound(s)) + + @property + def tier(self) -> int: + if not self.family_in_repo or not self.sinks: + return TIER_EXCLUDE + if not self.any_source: + return TIER_NO_SOURCE + if not self.open_sinks: + return TIER_CLEANSED + if self.access_declared_public or self.source_local or self.carried: + return TIER_OPEN_PUBLIC + return TIER_OPEN + + +def evidence_from_rows( + rows: Sequence[Mapping[str, object]] | None, + *, + entry: bool = False, + access_declared_public: bool = False, + carried: bool | None = None, + bound_names: Sequence[str] = (), +) -> Evidence | None: + """Read the query's evidence rows for one request. ``None`` when the query did not answer. + + A request that answered with no sinks still carries a ``summary`` row -- that is tier 0, and it must be + distinguishable from a request that failed, which carries nothing. + """ + if rows is None: + return None + summary = next((r for r in rows if isinstance(r, Mapping) and r.get("kind") == "summary"), None) + if summary is None: + return None + sinks = tuple( + SinkEvidence( + code=str(r.get("sink", "")), + line=_as_int(r.get("sinkLine")), + method=str(r.get("sinkMethod", "")), + arity=_as_int(r.get("sinkArity")), + arg0_literal=_as_bool(r.get("sinkArg0Literal")), + cleansed_on_call=_as_bool(r.get("cleansedOnCall")), + ) + for r in rows + if isinstance(r, Mapping) and r.get("kind") == "sink" + ) + return Evidence( + sinks=sinks, + family_in_repo=_as_bool(summary.get("familyInRepo"), default=True), + source_local=_as_bool(summary.get("sourceLocal")), + source_near=_as_bool(summary.get("sourceNear")), + entry=entry, + access_declared_public=access_declared_public, + carried=carried, + bound_names=tuple(bound_names), + ) + + +def _as_bool(value: object, *, default: bool = False) -> bool: + """A JSON boolean, or its string spelling. ``bool("False")`` is True, which would make every unsanitized + sink read as cleansed the moment a serialiser chose strings -- the kind of silent inversion this project + has paid for before.""" + if value is None: + return default + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"true", "1", "yes"} + + +def _as_int(value: object) -> int: + try: + return int(str(value)) + except (TypeError, ValueError): + return -1 + + +__all__ = [ + "TIER_CLEANSED", + "TIER_EXCLUDE", + "TIER_NO_SOURCE", + "TIER_OPEN", + "TIER_OPEN_PUBLIC", + "Evidence", + "SinkEvidence", + "evidence_from_rows", +] diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 2bbba2c..37f65c8 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Any from ..cpg.backend import CpgResult +from .evidence import evidence_from_rows from .candidates import enumerate_candidates from .config_value import request_params as config_params from .dominance import request_params as dominance_params @@ -64,6 +65,11 @@ class ScanBudget: max_model_calls: int = 200 max_regions: int = 500 + # Compute the evidence tier of every (region, family) pair and RECORD it. Phase 0 of flow-aware-ranking: + # the instrument runs on every real scan so later phases are measurable, and it changes nothing about + # what is asked or in what order. One extra query invocation without dataflow -- a JVM load, then + # milliseconds per pair. + tiering: bool = True @dataclass(frozen=True) @@ -87,6 +93,10 @@ class ModelScanResult: # would remove) or CPGQL evaluation (which it would not). That is the whole of the server-mode decision. query_seconds_by_kind: Mapping[str, float] = field(default_factory=dict) degradations: tuple[Mapping[str, object], ...] = () + # The evidence tier of each (path, function, family) pair the scan collected, and the count per tier. + # Recorded, not acted on: this is what "the ranker got better" will be measured against. + tiers: tuple[tuple[str, str, str, int], ...] = () + tier_counts: Mapping[int, int] = field(default_factory=dict) @property def arbitrate_seconds(self) -> float: @@ -176,11 +186,44 @@ def scan_repository( continue work.append((f"{len(work)}", region, spec)) + # Phase 1b: the evidence tier of every taint pair, before any dataflow is asked. Same query, `evidenceOnly`, + # no `reachableByFlows`. Recorded on the result and NOT used to reorder or skip anything -- that is a + # later phase, and it must be measured against this record before it is allowed to change behaviour. + tiers: list[tuple[str, str, str, int]] = [] + per_kind: dict[str, float] = {} + if limits.tiering and callable(getattr(cpg, "run_batch", None)): + grouped_taint = _grouped(work, hook_callbacks=hooks).get("taint", {}) + if grouped_taint: + evidence_requests = {rid: {**dict(params), "evidenceOnly": "true"} for rid, params in grouped_taint.items()} + tier_started = time.monotonic() + answered_evidence = cpg.run_batch("taint", evidence_requests) + per_kind["evidence"] = round(time.monotonic() - tier_started, 2) + if answered_evidence is None: + degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": len(evidence_requests)}) + else: + answered_evidence.pop("__census__", None) + by_rid = {rid: (region, spec) for rid, region, spec in work} + for rid, rows in answered_evidence.items(): + pair = by_rid.get(rid) + if pair is None: + continue + region, spec = pair + evidence = evidence_from_rows( + rows, + entry=_is_entry_point(region), + # Rank 1.0 is the public tier. Provenance -- declared by a route contract versus + # inferred from an absent decorator -- is not yet carried on ScanRegion, so this is + # the proxy phase 0 has; the brief names it as the field to thread through next. + access_declared_public=region.rank >= 1.0, + bound_names=getattr(spec, "safe_shape_sinks", ()), + ) + if evidence is not None: + tiers.append((region.path, region.function or "", getattr(spec, "family", ""), evidence.tier)) + # Phase 2: ONE invocation per query kind. JVM startup dominates a repository scan -- a call per region # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while # the queries themselves are milliseconds once the CPG is loaded. rows_by_id: dict[str, list[object]] = {} - per_kind: dict[str, float] = {} census_reported = False query_started = time.monotonic() for kind, requests in _grouped(work, hook_callbacks=hooks).items(): @@ -293,6 +336,8 @@ def scan_repository( query_seconds=query_seconds, query_seconds_by_kind=per_kind, degradations=tuple(degradations), + tiers=tuple(tiers), + tier_counts={tier: sum(1 for t in tiers if t[3] == tier) for tier in sorted({t[3] for t in tiers})}, ) diff --git a/tests/test_model_evidence.py b/tests/test_model_evidence.py new file mode 100644 index 0000000..44803d3 --- /dev/null +++ b/tests/test_model_evidence.py @@ -0,0 +1,99 @@ +"""The evidence vector and its tiers, on canned rows (flow-aware-ranking phase 0).""" + +from __future__ import annotations + +from openultrasast.model.evidence import ( + TIER_CLEANSED, + TIER_EXCLUDE, + TIER_NO_SOURCE, + TIER_OPEN, + TIER_OPEN_PUBLIC, + evidence_from_rows, +) + +SUMMARY = {"kind": "summary", "sinks": 1, "sourceLocal": False, "sourceNear": False, "familyInRepo": True} + + +def _sink(code: str, *, literal: bool = False, cleansed: bool = False, arity: int = 1) -> dict[str, object]: + return { + "kind": "sink", + "sink": code, + "sinkLine": "936", + "sinkMethod": "f", + "sinkArity": arity, + "sinkArg0Literal": literal, + "cleansedOnCall": cleansed, + } + + +def test_a_failed_query_is_not_a_tier() -> None: + """No rows at all means the query did not answer. That is a degradation, never tier 0.""" + assert evidence_from_rows(None) is None + assert evidence_from_rows([]) is None + + +def test_no_sink_in_scope_is_tier_zero_and_exact() -> None: + """The only tier that excludes. A family with no sink in reach cannot produce a flow, in any language.""" + e = evidence_from_rows([{**SUMMARY, "sinks": 0}], entry=True) + assert e is not None and e.tier == TIER_EXCLUDE + + +def test_no_sink_anywhere_in_the_repository_is_tier_zero_at_any_depth() -> None: + """PMPro never calls `unserialize`: all 487 deserialization requests were asked of nothing.""" + e = evidence_from_rows([{**SUMMARY, "familyInRepo": False}, _sink("x($y)")], entry=True) + assert e is not None and e.tier == TIER_EXCLUDE + + +def test_a_sink_with_no_source_is_asked_last() -> None: + e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_row("... $id")')]) + assert e is not None and e.tier == TIER_NO_SOURCE + + +def test_the_cve_shape_is_the_top_tier() -> None: + """CVE-2023-23488: an entry-point method whose sink concatenates a parameter into SQL, with `esc_sql` + present in the FUNCTION but wrapping a different statement -- so `cleansedOnCall` is false here.""" + e = evidence_from_rows( + [SUMMARY, _sink('$wpdb->get_var("... WHERE code = \'" . $code . "\'")', cleansed=False)], + entry=True, + ) + assert e is not None and e.tier == TIER_OPEN + assert len(e.open_sinks) == 1 + + +def test_cleansing_on_the_call_itself_lowers_the_tier() -> None: + """The fixed side of that pair wraps the value in `$wpdb->prepare`, which is an ancestor of the argument.""" + e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_var($wpdb->prepare("... %s", $code))', cleansed=True)], entry=True) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_a_literal_argument_cannot_carry_taint() -> None: + e = evidence_from_rows([SUMMARY, _sink('system("ls")', literal=True)], entry=True) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_a_bound_shape_is_the_arbiters_own_rule() -> None: + """`execute(sql, params)` binds where `execute(sql + x)` interpolates; same rule the taint verdict uses.""" + e = evidence_from_rows([SUMMARY, _sink("cursor.execute(sql, params)", arity=2)], entry=True, bound_names=("execute",)) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_declared_public_or_a_local_source_asks_very_first() -> None: + public = evidence_from_rows([SUMMARY, _sink("unlink($f)")], entry=True, access_declared_public=True) + local = evidence_from_rows([{**SUMMARY, "sourceLocal": True}, _sink("unlink($_GET['f'])")]) + assert public is not None and public.tier == TIER_OPEN_PUBLIC + assert local is not None and local.tier == TIER_OPEN_PUBLIC + + +def test_a_carried_field_counts_as_a_source() -> None: + """`_delete_files()` has no parameters; its taint arrives as `$this->attachments`. Stage one of the join + already knows that, and the vector must not demote the region for lacking a direct source.""" + e = evidence_from_rows([SUMMARY, _sink("unlink($file)")], carried=True) + assert e is not None and e.tier == TIER_OPEN_PUBLIC + + +def test_a_stringified_boolean_is_read_as_its_value_not_its_truthiness() -> None: + """`bool("False")` is True. A serialiser that chose strings would make every open sink read as cleansed.""" + rows = [{**SUMMARY, "sourceLocal": "False", "familyInRepo": "true"}, _sink("unlink($f)")] + rows[1]["cleansedOnCall"] = "False" + e = evidence_from_rows(rows, entry=True) + assert e is not None and e.tier == TIER_OPEN, "a 'False' string must not turn an open sink into a cleansed one" diff --git a/tests/test_model_scan.py b/tests/test_model_scan.py index ba09a74..979926d 100644 --- a/tests/test_model_scan.py +++ b/tests/test_model_scan.py @@ -388,7 +388,9 @@ def test_a_query_the_engine_could_not_answer_is_recorded_not_ignored() -> None: failures = [d for d in result.degradations if d.get("reason") == "query_failed"] assert failures, "a query that could not be asked must be reported" - assert failures[0]["kind"] == "taint" + kinds = {d["kind"] for d in failures} + assert "taint" in kinds, "the taint failure is recorded" + assert "evidence" in kinds, "and so is the evidence pass that runs before it -- phase 0 of flow-aware-ranking" assert result.findings == ()