diff --git a/.kiro/specs/contributor-scan/tasks.md b/.kiro/specs/contributor-scan/tasks.md index 8d0e4f8..612c38c 100644 --- a/.kiro/specs/contributor-scan/tasks.md +++ b/.kiro/specs/contributor-scan/tasks.md @@ -821,6 +821,13 @@ half is free (inside noise, and measured slightly faster), the hook half adds 7.5%, both add 22%. Removing them buys a fifth of the time and costs two of the three CVEs, which is not a trade worth making. + - **Corrected 2026-09-10, same day: most of that 6.63 s was never dataflow.** `sinkCalls` scanned every + call in the graph per request, and `reachableMethods` ran a call-graph BFS per request — both + independent of the region's family, both repeated 2,500 times. Memoised across the batch (found while + building flow-aware-ranking's evidence mode, which asks no dataflow and still took an hour), the same + 250 requests with both joins on: **2,026.3 s → 287.0 s, 8,105 → 1,148 ms per request, identical 5,779 + rows.** A 7× speedup with nothing about the analysis changed. The base cost of a taint request is about + 1.1 s, and the levers above still apply to it. - **The estimate this replaces was wrong and worth recording as wrong.** ">0.94s per request" came from assuming taint COMPLETED within its 2,400s ceiling. It timed out, so that was a floor presented as a figure. The measured cost is seven times higher. diff --git a/.kiro/specs/flow-aware-ranking/brief.md b/.kiro/specs/flow-aware-ranking/brief.md index 1e46900..1c9f7b6 100644 --- a/.kiro/specs/flow-aware-ranking/brief.md +++ b/.kiro/specs/flow-aware-ranking/brief.md @@ -55,6 +55,41 @@ A phase that cannot show that is not an improvement, it is a story. - **Phase 0 — the oracle and the baseline.** The evidence extractor (no model) and a scoring harness over the pinned CVEs, reporting each one's rank position. Gate: reproduces today's numbers — CVE-2023-23488 at position 390 of 4,463, 83.8% of regions in one tier. Without a baseline nothing later is measurable. + + **Done 2026-09-10.** `benchmarks/ranking/position.py`, committed as + `benchmarks/ranking/baselines/2026-09-10-access-rank.json`: + + | repository | regions | largest tier | CVE position | `budget_at_recall` | + |---|---|---|---|---| + | `pmpro` | 4,463 | 83.8% at 0.30 | 390 | 391 | + | `mwwpform` | 983 | 81.7% at 0.30 | **512** | **513** | + | `wpstatistics` | 1,673 | 75.3% at 0.30 | **503** | **504** | + | `vampi` | 25 | 36.0% at 1.00 | 15, 10 (SQLI reached transitively) | 16 | + + **And this corrects the cross-assessment above.** "Ranking's job is concentration, not inclusion" was + measured on `pmpro` alone, where the CVE is inside the budget. On the other two plugins the CVE sits at + 512 and 503 — **outside a 500-region budget** — so the current ranker never examines it at all, and the + slices found those CVEs only because a slice has too few regions to exclude anything. Both jobs are + real: inclusion on two repositories, concentration on all three. The single metric `budget_at_recall` + covers both, which is why it is the one that matters. + + The instrument is `evidenceOnly` mode on the taint query, recorded on every scan as `ModelScanResult.tiers` + and `tier_counts`, and acted on by nothing — so every later phase is measured against a record made by + scans that did not know they were being measured. + + **Measured cost of the instrument**, 637-file graph, the scan's own 500-region request set: + + 2,500 pairs in 98.9 s including JVM load — about 40 ms per pair + tier 0 (no sink of that family in reach): 2,161 of 2,500 = 86% + + Two attempts before that ran past an hour each without producing a payload, and the cause was mine both + times: repository-wide walks — the sink scan, the reachable-method BFS, `familyInRepo` — repeated per + request. Memoising them across the batch is what made "milliseconds per pair" true rather than claimed. + + **And the same memoisation reaches the dataflow mode.** The identical 250-request set that cost 2,026.3 s + (8,105 ms per request) with both joins on now costs **287.0 s — 1,148 ms per request — for the same 5,779 + rows.** Most of what contributor-scan 5.13 measured as "the base taint query" was the per-request scans, + not the dataflow. That correction is recorded there. - **Phase 1 — tier 0 only, no model.** Exact pruning. Gate: **53% fewer taint requests issued, measured** — and nothing more. 1,177 requests at 6.63 s is still 2.2 hours, so this phase does NOT make the scan finish; it halves the problem. Claiming completion here would send someone looking for a bug where there is none. diff --git a/benchmarks/measurements/2026-09-08-module-audit.json b/benchmarks/measurements/2026-09-08-module-audit.json index a1bd487..4d1e5a1 100644 --- a/benchmarks/measurements/2026-09-08-module-audit.json +++ b/benchmarks/measurements/2026-09-08-module-audit.json @@ -1,28 +1,24 @@ { + "modules": 98, "counts": { - "load_bearing": 90, + "load_bearing": 91, "standalone_capability": 7 }, - "generated": "2026-09-09 (regenerated after contributor-scan 2.10 added model/shipped.py)", - "kept_with_a_lever_stripped": { - "improve/evolve.py + improve/validator.py": "the design's Removed list named these whole, but each is a mix: the mechanism lever (removed) and the deterministic rule-status improve loop that `ousast improve` still runs from benchmark evidence, which has no LLM in it and is not what this feature supersedes. Deleting the files would have taken a retained capability with them." - }, - "modules": 97, - "note": "Classification of every source module after the group-1 subtraction. load_bearing = reachable from a gate, the CLI, or named by a surviving requirement; standalone_capability = its own entry point or protocol, deliberately off the scan path; orphaned = no importer, no entry point, no surviving requirement -- removed, never carried.", "orphaned": [], - "removed_in_this_pass": { - "learning/ (16 modules)": "the noise architecture: rounds, acceptance, proposer, journal, canaries, verifiers, difficulty, scoring, classify, detectors, split, publish. Taxonomy, candidates and endpoint were re-homed to model/ first.", - "semantic/loo.py": "leave-one-out scoring of the mechanism store", - "semantic/mechanisms.py": "the mechanism store and the shape search that fed the optimisation loop", - "semantic/prove_budget.py": "embedding-ordered promotion budget, driven by the mechanism store", - "semantic/seed.py": "the mechanism exporter", - "semantic/variant_search.py": "variant search over the mechanism store", - "semantic/variants.py": "the shape-derivation optimisation surface; its guard vocabulary was ported verbatim to model/specs.py first, and trailing_name re-homed to semantic/ir.py", - "vectorstore.py": "a vector-store bakeoff harness with no production caller and no CLI entry, kept alive only by its own test -- the one genuine orphan the audit found" + "standalone": { + "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", + "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", + "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", + "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", + "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", + "slot_contract": "the slot/contract protocol stage_processors implements", + "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" }, "rows": [ { + "module": "benchmark", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "gate", @@ -30,28 +26,28 @@ "map_gate", "mcp", "pairs" - ], - "module": "benchmark", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "calibration", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "calibration", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "cli", "classification": "load_bearing", + "reason": "entry point", "importers": [ "mcp" - ], - "module": "cli", - "reason": "entry point" + ] }, { + "module": "complexity", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity.hints", @@ -61,32 +57,32 @@ "regress.candidate", "semantic.prove_filter", "tool_hunter" - ], - "module": "complexity", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.hints", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.map" - ], - "module": "complexity.hints", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.ledger", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity", "complexity.map", "regress.candidate" - ], - "module": "complexity.ledger", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.map", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.hints", @@ -95,22 +91,22 @@ "regress.candidate", "semantic.prove_filter", "tool_hunter" - ], - "module": "complexity.map", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "complexity.signals", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "complexity", "complexity.hints", "complexity.map" - ], - "module": "complexity.signals", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "config", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "harness", @@ -120,45 +116,49 @@ "regress.candidate", "regress.recipes", "run" - ], - "module": "config", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "cpg", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4: the CPG seam package", "importers": [ + "cli", "cpg.backend", "model.config_value", "model.dominance", "model.judge", "model.pipeline", + "model.scan", "model.taint" - ], - "module": "cpg", - "reason": "model-grounded-detection Req 4: the CPG seam package" + ] }, { + "module": "cpg.backend", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary", "importers": [ + "cli", "model.config_value", "model.dominance", "model.judge", "model.pipeline", + "model.scan", "model.taint" - ], - "module": "cpg.backend", - "reason": "model-grounded-detection Req 4.1/4.5: the single Joern subprocess boundary" + ] }, { + "module": "cpg.capability", "classification": "load_bearing", + "reason": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion", "importers": [ "cpg.backend" - ], - "module": "cpg.capability", - "reason": "model-grounded-detection Req 4.2/4.3: the engine probe that degrades to suspicion" + ] }, { + "module": "findings", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "benchmark", "calibration", @@ -187,147 +187,148 @@ "tool_hunter", "verification", "verify_judge" - ], - "module": "findings", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "standalone_capability", - "importers": [ - "cli" - ], "module": "fusion", - "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone" - }, - { - "classification": "load_bearing", + "classification": "standalone_capability", + "reason": "two-panel adjudication engine; reached from the scan's report stage and used standalone", "importers": [ "cli" - ], - "module": "gate", - "reason": "entry point" + ] }, { + "module": "gate", "classification": "load_bearing", + "reason": "entry point", + "importers": [ + "cli" + ] + }, + { + "module": "harness", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "slot_contract" - ], - "module": "harness", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "harness_ext", "classification": "standalone_capability", + "reason": "HarnessX extension surface; loaded by name when the optional harness is present", "importers": [ "cli", "fusion", "hunter_harness", "stage_processors", "verify_judge" - ], - "module": "harness_ext", - "reason": "HarnessX extension surface; loaded by name when the optional harness is present" + ] }, { + "module": "hunter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_harness" - ], - "module": "hunter", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "hunter_harness", "classification": "standalone_capability", + "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp", "importers": [ "cli" - ], - "module": "hunter_harness", - "reason": "harness wiring for the tool hunter; used by the hunter path and by mcp" + ] }, { + "module": "hunter_tools", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "tool_hunter" - ], - "module": "hunter_tools", - "reason": "imported by tool_hunter" + ] }, { + "module": "improve", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve.evolve" - ], - "module": "improve", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "improve.evolve", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve" - ], - "module": "improve.evolve", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "improve", - "improve.evolve" - ], "module": "improve.journal", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve", "improve.evolve" - ], - "module": "improve.validator", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "improve.validator", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "improve", + "improve.evolve" + ] + }, + { + "module": "index", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "skills" - ], - "module": "index", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "map_gate", - "reason": "entry point" + "classification": "load_bearing", + "reason": "entry point", + "importers": [] }, { + "module": "mapping", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "gate", "hunter_tools", "improve.evolve", "map_gate", + "model.scan", "pairs", "preprocess", "stage_processors" - ], - "module": "mapping", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "mcp", "classification": "standalone_capability", + "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", "importers": [ "cli" - ], - "module": "mcp", - "reason": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration" + ] }, { + "module": "model", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.candidates", @@ -337,70 +338,83 @@ "model.judge", "model.pipeline", "model.regions", + "model.report", "model.scan", "model.specs", "model.taint" - ], - "module": "model", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.audit", - "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 3: this classifier; run by the maintainer, not by the scan", + "importers": [] }, { - "classification": "load_bearing", - "importers": [], "module": "model.calibrate", - "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 9: the corpus as the model's calibration set", + "importers": [] }, { + "module": "model.candidates", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.scan" - ], - "module": "model.candidates", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.pipeline" - ], "module": "model.config_value", - "reason": "model-grounded-detection Req 6.3: constant abstraction for the configuration families" - }, - { "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.3: constant abstraction for the configuration families", "importers": [ - "model.pipeline" - ], + "model.pipeline", + "model.scan" + ] + }, + { "module": "model.dominance", - "reason": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.2: guard dominance, the arbiter for bugs that never crash", + "importers": [ + "model.pipeline", + "model.scan" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.endpoint", - "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: the judge client, wired by model/judge.py in group 3", + "importers": [ + "cli" + ] }, { + "module": "model.evidence", "classification": "load_bearing", - "importers": [], + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "model.scan" + ] + }, + { "module": "model.execution", - "reason": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 10: the deferred execution tier seam, adopted not built", + "importers": [] }, { - "classification": "load_bearing", - "importers": [], "module": "model.judge", - "reason": "model-grounded-detection Req 8: one bounded question, checked against the model" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: one bounded question, checked against the model", + "importers": [] }, { + "module": "model.ladder", "classification": "load_bearing", + "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries", "importers": [ "model.config_value", "model.dominance", @@ -409,94 +423,110 @@ "model.pipeline", "model.scan", "model.taint" - ], - "module": "model.ladder", - "reason": "model-grounded-detection Req 5: the evidence ladder every finding carries" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.scan" - ], "module": "model.pipeline", - "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes" - }, - { "classification": "load_bearing", + "reason": "model-grounded-detection Req 8: the enumerator proposes, the LLM answers, the model disposes", "importers": [ "model.scan" - ], + ] + }, + { "module": "model.regions", - "reason": "contributor-scan Req 2: regions from entry points, with the families each admits" + "classification": "load_bearing", + "reason": "contributor-scan Req 2: regions from entry points, with the families each admits", + "importers": [ + "cli", + "model.scan" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.report", - "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap" + "classification": "load_bearing", + "reason": "model-grounded-detection Req 9.3/11.1: per-slice reporting with the overfitting gap", + "importers": [ + "cli" + ] }, { - "classification": "load_bearing", - "importers": [], "module": "model.scan", - "reason": "contributor-scan Req 3: the repository driver -- one CPG, one budget, ranked spend" + "classification": "load_bearing", + "reason": "contributor-scan Req 3: the repository driver -- one CPG, one budget, ranked spend", + "importers": [ + "cli" + ] }, { + "module": "model.shipped", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "cli" + ] + }, + { + "module": "model.specs", + "classification": "load_bearing", + "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by", "importers": [ "model.config_value", "model.dominance", "model.judge", "model.pipeline", "model.regions", + "model.report", "model.scan", "model.taint" - ], - "module": "model.specs", - "reason": "model-grounded-detection Req 7.2-7.4: the security vocabularies the CPG queries are parameterised by" + ] }, { + "module": "model.taint", "classification": "load_bearing", + "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter", "importers": [ "model.judge", - "model.pipeline" - ], - "module": "model.taint", - "reason": "model-grounded-detection Req 6.1: taint reachability over the CPG, the flow-family arbiter" + "model.pipeline", + "model.scan" + ] }, { + "module": "model.taxonomy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "model.candidates", + "model.report", "model.scan", "model.specs" - ], - "module": "model.taxonomy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "pair_gate", "classification": "load_bearing", + "reason": "entry point", "importers": [ "cli" - ], - "module": "pair_gate", - "reason": "entry point" + ] }, { + "module": "pairs", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve", "improve.evolve", "model.candidates", "pair_gate" - ], - "module": "pairs", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "policy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "findings", @@ -507,20 +537,20 @@ "scoring.project_score", "semantic.overlay", "stage_processors" - ], - "module": "policy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "policy.verycode", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "policy" - ], - "module": "policy.verycode", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "preprocess", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "complexity.hints", @@ -536,44 +566,45 @@ "map_gate", "mapping", "model.candidates", + "model.scan", "pairs", "rank", "regress.candidate", "semantic.overlay", "skills", "stage_processors" - ], - "module": "preprocess", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "provenance", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "provenance", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "model.endpoint", - "tool_hunter" - ], "module": "provider", - "reason": "imported by model.endpoint, tool_hunter" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.endpoint", "tool_hunter" - ], - "module": "provider.openrouter", - "reason": "imported by model.endpoint, tool_hunter" + ] }, { + "module": "provider.openrouter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "model.endpoint", + "tool_hunter" + ] + }, + { + "module": "rank", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "calibration", "cli", @@ -586,76 +617,84 @@ "map_gate", "pairs", "stage_processors" - ], - "module": "rank", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "redaction", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "harness", "model.judge", "model.pipeline", "reports", "semantic.obligations.intent" - ], - "module": "redaction", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "regress.candidate" - ], - "module": "regress", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress.candidate", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress" - ], - "module": "regress.candidate", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { - "classification": "load_bearing", - "importers": [ - "regress", - "regress.candidate" - ], "module": "regress.recipes", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress", "regress.candidate" - ], + ] + }, + { "module": "regress.safety", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" - }, - { "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress", "regress.candidate" - ], - "module": "regress.verdict", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "regress.verdict", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "regress", + "regress.candidate" + ] + }, + { + "module": "reports", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "reports", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "repos", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", + "importers": [ + "cli" + ] + }, + { + "module": "ruleset", + "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "findings", @@ -665,83 +704,84 @@ "improve.validator", "pairs", "stage_processors" - ], - "module": "ruleset", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "ruleset.store", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "improve.validator", "ruleset" - ], - "module": "ruleset.store", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "run", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "reports" - ], - "module": "run", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "regress.candidate", "regress.recipes", "regress.safety", "regress.verdict" - ], - "module": "sandbox", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox.probe", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "sandbox" - ], - "module": "sandbox.probe", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "sandbox.runner", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "regress.candidate", "regress.recipes", "regress.safety", "sandbox" - ], - "module": "sandbox.runner", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "scoring", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "improve.evolve", "stage_processors" - ], - "module": "scoring", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "scoring.project_score", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "scoring" - ], - "module": "scoring.project_score", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "cpg.capability", "hunter_tools", + "mapping", "model.candidates", "model.specs", "pairs", @@ -755,43 +795,44 @@ "semantic.overlay", "semantic.prove_filter", "semantic.taint" - ], - "module": "semantic", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.cst", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.ir" - ], - "module": "semantic.cst", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.engines", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cpg.capability", "semantic", "semantic.overlay" - ], - "module": "semantic.engines", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.extra", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic", "semantic.cst", "semantic.engines" - ], - "module": "semantic.extra", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.facts", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", + "mapping", "model.candidates", "model.specs", "pairs", @@ -801,23 +842,23 @@ "semantic.obligations.shapes", "semantic.overlay", "semantic.taint" - ], - "module": "semantic.facts", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.functions", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "hunter_tools", "pairs", "semantic.obligations.shapes", "semantic.overlay" - ], - "module": "semantic.functions", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.ir", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", @@ -830,12 +871,12 @@ "semantic.obligations.shapes", "semantic.overlay", "semantic.taint" - ], - "module": "semantic.ir", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", @@ -849,33 +890,33 @@ "semantic.obligations.policy", "semantic.obligations.shapes", "semantic.obligations.siblings" - ], - "module": "semantic.obligations", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.check", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "semantic.obligations", "semantic.obligations.intent" - ], - "module": "semantic.obligations.check", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.dominance", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "hunter_tools", "pairs", "semantic.obligations.check" - ], - "module": "semantic.obligations.dominance", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.facts", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.specs", "pairs", @@ -885,148 +926,136 @@ "semantic.obligations.policy", "semantic.obligations.shapes", "semantic.obligations.siblings" - ], - "module": "semantic.obligations.facts", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.intent", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "semantic.obligations.intent", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.operations", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.candidates", "semantic.obligations.check", "semantic.obligations.dominance", "semantic.obligations.siblings" - ], - "module": "semantic.obligations.operations", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.policy", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli", "semantic.obligations.check" - ], - "module": "semantic.obligations.policy", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.shapes", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.obligations", "semantic.obligations.operations" - ], - "module": "semantic.obligations.shapes", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.obligations.siblings", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.obligations.check" - ], - "module": "semantic.obligations.siblings", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.overlay", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic", "semantic.prove_filter" - ], - "module": "semantic.overlay", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.prove_filter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic" - ], - "module": "semantic.prove_filter", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "semantic.taint", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "semantic.overlay" - ], - "module": "semantic.taint", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "skills", "classification": "standalone_capability", + "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", "importers": [ "hunter", "hunter_harness" - ], - "module": "skills", - "reason": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path" + ] }, { + "module": "slot_contract", "classification": "standalone_capability", + "reason": "the slot/contract protocol stage_processors implements", "importers": [ "stage_processors" - ], - "module": "slot_contract", - "reason": "the slot/contract protocol stage_processors implements" + ] }, { - "classification": "standalone_capability", - "importers": [], "module": "stage_processors", - "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" + "classification": "standalone_capability", + "reason": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate", + "importers": [] }, { + "module": "stages", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "stages", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "tool_hunter", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "model.endpoint" - ], - "module": "tool_hunter", - "reason": "imported by model.endpoint" + ] }, { + "module": "verification", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "calibration", "cli", "fusion", "reports", "verify_judge" - ], - "module": "verification", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] }, { + "module": "verify_judge", "classification": "load_bearing", + "reason": "reachable from gate, map_gate, pair_gate or the CLI", "importers": [ "cli" - ], - "module": "verify_judge", - "reason": "reachable from gate, map_gate, pair_gate or the CLI" + ] } - ], - "spec": "model-grounded-detection Req 3 (the outgrowth audit)", - "standalone": { - "fusion": "two-panel adjudication engine; reached from the scan's report stage and used standalone", - "harness_ext": "HarnessX extension surface; loaded by name when the optional harness is present", - "hunter_harness": "harness wiring for the tool hunter; used by the hunter path and by mcp", - "mcp": "narrow MCP server over stdio; its own `ousast mcp` entry point and OpenCode integration", - "skills": "skill router; consumed by mcp and by the OpenCode integration, not by the scan path", - "slot_contract": "the slot/contract protocol stage_processors implements", - "stage_processors": "HarnessX slot-contracted scan stages; pinned by the zero-dependency guard in test_gate" - } + ] } diff --git a/benchmarks/ranking/baselines/2026-09-10-access-rank.json b/benchmarks/ranking/baselines/2026-09-10-access-rank.json new file mode 100644 index 0000000..8bbd6d8 --- /dev/null +++ b/benchmarks/ranking/baselines/2026-09-10-access-rank.json @@ -0,0 +1,96 @@ +[ + { + "repo": "mwwpform", + "regions": 983, + "distinct_ranks": 5, + "largest_tier_share": 0.817, + "largest_tier_rank": 0.3, + "budget_at_recall": 513, + "known": [ + { + "id": "CVE-2023-6559", + "family": "path", + "site": "classes/services/class.mail.php:259:_delete_files", + "position": 512, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "pmpro", + "regions": 4463, + "distinct_ranks": 5, + "largest_tier_share": 0.838, + "largest_tier_rank": 0.3, + "budget_at_recall": 391, + "known": [ + { + "id": "CVE-2023-23488", + "family": "injection", + "site": "classes/class.memberorder.php:936:getMemberOrderByCode", + "position": 390, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "vampi", + "regions": 25, + "distinct_ranks": 5, + "largest_tier_share": 0.36, + "largest_tier_rank": 1.0, + "budget_at_recall": 16, + "known": [ + { + "id": "VAmPI-SQLI", + "family": "injection", + "site": "models/user_model.py:73:get_user", + "position": null, + "rank": null, + "in_scope": true, + "reached_transitively": true + }, + { + "id": "VAmPI-BOLA", + "family": "access_control", + "site": "api_views/users.py:187:update_password", + "position": 15, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + }, + { + "id": "VAmPI-BOLA-BOOKS", + "family": "access_control", + "site": "api_views/books.py:51:get_by_title", + "position": 10, + "rank": 0.8, + "in_scope": true, + "reached_transitively": false + } + ] + }, + { + "repo": "wpstatistics", + "regions": 1673, + "distinct_ranks": 6, + "largest_tier_share": 0.753, + "largest_tier_rank": 0.3, + "budget_at_recall": 504, + "known": [ + { + "id": "CVE-2022-25148", + "family": "injection", + "site": "includes/class-wp-statistics-pages.php:225:record", + "position": 503, + "rank": 0.3, + "in_scope": true, + "reached_transitively": false + } + ] + } +] diff --git a/benchmarks/ranking/position.py b/benchmarks/ranking/position.py new file mode 100644 index 0000000..5b9c15b --- /dev/null +++ b/benchmarks/ranking/position.py @@ -0,0 +1,121 @@ +"""Phase 0 of flow-aware-ranking: where does each pinned CVE sit in the order the scan actually examines? + +The ranker's success metric is not "is the CVE in the budget" -- on Paid Memberships Pro it already is, at +position 390 of 4,463 -- but **how small the budget can be while every CVE stays in it**. That number is +`budget_at_recall`, and until it is written down nothing later can claim to have improved it. + +This is the baseline. It runs no engine and asks no model: it is the region pipeline exactly as the scan +uses it, sorted exactly as the scan sorts it, with each known vulnerability's position read off. Offline, +against pinned checkouts, so two runs of one repository agree. + + ousast-venv/bin/python benchmarks/ranking/position.py # every pinned recipe with a checkout + ousast-venv/bin/python benchmarks/ranking/position.py --json # the same, as an artifact +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + +from openultrasast.mapping import analyze_entry_points # noqa: E402 +from openultrasast.model.regions import ScanRegion, regions_for # noqa: E402 +from openultrasast.model.shipped import declared_sources # noqa: E402 +from openultrasast.preprocess import preprocess_repository # noqa: E402 +from openultrasast.repos import checkout_path, load_repo_recipes # noqa: E402 + + +def ordered_regions(root: Path) -> list[ScanRegion]: + """The regions in the order `scan_repository` examines them. One definition, imported, never copied.""" + _, targets = preprocess_repository(root) + regions = regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root)) + # The scan's own total order. If this drifts from scan.py the baseline measures the wrong thing, which + # is why the sort key is the one thing here worth a test against the driver. + return sorted(regions, key=lambda r: (not r.shipped, -r.rank, r.path, r.function or "")) + + +def measure(name: str, root: Path, known: list[dict[str, object]]) -> dict[str, object]: + regions = ordered_regions(root) + tiers = Counter(r.rank for r in regions) + largest = max(tiers.values()) if tiers else 0 + rows: list[dict[str, object]] = [] + for item in known: + # A known vulnerability names a file and a function. Match on both, path by suffix because a recipe + # writes it relative to the repository and a region may carry it the same way or deeper. + hit = next( + (i for i, r in enumerate(regions) if r.path.endswith(str(item["file"])) and r.function == item["function"]), + None, + ) + rows.append( + { + "id": item["id"], + "family": item["family"], + "site": f"{item['file']}:{item['line']}:{item['function']}", + # Zero-based. None means no region is named after this function: the site is a SINK that an + # entry-point region elsewhere reaches through `callDepth` -- VAmPI's SQL injection lives in + # `models/user_model.py:get_user` and is found from the handler that calls it. Its true + # position is that handler's, which only the flow can say, so it is reported and left out of + # `budget_at_recall` rather than counted as a miss it is not. + "position": hit, + "rank": regions[hit].rank if hit is not None else None, + "in_scope": item["in_scope"], + "reached_transitively": hit is None, + } + ) + positioned = [r["position"] for r in rows if r["position"] is not None and r["in_scope"]] + return { + "repo": name, + "regions": len(regions), + "distinct_ranks": len(tiers), + "largest_tier_share": round(largest / len(regions), 3) if regions else 0.0, + "largest_tier_rank": max(tiers, key=lambda k: tiers[k]) if tiers else None, + # The smallest examined-region budget that still holds every in-scope known vulnerability. + "budget_at_recall": (max(positioned) + 1) if positioned else None, + "known": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--json", action="store_true", help="emit one JSON object per repository") + parser.add_argument("--repo", help="only this recipe") + args = parser.parse_args() + + results: list[dict[str, object]] = [] + for recipe in load_repo_recipes(): + if args.repo and recipe.name != args.repo: + continue + if "detection" not in recipe.measures: + continue + root = checkout_path(recipe) + if not root.is_dir(): + print(f"{recipe.name}: not checked out, skipping", file=sys.stderr) + continue + known = [ + {"id": k.id, "family": k.family, "file": k.file, "function": k.function, "line": k.line, "in_scope": k.in_scope} + for k in recipe.known + ] + results.append(measure(recipe.name, root, known)) + + if args.json: + print(json.dumps(results, indent=2)) + return 0 + for r in results: + print( + f"{r['repo']}: {r['regions']} regions, {r['distinct_ranks']} distinct ranks, " + f"{100 * float(r['largest_tier_share']):.1f}% share rank {r['largest_tier_rank']}" + ) + for k in r["known"]: + where = "reached via another region" if k["position"] is None else f"position {k['position']} (rank {k['rank']})" + scope = "" if k["in_scope"] else " [out of scope]" + print(f" {k['id']:16} {where:28} {k['site']}{scope}") + print(f" budget_at_recall = {r['budget_at_recall']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/openultrasast/cpg/queries/taint.sc b/src/openultrasast/cpg/queries/taint.sc index 855875e..772ef4b 100644 --- a/src/openultrasast/cpg/queries/taint.sc +++ b/src/openultrasast/cpg/queries/taint.sc @@ -49,6 +49,7 @@ boundedSinks: String = "", parameterSources: String = "false", fieldSources: String = "true", + evidenceOnly: String = "false", hookCallbacks: String = "", dispatchApply: String = "", requests: String = "", @@ -65,6 +66,12 @@ def split(raw: String): List[String] = raw.split(",").map(_.trim).filter(_.nonEmpty).toList + // Per-family answers that do not depend on the region, kept across the whole batch. + val familyInRepoMemo = scala.collection.mutable.Map.empty[String, Boolean] + val sinkCandidatesMemo = scala.collection.mutable.Map.empty[String, List[io.shiftleft.codepropertygraph.generated.nodes.Call]] + val reachableMemo = scala.collection.mutable.Map.empty[(String, String, Int), Set[String]] + val methodSourceMemo = scala.collection.mutable.Map.empty[String, Boolean] + def rowsFor( sourcesS: String, sinksS: String, @@ -74,6 +81,7 @@ functionS: String, paramSrc: String, fieldSrc: String, + evidenceS: String, fileS: String, depthS: String, boundedS: String @@ -132,7 +140,9 @@ // Bounded by `callDepth`, because "every method transitively reachable from an entry point" is most of a // repository, and an unbounded question is how this becomes slow again. val depth = scala.util.Try(depthS.toInt).getOrElse(0) - lazy val reachableMethods: Set[String] = + lazy val reachableMethods: Set[String] = reachableMemo.getOrElseUpdate((function, fileS, depth), computeReachable) + + def computeReachable: Set[String] = if (depth <= 0 || labeledMethods.isEmpty) Set.empty else { var frontier = labeledMethods.toSet @@ -502,7 +512,66 @@ byName || sanitizerNames.exists(n => code.startsWith(n + "(") || code.contains("'" + n + "'") || code.contains("\"" + n + "\"")) } - def sinkCalls = cpg.call.filter(c => sinkNames.exists(n => sinkMatches(c, n))).filter(c => inScope(c.method)) + // The candidate sink calls for a FAMILY do not depend on the region, and scanning every call in the graph + // for each of 2,487 requests was a large share of what made a request cost seconds -- in evidence mode, + // which asks for no dataflow at all, it alone took a query past an hour. Scanned once per sink list and + // kept for the batch; the per-request work is the scope filter over that list. + def sinkCalls = + sinkCandidatesMemo + .getOrElseUpdate(sinksS, cpg.call.filter(c => sinkNames.exists(n => sinkMatches(c, n))).l) + .filter(c => inScope(c.method)) + + // ---- EVIDENCE MODE: the same question without the dataflow (flow-aware-ranking, phase 0) -------------- + // + // Tiering is the CPG query WITHOUT dataflow; arbitration is the query WITH it. Same graph, same facts, two + // costs. This branch answers "what is in scope" -- which sink calls, what shape they take, whether a + // cleansing call wraps THEIR argument, whether a source is local or reachable -- and returns before + // `reachableByFlows` is ever called. Milliseconds per call, where a flow costs 6.63 seconds. + // + // Every field is stated in fact-table terms, so the vector means the same thing in every language the + // graph can hold. A `summary` row is always emitted, even when there are no sinks, because "no sinks" is + // tier 0 and must be distinguishable from "the query failed". + if (evidenceS == "true") { + // Whether a method holds a modelled source, memoised by full name across the batch: the same helper + // is reachable from hundreds of regions, and re-walking its AST for each of them is what made a + // no-dataflow query take minutes. `cpg.method.filter(...)` over every method per request was the + // other half of that, replaced by a direct lookup of the reachable names. + def methodHasSource(m: io.shiftleft.codepropertygraph.generated.nodes.Method): Boolean = + methodSourceMemo.getOrElseUpdate(m.fullName, m.ast.isCall.exists(c => sourcePatterns.exists(p => c.code.contains(p)))) + val sinkList = sinkCalls.l + val sourceLocal = labeledMethods.exists(methodHasSource) + val sourceNear = depth > 0 && cpg.method.fullNameExact(reachableMethods.toSeq: _*).exists(methodHasSource) + // Anywhere in the repository at all. Exact at every callDepth, and the cheapest cut there is: a family + // with no sink anywhere cannot produce a flow for any region. Memoised on the sink list, because it is + // the same answer for every request of one family and a repository-wide walk per request turned a + // milliseconds-per-pair query into minutes. + val familyInRepo = familyInRepoMemo.getOrElseUpdate(sinksS, cpg.call.exists(c => sinkNames.exists(n => sinkMatches(c, n)))) + val summary = ujson.Obj( + "kind" -> "summary", + "sinks" -> sinkList.size, + "sourceLocal" -> sourceLocal, + "sourceNear" -> sourceNear, + "familyInRepo" -> familyInRepo + ) + val perSink = sinkList.map { sink => + val realArgs = sink.argument.argumentIndexGt(0).l + ujson.Obj( + "kind" -> "sink", + "sink" -> sink.code.take(200), + "sinkLine" -> sink.lineNumber.getOrElse(-1).toString, + "sinkMethod" -> sink.method.name, + "sinkFile" -> sink.method.filename.split("/").last, + "sinkArity" -> realArgs.size, + "sinkArg0Literal" -> realArgs.headOption.map(a => a.isLiteral).getOrElse(false), + // A sanitizer of this family is an ANCESTOR of an argument in the AST -- it wraps what flows in -- + // as opposed to merely appearing somewhere in the same function. This is the field a text scan + // gets wrong, and the one that would have misranked CVE-2023-23488: `esc_sql` is in that function, + // on a different statement. + "cleansedOnCall" -> sink.argument.ast.l.exists(node => sanitizesHere(node)) + ) + } + return summary :: perSink + } val rows = sinkCalls.l.flatMap { sink => // Data flows into the arguments; asking the call node itself finds nothing. @@ -576,6 +645,7 @@ def field(name: String): String = req.obj.get(name).map(_.str).getOrElse("") val paramSrc = req.obj.get("parameterSources").map(_.str).getOrElse("false") val fieldSrc = req.obj.get("fieldSources").map(_.str).getOrElse("true") + val evidence = req.obj.get("evidenceOnly").map(_.str).getOrElse("false") id -> ujson.Arr( rowsFor( field("sources"), @@ -589,6 +659,7 @@ field("function"), paramSrc, fieldSrc, + evidence, field("file"), field("callDepth"), field("boundedSinks") @@ -604,7 +675,7 @@ println(ujson.write(ujson.Obj.from(answers.toSeq :+ ("__census__" -> census)))) } else { println( - ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, hookCallbacks, dispatchApply, function, parameterSources, fieldSources, file, callDepth, boundedSinks): _*)) + ujson.write(ujson.Arr(rowsFor(sources, sinks, sanitizers, hookCallbacks, dispatchApply, function, parameterSources, fieldSources, evidenceOnly, file, callDepth, boundedSinks): _*)) ) } println("---OUSAST-CPG-END---") diff --git a/src/openultrasast/model/evidence.py b/src/openultrasast/model/evidence.py new file mode 100644 index 0000000..d553154 --- /dev/null +++ b/src/openultrasast/model/evidence.py @@ -0,0 +1,150 @@ +"""The evidence vector: what a (region, family) pair carries BEFORE any dataflow is asked (flow-aware-ranking). + +Tiering is the CPG query without dataflow; arbitration is the query with it. Same graph, same facts, two costs. +This module turns the evidence rows that `taint.sc` emits in `evidenceOnly` mode into one vector per pair, +and assigns the tier. Every field is stated in fact-table terms -- source, sink, sanitizer -- so a heuristic +that mentions a language's own syntax has, by construction, left this module. + +Only tier 0 excludes. Its predicate has a proof -- a family with no sink in reach cannot produce a flow, in any +language, under any dataflow model -- and it is the only place exactness is required. Everything above orders, +where being wrong costs position, and `regions_truncated` reports what position cost. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field + +TIER_EXCLUDE = 0 # no sink of this family in reach: never ask +TIER_NO_SOURCE = 1 # sinks, but nothing untrusted can arrive: ask last +TIER_CLEANSED = 2 # a source exists, but every sink call is bound or cleansed on the call: ask late +TIER_OPEN = 3 # a source exists and some sink call is computed and uncleansed: ask first +TIER_OPEN_PUBLIC = 4 # tier 3 and declared public, or a source on the same statement: ask very first + + +@dataclass(frozen=True) +class SinkEvidence: + """One sink call of the family, as the graph saw it, with no flow asked.""" + + code: str + line: int + method: str + arity: int + arg0_literal: bool + cleansed_on_call: bool # a sanitizer of this family wraps an ARGUMENT -- not merely present in the function + + @property + def shape(self) -> str: + """``literal`` cannot carry taint; ``computed`` is the residue that can.""" + return "literal" if self.arg0_literal else "computed" + + +@dataclass(frozen=True) +class Evidence: + """The vector for one (region, family) pair.""" + + sinks: tuple[SinkEvidence, ...] = () + family_in_repo: bool = True # False is the repository-wide tier 0, exact at every callDepth + source_local: bool = False # a modelled source in the region's own body + source_near: bool = False # a modelled source within the arbiter's callDepth + entry: bool = False # parameters untrusted by contract + access_declared_public: bool = False # declared, not inferred -- "no decorator found" is not "public" + carried: bool | None = None # stage one of the two-stage join; None where it was not computed + bound_names: tuple[str, ...] = field(default=()) # sink names the shape test treats as bound at arity >= 2 + + @property + def any_source(self) -> bool: + return self.source_local or self.source_near or self.entry or bool(self.carried) + + def is_bound(self, sink: SinkEvidence) -> bool: + """The safe-shape rule the arbiter already applies: a named sink with two or more arguments binds.""" + return sink.arity >= 2 and any(name in sink.code for name in self.bound_names) + + @property + def open_sinks(self) -> tuple[SinkEvidence, ...]: + """Sink calls that could carry an uncleansed, unbound, computed value.""" + return tuple(s for s in self.sinks if s.shape == "computed" and not s.cleansed_on_call and not self.is_bound(s)) + + @property + def tier(self) -> int: + if not self.family_in_repo or not self.sinks: + return TIER_EXCLUDE + if not self.any_source: + return TIER_NO_SOURCE + if not self.open_sinks: + return TIER_CLEANSED + if self.access_declared_public or self.source_local or self.carried: + return TIER_OPEN_PUBLIC + return TIER_OPEN + + +def evidence_from_rows( + rows: Sequence[Mapping[str, object]] | None, + *, + entry: bool = False, + access_declared_public: bool = False, + carried: bool | None = None, + bound_names: Sequence[str] = (), +) -> Evidence | None: + """Read the query's evidence rows for one request. ``None`` when the query did not answer. + + A request that answered with no sinks still carries a ``summary`` row -- that is tier 0, and it must be + distinguishable from a request that failed, which carries nothing. + """ + if rows is None: + return None + summary = next((r for r in rows if isinstance(r, Mapping) and r.get("kind") == "summary"), None) + if summary is None: + return None + sinks = tuple( + SinkEvidence( + code=str(r.get("sink", "")), + line=_as_int(r.get("sinkLine")), + method=str(r.get("sinkMethod", "")), + arity=_as_int(r.get("sinkArity")), + arg0_literal=_as_bool(r.get("sinkArg0Literal")), + cleansed_on_call=_as_bool(r.get("cleansedOnCall")), + ) + for r in rows + if isinstance(r, Mapping) and r.get("kind") == "sink" + ) + return Evidence( + sinks=sinks, + family_in_repo=_as_bool(summary.get("familyInRepo"), default=True), + source_local=_as_bool(summary.get("sourceLocal")), + source_near=_as_bool(summary.get("sourceNear")), + entry=entry, + access_declared_public=access_declared_public, + carried=carried, + bound_names=tuple(bound_names), + ) + + +def _as_bool(value: object, *, default: bool = False) -> bool: + """A JSON boolean, or its string spelling. ``bool("False")`` is True, which would make every unsanitized + sink read as cleansed the moment a serialiser chose strings -- the kind of silent inversion this project + has paid for before.""" + if value is None: + return default + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"true", "1", "yes"} + + +def _as_int(value: object) -> int: + try: + return int(str(value)) + except (TypeError, ValueError): + return -1 + + +__all__ = [ + "TIER_CLEANSED", + "TIER_EXCLUDE", + "TIER_NO_SOURCE", + "TIER_OPEN", + "TIER_OPEN_PUBLIC", + "Evidence", + "SinkEvidence", + "evidence_from_rows", +] diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 2bbba2c..37f65c8 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Any from ..cpg.backend import CpgResult +from .evidence import evidence_from_rows from .candidates import enumerate_candidates from .config_value import request_params as config_params from .dominance import request_params as dominance_params @@ -64,6 +65,11 @@ class ScanBudget: max_model_calls: int = 200 max_regions: int = 500 + # Compute the evidence tier of every (region, family) pair and RECORD it. Phase 0 of flow-aware-ranking: + # the instrument runs on every real scan so later phases are measurable, and it changes nothing about + # what is asked or in what order. One extra query invocation without dataflow -- a JVM load, then + # milliseconds per pair. + tiering: bool = True @dataclass(frozen=True) @@ -87,6 +93,10 @@ class ModelScanResult: # would remove) or CPGQL evaluation (which it would not). That is the whole of the server-mode decision. query_seconds_by_kind: Mapping[str, float] = field(default_factory=dict) degradations: tuple[Mapping[str, object], ...] = () + # The evidence tier of each (path, function, family) pair the scan collected, and the count per tier. + # Recorded, not acted on: this is what "the ranker got better" will be measured against. + tiers: tuple[tuple[str, str, str, int], ...] = () + tier_counts: Mapping[int, int] = field(default_factory=dict) @property def arbitrate_seconds(self) -> float: @@ -176,11 +186,44 @@ def scan_repository( continue work.append((f"{len(work)}", region, spec)) + # Phase 1b: the evidence tier of every taint pair, before any dataflow is asked. Same query, `evidenceOnly`, + # no `reachableByFlows`. Recorded on the result and NOT used to reorder or skip anything -- that is a + # later phase, and it must be measured against this record before it is allowed to change behaviour. + tiers: list[tuple[str, str, str, int]] = [] + per_kind: dict[str, float] = {} + if limits.tiering and callable(getattr(cpg, "run_batch", None)): + grouped_taint = _grouped(work, hook_callbacks=hooks).get("taint", {}) + if grouped_taint: + evidence_requests = {rid: {**dict(params), "evidenceOnly": "true"} for rid, params in grouped_taint.items()} + tier_started = time.monotonic() + answered_evidence = cpg.run_batch("taint", evidence_requests) + per_kind["evidence"] = round(time.monotonic() - tier_started, 2) + if answered_evidence is None: + degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": len(evidence_requests)}) + else: + answered_evidence.pop("__census__", None) + by_rid = {rid: (region, spec) for rid, region, spec in work} + for rid, rows in answered_evidence.items(): + pair = by_rid.get(rid) + if pair is None: + continue + region, spec = pair + evidence = evidence_from_rows( + rows, + entry=_is_entry_point(region), + # Rank 1.0 is the public tier. Provenance -- declared by a route contract versus + # inferred from an absent decorator -- is not yet carried on ScanRegion, so this is + # the proxy phase 0 has; the brief names it as the field to thread through next. + access_declared_public=region.rank >= 1.0, + bound_names=getattr(spec, "safe_shape_sinks", ()), + ) + if evidence is not None: + tiers.append((region.path, region.function or "", getattr(spec, "family", ""), evidence.tier)) + # Phase 2: ONE invocation per query kind. JVM startup dominates a repository scan -- a call per region # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while # the queries themselves are milliseconds once the CPG is loaded. rows_by_id: dict[str, list[object]] = {} - per_kind: dict[str, float] = {} census_reported = False query_started = time.monotonic() for kind, requests in _grouped(work, hook_callbacks=hooks).items(): @@ -293,6 +336,8 @@ def scan_repository( query_seconds=query_seconds, query_seconds_by_kind=per_kind, degradations=tuple(degradations), + tiers=tuple(tiers), + tier_counts={tier: sum(1 for t in tiers if t[3] == tier) for tier in sorted({t[3] for t in tiers})}, ) diff --git a/tests/test_model_evidence.py b/tests/test_model_evidence.py new file mode 100644 index 0000000..44803d3 --- /dev/null +++ b/tests/test_model_evidence.py @@ -0,0 +1,99 @@ +"""The evidence vector and its tiers, on canned rows (flow-aware-ranking phase 0).""" + +from __future__ import annotations + +from openultrasast.model.evidence import ( + TIER_CLEANSED, + TIER_EXCLUDE, + TIER_NO_SOURCE, + TIER_OPEN, + TIER_OPEN_PUBLIC, + evidence_from_rows, +) + +SUMMARY = {"kind": "summary", "sinks": 1, "sourceLocal": False, "sourceNear": False, "familyInRepo": True} + + +def _sink(code: str, *, literal: bool = False, cleansed: bool = False, arity: int = 1) -> dict[str, object]: + return { + "kind": "sink", + "sink": code, + "sinkLine": "936", + "sinkMethod": "f", + "sinkArity": arity, + "sinkArg0Literal": literal, + "cleansedOnCall": cleansed, + } + + +def test_a_failed_query_is_not_a_tier() -> None: + """No rows at all means the query did not answer. That is a degradation, never tier 0.""" + assert evidence_from_rows(None) is None + assert evidence_from_rows([]) is None + + +def test_no_sink_in_scope_is_tier_zero_and_exact() -> None: + """The only tier that excludes. A family with no sink in reach cannot produce a flow, in any language.""" + e = evidence_from_rows([{**SUMMARY, "sinks": 0}], entry=True) + assert e is not None and e.tier == TIER_EXCLUDE + + +def test_no_sink_anywhere_in_the_repository_is_tier_zero_at_any_depth() -> None: + """PMPro never calls `unserialize`: all 487 deserialization requests were asked of nothing.""" + e = evidence_from_rows([{**SUMMARY, "familyInRepo": False}, _sink("x($y)")], entry=True) + assert e is not None and e.tier == TIER_EXCLUDE + + +def test_a_sink_with_no_source_is_asked_last() -> None: + e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_row("... $id")')]) + assert e is not None and e.tier == TIER_NO_SOURCE + + +def test_the_cve_shape_is_the_top_tier() -> None: + """CVE-2023-23488: an entry-point method whose sink concatenates a parameter into SQL, with `esc_sql` + present in the FUNCTION but wrapping a different statement -- so `cleansedOnCall` is false here.""" + e = evidence_from_rows( + [SUMMARY, _sink('$wpdb->get_var("... WHERE code = \'" . $code . "\'")', cleansed=False)], + entry=True, + ) + assert e is not None and e.tier == TIER_OPEN + assert len(e.open_sinks) == 1 + + +def test_cleansing_on_the_call_itself_lowers_the_tier() -> None: + """The fixed side of that pair wraps the value in `$wpdb->prepare`, which is an ancestor of the argument.""" + e = evidence_from_rows([SUMMARY, _sink('$wpdb->get_var($wpdb->prepare("... %s", $code))', cleansed=True)], entry=True) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_a_literal_argument_cannot_carry_taint() -> None: + e = evidence_from_rows([SUMMARY, _sink('system("ls")', literal=True)], entry=True) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_a_bound_shape_is_the_arbiters_own_rule() -> None: + """`execute(sql, params)` binds where `execute(sql + x)` interpolates; same rule the taint verdict uses.""" + e = evidence_from_rows([SUMMARY, _sink("cursor.execute(sql, params)", arity=2)], entry=True, bound_names=("execute",)) + assert e is not None and e.tier == TIER_CLEANSED + + +def test_declared_public_or_a_local_source_asks_very_first() -> None: + public = evidence_from_rows([SUMMARY, _sink("unlink($f)")], entry=True, access_declared_public=True) + local = evidence_from_rows([{**SUMMARY, "sourceLocal": True}, _sink("unlink($_GET['f'])")]) + assert public is not None and public.tier == TIER_OPEN_PUBLIC + assert local is not None and local.tier == TIER_OPEN_PUBLIC + + +def test_a_carried_field_counts_as_a_source() -> None: + """`_delete_files()` has no parameters; its taint arrives as `$this->attachments`. Stage one of the join + already knows that, and the vector must not demote the region for lacking a direct source.""" + e = evidence_from_rows([SUMMARY, _sink("unlink($file)")], carried=True) + assert e is not None and e.tier == TIER_OPEN_PUBLIC + + +def test_a_stringified_boolean_is_read_as_its_value_not_its_truthiness() -> None: + """`bool("False")` is True. A serialiser that chose strings would make every open sink read as cleansed.""" + rows = [{**SUMMARY, "sourceLocal": "False", "familyInRepo": "true"}, _sink("unlink($f)")] + rows[1]["cleansedOnCall"] = "False" + e = evidence_from_rows(rows, entry=True) + assert e is not None and e.tier == TIER_OPEN, "a 'False' string must not turn an open sink into a cleansed one" diff --git a/tests/test_model_scan.py b/tests/test_model_scan.py index ba09a74..979926d 100644 --- a/tests/test_model_scan.py +++ b/tests/test_model_scan.py @@ -388,7 +388,9 @@ def test_a_query_the_engine_could_not_answer_is_recorded_not_ignored() -> None: failures = [d for d in result.degradations if d.get("reason") == "query_failed"] assert failures, "a query that could not be asked must be reported" - assert failures[0]["kind"] == "taint" + kinds = {d["kind"] for d in failures} + assert "taint" in kinds, "the taint failure is recorded" + assert "evidence" in kinds, "and so is the evidence pass that runs before it -- phase 0 of flow-aware-ranking" assert result.findings == ()