feat(model): phase 2 of flow-aware-ranking -- spend the region budget by evidence, weights published

`Evidence.score` orders pairs within a tier by one table of weights (open sinks, cleansed sinks, local /
carried / near sources, entry, declared public), capped so volume stops counting, and zero for tier 0 so
the weights can rank but never manufacture. `ScanBudget.order_by_evidence` runs the evidence pass over
every region and cuts `max_regions` from each region's best (tier, score); `order_by_evidence()` is the
one ordering function, used by the scan and by `benchmarks/ranking/position.py --evidence`. Off by
default until the position benchmark says where the pinned CVEs land under it.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_model_evidence.py

@@ -97,3 +97,29 @@ def test_a_stringified_boolean_is_read_as_its_value_not_its_truthiness() -> None
     rows[1]["cleansedOnCall"] = "False"
     e = evidence_from_rows(rows, entry=True)
     assert e is not None and e.tier == TIER_OPEN, "a 'False' string must not turn an open sink into a cleansed one"
+
+
+def test_the_score_orders_within_a_tier_and_never_across_one() -> None:
+    """Phase 2: the published weights rank pairs inside a tier. They cannot lift a pair out of tier 0 --
+    a pair with no sink scores nothing whatever the weights say -- and counts are capped so a 346-sink
+    file-scope region does not outrank everything by volume alone."""
+    from openultrasast.model.evidence import OPEN_SINK_CAP, WEIGHTS, Evidence, SinkEvidence
+
+    def sink(n: int, cleansed: bool = False) -> SinkEvidence:
+        return SinkEvidence(code=f"q({n})", line=n, method="m", arity=1, arg0_literal=False, cleansed_on_call=cleansed)
+
+    nothing = Evidence(sinks=(), source_local=True)
+    assert nothing.tier == 0 and nothing.score == 0.0, "no sink, no score: the weights cannot manufacture a pair"
+
+    one = Evidence(sinks=(sink(1),), source_near=True)
+    local = Evidence(sinks=(sink(1),), source_local=True)
+    assert local.tier > one.tier, "a local source is the higher tier, not merely the higher score"
+    assert local.score == WEIGHTS["open_sink"] + WEIGHTS["source_local"]
+
+    many = Evidence(sinks=tuple(sink(i) for i in range(50)), source_near=True)
+    assert many.tier == one.tier
+    assert many.score == WEIGHTS["open_sink"] * OPEN_SINK_CAP + WEIGHTS["source_near"], "capped: volume stops counting"
+
+    cleansed = Evidence(sinks=(sink(1), sink(2, cleansed=True)), source_near=True)
+    assert cleansed.score < one.score, "a cleansed sink beside an open one counts against the pair"
+    assert sorted([many, one, local], key=lambda e: e.order_key, reverse=True) == [local, many, one]

tests/test_model_scan.py

@@ -472,3 +472,67 @@ def test_tier_zero_pairs_are_not_asked_and_nothing_else_is_skipped() -> None:
     assert taint["count"] == 1, "only the pair with a sink in reach is asked"
     assert result.requests_pruned == 2, "and the two tier-0 pairs are counted, not silently dropped"
     assert not any(d.get("reason") == "query_failed" for d in result.degradations)
+
+
+def test_the_budget_is_spent_by_evidence_when_asked() -> None:
+    """Phase 2 of flow-aware-ranking. With `order_by_evidence`, the evidence pass runs over every region and
+    the region cap is cut from the (tier, score) order, so a low-ranked region with an open sink and a
+    local source is examined before a top-ranked region with no sink in reach. Off, the static order holds."""
+    from openultrasast.model.regions import ScanRegion
+    from openultrasast.model.scan import ScanBudget, scan_repository
+
+    class _Backend:
+        def __init__(self) -> None:
+            self.taint_files: list[list[str]] = []
+            self.evidence_sizes: list[int] = []
+
+        def available(self) -> bool:
+            return True
+
+        def build(self, root, *, language=""):  # type: ignore[no-untyped-def]
+            from openultrasast.cpg.backend import CpgResult
+
+            def batch(kind, requests):  # type: ignore[no-untyped-def]
+                if any(p.get("evidenceOnly") == "true" for p in requests.values()):
+                    self.evidence_sizes.append(len(requests))
+                    out = {}
+                    for rid, params in requests.items():
+                        hot = str(params.get("file", "")).endswith("low.py")
+                        out[rid] = [{"kind": "summary", "sinks": 1 if hot else 0, "sourceLocal": hot, "familyInRepo": True}] + (
+                            [
+                                {
+                                    "kind": "sink",
+                                    "sink": "os.system(x)",
+                                    "sinkLine": "4",
+                                    "sinkMethod": "run",
+                                    "sinkArity": 1,
+                                    "sinkArg0Literal": False,
+                                    "cleansedOnCall": False,
+                                }
+                            ]
+                            if hot
+                            else []
+                        )
+                    return out
+                if kind == "taint":
+                    self.taint_files.append(sorted({str(p.get("file", "")) for p in requests.values()}))
+                return {rid: [] for rid in requests}
+
+            return CpgResult(cpg_path=Path("cpg.bin"), run=lambda q, p: [], run_batch=batch)
+
+    top = ScanRegion(path="top.py", function="handle", language="python", families=("injection",), rank=1.0, source="entry_point")
+    low = ScanRegion(path="low.py", function="run", language="python", families=("injection",), rank=0.3, source="entry_point")
+
+    static = _Backend()
+    scan_repository(Path("."), [top, low], backend=static, budget=ScanBudget(max_model_calls=0, max_regions=1))
+    assert static.evidence_sizes == [1] and static.taint_files == [], (
+        "static order: the budget holds `top`, whose pair is tier 0 and pruned"
+    )
+
+    ordered = _Backend()
+    result = scan_repository(
+        Path("."), [top, low], backend=ordered, budget=ScanBudget(max_model_calls=0, max_regions=1, order_by_evidence=True)
+    )
+    assert ordered.evidence_sizes == [2], "the evidence pass covers every region, not only the budget"
+    assert ordered.taint_files == [["low.py"]], "and the budget is spent on the region the evidence points at"
+    assert result.tier_counts == {0: 1, 4: 1}