From 546db24b8e5ec0c7b96e6a42b972994c985f3d45 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 13 Sep 2026 10:16:22 +0200 Subject: [PATCH] feat(pre-push-safety-net): preserve exact ranker scope and outcomes --- .../implementation-group-3.md | 29 ++ .kiro/specs/pre-push-safety-net/spec.json | 7 +- .kiro/specs/pre-push-safety-net/tasks.md | 5 +- .../2026-09-13-ranker-scope-smoke.json | 294 ++++++++++++++++++ ops/README.md | 7 + ops/smoke_scope.py | 72 +++++ src/openultrasast/cli.py | 16 +- src/openultrasast/model/contracts.py | 48 +++ src/openultrasast/model/scan.py | 217 +++++++++++-- tests/test_model_scope.py | 273 ++++++++++++++++ 10 files changed, 931 insertions(+), 37 deletions(-) create mode 100644 benchmarks/measurements/2026-09-13-ranker-scope-smoke.json create mode 100644 ops/smoke_scope.py create mode 100644 tests/test_model_scope.py diff --git a/.kiro/specs/pre-push-safety-net/implementation-group-3.md b/.kiro/specs/pre-push-safety-net/implementation-group-3.md index 427133f..406c8dc 100644 --- a/.kiro/specs/pre-push-safety-net/implementation-group-3.md +++ b/.kiro/specs/pre-push-safety-net/implementation-group-3.md @@ -61,3 +61,32 @@ inside the driver; this does not change the ranker. Global mypy now checks all nine skipped, 96.85 seconds; global Ruff, formatting (219 files), mypy and diff checks passed. Verification status: VERIFIED, claim type TASK. This completes 8/27 tasks; tasks 3.2–3.4 remain in progress for this group. + +## Task 3.2 evidence + +Independent review APPROVED; verification status VERIFIED, claim type TASK. +The fresh reviewer suite passed 1,105 tests with nine skipped in 101.18 seconds. +Global Ruff, formatting (221 files), mypy (108 source files) and diff checks passed. +The reviewer also exercised 24 combinations of order, budget and failed query +families, retaining all 12 declared questions and reconciling coverage totals. + +The driver accepts explicit `ranking_mode`, `unit` and `population_complete` while +preserving ordinary defaults. Scope and execution requests are produced together +from the existing ranker's final order. Selected/deferred identities, normalized +evidence, unchanged tier/score, per-question outcomes and per-family completion +travel to the CLI directly. Missing evidence stays unknown. Partial graphs cannot +justify tier-zero pruning or completed coverage. Raw query answers survive a later +timeout even when arbitration cannot complete. A completed outcome denotes query +and arbitration completion within the modeled boundary, not a safety assertion. + +`benchmarks/measurements/2026-09-13-ranker-scope-smoke.json` records a rebuilt image +whose backend, driver, contracts and CLI hashes match the source. Real PHP (78 bytes) +and JavaScript (69 bytes) fixtures each selected `risky` over the initially higher +static-ranked `safe`. Both selected questions had tier 4 and score 2.5; execution +outcome IDs matched selection, and each family reported one selected/completed, +one deferred and zero unanswered questions. Each produced one controlled finding. +PHP took 61.57 seconds and JavaScript 51.27 seconds under one 900-second lab budget. +This manually declared entry-region smoke does not validate mapper discovery, +PMPro/NodeGoat replay, capability admission or hook latency. + +Nine of 27 tasks are now complete; tasks 3.3–3.4 remain for this group. diff --git a/.kiro/specs/pre-push-safety-net/spec.json b/.kiro/specs/pre-push-safety-net/spec.json index c12de36..9eb113c 100644 --- a/.kiro/specs/pre-push-safety-net/spec.json +++ b/.kiro/specs/pre-push-safety-net/spec.json @@ -1,7 +1,7 @@ { "feature_name": "pre-push-safety-net", "created_at": "2026-09-12T15:35:00Z", - "updated_at": "2026-09-13T08:02:59.282414+00:00", + "updated_at": "2026-09-13T08:16:14.274231+00:00", "language": "en", "phase": "implementation", "approvals": { @@ -70,9 +70,10 @@ "2.1", "2.2", "2.3", - "3.1" + "3.1", + "3.2" ], - "remaining_tasks": 19, + "remaining_tasks": 18, "verification_record": "implementation-group-3.md", "verification_records": [ "implementation-group-1.md", diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index d3d7f02..2eab9ff 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -1,7 +1,7 @@ # Implementation Plan: pre-push-safety-net Status: task plan approved by the maintainer's "approve." on 2026-09-12. Groups 1 and 2 (tasks 1.1–2.3) -are complete after independent review and fresh verification. Group 3 is in progress (3.1 verified); groups 4–8 remain pending; +are complete after independent review and fresh verification. Group 3 is in progress (3.1–3.2 verified); groups 4–8 remain pending; the hook and capability admission are not yet implemented. See `implementation-group-1.md` and `implementation-group-2.md`. ## Execution contract @@ -87,13 +87,14 @@ the hook and capability admission are not yet implemented. See `implementation-g - _Depends: 1.4_ - _Requirements: 4.1, 4.2, 7.3_ - _Verified 2026-09-13:_ independent review APPROVED; fresh suite 1,089 passed, nine skipped; global Ruff/format and mypy (108 files) passed. Real hanging probe/frontend/overlay/census/query processes obey the shared deadline and cancellation allowance; retries do not reset it. Non-root packaged PHP/JavaScript graph/query smoke passed with matching source hashes under one 900-second lab budget. See `implementation-group-3.md` and `benchmarks/measurements/2026-09-13-engine-budget-smoke.json`. Runtime readiness only; no hook latency claim. Former three mypy errors resolved by a behavior-preserving variable rename. -- [ ] 3.2 Return the ranker's exact selected and deferred work +- [x] 3.2 Return the ranker's exact selected and deferred work - Integrate the existing ranker through explicit mode selection and shared question identity; record selected/deferred IDs, evidence and reasons directly from the final order. - Preserve unanswered evidence as unknown rather than tier zero; report per-family coverage so taint ranking cannot masquerade as authorization/configuration coverage. - Add paired PHP/JavaScript normalized-evidence contract tests asserting identical tier/score and selection decisions for equivalent input and budgets, without repository/language priority exceptions. - Done when changing order produces matching execution and manifest IDs, with no reconstruction from an earlier list, and normalized cross-language equivalence tests pass. - _Boundary: Ranker Scope Contract, generic driver integration_ - _Requirements: 2.1, 2.2, 6.4, 7.3, 8.1, 8.2_ + - _Verified 2026-09-13:_ independent review APPROVED; 1,105 passed, nine skipped; global Ruff/format/mypy passed. Twenty-four independent order/budget/missing-family combinations retained all question identities. Packaged PHP/JavaScript smoke selected the same controlled risky operation at tier 4/score 2.5, with exact execution/outcome IDs and reconciled family coverage. Missing evidence, partial graphs and interrupted arbitration remain explicit; CLI consumes authoritative scope. Evidence: `benchmarks/measurements/2026-09-13-ranker-scope-smoke.json`. No new ranking weights or hook admission claim. - [ ] 3.3 Connect change context to supported first-party relationships - Use existing entry/callee, field, hook and guard relationships to supply affected context to ranked questions, including unchanged sinks after source/guard changes. - Preserve current abstract semantics; absent dynamic or external relationships remain unresolved rather than guessed safe. A required new feature or fact is reported to its owning spec. diff --git a/benchmarks/measurements/2026-09-13-ranker-scope-smoke.json b/benchmarks/measurements/2026-09-13-ranker-scope-smoke.json new file mode 100644 index 0000000..b6d6c9d --- /dev/null +++ b/benchmarks/measurements/2026-09-13-ranker-scope-smoke.json @@ -0,0 +1,294 @@ +{ + "recorded_at": "2026-09-13T08:14:51.777108+00:00", + "task": "3.2", + "image_id": "sha256:0078b93b0caf7cb2587e47fa922225852bad2d42a594344a0de0161cf178b690", + "build_exit_code": 0, + "smoke_exit_code": 0, + "packaged_source_hashes": { + "openultrasast.cpg.backend": "2e327d26958efd592932bbe5a84aa122d969c0e49a1c13469a4d956b621a9a34", + "openultrasast.model.scan": "ceeb9aa2e1419565642ed11e5de6be6a80ca242f574eb533baa0b11aa79ff651", + "openultrasast.model.contracts": "0965fb3f0a365f91f2d0984666dc0fcf772344da51741367ea3b74fd7ed885d9", + "openultrasast.cli": "0a881f981d6a50d0c030ce75ddeb773a9434c6f93a9a4e059149d77f2c1f7736" + }, + "packaged_sources_match_workspace": true, + "smoke_script_sha256": "5183231d42ae4acea57399bc4de3285f5510c0d95466bf8f289e35dd5289280d", + "runtime": { + "purpose": "Real PHP/JavaScript scope smoke; declared entry regions, no admission or latency claim", + "shared_budget_seconds": 900, + "languages": { + "php": { + "source_bytes": 78, + "source_sha256": "9f73839b8b5b3861611a4a627e7fb1e252e2450b861b187d43bd9e8bfd9a5f9a", + "selected_question_ids": [ + "df64a78dd4ad131c49dd84b93e5371d01cbf9dd555c172d527f809b5ddebf474" + ], + "result": { + "findings": [ + { + "site": "api.php:3:risky", + "family": "injection", + "rung": "model_entailed", + "witness": "$req -> eval($req) in api.php (line 3, 2 steps)", + "contradiction": "", + "reached_from": 1 + } + ], + "by_rung": { + "suspicion": 0, + "model_corroborated": 0, + "model_entailed": 1, + "execution_confirmed": 0 + }, + "regions_scanned": 1, + "regions_unjudged": 1, + "regions_unasked": 0, + "model_calls": 0, + "cost_usd": 0.0, + "seconds": 61.57, + "build_seconds": 28.68, + "query_seconds": 16.09, + "query_seconds_by_kind": { + "evidence": 16.75, + "taint": 16.09 + }, + "degradations": [ + { + "stage": "model", + "reason": "regions_truncated", + "examined": 1, + "total": 2 + } + ], + "tiers": [ + [ + "api.php", + "safe", + "injection", + 0 + ], + [ + "api.php", + "risky", + "injection", + 4 + ] + ], + "tier_counts": { + "0": 1, + "4": 1 + }, + "requests_pruned": 0, + "scope": { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "scope-smoke", + "language": "php", + "path": "api.php", + "function": "risky", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"mysqli_prepare\",\"mysqli_stmt_bind_param\",\"pg_execute\",\"pg_prepare\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval($req)\",\"line\":3,\"method\":\"risky\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [ + { + "identity": { + "unit": "scope-smoke", + "language": "php", + "path": "api.php", + "function": "safe", + "family": "injection" + }, + "reason": "region_budget", + "evidence": [ + "static_rank=2.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"mysqli_prepare\",\"mysqli_stmt_bind_param\",\"pg_execute\",\"pg_prepare\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[],\"source_local\":false,\"source_near\":false}", + "tier=0", + "score=0.0" + ] + } + ], + "unresolved_boundaries": [] + }, + "question_outcomes": [ + { + "identity": { + "unit": "scope-smoke", + "language": "php", + "path": "api.php", + "function": "risky", + "family": "injection" + }, + "status": "completed", + "reason": "query_answered_and_arbitrated", + "raw_rows_json": "[{\"bound\":\"\",\"bounded\":false,\"inLabeledScope\":true,\"length\":2,\"paths\":1,\"sanitized\":false,\"sink\":\"eval($req)\",\"sinkArg0Literal\":false,\"sinkArity\":1,\"sinkFile\":\"api.php\",\"sinkLine\":\"3\",\"sinkMethod\":\"risky\",\"source\":\"$req\",\"sourceKind\":\"parameter\"}]" + } + ], + "family_coverage": [ + { + "unit": "scope-smoke", + "language": "php", + "family": "injection", + "selected": 1, + "completed": 1, + "unanswered": 0, + "deferred": 1, + "unsupported": 0 + } + ] + } + }, + "javascript": { + "source_bytes": 69, + "source_sha256": "d3e956e2b4b4f29c51afb9c1ba45fe207dae93c11fd3b7d50171121a121c85c6", + "selected_question_ids": [ + "f29db1d7c33c22f09e10d8915be1772af4d2ff7856807e59b85678306d5b1269" + ], + "result": { + "findings": [ + { + "site": "api.js:2:risky", + "family": "injection", + "rung": "model_entailed", + "witness": "req -> eval(req) in api.js (line 2, 2 steps)", + "contradiction": "", + "reached_from": 1 + } + ], + "by_rung": { + "suspicion": 0, + "model_corroborated": 0, + "model_entailed": 1, + "execution_confirmed": 0 + }, + "regions_scanned": 1, + "regions_unjudged": 1, + "regions_unasked": 0, + "model_calls": 0, + "cost_usd": 0.0, + "seconds": 51.27, + "build_seconds": 6.47, + "query_seconds": 23.0, + "query_seconds_by_kind": { + "evidence": 21.76, + "taint": 23.0 + }, + "degradations": [ + { + "stage": "model", + "reason": "regions_truncated", + "examined": 1, + "total": 2 + } + ], + "tiers": [ + [ + "api.js", + "safe", + "injection", + 0 + ], + [ + "api.js", + "risky", + "injection", + 4 + ] + ], + "tier_counts": { + "0": 1, + "4": 1 + }, + "requests_pruned": 0, + "scope": { + "ranker_version": "existing-static-evidence-v1", + "ranking_mode": "evidence", + "population_complete": true, + "selected": [ + { + "identity": { + "unit": "scope-smoke", + "language": "javascript", + "path": "api.js", + "function": "risky", + "family": "injection" + }, + "priority": 0.0, + "evidence": [ + "static_rank=1.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"execute\",\"query\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[{\"arg0_literal\":false,\"arity\":1,\"cleansed_on_call\":false,\"code\":\"eval(req)\",\"line\":2,\"method\":\"risky\"}],\"source_local\":false,\"source_near\":false}", + "tier=4", + "score=2.5" + ], + "tier": 4, + "score": 2.5 + } + ], + "deferred": [ + { + "identity": { + "unit": "scope-smoke", + "language": "javascript", + "path": "api.js", + "function": "safe", + "family": "injection" + }, + "reason": "region_budget", + "evidence": [ + "static_rank=2.0", + "normalized_evidence={\"access_declared_public\":true,\"bound_names\":[\"execute\",\"query\"],\"carried\":false,\"carried_weak\":false,\"entry\":true,\"family_in_repo\":true,\"shipped\":true,\"sinks\":[],\"source_local\":false,\"source_near\":false}", + "tier=0", + "score=0.0" + ] + } + ], + "unresolved_boundaries": [] + }, + "question_outcomes": [ + { + "identity": { + "unit": "scope-smoke", + "language": "javascript", + "path": "api.js", + "function": "risky", + "family": "injection" + }, + "status": "completed", + "reason": "query_answered_and_arbitrated", + "raw_rows_json": "[{\"bound\":\"\",\"bounded\":false,\"inLabeledScope\":true,\"length\":2,\"paths\":1,\"sanitized\":false,\"sink\":\"eval(req)\",\"sinkArg0Literal\":false,\"sinkArity\":1,\"sinkFile\":\"api.js\",\"sinkLine\":\"2\",\"sinkMethod\":\"risky\",\"source\":\"req\",\"sourceKind\":\"parameter\"}]" + } + ], + "family_coverage": [ + { + "unit": "scope-smoke", + "language": "javascript", + "family": "injection", + "selected": 1, + "completed": 1, + "unanswered": 0, + "deferred": 1, + "unsupported": 0 + } + ] + } + } + } + }, + "limits": [ + "Manually declared controlled entry regions, not source-mapper validation.", + "Not PMPro/NodeGoat replay, detector admission, or hook latency.", + "Selected means scheduled; completed means answered and arbitrated, not safe or admitted." + ] +} diff --git a/ops/README.md b/ops/README.md index 4441972..11bd2d3 100644 --- a/ops/README.md +++ b/ops/README.md @@ -90,6 +90,13 @@ To exercise the same smoke through one shared backend deadline, add all census/witness queries consume that same budget. The generous smoke allowance verifies runtime integration; it does not represent the hook's target latency. +`ops/smoke_scope.py` exercises the generic driver with real PHP and JavaScript graphs, +explicit evidence ranking and two declared entry regions per language. Run it with the +same Docker command, substituting `smoke_scope.py` for `smoke_engine.py`. It verifies +that the lower static-ranked risky function is selected, that the harmless function is +recorded as deferred, and that selected IDs match completed query outcomes and per-family +coverage. It uses one 900-second lab budget; it is not a mapper or hook-quality evaluation. + Why this exists: Joern was first installed here by hand with `curl` and `unzip`, which is fine once and unreproducible thereafter. The engine decides whether a finding is a `suspicion` or a `model_entailed`, and every ceiling number in `benchmarks/measurements` is only comparable against a fixed version — so the version diff --git a/ops/smoke_scope.py b/ops/smoke_scope.py new file mode 100644 index 0000000..4dd92d6 --- /dev/null +++ b/ops/smoke_scope.py @@ -0,0 +1,72 @@ +"""Real engine scope smoke with declared entry regions; not a hook quality benchmark.""" + +from __future__ import annotations + +import hashlib +import json +import tempfile +import time +from dataclasses import asdict +from pathlib import Path + +from openultrasast.cpg.backend import JoernBackend +from openultrasast.model.contracts import ExecutionBudget +from openultrasast.model.regions import ScanRegion +from openultrasast.model.scan import ScanBudget, scan_repository + + +def main() -> None: + budget = ExecutionBudget(time.monotonic() + 900, 2.0) + report: dict[str, object] = { + "purpose": "Real PHP/JavaScript scope smoke; declared entry regions, no admission or latency claim", + "shared_budget_seconds": 900, + "languages": {}, + } + languages = report["languages"] + assert isinstance(languages, dict) + with tempfile.TemporaryDirectory(prefix="ousast-scope-smoke-") as temporary: + for language, filename, content in ( + ("php", "api.php", " str: diff --git a/src/openultrasast/model/contracts.py b/src/openultrasast/model/contracts.py index 5639c33..1989405 100644 --- a/src/openultrasast/model/contracts.py +++ b/src/openultrasast/model/contracts.py @@ -145,6 +145,8 @@ class RankedQuestion(Contract): identity: QuestionIdentity priority: float evidence: tuple[str, ...] + tier: int | None = None + score: float | None = None @dataclass(frozen=True) @@ -169,3 +171,49 @@ class ScopeDecision(Contract): identities = [item.identity for item in self.selected] + [item.identity for item in self.deferred] if len(set(identities)) != len(identities): raise ValueError("question identities must be unique across selected and deferred scope") + + +@dataclass(frozen=True) +class QuestionOutcome(Contract): + """Execution, not vulnerability disposition; raw answers survive interrupted arbitration.""" + + identity: QuestionIdentity + status: Literal["completed", "unanswered", "unresolved", "not_arbitrated"] + reason: str + raw_rows_json: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + if self.raw_rows_json is not None: + try: + rows = json.loads(self.raw_rows_json) + except (ValueError, TypeError) as error: + raise ValueError("raw_rows_json must contain a JSON row list") from error + if not isinstance(rows, list): + raise ValueError("raw_rows_json must contain a JSON row list") + if self.status in {"completed", "not_arbitrated"} and self.raw_rows_json is None: + raise ValueError("answered outcome requires raw rows") + if self.status == "unanswered" and self.raw_rows_json is not None: + raise ValueError("unanswered outcome cannot carry an answer") + + +@dataclass(frozen=True) +class FamilyCoverage(Contract): + """Counts over the supplied population; completed never means safe or admitted.""" + + unit: str + language: str + family: str + selected: int + completed: int + unanswered: int + deferred: int + unsupported: int + + def __post_init__(self) -> None: + super().__post_init__() + counts = (self.selected, self.completed, self.unanswered, self.deferred, self.unsupported) + if any(count < 0 for count in counts): + raise ValueError("coverage counts must be nonnegative") + if self.completed + self.unanswered != self.selected or self.unsupported > self.deferred: + raise ValueError("inconsistent family coverage counts") diff --git a/src/openultrasast/model/scan.py b/src/openultrasast/model/scan.py index 9de45e6..f2c99c9 100644 --- a/src/openultrasast/model/scan.py +++ b/src/openultrasast/model/scan.py @@ -19,17 +19,18 @@ the region's risk score said beforehand. from __future__ import annotations import inspect +import json import logging import time from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field, replace +from dataclasses import asdict, dataclass, field, replace from pathlib import Path from typing import Any from ..cpg.backend import CpgResult from .candidates import enumerate_candidates from .config_value import request_params as config_params -from .contracts import ExecutionBudget +from .contracts import DeferredQuestion, ExecutionBudget, FamilyCoverage, QuestionIdentity, QuestionOutcome, RankedQuestion, ScopeDecision from .dominance import request_params as dominance_params from .evidence import TIER_EXCLUDE, Evidence, evidence_from_rows from .ladder import Rung @@ -59,6 +60,7 @@ ENTRY_POINT_CALL_DEPTH = 3 logger = logging.getLogger(__name__) + _RUNG_ORDER = {Rung.ENTAILED: 0, Rung.CORROBORATED: 1, Rung.SUSPICION: 2, Rung.EXECUTION_CONFIRMED: -1} @@ -115,6 +117,10 @@ class ModelScanResult: # could have been found was skipped. Reported so the request count is legible. requests_pruned: int = 0 + scope: ScopeDecision | None = None + question_outcomes: tuple[QuestionOutcome, ...] = () + family_coverage: tuple[FamilyCoverage, ...] = () + @property def arbitrate_seconds(self) -> float: """What is left once the CPG is built and queried: the judge calls and the arbiters themselves.""" @@ -130,8 +136,21 @@ def scan_repository( model: str = "", budget: ScanBudget | None = None, execution_budget: ExecutionBudget | None = None, + ranking_mode: str | None = None, + unit: str = "repository", + population_complete: bool = False, ) -> ModelScanResult: """Run with an optional transaction deadline; graph ownership ends on every exit.""" + if ranking_mode not in (None, "static", "evidence"): + raise ValueError("ranking_mode must be static or evidence") + limits = budget if budget is not None else ScanBudget() + if ranking_mode is not None: + limits = replace(limits, order_by_evidence=ranking_mode == "evidence") + mode = "evidence" if limits.order_by_evidence else "static" + # Reject duplicate canonical questions before any engine work or resource ownership. + identities = [_identity(unit, r, f) for r in regions for f in r.families] + if len(set(identities)) != len(identities): + raise ValueError("duplicate question identity") started = time.monotonic() owned: list[Any] = [] try: @@ -141,7 +160,10 @@ def scan_repository( backend=backend, client=client if execution_budget is None else None, model=model, - budget=budget, + budget=limits, + ranking_mode=mode, + unit=unit, + population_complete=population_complete, execution_budget=execution_budget, owned=owned, ) @@ -172,6 +194,9 @@ def _scan_repository_impl( budget: ScanBudget | None = None, execution_budget: ExecutionBudget | None = None, owned: list[Any], + ranking_mode: str, + unit: str, + population_complete: bool, ) -> ModelScanResult: """Arbitrate a repository's regions under one CPG and one budget.""" limits = budget if budget is not None else ScanBudget() @@ -188,7 +213,10 @@ def _scan_repository_impl( cpg = _build(backend, root, dominant, execution_budget=execution_budget) build_seconds = round(time.monotonic() - build_started, 2) if cpg is None: + scope, _ = _scope_work(regions, {}, limits, unit, ranking_mode, population_complete, failure="cpg_build_failed") return ModelScanResult( + scope=scope, + family_coverage=_family_coverage(scope, ()), by_rung=_empty_tally(), regions_unjudged=len(regions), seconds=round(time.monotonic() - started, 2), @@ -252,7 +280,7 @@ def _scan_repository_impl( else ordered_regions[: limits.max_regions] ) tier_started = time.monotonic() - evidence_by_pair, failed = _evidence_pass(cpg, _collect(evidence_candidates), hooks) + evidence_by_pair, failed = _evidence_pass(cpg, _collect(evidence_candidates), hooks, degradations=degradations) per_kind["evidence"] = round(time.monotonic() - tier_started, 2) if failed: degradations.append({"stage": "model", "reason": "query_failed", "kind": "evidence", "requests": failed}) @@ -260,11 +288,15 @@ def _scan_repository_impl( if limits.order_by_evidence and evidence_by_pair: ordered_regions = order_by_evidence(ordered_regions, evidence_by_pair) - # Phase 1: collect the whole scan's questions, for at most `max_regions` regions. Nothing is asked yet. - work = _collect(ordered_regions[: limits.max_regions]) - tier_by_rid = { - rid: evidence_by_pair[_pair_key(region, spec)].tier for rid, region, spec in work if _pair_key(region, spec) in evidence_by_pair - } + # Authoritative selection is made once, from the actual final ranker order. + # Stable question IDs are the IDs sent to every execution request below. + scope_limits = ( + replace(limits, prune_tier0=False) + if any(d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"} for d in degradations) + else limits + ) + scope, work = _scope_work(ordered_regions, evidence_by_pair, scope_limits, unit, ranking_mode, population_complete) + outcomes: dict[str, QuestionOutcome] = {} # Phase 2: ONE invocation per query kind. JVM startup dominates a repository scan -- a call per region # per family put a ten-line file at four minutes and a thousand regions at roughly fifty hours -- while @@ -272,19 +304,10 @@ def _scan_repository_impl( rows_by_id: dict[str, list[object]] = {} census_reported = False query_started = time.monotonic() - pruned = 0 + pruned = sum(q.reason == "tier_zero" for q in scope.deferred) for kind, requests in _grouped(work, hook_callbacks=hooks).items(): if not _within_deadline(execution_budget): break - if kind == "taint" and limits.prune_tier0 and tier_by_rid: - # Tier 0 is exact: a family with no sink in reach cannot yield a flow. Measured on a 637-file - # plugin, that is 2,161 of 2,500 requests -- 86% of a query that timed out at forty minutes. - keep = {rid: params for rid, params in requests.items() if tier_by_rid.get(rid, TIER_EXCLUDE + 1) != TIER_EXCLUDE} - pruned = len(requests) - len(keep) - requests = keep - if not requests: - per_kind[kind] = 0.0 - continue kind_started = time.monotonic() batch = getattr(cpg, "run_batch", None) if callable(batch): @@ -314,7 +337,11 @@ def _scan_repository_impl( if shards > 1: degradations.append({"stage": "model", "reason": "cpg_sharded", "shards": shards}) census_reported = True - rows_by_id.update(answered_batch) + valid = {rid: rows for rid, rows in answered_batch.items() if rid in requests and isinstance(rows, list)} + rows_by_id.update(valid) + missing = len(requests) - len(valid) + if missing: + degradations.append({"stage": "model", "reason": "query_failed", "kind": kind, "requests": missing}) else: # a backend without batching still works, one call at a time for rid, params in requests.items(): if not _within_deadline(execution_budget): @@ -322,8 +349,6 @@ def _scan_repository_impl( answered = cpg.run(kind, params) if isinstance(answered, list): rows_by_id[rid] = list(answered) - elif execution_budget is None: - rows_by_id[rid] = [] else: degradations.append({"stage": "model", "reason": "query_failed", "kind": kind, "requests": 1}) per_kind[kind] = round(time.monotonic() - kind_started, 2) @@ -345,9 +370,8 @@ def _scan_repository_impl( for rid, region, spec in work: if not _within_deadline(execution_budget): break - # Unanswered requests are deferred on the bounded path, never adjudicated - # from an invented empty response. Exact identities are added in task 3.2. - if execution_budget is not None and rid not in rows_by_id: + # Unanswered requests are never adjudicated from invented empty responses. + if rid not in rows_by_id: continue exhausted = counted is not None and counted.calls >= limits.max_model_calls family = getattr(spec, "family", "") @@ -379,7 +403,13 @@ def _scan_repository_impl( except Exception as exc: # noqa: BLE001 -- one bad region must not end the scan logger.warning("model scan failed for %s: %s", region.path, exc) degradations.append({"stage": "model", "reason": "region_failed", "path": region.path}) + outcomes[rid] = QuestionOutcome( + _identity(unit, region, family), "unresolved", "arbitration_failed", _rows_json(rows_by_id[rid]) + ) continue + outcomes[rid] = QuestionOutcome( + _identity(unit, region, family), "completed", "query_answered_and_arbitrated", _rows_json(rows_by_id[rid]) + ) judged.add((region.path, region.function)) if exhausted: unasked.add((region.path, region.function)) @@ -388,6 +418,21 @@ def _scan_repository_impl( if unasked: degradations.append({"stage": "model", "reason": "budget_exhausted", "regions_unasked": len(unasked)}) + graph_gaps = tuple(str(d["reason"]) for d in degradations if d.get("reason") in {"files_unparsed", "cpg_empty", "cpg_sharded"}) + for question in scope.selected: + rid = question.identity.question_id + if rid not in outcomes: + answered = rid in rows_by_id + outcomes[rid] = QuestionOutcome( + question.identity, + "not_arbitrated" if answered else "unanswered", + "deadline_exhausted" if not _within_deadline(execution_budget) else "query_unanswered", + _rows_json(rows_by_id[rid]) if answered else None, + ) + if graph_gaps and outcomes[rid].status == "completed": + outcomes[rid] = replace(outcomes[rid], status="unresolved", reason="graph_incomplete") + scope = replace(scope, unresolved_boundaries=tuple(dict.fromkeys((*scope.unresolved_boundaries, *graph_gaps)))) + ordered_outcomes = tuple(outcomes[q.identity.question_id] for q in scope.selected) scanned = len(judged) findings = _ordered(_deduplicated(collected)) @@ -409,6 +454,100 @@ def _scan_repository_impl( tiers=tuple(tiers), tier_counts={tier: sum(1 for t in tiers if t[3] == tier) for tier in sorted({t[3] for t in tiers})}, requests_pruned=pruned, + scope=scope, + question_outcomes=ordered_outcomes, + family_coverage=_family_coverage(scope, ordered_outcomes), + ) + + +def _identity(unit: str, region: ScanRegion, family: str) -> QuestionIdentity: + return QuestionIdentity(unit, region.language, region.path, region.function, family) + + +def _rows_json(rows: object) -> str: + return json.dumps(rows, sort_keys=True, separators=(",", ":")) + + +def _scope_work( + regions: Sequence[ScanRegion], + evidence: Mapping[tuple[str, str, str], Evidence], + limits: ScanBudget, + unit: str, + mode: str, + population_complete: bool, + *, + failure: str | None = None, +) -> tuple[ScopeDecision, list[tuple[str, ScanRegion, ArbiterSpec]]]: + """Consume the ranker's final order once; the returned work IS selected scope.""" + selected: list[RankedQuestion] = [] + deferred: list[DeferredQuestion] = [] + work: list[tuple[str, ScanRegion, ArbiterSpec]] = [] + boundaries: list[str] = [] if population_complete else ["population_not_asserted_complete"] + if failure: + boundaries.append(failure) + for index, region in enumerate(regions): + for family in region.families: + identity = _identity(unit, region, family) + spec = _spec_for(family, region.language) + vector = evidence.get((region.path, region.function or "", family)) + facts: tuple[str, ...] = ("static_rank=" + str(region.rank),) + if vector is not None: + facts += ("normalized_evidence=" + _rows_json(asdict(vector)), "tier=" + str(vector.tier), "score=" + str(vector.score)) + elif isinstance(spec, TaintSpec): + facts += ("evidence_unknown",) + boundaries.append("evidence_unknown:" + identity.question_id) + else: + facts += ("taint_evidence_not_applicable",) + reason = ( + "unsupported_family" + if spec is None + else failure + or ( + "region_budget" + if index >= limits.max_regions + else "tier_zero" + if limits.prune_tier0 and isinstance(spec, TaintSpec) and vector is not None and vector.tier == TIER_EXCLUDE + else None + ) + ) + if reason: + deferred.append(DeferredQuestion(identity, reason, facts)) + if reason == "unsupported_family": + boundaries.append("unsupported_family:" + identity.question_id) + else: + assert spec is not None + selected.append( + RankedQuestion( + identity, + float(-len(selected)), + facts, + vector.tier if vector is not None else None, + vector.score if vector is not None else None, + ) + ) + work.append((identity.question_id, region, spec)) + return ScopeDecision( + "existing-static-evidence-v1", mode, population_complete, tuple(selected), tuple(deferred), tuple(boundaries) + ), work + + +def _family_coverage(scope: ScopeDecision, outcomes: tuple[QuestionOutcome, ...]) -> tuple[FamilyCoverage, ...]: + population: list[RankedQuestion | DeferredQuestion] = [*scope.selected, *scope.deferred] + keys = sorted({(q.identity.unit, q.identity.language, q.identity.family) for q in population}) + + def matches(identity: QuestionIdentity, key: tuple[str, str, str]) -> bool: + return (identity.unit, identity.language, identity.family) == key + + return tuple( + FamilyCoverage( + *key, + selected=sum(matches(q.identity, key) for q in scope.selected), + completed=sum(matches(q.identity, key) and q.status == "completed" for q in outcomes), + unanswered=sum(matches(q.identity, key) and q.status != "completed" for q in outcomes), + deferred=sum(matches(q.identity, key) for q in scope.deferred), + unsupported=sum(matches(q.identity, key) and q.reason == "unsupported_family" for q in scope.deferred), + ) + for key in keys ) @@ -429,7 +568,7 @@ def _collect(regions: Sequence[ScanRegion]) -> list[tuple[str, ScanRegion, Arbit def _evidence_pass( - cpg: Any, work: Sequence[tuple[str, ScanRegion, ArbiterSpec]], hooks: str + cpg: Any, work: Sequence[tuple[str, ScanRegion, ArbiterSpec]], hooks: str, *, degradations: list[Mapping[str, object]] | None = None ) -> tuple[dict[tuple[str, str, str], Evidence], int]: """The evidence vector of every taint pair in ``work``, keyed by (path, function, family). @@ -443,12 +582,23 @@ def _evidence_pass( answered = cpg.run_batch("taint", requests) if answered is None: return {}, len(requests) - answered.pop("__census__", None) + census = answered.pop("__census__", None) + if census and degradations is not None: + entry = census[0] if isinstance(census[0], Mapping) else {} + try: + methods = int(str(entry.get("methods", "0"))) + shards = int(str(entry.get("shards", "1"))) + except ValueError: + methods, shards = 0, 1 + if methods <= 0: + degradations.append({"stage": "model", "reason": "cpg_empty"}) + if shards > 1: + degradations.append({"stage": "model", "reason": "cpg_sharded", "shards": shards}) by_rid = {rid: (region, spec) for rid, region, spec in work} out: dict[tuple[str, str, str], Evidence] = {} for rid, rows in answered.items(): pair = by_rid.get(rid) - if pair is None: + if pair is None or rid not in requests or not isinstance(rows, list): continue region, spec = pair evidence = evidence_from_rows( @@ -463,7 +613,16 @@ def _evidence_pass( ) if evidence is not None: out[_pair_key(region, spec)] = evidence - return out, 0 + # Legacy ranker keys omit language. An overlapping frontend population cannot + # borrow another language's vector; partitioned callers retain distinct IDs. + languages_by_pair: dict[tuple[str, str, str], set[str]] = {} + for rid, region, spec in work: + if rid in requests: + languages_by_pair.setdefault(_pair_key(region, spec), set()).add(region.language) + for key, languages in languages_by_pair.items(): + if len(languages) > 1: + out.pop(key, None) + return out, len(requests) - len(out) def order_by_evidence(regions: Sequence[ScanRegion], evidence: Mapping[tuple[str, str, str], Evidence]) -> list[ScanRegion]: diff --git a/tests/test_model_scope.py b/tests/test_model_scope.py new file mode 100644 index 0000000..6c89234 --- /dev/null +++ b/tests/test_model_scope.py @@ -0,0 +1,273 @@ +"""Exact ranker execution scope, uncertainty and cross-language transfer (task 3.2).""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openultrasast.model.regions import ScanRegion +from openultrasast.model.scan import ScanBudget, scan_repository + + +def region(path, language="python", families=("injection",), rank=0.5): + return ScanRegion(path, "run", language, families, rank, "entry_point") + + +class Backend: + def __init__(self, evidence=None, missing=(), fail=False): + self.evidence = evidence or {} + self.missing = missing + self.fail = fail + self.executed = [] + + def build(self, root, **kwargs): + # Prove these tests provide real readable source to the driver boundary. + assert any(p.read_bytes() for p in root.iterdir() if p.is_file()) + if self.fail: + return None + return SimpleNamespace(cpg_path=Path("test.cpg"), run_batch=self.batch, cleanup=lambda: None) + + def batch(self, kind, requests): + if any(p.get("evidenceOnly") == "true" for p in requests.values()): + return {rid: self.evidence.get(p["file"], []) for rid, p in requests.items()} + self.executed.extend((rid, kind, p["file"]) for rid, p in requests.items()) + return {rid: [] for rid, p in requests.items() if kind not in self.missing} + + +def source(tmp_path, *paths): + for path in paths: + (tmp_path / path).write_text("readable source\n") + return tmp_path + + +def vector(open_sink=True): + rows = [{"kind": "summary", "familyInRepo": True, "sourceLocal": True}] + if open_sink: + rows += [{"kind": "sink", "sink": "operation(value)", "sinkLine": 1, "sinkMethod": "run", "sinkArity": 1}] + return rows + + +def test_final_order_is_the_scope_and_execution_order(tmp_path): + root = source(tmp_path, "high.py", "low.py") + backend = Backend({"high.py": vector(False), "low.py": vector()}) + result = scan_repository( + root, + [region("high.py", rank=1.0), region("low.py")], + backend=backend, + ranking_mode="evidence", + unit="head-unit", + population_complete=True, + budget=ScanBudget(max_regions=1), + ) + assert [q.identity.path for q in result.scope.selected] == ["low.py"] + assert [q.identity.path for q in result.scope.deferred] == ["high.py"] + assert backend.executed == [(result.scope.selected[0].identity.question_id, "taint", "low.py")] + assert result.scope.ranking_mode == "evidence" + assert result.scope.population_complete + assert result.question_outcomes[0].status == "completed" + + +def test_unknown_evidence_and_family_completion_remain_separate(tmp_path): + root = source(tmp_path, "api.py") + result = scan_repository( + root, + [region("api.py", families=("injection", "access_control", "config_secrets", "unmodeled"))], + backend=Backend(missing=("dominance", "config")), + ranking_mode="evidence", + ) + selected = {q.identity.family: q for q in result.scope.selected} + assert selected["injection"].tier is None + assert selected["access_control"].tier is None + assert selected["config_secrets"].tier is None + coverage = {c.family: c for c in result.family_coverage} + assert coverage["injection"].completed == 1 + assert coverage["access_control"].completed == coverage["config_secrets"].completed == 0 + assert coverage["unmodeled"].unsupported == 1 + assert any(q.reason == "unsupported_family" for q in result.scope.deferred) + assert any("evidence" in boundary for boundary in result.scope.unresolved_boundaries) + + +@pytest.mark.parametrize("mode", ["static", "evidence"]) +def test_php_javascript_equivalent_vectors_produce_same_decisions(tmp_path, mode): + results = [] + for language, suffix in [("php", "php"), ("javascript", "js")]: + root = tmp_path / language + root.mkdir() + source(root, f"a.{suffix}", f"b.{suffix}") + backend = Backend({f"a.{suffix}": vector(False), f"b.{suffix}": vector()}) + result = scan_repository( + root, + [region(f"a.{suffix}", language), region(f"b.{suffix}", language)], + backend=backend, + ranking_mode=mode, + budget=ScanBudget(max_regions=1), + ) + results.append(result) + assert [(q.identity.function, q.tier, q.score) for q in results[0].scope.selected] == [ + (q.identity.function, q.tier, q.score) for q in results[1].scope.selected + ] + assert [(q.reason, q.identity.path.split(".")[0]) for q in results[0].scope.deferred] == [ + (q.reason, q.identity.path.split(".")[0]) for q in results[1].scope.deferred + ] + + +def test_failed_build_retains_exact_unanswered_population(tmp_path): + result = scan_repository(source(tmp_path, "a.py"), [region("a.py")], backend=Backend(fail=True), ranking_mode="evidence") + assert result.scope.selected == () + assert result.scope.deferred[0].reason == "cpg_build_failed" + assert result.family_coverage[0].completed == 0 + assert not result.scope.population_complete + + +def test_explicit_static_mode_overrides_legacy_evidence_toggle(tmp_path): + root = source(tmp_path, "a.py", "b.py") + result = scan_repository( + root, + [region("a.py", rank=1.0), region("b.py")], + backend=Backend(), + ranking_mode="static", + budget=ScanBudget(max_regions=1, order_by_evidence=True), + ) + assert result.scope.selected[0].identity.path == "a.py" + assert result.scope.ranking_mode == "static" + + +def test_scope_payload_is_json_serializable(tmp_path): + result = scan_repository(source(tmp_path, "a.py"), [region("a.py")], backend=Backend(), ranking_mode="evidence") + from openultrasast.cli import _model_payload + + payload = _model_payload(result) + assert json.loads(json.dumps(payload))["scope"]["selected"][0]["identity"]["path"] == "a.py" + assert payload["family_coverage"][0]["completed"] == 1 + + +def test_partial_evidence_batch_is_unknown_not_tier_zero(tmp_path): + backend = Backend({"a.py": vector(False)}) + result = scan_repository(source(tmp_path, "a.py", "b.py"), [region("a.py"), region("b.py")], backend=backend, ranking_mode="evidence") + assert [(q.identity.path, q.reason) for q in result.scope.deferred] == [("a.py", "tier_zero")] + assert result.scope.selected[0].identity.path == "b.py" + assert result.scope.selected[0].tier is None + assert any(d.get("kind") == "evidence" and d.get("requests") == 1 for d in result.degradations) + assert result.family_coverage[0].completed == 1 + assert result.family_coverage[0].deferred == 1 + + +def test_later_timeout_retains_raw_answer_without_claiming_arbitration(tmp_path): + import time + + from openultrasast.model.contracts import ExecutionBudget + + class Slow(Backend): + def batch(self, kind, requests): + rows = super().batch(kind, requests) + if kind == "taint" and not any(p.get("evidenceOnly") for p in requests.values()): + time.sleep(0.08) + return {rid: [{"kind": "retained", "value": 42}] for rid in requests} + return rows + + result = scan_repository( + source(tmp_path, "a.py"), + [region("a.py", families=("injection", "access_control"))], + backend=Slow(), + ranking_mode="static", + execution_budget=ExecutionBudget(time.monotonic() + 0.05, 0.2), + ) + outcomes = {q.identity.family: q for q in result.question_outcomes} + assert outcomes["injection"].status == "not_arbitrated" + assert json.loads(outcomes["injection"].raw_rows_json) == [{"kind": "retained", "value": 42}] + assert outcomes["access_control"].status == "unanswered" + assert all(c.completed == 0 for c in result.family_coverage) + + +def test_missing_graph_files_disable_safe_pruning_and_completion(tmp_path): + class Partial(Backend): + def build(self, root, **kwargs): + cpg = super().build(root, **kwargs) + cpg.unparsed = ("missing.py",) + return cpg + + result = scan_repository(source(tmp_path, "a.py"), [region("a.py")], backend=Partial({"a.py": vector(False)}), ranking_mode="evidence") + assert len(result.scope.selected) == 1 + assert result.scope.deferred == () + assert result.question_outcomes[0].status == "unresolved" + assert result.family_coverage[0].completed == 0 + + +def test_cli_sample_uses_deferred_order_not_scanned_count(tmp_path): + from openultrasast.cli import _model_payload + + backend = Backend({"high.py": vector(False), "low.py": vector()}) + result = scan_repository( + source(tmp_path, "high.py", "low.py"), + [region("high.py", rank=1), region("low.py")], + backend=backend, + ranking_mode="evidence", + budget=ScanBudget(max_regions=1), + ) + assert _model_payload(result)["unjudged_sample"] == ["high.py"] + + +def test_invalid_mode_and_duplicate_identity_fail_before_build(tmp_path): + backend = Backend() + root = source(tmp_path, "a.py") + with pytest.raises(ValueError, match="ranking_mode"): + scan_repository(root, [region("a.py")], backend=backend, ranking_mode="invented") + with pytest.raises(ValueError, match="duplicate question"): + scan_repository(root, [region("a.py"), region("a.py")], backend=backend) + assert backend.executed == [] + + +def test_empty_evidence_graph_census_cannot_prune_every_check(tmp_path): + class Empty(Backend): + def batch(self, kind, requests): + result = super().batch(kind, requests) + result["__census__"] = [{"methods": "0", "files": "0"}] + return result + + result = scan_repository(source(tmp_path, "a.py"), [region("a.py")], backend=Empty({"a.py": vector(False)}), ranking_mode="evidence") + assert len(result.scope.selected) == 1 + assert result.family_coverage[0].completed == 0 + assert "cpg_empty" in result.scope.unresolved_boundaries + + +def test_overlapping_frontend_paths_keep_ids_distinct_and_evidence_unknown(tmp_path): + result = scan_repository( + source(tmp_path, "a.txt"), + [region("a.txt", "php"), region("a.txt", "javascript")], + backend=Backend({"a.txt": vector()}), + ranking_mode="evidence", + ) + assert len({q.identity.question_id for q in result.scope.selected}) == 2 + assert all(q.tier is None for q in result.scope.selected) + assert len(result.family_coverage) == 2 + + +def test_outcome_and_coverage_contracts_reject_forged_completion(): + from openultrasast.model.contracts import FamilyCoverage, QuestionIdentity, QuestionOutcome + + identity = QuestionIdentity("unit", "php", "api.php", "run", "injection") + for raw in ("{}", "not json"): + with pytest.raises(ValueError, match="JSON row list"): + QuestionOutcome(identity, "completed", "answered", raw) + with pytest.raises(ValueError, match="requires raw rows"): + QuestionOutcome(identity, "completed", "answered") + with pytest.raises(ValueError, match="cannot carry"): + QuestionOutcome(identity, "unanswered", "failed", "[]") + with pytest.raises(ValueError, match="nonnegative"): + FamilyCoverage("unit", "php", "injection", -1, 0, -1, 0, 0) + with pytest.raises(ValueError, match="inconsistent"): + FamilyCoverage("unit", "php", "injection", 1, 1, 1, 0, 0) + outcome = QuestionOutcome(identity, "completed", "answered", "[]") + assert QuestionOutcome.from_payload(outcome.to_payload()) == outcome + + +def test_unbatched_failure_is_not_a_completed_empty_answer(tmp_path): + class Single(Backend): + def build(self, root, **kwargs): + return SimpleNamespace(cpg_path=Path("cpg"), run=lambda *a: None, cleanup=lambda: None) + + result = scan_repository(source(tmp_path, "a.py"), [region("a.py")], backend=Single(), ranking_mode="static") + assert result.question_outcomes[0].status == "unanswered" + assert result.regions_scanned == 0 + assert result.family_coverage[0].completed == 0