From 0ab5d56618cd10b00772f0706cc451540bfadc97 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 13 Sep 2026 16:16:19 +0200 Subject: [PATCH] feat(pre-push-safety-net): key evidence reuse by compatible semantics --- .../implementation-group-6.md | 34 ++++++ .kiro/specs/pre-push-safety-net/tasks.md | 2 +- src/openultrasast/push/cache.py | 55 +++++++++- src/openultrasast/push/policy.py | 100 +++++++++++++++++- tests/test_push_cache_semantics.py | 62 +++++++++++ tests/test_push_policy.py | 27 +++++ 6 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 tests/test_push_cache_semantics.py diff --git a/.kiro/specs/pre-push-safety-net/implementation-group-6.md b/.kiro/specs/pre-push-safety-net/implementation-group-6.md index ab10ca0..c8c0a3a 100644 --- a/.kiro/specs/pre-push-safety-net/implementation-group-6.md +++ b/.kiro/specs/pre-push-safety-net/implementation-group-6.md @@ -69,3 +69,37 @@ No graph node mutation, executable deserialization or repository writes are intr - CLAIM: Task 6.2 bounded local artifact publication and eviction. - EVIDENCE: nine cache contract tests and full suite above; production cache implementation, not a mocked storage path. - GAPS: Semantic keys and live replay integration follow in 6.3 and 6.4; no warm-latency claim. + +## Task 6.3 brief + +Requirements 3.4, 4.3, 4.4, 6.4, 7.2 and 7.3; design: Cache and Engine Boundary +and Admission Policy. Query keys include graph, actual question parameters/context/depth, +facts/sanitizers, query versions and configuration. Comparison/final keys additionally bind +ranking, admission, optional model/prompt, eligibility and mode; comparisons include both +revisions and actual completed evidence. Ranking-only changes preserve query keys. + +Comparison reuse is integrated into the existing policy entry point. Only complete, comparable +evidence can populate/look up comparisons. Unknown outcomes never qualify. Loaded CandidateDelta +contracts must match revision and semantics ownership. Current admission still runs afterwards; +the cache never confers eligibility. JSON decoding is non-executable and bounded to 32 MiB. + +## Status Report +- STATUS: READY_FOR_REVIEW +- TASK: 6.3 +- RED_PHASE_OUTPUT: `/tmp/group6-3-red.log`: two failing semantic-key tests with switch OFF; `/tmp/group6-3-policy-red.log`: missing policy cache integration failed. Enabled implementation passed; switch removed. +- TESTS_RUN: 71 focused policy/admission/cache tests passed; full suite 1260 passed, 9 skipped in 103.17s. Final ruff, formatting and mypy passed. +- EVIDENCE: real on-disk comparison hit returns identical supported delta; policy/revision changes miss; unanswered comparisons do not populate complete entries; exhaustive key-dimension and final-scope invalidation tests. + +## Review Verdict +- VERDICT: APPROVED +- TASK: 6.3 +- MECHANICAL_RESULTS: full regression suite, 71 focused tests, static checks and whitespace checks passed; no new placeholders or secrets. +- FINDINGS: cache does not store or replay push permission. Comparison data remains separate from current admission. Query reuse deliberately ignores ranking-only/mode changes, whereas comparison/final keys include them. Decoder and deadline checks were tightened during review. +- SUMMARY: Semantic identities preserve compatible evidence without transferring obsolete policy eligibility. + +## Verification Result +- STATUS: VERIFIED +- CLAIM_TYPE: TASK +- CLAIM: Task 6.3 semantic keys and completed comparison evidence reuse. +- EVIDENCE: production policy integration and cache key tests, 1260 passing suite tests and final static checks. +- GAPS: Graph/query/preparation reuse must still be connected to replay under 6.4; no rollout claim. diff --git a/.kiro/specs/pre-push-safety-net/tasks.md b/.kiro/specs/pre-push-safety-net/tasks.md index 842cb99..a624617 100644 --- a/.kiro/specs/pre-push-safety-net/tasks.md +++ b/.kiro/specs/pre-push-safety-net/tasks.md @@ -174,7 +174,7 @@ hook latency is admitted. - _Boundary: Cache and Engine Boundary, push cache integration_ - _Depends: 6.1_ - _Requirements: 4.1, 4.3, 4.4, 7.3_ -- [ ] 6.3 Key query and comparison results by all relevant semantics +- [x] 6.3 Key query and comparison results by all relevant semantics - Include graph identity, question/context/depth, facts/sanitizers/query versions and configuration; include ranking/admission/model provenance for final results and both revision identities for comparisons. - Keep incomplete query/comparison results out of complete lookups; preserve current eligibility checks after reuse. - Done when source, context, fact, query, mode, policy and optional model changes invalidate the appropriate layer, while an identical compatible request returns the same evidence and provenance. diff --git a/src/openultrasast/push/cache.py b/src/openultrasast/push/cache.py index cf3296f..ef8c306 100644 --- a/src/openultrasast/push/cache.py +++ b/src/openultrasast/push/cache.py @@ -12,10 +12,11 @@ import tempfile import time from collections.abc import Iterator from contextlib import ExitStack, contextmanager -from dataclasses import dataclass +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any +from openultrasast.contracts import Contract from openultrasast.cpg.artifact import digest_value, graph_bytes, read_bytes from openultrasast.cpg.backend import _DeadlineExpired, _remove_owned_tree from openultrasast.model.contracts import ExecutionBudget @@ -123,6 +124,24 @@ class ArtifactCache: self.hits += 1 yield entry + def get_json(self, key: str, *, kind: str, budget: ExecutionBudget) -> Any: + with self.lookup(key, budget) as entry: + if entry is None or entry.metadata != {"kind": kind} or entry.path.stat().st_size > 32 * 1024**2: + return None + try: + return json.loads(read_bytes(entry.path, budget.deadline_monotonic)) + except (OSError, ValueError, TimeoutError): + return None + + def put_json(self, key: str, payload: object, *, kind: str, budget: ExecutionBudget, complete: bool) -> bool: + if not complete or time.monotonic() >= budget.deadline_monotonic: + return False + try: + data = json.dumps(payload, sort_keys=True, allow_nan=False).encode() + except (TypeError, ValueError): + return False + return self.publish(key, data, {"kind": kind}, budget, complete=complete) + def _remove(self, root: Path, budget: ExecutionBudget) -> None: _remove_owned_tree(root, budget.deadline_monotonic) @@ -215,3 +234,37 @@ class ArtifactCache: _remove_owned_tree(pending, budget.deadline_monotonic + budget.cancellation_allowance_seconds) except (OSError, _DeadlineExpired): self.last_reason = "cache_partial_cleanup_incomplete" + + +@dataclass(frozen=True) +class SemanticKeys(Contract): + facts: str + queries: str + configuration: str + ranking: str + admission: str + model: str + eligibility: str + mode: str + + def query(self, *, graph: str, kind: str, request: object, context: object) -> str: + return digest_value( + { + "layer": "query-v1", + "graph": graph, + "kind": kind, + "request": request, + "context": context, + "facts": self.facts, + "queries": self.queries, + "configuration": self.configuration, + } + ) + + def comparison(self, *, base: str, head: str, context: object, evidence: object) -> str: + return digest_value( + {"layer": "comparison-v1", "base": base, "head": head, "context": context, "evidence": evidence, "semantics": asdict(self)} + ) + + def result(self, *, comparisons: object, scope: object) -> str: + return digest_value({"layer": "result-v1", "comparisons": comparisons, "scope": scope, "semantics": asdict(self)}) diff --git a/src/openultrasast/push/policy.py b/src/openultrasast/push/policy.py index a0b7c89..af79a96 100644 --- a/src/openultrasast/push/policy.py +++ b/src/openultrasast/push/policy.py @@ -6,7 +6,7 @@ import hashlib import json import time from collections.abc import Sequence -from dataclasses import dataclass, replace +from dataclasses import asdict, dataclass, replace from pathlib import Path, PurePosixPath from string import Formatter from typing import Any, Literal @@ -20,6 +20,7 @@ from openultrasast.model.pipeline import ModelFinding, _arbitrate_all from openultrasast.model.regions import ScanRegion from openultrasast.model.scan import ModelScanResult, ScanBudget, _prefetched, _spec_for, scan_repository from openultrasast.model.specs import ConfigSpec, DominanceSpec, TaintSpec +from openultrasast.push.cache import ArtifactCache, SemanticKeys from openultrasast.push.contracts import ComparisonAnalysis, PushComparison, PushResult @@ -247,6 +248,92 @@ def compare_evidence( head_semantics: str, base_semantics: str, execution_budget: ExecutionBudget | None = None, + cache: ArtifactCache | None = None, + cache_semantics: SemanticKeys | None = None, +) -> tuple[CandidateDelta, ...]: + """Reuse completed comparison evidence; alert eligibility is always applied later.""" + + def complete(scan: ModelScanResult | None) -> bool: + return bool( + scan + and scan.scope + and scan.scope.population_complete + and not scan.degradations + and not scan.scope.unresolved_boundaries + and all(item.reason == "tier_zero" for item in scan.scope.deferred) + and all(item.status == "completed" for item in scan.question_outcomes) + and {item.identity for item in scan.scope.selected} == {item.identity for item in scan.question_outcomes} + ) + + key = None + if ( + cache is not None + and cache_semantics is not None + and execution_budget is not None + and time.monotonic() < execution_budget.deadline_monotonic + and context.base_revision is not None + and not context.unresolved_boundaries + and head_semantics == base_semantics + and complete(head) + and complete(base) + ): + from openultrasast.cpg.artifact import read_bytes + + def evidence(scan: ModelScanResult) -> dict[str, object]: + return { + name: value + for name, value in asdict(scan).items() + if name not in {"seconds", "build_seconds", "query_seconds", "query_seconds_by_kind"} + } + + assert base is not None + try: + key = cache_semantics.comparison( + base=context.base_revision, + head=context.head_revision, + context=context.to_payload(), + evidence={ + "head": evidence(head), + "base": evidence(base), + "semantics": head_semantics, + "policy": hashlib.sha256(read_bytes(Path(__file__), execution_budget.deadline_monotonic)).hexdigest(), + }, + ) + raw = cache.get_json(key, kind="comparison", budget=execution_budget) + if isinstance(raw, list): + decoded = tuple(CandidateDelta.from_payload(item) for item in raw) + if time.monotonic() < execution_budget.deadline_monotonic and all( + item.novelty != "unknown" + and item.base_revision == context.base_revision + and item.head_revision == context.head_revision + and item.semantics == head_semantics + for item in decoded + ): + return decoded + except (OSError, ValueError, TimeoutError): + key = None + result = _compare_evidence( + head, base, context=context, head_semantics=head_semantics, base_semantics=base_semantics, execution_budget=execution_budget + ) + if key is not None and cache is not None and execution_budget is not None: + cache.put_json( + key, + [item.to_payload() for item in result], + kind="comparison", + budget=execution_budget, + complete=all(item.novelty != "unknown" for item in result), + ) + return result + + +def _compare_evidence( + head: ModelScanResult, + base: ModelScanResult | None, + *, + context: ChangeContext, + head_semantics: str, + base_semantics: str, + execution_budget: ExecutionBudget | None = None, ) -> tuple[CandidateDelta, ...]: """Absence counts only in completed comparable scope, never from scan silence.""" head_ops, base_ops = _operations(head, execution_budget), _operations(base, execution_budget) if base else () @@ -356,6 +443,8 @@ def compare_targeted_base( scan_budget: ScanBudget, head_semantics: str, base_semantics: str, + cache: ArtifactCache | None = None, + cache_semantics: SemanticKeys | None = None, ) -> DeltaComparison: """Re-run counterparts of the ranker's selected head questions through the existing driver. @@ -398,7 +487,14 @@ def compare_targeted_base( except Exception as error: # noqa: BLE001 -- comparison failure is coverage, not a new defect reasons.append(f"base_scan_failed:{type(error).__name__}") candidates = compare_evidence( - head, base, context=context, head_semantics=head_semantics, base_semantics=base_semantics, execution_budget=execution_budget + head, + base, + context=context, + head_semantics=head_semantics, + base_semantics=base_semantics, + execution_budget=execution_budget, + cache=cache, + cache_semantics=cache_semantics, ) reasons.extend(c.reason for c in candidates if c.novelty == "unknown") return DeltaComparison(candidates, base, head_semantics, tuple(dict.fromkeys(reasons))) diff --git a/tests/test_push_cache_semantics.py b/tests/test_push_cache_semantics.py new file mode 100644 index 0000000..9af67cf --- /dev/null +++ b/tests/test_push_cache_semantics.py @@ -0,0 +1,62 @@ +"""Semantic layers invalidate independently and comparisons retain revision ownership.""" + +import time +from dataclasses import replace + +from openultrasast.model.contracts import ExecutionBudget +from openultrasast.push.cache import ArtifactCache, SemanticKeys + + +def keys(): + return SemanticKeys("facts", "queries", "configuration", "ranking", "admission", "model-disabled", "eligibility", "advisory") + + +def test_query_and_result_invalidation_layers(): + original = keys() + + def query(k): + return k.query(graph="graph", kind="taint", request={"file": "a.js", "depth": 3}, context="context") + + def result(k): + return k.comparison(base="base", head="head", context="context", evidence="evidence") + + for field in ("facts", "queries", "configuration"): + changed = replace(original, **{field: "changed"}) + assert query(changed) != query(original) + assert result(changed) != result(original) + for field in ("ranking", "admission", "model", "eligibility", "mode"): + changed = replace(original, **{field: "changed"}) + assert query(changed) == query(original) + assert result(changed) != result(original) + for field in ("graph", "kind", "request", "context"): + args = {"graph": "graph", "kind": "taint", "request": {"file": "a.js", "depth": 3}, "context": "context"} + args[field] = "changed" + assert original.query(**args) != query(original) + for field in ("base", "head", "context", "evidence"): + args = {"base": "base", "head": "head", "context": "context", "evidence": "evidence"} + args[field] = "changed" + assert original.comparison(**args) != result(original) + + +def test_json_evidence_retains_provenance_and_rejects_incomplete(tmp_path): + cache = ArtifactCache(tmp_path / "cache", max_bytes=10000) + budget = ExecutionBudget(time.monotonic() + 5, 0.2) + key = keys().query(graph="graph", kind="taint", request={}, context="context") + payload = {"rows": [], "provenance": {"facts": "facts", "query": "queries"}} + assert not cache.put_json(key, payload, kind="query", budget=budget, complete=False) + assert cache.get_json(key, kind="query", budget=budget) is None + assert cache.put_json(key, payload, kind="query", budget=budget, complete=True) + assert cache.get_json(key, kind="query", budget=budget) == payload + assert cache.get_json(key, kind="comparison", budget=budget) is None + + +def test_final_results_bind_scope_and_current_admission_provenance(): + original = keys() + + def result(k, scope="scope"): + return k.result(comparisons=["base-head"], scope=scope) + + for field in ("facts", "queries", "configuration", "ranking", "admission", "model", "eligibility", "mode"): + assert result(replace(original, **{field: "changed"})) != result(original) + assert result(original, "different-scope") != result(original) + assert original.result(comparisons=["another-base-head"], scope="scope") != result(original) diff --git a/tests/test_push_policy.py b/tests/test_push_policy.py index e6dcd4f..336ef19 100644 --- a/tests/test_push_policy.py +++ b/tests/test_push_policy.py @@ -238,3 +238,30 @@ def test_proven_tier_zero_other_families_do_not_invalidate_base_answer(): assert delta(scan(), base).novelty == "new" base = replace(base, scope=replace(base.scope, deferred=(DeferredQuestion(other, "budget_exhausted", ()),))) assert delta(scan(), base).novelty == "unknown" + + +def test_comparison_cache_reuses_only_completed_compatible_evidence(tmp_path, monkeypatch): + import openultrasast.push.policy as policy + from openultrasast.push.cache import ArtifactCache, SemanticKeys + + cache = ArtifactCache(tmp_path / "cache", max_bytes=100000) + keys = SemanticKeys("facts", "queries", "config", "rank", "admission", "disabled", "empty", "advisory") + budget = ExecutionBudget(time.monotonic() + 10, 0.2) + args = dict(context=context(), head_semantics="same", base_semantics="same", execution_budget=budget, cache=cache, cache_semantics=keys) + expected = policy.compare_evidence(scan(), scan(line=5, sanitized=True, findings=False), **args) + assert expected[0].novelty == "worsened" + hits = cache.hits + assert policy.compare_evidence(scan(), scan(line=5, sanitized=True, findings=False), **args) == expected + assert cache.hits > hits + args["cache_semantics"] = replace(keys, admission="changed") + hits = cache.hits + policy.compare_evidence(scan(), scan(line=5, sanitized=True, findings=False), **args) + assert cache.hits == hits + args["context"] = replace(context(), base_revision="another-base") + changed = policy.compare_evidence(scan(), scan(line=5, sanitized=True, findings=False), **args) + assert changed[0].base_revision == "another-base" + incomplete = policy.compare_evidence(scan(), scan(status="unanswered"), **args) + assert incomplete[0].novelty == "unknown" + hits = cache.hits + policy.compare_evidence(scan(), scan(status="unanswered"), **args) + assert cache.hits == hits