feat(pre-push-safety-net): reuse compatible artifacts throughout replay

src/openultrasast/cli.py

@@ -115,6 +115,9 @@ def main(argv: list[str] | None = None) -> int:
     replay_parser.add_argument("--base", required=True)
     replay_parser.add_argument("--head", required=True)
     replay_parser.add_argument("--artifact", type=Path, required=True)
+    replay_parser.add_argument(
+        "--cache-dir", type=Path, help="reuse compatible local artifacts in a private directory outside the repository"
+    )
     replay_parser.add_argument("--deadline", type=float, default=30.0)
     replay_parser.add_argument("--cancellation-allowance", type=float, default=2.0)
     replay_parser.add_argument("--max-regions", type=int, default=500)
@@ -212,7 +215,13 @@ def main(argv: list[str] | None = None) -> int:
                 incomplete_coverage_policy=args.incomplete_coverage,
             )
             delivery = replay(
-                args.path, base=args.base, head=args.head, artifact=args.artifact, config=settings, max_regions=args.max_regions
+                args.path,
+                base=args.base,
+                head=args.head,
+                artifact=args.artifact,
+                config=settings,
+                max_regions=args.max_regions,
+                cache_dir=args.cache_dir,
             )
         except ValueError as error:
             parser.error(str(error))

src/openultrasast/cpg/backend.py

@@ -266,6 +266,41 @@ class NullBackend:
         return None
 
 
+def engine_runtime_identity(
+    *, include_tests: bool = True, language: str | None = None, execution_budget: ExecutionBudget | None = None
+) -> str:
+    """Installed frontend/adaptation bytes, also used by comparison eligibility provenance."""
+    from .artifact import digest_value, graph_bytes
+
+    budget = execution_budget
+    deadline = budget.deadline_monotonic if budget else float("inf")
+    launcher = shutil.which("joern")
+    version = joern_version()
+    installation = []
+    if launcher is not None and version is not None:
+        home = Path(launcher).resolve().parent
+        engine = ".".join(map(str, version))
+        frontends = {_FRONTENDS.get(language, "joern-parse")} if language else set(_FRONTENDS.values())
+        candidates = {home / f"lib/io.joern.joern-cli-{engine}.jar", Path(launcher).resolve()}
+        for frontend in frontends:
+            candidates.add(home / f"frontends/{frontend}/lib/io.joern.{frontend}-{engine}.jar")
+            frontend_launcher = shutil.which(frontend)
+            if frontend_launcher is not None:
+                candidates.add(Path(frontend_launcher).resolve())
+        if "jssrc2cpg" in frontends:
+            candidates.update({home / "ousast-frontend-retention-v1/manifest.json", home / "frontends/jssrc2cpg/bin/astgen/astgen-linux"})
+        for candidate in sorted(candidates):
+            installation.append((candidate.name, graph_bytes(candidate, deadline) if candidate.is_file() else "absent"))
+    return digest_value(
+        {
+            "version": version,
+            "files": installation,
+            "include_tests": include_tests,
+            "java_options": os.environ.get("JAVA_TOOL_OPTIONS", ""),
+        }
+    )
+
+
 @dataclass
 class JoernBackend:
     """``joern-parse`` then ``joern --script``, both out of process and both timeout-bounded."""
@@ -324,7 +359,8 @@ class JoernBackend:
     ) -> GraphIdentity:
         from .artifact import digest_value, read_bytes, source_inventory
 
-        deadline = execution_budget.deadline_monotonic if execution_budget else float("inf")
+        budget = execution_budget or self.execution_budget
+        deadline = budget.deadline_monotonic if budget else float("inf")
         version = joern_version()
         if version is None:
             raise ValueError("engine_identity_unknown")
@@ -342,7 +378,16 @@ class JoernBackend:
             engine,
             (DATAFLOW_OVERLAY,),
             (("implementation", code),),
-            (("include_tests", str(self.include_tests)), ("java_options", os.environ.get("JAVA_TOOL_OPTIONS", "default") or "default")),
+            (
+                ("include_tests", str(self.include_tests)),
+                (
+                    "installation",
+                    engine_runtime_identity(
+                        include_tests=self.include_tests, language=language, execution_budget=execution_budget or self.execution_budget
+                    ),
+                ),
+                ("java_options", os.environ.get("JAVA_TOOL_OPTIONS", "default") or "default"),
+            ),
         )
 
     def describe_graph(

src/openultrasast/push/reuse.py

@@ -0,0 +1,202 @@
+"""Compatible replay reuse; the existing ranker and detector remain decision owners."""
+
+from __future__ import annotations
+
+import math
+import os
+import time
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import asdict, replace
+from pathlib import Path
+from typing import Any
+
+from openultrasast.cpg.artifact import GraphArtifact, digest_value
+from openultrasast.cpg.backend import CpgResult, JoernBackend
+from openultrasast.model.contracts import ExecutionBudget
+from openultrasast.model.regions import ScanRegion
+from openultrasast.push.cache import ArtifactCache, SemanticKeys
+from openultrasast.push.contracts import SnapshotManifest
+
+
+class ReusingBackend:
+    def __init__(self, backend: JoernBackend, cache: ArtifactCache, semantics: SemanticKeys, *, declarations: str, exclusions: str) -> None:
+        self.backend = backend
+        self.cache = cache
+        self.semantics = semantics
+        self.declarations = declarations
+        self.exclusions = exclusions
+        self.last_failure = ""
+        self.graph_hits = 0
+        self.query_hits = 0
+
+    def build(self, root: Path, *, language: str, exclude: Sequence[str], execution_budget: ExecutionBudget) -> CpgResult | None:
+        identity = None
+        artifact = None
+        result = None
+        try:
+            identity = self.backend.graph_identity(
+                root,
+                language=language,
+                declarations_digest=self.declarations,
+                exclusions_digest=digest_value([self.exclusions, sorted(exclude)]),
+                execution_budget=execution_budget,
+            )
+            key = digest_value({"layer": "graph-v1", "identity": identity.to_payload()})
+            with self.cache.lookup(key, execution_budget) as entry:
+                if entry is not None:
+                    artifact = GraphArtifact.from_payload(entry.metadata)
+                    result = self.backend.load_graph(entry.path, artifact, identity, execution_budget=execution_budget)
+                    if result is not None:
+                        self.graph_hits += 1
+        except (OSError, ValueError, TimeoutError):
+            artifact = None
+        if result is None:
+            artifact = None
+            result = self.backend.build(root, language=language, exclude=exclude, execution_budget=execution_budget)
+            self.last_failure = self.backend.last_failure
+            if result is None:
+                return None
+            if identity is not None and time.monotonic() < execution_budget.deadline_monotonic:
+                artifact = self.backend.describe_graph(result, root, identity, execution_budget=execution_budget)
+                if artifact is not None:
+                    self.cache.publish(key, result.cpg_path, artifact.to_payload(), execution_budget)
+        if artifact is None:
+            return result
+        return self._answers(result, artifact, execution_budget)
+
+    def _answers(self, result: CpgResult, artifact: GraphArtifact, budget: ExecutionBudget) -> CpgResult:
+        def normalize(value: Any) -> Any:
+            if isinstance(value, str):
+                return value.replace(artifact.source_root + "/", "")
+            if isinstance(value, list):
+                return [normalize(item) for item in value]
+            if isinstance(value, dict):
+                return {key: normalize(item) for key, item in value.items()}
+            return value
+
+        def batch(kind: str, requests: Mapping[str, Mapping[str, object]]) -> dict[str, list[object]] | None:
+            answers: dict[str, list[object]] = {}
+            missing = {}
+            keys = {}
+            for rid, request in requests.items():
+                if time.monotonic() >= budget.deadline_monotonic:
+                    break
+                key = self.semantics.query(
+                    graph=digest_value({"identity": artifact.identity.to_payload(), "bytes": artifact.graph_digest}),
+                    kind=kind,
+                    request=request,
+                    context={"paths": artifact.source_paths, "declarations": artifact.identity.declarations_digest},
+                )
+                keys[rid] = key
+                cached = self.cache.get_json(key, kind="query", budget=budget)
+                if isinstance(cached, dict) and isinstance(cached.get("rows"), list) and self._census(cached.get("census"), artifact):
+                    answers[rid] = cached["rows"]
+                    answers["__census__"] = cached["census"]
+                    self.query_hits += 1
+                else:
+                    missing[rid] = request
+            if missing and time.monotonic() < budget.deadline_monotonic:
+                raw: Any = result.run_batch(kind, missing) if result.run_batch else None
+                if raw is None and result.run_batch is None:
+                    raw = {rid: result.run(kind, params) for rid, params in missing.items()}
+                if isinstance(raw, dict):
+                    normalized = normalize(raw)
+                    census = normalized.get("__census__")
+                    valid = self._census(census, artifact)
+                    if not valid:
+                        return answers or None
+                    answers["__census__"] = census
+                    for rid in missing:
+                        rows = normalized.get(rid)
+                        if isinstance(rows, list):
+                            answers[rid] = rows
+                            self.cache.put_json(keys[rid], {"rows": rows, "census": census}, kind="query", budget=budget, complete=valid)
+            return answers or None
+
+        def run(kind: str, params: Mapping[str, object]) -> object | None:
+            if kind == "census":
+                value: object = normalize(result.run(kind, params))
+                return value
+            rows = batch(kind, {"one": params})
+            return rows.get("one") if rows is not None else None
+
+        return replace(result, run=run, run_batch=batch)
+
+    @staticmethod
+    def _census(value: object, artifact: GraphArtifact) -> bool:
+        if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict):
+            return False
+        row = value[0]
+        names = row.get("file_names")
+        try:
+            return (
+                isinstance(names, list)
+                and all(isinstance(name, str) for name in names)
+                and set(artifact.source_paths) <= set(names)
+                and int(str(row.get("methods", 0))) > 0
+                and int(str(row.get("files", 0))) >= len(artifact.source_paths)
+                and int(str(row.get("shards", 1))) == 1
+            )
+        except (ValueError, TypeError):
+            return False
+
+
+def discovery(
+    root: Path,
+    manifest: SnapshotManifest,
+    *,
+    cache: ArtifactCache | None,
+    identity: str,
+    budget: ExecutionBudget,
+    discover: Callable[[Path], tuple[tuple[ScanRegion, ...], tuple[bytes, ...]]],
+) -> tuple[tuple[ScanRegion, ...], tuple[bytes, ...], bool]:
+    """Persist pure discovery only after the adapter proves the complete snapshot readable."""
+    key = digest_value(
+        {
+            "layer": "discovery-v1",
+            "files": [item.to_payload() for item in manifest.files],
+            "object_format": manifest.object_format,
+            "identity": identity,
+        }
+    )
+    raw = cache.get_json(key, kind="discovery", budget=budget) if cache and manifest.complete else None
+    if isinstance(raw, dict):
+        try:
+            regions = []
+            for item in raw["regions"]:
+                values = dict(item)
+                values["families"] = tuple(values["families"])
+                region = ScanRegion(**values)
+                if (
+                    not all(isinstance(v, str) for v in (region.path, region.language, region.source, *region.families))
+                    or region.function is not None
+                    and not isinstance(region.function, str)
+                    or type(region.shipped) is not bool
+                    or type(region.rank) not in (int, float)
+                    or not math.isfinite(region.rank)
+                ):
+                    raise ValueError("invalid_discovery_region")
+                if os.fsencode(region.path) not in {item.path_bytes for item in manifest.files}:
+                    raise ValueError("discovery_path_outside_snapshot")
+                regions.append(region)
+            declarations = tuple(bytes.fromhex(item) for item in raw["declarations"])
+            if not set(declarations) <= {item.path_bytes for item in manifest.files}:
+                raise ValueError("invalid_declarations")
+            return tuple(regions), declarations, True
+        except (KeyError, ValueError, TypeError):
+            pass
+    found, declarations = discover(root)
+    if cache and manifest.complete:
+        cache.put_json(
+            key,
+            {"regions": [asdict(region) for region in found], "declarations": [item.hex() for item in declarations]},
+            kind="discovery",
+            budget=budget,
+            complete=True,
+        )
+    return found, declarations, False
+
+
+def declaration_identity(manifest: SnapshotManifest, paths: tuple[bytes, ...]) -> str:
+    declared = set(paths)
+    return digest_value([item.to_payload() for item in manifest.files if item.path_bytes in declared])

src/openultrasast/push/runner.py

@@ -9,18 +9,22 @@ import pickle
 import selectors
 import time
 from collections.abc import Callable
+from contextlib import nullcontext
 from dataclasses import asdict, replace
 from pathlib import Path
 from typing import Any, Literal, TypeVar, cast
 
 from openultrasast.config import PushConfig
-from openultrasast.cpg.backend import JoernBackend, joern_version
+from openultrasast.cpg.artifact import digest_value
+from openultrasast.cpg.backend import JoernBackend, engine_runtime_identity, joern_version
 from openultrasast.mapping import analyze_entry_points
 from openultrasast.model.contracts import ExecutionBudget
+from openultrasast.model.layout import layout_facts
 from openultrasast.model.regions import ScanRegion, regions_for
 from openultrasast.model.scan import ScanBudget, scan_repository
 from openultrasast.model.shipped import declared_sources
 from openultrasast.preprocess import build_file_target, enumerate_source_files
+from openultrasast.push.cache import ArtifactCache, SemanticKeys
 from openultrasast.push.contracts import ComparisonAnalysis, SnapshotManifest
 from openultrasast.push.policy import (
     AdmissionCandidate,
@@ -32,6 +36,7 @@ from openultrasast.push.policy import (
     semantics_digest,
 )
 from openultrasast.push.report import PushReport, ReportDelivery, ScanRecord, _completed_scan, deliver_report, render_report
+from openultrasast.push.reuse import ReusingBackend, declaration_identity, discovery
 from openultrasast.push.snapshot import SnapshotAdapter
 
 _T = TypeVar("_T")
@@ -124,14 +129,25 @@ def _provenance(repository: Path, scan_budget: ScanBudget) -> dict[str, str]:
 
     version = joern_version()
     engine = "joern-" + (".".join(map(str, version)) if version else "unavailable_or_unidentified")
+    runtime = engine_runtime_identity()
+    integration = digest(list((root / "cpg").glob("*.py")))
     return {
         "repository": str(repository.absolute()),
+        "engine_runtime": runtime,
         "engine": engine,
         "facts": digest(list((root / "ruleset").rglob("*.toml"))),
+        "graph_layout": digest_value(
+            {
+                "facts": [asdict(row) for row in layout_facts()],
+                "implementation": digest([root / "model/layout.py", root / "model/partitions.py"]),
+            }
+        ),
         "queries": digest(list((root / "cpg/queries").glob("*.sc"))),
         "policy": digest(list((root / "push").glob("*.py"))),
         "core": digest(list((root / "model").glob("*.py")) + list((root / "cpg").glob("*.py"))),
-        "semantics": semantics_digest(engine_identity=engine, options=asdict(scan_budget)),
+        "semantics": semantics_digest(
+            engine_identity=engine, options={**asdict(scan_budget), "engine_runtime": runtime, "engine_integration": integration}
+        ),
         "ranking_mode": "evidence",
         "model": "disabled",
         "capabilities": "empty_registry_pending_task_8.3",
@@ -139,6 +155,13 @@ def _provenance(repository: Path, scan_budget: ScanBudget) -> dict[str, str]:
     }
 
 
+def _discovery_identity() -> str:
+    root = Path(__file__).resolve().parents[1]
+    # Discovery invokes the shipped mapper, preprocessing, layout and family specs.
+    # Hash code bytes, never a mutable import name or a repository-provided version.
+    return digest_value([(p.relative_to(root).as_posix(), hashlib.sha256(p.read_bytes()).hexdigest()) for p in sorted(root.rglob("*.py"))])
+
+
 def replay(
     repository: Path,
     *,
@@ -148,16 +171,22 @@ def replay(
     config: PushConfig | None = None,
     backend: Any = None,
     max_regions: int = 500,
+    cache_dir: Path | None = None,
 ) -> ReportDelivery:
     """Run a single explicit comparison; exit policy never stands for coverage."""
     config = config or PushConfig()
     if max_regions < 1:
         raise ValueError("max_regions must be positive")
+    if cache_dir is not None and cache_dir.resolve().is_relative_to(repository.resolve()):
+        raise ValueError("cache directory must be outside the analyzed repository")
     started = time.monotonic()
     budget = ExecutionBudget(started + config.deadline_seconds, config.cancellation_allowance_seconds)
     scan_budget = ScanBudget(max_model_calls=0, max_regions=max_regions, order_by_evidence=True)
     if backend is None:
         backend = JoernBackend()
+    cache = None
+    cache_semantics = None
+    reused: list[ReusingBackend] = []
     records: list[ScanRecord] = []
     manifests: list[SnapshotManifest] = []
     reasons: list[str] = []
@@ -179,16 +208,66 @@ def replay(
         stage, stage_started = "provenance", time.monotonic()
         provenance.update(_prepare(lambda: _provenance(repository, scan_budget), budget))
         timings[stage + "_seconds"] = time.monotonic() - stage_started
+        if cache_dir is not None:
+            cache = ArtifactCache(cache_dir, max_bytes=config.cache_max_bytes)
+            cache_semantics = SemanticKeys(
+                provenance["facts"],
+                provenance["queries"],
+                digest_value({"query_parameters": "complete-joern-request-v1"}),
+                digest_value({"ranking_mode": "evidence", "options": asdict(scan_budget)}),
+                provenance["policy"],
+                provenance["model"],
+                provenance["capabilities"],
+                config.mode,
+            )
+            provenance["cache"] = "validated_local_reuse"
+        discovery_identity = digest_value({"code": _prepare(lambda: _discovery_identity(), budget), "facts": provenance["facts"]})
         stage, stage_started = "preparation", time.monotonic()
         with adapter.materialize(comparison.head_oid, budget=budget) as tip:
             manifests.append(tip.manifest)
             assert comparison.base_oid is not None
-            with adapter.materialize(comparison.base_oid, budget=budget) as old:
+            old_context = (
+                nullcontext(tip) if comparison.base_oid == comparison.head_oid else adapter.materialize(comparison.base_oid, budget=budget)
+            )
+            with old_context as old:
                 manifests.append(old.manifest)
                 for manifest in manifests:
                     reasons.extend("snapshot:" + boundary.reason for boundary in manifest.boundaries)
-                head_regions, head_declarations = _prepare(lambda: _discover(tip.root), budget)
-                base_regions, base_declarations = _prepare(lambda: _discover(old.root), budget)
+                timings["snapshot_seconds"] = time.monotonic() - stage_started
+                timings["snapshot_reuses"] = int(comparison.base_oid == comparison.head_oid)
+                discovery_started = time.monotonic()
+                head_regions, head_declarations, head_hit = _prepare(
+                    lambda: discovery(tip.root, tip.manifest, cache=cache, identity=discovery_identity, budget=budget, discover=_discover),
+                    budget,
+                )
+                base_regions, base_declarations, base_hit = _prepare(
+                    lambda: discovery(old.root, old.manifest, cache=cache, identity=discovery_identity, budget=budget, discover=_discover),
+                    budget,
+                )
+                timings["discovery_seconds"] = time.monotonic() - discovery_started
+                timings["discovery_hits"] = int(head_hit) + int(base_hit)
+                head_backend = base_backend = backend
+                if (
+                    cache is not None
+                    and cache_semantics is not None
+                    and isinstance(backend, JoernBackend)
+                    and all(m.complete for m in manifests)
+                ):
+                    head_backend = ReusingBackend(
+                        backend,
+                        cache,
+                        cache_semantics,
+                        declarations=declaration_identity(tip.manifest, head_declarations),
+                        exclusions=provenance["graph_layout"],
+                    )
+                    base_backend = ReusingBackend(
+                        backend,
+                        cache,
+                        cache_semantics,
+                        declarations=declaration_identity(old.manifest, base_declarations),
+                        exclusions=provenance["graph_layout"],
+                    )
+                    reused.extend((head_backend, base_backend))
                 context = adapter.compare(comparison, declaration_paths=tuple(set(head_declarations + base_declarations)), budget=budget)
                 context = replace(context, unresolved_boundaries=tuple(dict.fromkeys((*context.unresolved_boundaries, *reasons))))
                 reasons.extend(context.unresolved_boundaries)
@@ -197,7 +276,7 @@ def replay(
                 head_scan = scan_repository(
                     tip.root,
                     head_regions,
-                    backend=backend,
+                    backend=head_backend,
                     budget=scan_budget,
                     execution_budget=budget,
                     ranking_mode="evidence",
@@ -214,11 +293,13 @@ def replay(
                     head_regions=head_regions,
                     base_regions=base_regions,
                     context=context,
-                    backend=backend,
+                    backend=base_backend,
                     execution_budget=budget,
                     scan_budget=scan_budget,
                     head_semantics=provenance["semantics"],
                     base_semantics=provenance["semantics"],
+                    cache=cache,
+                    cache_semantics=cache_semantics,
                 )
                 if delta.base_scan is not None:
                     records.append(ScanRecord(comparison, "base", delta.base_scan))
@@ -251,6 +332,8 @@ def replay(
         timings[stage + "_seconds"] = time.monotonic() - stage_started
     if time.monotonic() >= budget.deadline_monotonic:
         reasons.append("deadline_exhausted")
+    timings["graph_hits"] = sum(item.graph_hits for item in reused)
+    timings["query_hits"] = sum(item.query_hits for item in reused)
     head_records = [r.scan for r in records if r.side == "head"]
     for record in records:
         if record.scan.scope is not None: