mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 12:09:27 +00:00
Marius CiepluchandClaude Opus 5
45f9d6562e
Three PHP investigations were spent reading `cpg batch taint failed: timeout` as a slow query. The query is not slow: 4,425 evidence requests answer in 30 s, 50 dataflow requests in 24 s, and 20 entry-point requests at call depth 3 with parameter sources in 24.7 s. The production PHP build takes 79 s and does not shard. None of that explains a case running 26 minutes without finishing. The reason it could not be diagnosed is that nothing reported where the budget went. A run that is killed, or that blames a later stage's timeout, left no account of what had already consumed the transaction, and a subprocess bounded by the remaining time reports its own timeout whatever spent it. Each stage now records and logs what it cost as it finishes, so a killed run still says where it was. A control asserts the stages report and that each line carries its duration. 1,410 tests passed, nine skipped. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
605 lines
27 KiB
Python
605 lines
27 KiB
Python
"""Bounded, local base/head replay through the existing repository driver."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import multiprocessing
|
|
import os
|
|
import pickle
|
|
import selectors
|
|
import time
|
|
from collections.abc import Callable, Sequence
|
|
from contextlib import nullcontext
|
|
from dataclasses import asdict, replace
|
|
from pathlib import Path
|
|
from typing import Any, Literal, TypeVar, cast
|
|
|
|
from openultrasast.config import PushConfig
|
|
from openultrasast.cpg.artifact import digest_value
|
|
from openultrasast.cpg.backend import JoernBackend, engine_runtime_identity, joern_version
|
|
from openultrasast.mapping import analyze_entry_points
|
|
from openultrasast.model.contracts import ChangeContext, ExecutionBudget
|
|
from openultrasast.model.layout import layout_facts
|
|
from openultrasast.model.regions import ScanRegion, regions_for
|
|
from openultrasast.model.scan import ModelScanResult, ScanBudget, scan_repository
|
|
from openultrasast.model.shipped import declared_sources
|
|
from openultrasast.preprocess import build_file_target, enumerate_source_files
|
|
from openultrasast.push.cache import ArtifactCache, SemanticKeys
|
|
from openultrasast.push.contracts import ComparisonAnalysis, PushComparison, PushResult, SnapshotManifest
|
|
from openultrasast.push.eligibility import fingerprint, load_registry
|
|
from openultrasast.push.policy import (
|
|
ActionableDefect,
|
|
AdmissionCandidate,
|
|
AdmissionResult,
|
|
CapabilityKey,
|
|
DeltaComparison,
|
|
admission_push_result,
|
|
admit_candidates,
|
|
compare_targeted_base,
|
|
dependency_gaps,
|
|
semantics_digest,
|
|
)
|
|
from openultrasast.push.report import PushReport, ReportDelivery, ScanRecord, _completed_scan, deliver_report, render_report
|
|
from openultrasast.push.reuse import ReusingBackend, declaration_identity, discovery
|
|
from openultrasast.push.snapshot import SnapshotAdapter
|
|
from openultrasast.push.vetoes import FLAG, LABEL, load_declarations, record_vetoes
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_T = TypeVar("_T")
|
|
|
|
|
|
def _prepare(function: Callable[[], _T], budget: ExecutionBudget) -> _T:
|
|
"""Bound owned discovery/metadata or explicit HTTP assistance on Linux.
|
|
|
|
Read incrementally while the child writes; no pipe-buffer deadlock or unbounded
|
|
blocking recv. The child never launches engines, subprocesses or project code.
|
|
"""
|
|
if time.monotonic() >= budget.deadline_monotonic:
|
|
raise TimeoutError("deadline_exhausted")
|
|
reader, writer = os.pipe()
|
|
|
|
def child() -> None:
|
|
os.close(reader)
|
|
try:
|
|
payload: tuple[bool, Any]
|
|
try:
|
|
payload = (True, function())
|
|
except Exception as error:
|
|
payload = (False, type(error).__name__)
|
|
with os.fdopen(writer, "wb") as stream:
|
|
pickle.dump(payload, stream, protocol=5)
|
|
finally:
|
|
os._exit(0)
|
|
|
|
process = multiprocessing.get_context("fork").Process(target=child)
|
|
data = bytearray()
|
|
try:
|
|
process.start()
|
|
os.close(writer)
|
|
writer = -1
|
|
with selectors.DefaultSelector() as selector:
|
|
selector.register(reader, selectors.EVENT_READ)
|
|
while selector.get_map():
|
|
remaining = budget.deadline_monotonic - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError("deadline_exhausted")
|
|
for key, _ in selector.select(min(0.05, remaining)):
|
|
chunk = os.read(key.fd, 65536)
|
|
if not chunk:
|
|
selector.unregister(key.fileobj)
|
|
else:
|
|
data.extend(chunk)
|
|
if len(data) > 32 * 1024**2:
|
|
raise ValueError("preparation_output_limit")
|
|
process.join(max(0, budget.deadline_monotonic - time.monotonic()))
|
|
if process.is_alive() or process.exitcode != 0 or not data:
|
|
raise RuntimeError("preparation_worker_failed")
|
|
success, result = pickle.loads(data) # trusted child, never repository-provided pickle
|
|
if not success:
|
|
raise RuntimeError("preparation_failed:" + result)
|
|
return cast(_T, result)
|
|
finally:
|
|
os.close(reader)
|
|
if writer >= 0:
|
|
os.close(writer)
|
|
if process.pid is not None:
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join(max(0, budget.deadline_monotonic + budget.cancellation_allowance_seconds - time.monotonic()))
|
|
if not process.is_alive():
|
|
process.close()
|
|
|
|
|
|
def _discover(root: Path) -> tuple[tuple[ScanRegion, ...], tuple[bytes, ...]]:
|
|
# Same proposer and declaration-based shipping rules as ordinary scans. Do not
|
|
# call preprocess's live-HEAD probe on a materialized, intentionally Git-free tree.
|
|
files = enumerate_source_files(root)
|
|
targets = [build_file_target(root, path) for path in files]
|
|
regions = regions_for(analyze_entry_points(root, targets), targets, shipped=declared_sources(root))
|
|
# Conservatively seed every tracked declaration/non-source file. This is change
|
|
# context only, never a graph allowlist or a second scope selector.
|
|
sources = set(files)
|
|
declarations = tuple(os.fsencode(str(p.relative_to(root))) for p in root.rglob("*") if p.is_file() and p not in sources)
|
|
return regions, declarations
|
|
|
|
|
|
def _provenance(repository: Path, scan_budget: ScanBudget) -> dict[str, str]:
|
|
root = Path(__file__).resolve().parents[1]
|
|
|
|
def digest(paths: list[Path]) -> str:
|
|
hasher = hashlib.sha256()
|
|
for path in sorted(paths):
|
|
hasher.update(str(path.relative_to(root)).encode())
|
|
hasher.update(path.read_bytes())
|
|
return hasher.hexdigest()
|
|
|
|
version = joern_version()
|
|
engine = "joern-" + (".".join(map(str, version)) if version else "unavailable_or_unidentified")
|
|
runtime = engine_runtime_identity()
|
|
integration = digest(list((root / "cpg").glob("*.py")))
|
|
return {
|
|
"repository": str(repository.absolute()),
|
|
"engine_runtime": runtime,
|
|
"engine": engine,
|
|
"facts": digest(list((root / "ruleset").rglob("*.toml"))),
|
|
"graph_layout": digest_value(
|
|
{
|
|
"facts": [asdict(row) for row in layout_facts()],
|
|
"implementation": digest([root / "model/layout.py", root / "model/partitions.py"]),
|
|
}
|
|
),
|
|
"queries": digest(list((root / "cpg/queries").glob("*.sc"))),
|
|
"policy": digest(list((root / "push").glob("*.py"))),
|
|
"core": digest(list((root / "model").glob("*.py")) + list((root / "cpg").glob("*.py"))),
|
|
"semantics": semantics_digest(
|
|
engine_identity=engine, options={**asdict(scan_budget), "engine_runtime": runtime, "engine_integration": integration}
|
|
),
|
|
"ranking_mode": "evidence",
|
|
"model": "disabled",
|
|
"capabilities": fingerprint(),
|
|
"cache": "cold_no_reuse",
|
|
}
|
|
|
|
|
|
def _admit(candidates: tuple[AdmissionCandidate, ...], provenance: dict[str, str]) -> AdmissionResult:
|
|
registry = load_registry(current=provenance)
|
|
result = admit_candidates(candidates, capabilities=registry.capabilities)
|
|
if candidates and registry.reasons:
|
|
result = replace(
|
|
result,
|
|
coverage_reasons=tuple(dict.fromkeys((*result.coverage_reasons, *("eligibility:" + reason for reason in registry.reasons)))),
|
|
)
|
|
return result
|
|
|
|
|
|
def _record(
|
|
head: ModelScanResult,
|
|
delta: DeltaComparison,
|
|
context: ChangeContext,
|
|
comparison: PushComparison,
|
|
reasons: Sequence[str],
|
|
admission: AdmissionResult,
|
|
provenance: dict[str, str],
|
|
declarations: Path | None = None,
|
|
) -> dict[str, object]:
|
|
"""Experimental M1a: the same policy over lifted evidence, reported only, never admitted.
|
|
|
|
`declarations` supplies templates for the experimental evaluation rendering ONLY. They are
|
|
never passed to admission, so nothing read from that file can enable a capability.
|
|
"""
|
|
registry = load_registry(current=provenance)
|
|
supplied = load_declarations(declarations) if declarations is not None else ()
|
|
report = record_vetoes(
|
|
head=head,
|
|
base=delta.base_scan,
|
|
context=context,
|
|
comparison=comparison,
|
|
semantics=delta.semantics,
|
|
production=delta.candidates,
|
|
production_admission={d.candidate.delta.defect_id: d.reasons for d in admission.dispositions},
|
|
coverage_reasons=tuple(dict.fromkeys(reasons)),
|
|
capabilities=registry.capabilities,
|
|
declarations=supplied,
|
|
)
|
|
return report.to_payload()
|
|
|
|
|
|
def _discovery_identity() -> str:
|
|
root = Path(__file__).resolve().parents[1]
|
|
# Discovery invokes the shipped mapper, preprocessing, layout and family specs.
|
|
# Hash code bytes, never a mutable import name or a repository-provided version.
|
|
return digest_value([(p.relative_to(root).as_posix(), hashlib.sha256(p.read_bytes()).hexdigest()) for p in sorted(root.rglob("*.py"))])
|
|
|
|
|
|
def _stage(timings: dict[str, float], stage: str, started: float) -> float:
|
|
"""Close one stage, record its cost and say so.
|
|
|
|
A run that is killed, or that reports a later stage's timeout, otherwise leaves no account of
|
|
where its budget went. Three PHP investigations were spent reading `cpg batch taint failed:
|
|
timeout` as a slow query when the query answers 20 requests in 24 seconds, because nothing said
|
|
what had already consumed the transaction.
|
|
"""
|
|
now = time.monotonic()
|
|
timings[stage + "_seconds"] = now - started
|
|
logger.info("push stage %s finished in %.1fs", stage, now - started)
|
|
return now
|
|
|
|
|
|
def _analyze(
|
|
repository: Path,
|
|
*,
|
|
base: str,
|
|
head: str,
|
|
artifact: Path,
|
|
config: PushConfig | None = None,
|
|
backend: Any = None,
|
|
max_regions: int = 500,
|
|
cache_dir: Path | None = None,
|
|
execution_budget: ExecutionBudget,
|
|
resolved: PushComparison | None = None,
|
|
record_vetoes: bool = False,
|
|
declarations: Path | None = None,
|
|
) -> PushReport:
|
|
"""Run a single explicit comparison; exit policy never stands for coverage.
|
|
|
|
`record_vetoes` is the task-local M1a evaluation flag: after the production result is
|
|
decided, the same policy is rerun over lifted evidence and every veto is recorded in
|
|
the artifact's labeled `experimental` section. It changes no result, admission or
|
|
eligibility, and the runner rejects it outside explicit replay.
|
|
"""
|
|
config = config or PushConfig()
|
|
if max_regions < 1:
|
|
raise ValueError("max_regions must be positive")
|
|
if cache_dir is not None and cache_dir.resolve().is_relative_to(repository.resolve()):
|
|
raise ValueError("cache directory must be outside the analyzed repository")
|
|
started = time.monotonic()
|
|
budget = execution_budget
|
|
scan_budget = ScanBudget(max_model_calls=0, max_regions=max_regions, order_by_evidence=True)
|
|
if backend is None:
|
|
backend = JoernBackend()
|
|
cache = None
|
|
cache_semantics = None
|
|
reused: list[ReusingBackend] = []
|
|
records: list[ScanRecord] = []
|
|
manifests: list[SnapshotManifest] = []
|
|
reasons: list[str] = []
|
|
admission = AdmissionResult((), (), ())
|
|
comparison = resolved
|
|
context = None
|
|
experimental: dict[str, object] = {"label": LABEL, "flag": FLAG, "status": "not_evaluated"} if record_vetoes else {}
|
|
timings: dict[str, float] = {
|
|
"deadline_seconds": config.deadline_seconds,
|
|
"cancellation_allowance_seconds": config.cancellation_allowance_seconds,
|
|
}
|
|
provenance = {key: "unavailable_preparation_incomplete" for key in ("engine", "facts", "queries", "policy")}
|
|
provenance.update(repository=str(repository.absolute()), model="disabled", requested_base=base, requested_head=head)
|
|
stage = "resolution"
|
|
stage_started = started
|
|
try:
|
|
adapter = SnapshotAdapter(repository, execution_budget=budget)
|
|
comparison = resolved or adapter.resolve_replay(base, head)
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
stage = "provenance"
|
|
provenance.update(_prepare(lambda: _provenance(repository, scan_budget), budget))
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
if cache_dir is not None:
|
|
cache = ArtifactCache(cache_dir, max_bytes=config.cache_max_bytes)
|
|
cache_semantics = SemanticKeys(
|
|
provenance["facts"],
|
|
provenance["queries"],
|
|
digest_value({"query_parameters": "complete-joern-request-v1"}),
|
|
digest_value({"ranking_mode": "evidence", "options": asdict(scan_budget)}),
|
|
provenance["policy"],
|
|
provenance["model"],
|
|
provenance["capabilities"],
|
|
config.mode,
|
|
)
|
|
provenance["cache"] = "validated_local_reuse"
|
|
discovery_identity = digest_value({"code": _prepare(lambda: _discovery_identity(), budget), "facts": provenance["facts"]})
|
|
stage, stage_started = "preparation", time.monotonic()
|
|
with adapter.materialize(comparison.head_oid, budget=budget) as tip:
|
|
manifests.append(tip.manifest)
|
|
assert comparison.base_oid is not None
|
|
old_context = (
|
|
nullcontext(tip) if comparison.base_oid == comparison.head_oid else adapter.materialize(comparison.base_oid, budget=budget)
|
|
)
|
|
with old_context as old:
|
|
manifests.append(old.manifest)
|
|
for manifest in manifests:
|
|
reasons.extend("snapshot:" + boundary.reason for boundary in manifest.boundaries)
|
|
timings["snapshot_seconds"] = time.monotonic() - stage_started
|
|
timings["snapshot_reuses"] = int(comparison.base_oid == comparison.head_oid)
|
|
discovery_started = time.monotonic()
|
|
head_regions, head_declarations, head_hit = _prepare(
|
|
lambda: discovery(tip.root, tip.manifest, cache=cache, identity=discovery_identity, budget=budget, discover=_discover),
|
|
budget,
|
|
)
|
|
base_regions, base_declarations, base_hit = _prepare(
|
|
lambda: discovery(old.root, old.manifest, cache=cache, identity=discovery_identity, budget=budget, discover=_discover),
|
|
budget,
|
|
)
|
|
timings["discovery_seconds"] = time.monotonic() - discovery_started
|
|
timings["discovery_hits"] = int(head_hit) + int(base_hit)
|
|
head_backend = base_backend = backend
|
|
if (
|
|
cache is not None
|
|
and cache_semantics is not None
|
|
and isinstance(backend, JoernBackend)
|
|
and all(m.complete for m in manifests)
|
|
):
|
|
head_backend = ReusingBackend(
|
|
backend,
|
|
cache,
|
|
cache_semantics,
|
|
declarations=declaration_identity(tip.manifest, head_declarations),
|
|
exclusions=provenance["graph_layout"],
|
|
)
|
|
base_backend = ReusingBackend(
|
|
backend,
|
|
cache,
|
|
cache_semantics,
|
|
declarations=declaration_identity(old.manifest, base_declarations),
|
|
exclusions=provenance["graph_layout"],
|
|
)
|
|
reused.extend((head_backend, base_backend))
|
|
context = adapter.compare(comparison, declaration_paths=tuple(set(head_declarations + base_declarations)), budget=budget)
|
|
context = replace(context, unresolved_boundaries=tuple(dict.fromkeys((*context.unresolved_boundaries, *reasons))))
|
|
reasons.extend(context.unresolved_boundaries)
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
stage = "head"
|
|
head_scan = scan_repository(
|
|
tip.root,
|
|
head_regions,
|
|
backend=head_backend,
|
|
budget=scan_budget,
|
|
execution_budget=budget,
|
|
ranking_mode="evidence",
|
|
unit="repository",
|
|
population_complete=all(m.complete for m in manifests),
|
|
change_context=context,
|
|
)
|
|
records.append(ScanRecord(comparison, "head", head_scan))
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
stage = "base"
|
|
delta = compare_targeted_base(
|
|
old.root,
|
|
head=head_scan,
|
|
head_regions=head_regions,
|
|
base_regions=base_regions,
|
|
context=context,
|
|
backend=base_backend,
|
|
execution_budget=budget,
|
|
scan_budget=scan_budget,
|
|
head_semantics=provenance["semantics"],
|
|
base_semantics=provenance["semantics"],
|
|
cache=cache,
|
|
cache_semantics=cache_semantics,
|
|
)
|
|
if delta.base_scan is not None:
|
|
records.append(ScanRecord(comparison, "base", delta.base_scan))
|
|
reasons.extend(delta.coverage_reasons)
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
stage = "admission"
|
|
candidates = tuple(
|
|
AdmissionCandidate(
|
|
c,
|
|
CapabilityKey(
|
|
c.head_operation.question.language if c.head_operation else "unknown",
|
|
"unspecified",
|
|
"unspecified",
|
|
c.family,
|
|
c.head_operation.mechanism if c.head_operation else "taint",
|
|
"unreviewed",
|
|
delta.semantics,
|
|
),
|
|
comparison,
|
|
dependency_gaps(c, context=context, head=head_scan, base=delta.base_scan),
|
|
)
|
|
for c in delta.candidates
|
|
)
|
|
admission = _prepare(lambda: _admit(candidates, provenance), budget)
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
if record_vetoes:
|
|
stage, stage_started = "recorded_vetoes", time.monotonic()
|
|
assert context is not None
|
|
recorded = _prepare(
|
|
lambda: _record(head_scan, delta, context, comparison, tuple(reasons), admission, provenance, declarations),
|
|
budget,
|
|
)
|
|
experimental = {**recorded, "status": "evaluated"}
|
|
stage_started = _stage(timings, stage, stage_started)
|
|
stage = "cleanup"
|
|
timings[stage + "_seconds"] = time.monotonic() - stage_started
|
|
except Exception as error:
|
|
reasons.append(stage + "_failed:" + type(error).__name__)
|
|
timings[stage + "_seconds"] = time.monotonic() - stage_started
|
|
if experimental.get("status") == "not_evaluated":
|
|
experimental["reason"] = reasons[-1]
|
|
if time.monotonic() >= budget.deadline_monotonic:
|
|
reasons.append("deadline_exhausted")
|
|
if experimental.get("status") == "not_evaluated":
|
|
experimental.setdefault("reason", "deadline_exhausted")
|
|
timings["graph_hits"] = sum(item.graph_hits for item in reused)
|
|
timings["query_hits"] = sum(item.query_hits for item in reused)
|
|
head_records = [r.scan for r in records if r.side == "head"]
|
|
for record in records:
|
|
if record.scan.scope is not None:
|
|
reasons.extend(record.scan.scope.unresolved_boundaries)
|
|
if not records or not all(_completed_scan(r.scan) for r in records):
|
|
reasons.append("analysis_incomplete")
|
|
if not any(r.side == "base" for r in records):
|
|
reasons.append("base_comparison_unavailable")
|
|
admission = replace(admission, coverage_reasons=tuple(dict.fromkeys((*admission.coverage_reasons, *reasons))))
|
|
analyses: tuple[ComparisonAnalysis, ...] = ()
|
|
if comparison is not None:
|
|
scopes = tuple(scan.scope for scan in head_records if scan.scope is not None)
|
|
completed = tuple(outcome.identity for scan in head_records for outcome in scan.question_outcomes if outcome.status == "completed")
|
|
coverage: Literal["unavailable", "incomplete", "complete_within_scope"] = (
|
|
"unavailable" if not records else "incomplete" if admission.coverage_reasons else "complete_within_scope"
|
|
)
|
|
analyses = (ComparisonAnalysis(comparison, scopes, completed, coverage),)
|
|
result = admission_push_result(admission, analyses, config=config)
|
|
timings["total_seconds"] = time.monotonic() - started
|
|
return PushReport(
|
|
result,
|
|
admission,
|
|
tuple(records),
|
|
provenance,
|
|
timings,
|
|
snapshots=tuple(manifests),
|
|
change_context=context,
|
|
experimental=experimental,
|
|
)
|
|
|
|
|
|
def _deliver(repository: Path, report: PushReport, artifact: Path, budget: ExecutionBudget) -> ReportDelivery:
|
|
if artifact.resolve().is_relative_to(repository.resolve()):
|
|
error = "artifact_inside_repository"
|
|
return ReportDelivery(report.result, render_report(report, artifact=None, error=error), None, error)
|
|
return deliver_report(report, artifact, execution_budget=budget)
|
|
|
|
|
|
def replay(
|
|
repository: Path,
|
|
*,
|
|
base: str,
|
|
head: str,
|
|
artifact: Path,
|
|
config: PushConfig | None = None,
|
|
backend: Any = None,
|
|
max_regions: int = 500,
|
|
cache_dir: Path | None = None,
|
|
model_config: Path | None = None,
|
|
record_vetoes: bool = False,
|
|
declarations: Path | None = None,
|
|
) -> ReportDelivery:
|
|
config = config or PushConfig()
|
|
if declarations is not None and not record_vetoes:
|
|
raise ValueError("experimental declarations require the recorded-veto evaluation flag")
|
|
budget = ExecutionBudget(time.monotonic() + config.deadline_seconds, config.cancellation_allowance_seconds)
|
|
report = _analyze(
|
|
repository,
|
|
base=base,
|
|
head=head,
|
|
artifact=artifact,
|
|
config=config,
|
|
backend=backend,
|
|
max_regions=max_regions,
|
|
cache_dir=cache_dir,
|
|
execution_budget=budget,
|
|
record_vetoes=record_vetoes,
|
|
declarations=declarations,
|
|
)
|
|
from openultrasast.push.assistance import assist
|
|
|
|
report = assist(report, config=model_config, budget=budget)
|
|
return _deliver(repository, report, artifact, budget)
|
|
|
|
|
|
def push(
|
|
repository: Path,
|
|
*,
|
|
updates: str | Callable[[], str],
|
|
remote_name: str,
|
|
remote_url: str,
|
|
artifact: Path,
|
|
config: PushConfig | None = None,
|
|
backend: Any = None,
|
|
max_regions: int = 500,
|
|
cache_dir: Path | None = None,
|
|
model_config: Path | None = None,
|
|
execution_budget: ExecutionBudget | None = None,
|
|
) -> ReportDelivery:
|
|
"""Consume one Git transaction. Resolution, every comparison and reporting share a deadline."""
|
|
config = config or PushConfig()
|
|
if max_regions < 1:
|
|
raise ValueError("max_regions must be positive")
|
|
if cache_dir is not None and cache_dir.resolve().is_relative_to(repository.resolve()):
|
|
raise ValueError("cache directory must be outside the analyzed repository")
|
|
started = time.monotonic()
|
|
budget = execution_budget or ExecutionBudget(started + config.deadline_seconds, config.cancellation_allowance_seconds)
|
|
reports: list[PushReport] = []
|
|
analyses: list[ComparisonAnalysis] = []
|
|
reasons: list[str] = []
|
|
resolution = None
|
|
try:
|
|
data = _prepare(updates, budget) if callable(updates) else updates
|
|
if len(data.encode("utf-8")) > 1024 * 1024:
|
|
raise ValueError("push_input_limit")
|
|
adapter = SnapshotAdapter(repository, comparison_base=config.comparison_base, execution_budget=budget)
|
|
resolution = adapter.resolve_updates(data)
|
|
reasons.extend("resolution:" + row.disposition for row in resolution.updates if row.disposition not in ("ready", "deleted"))
|
|
for comparison in resolution.comparisons:
|
|
if comparison.base_oid is None or time.monotonic() >= budget.deadline_monotonic:
|
|
analyses.append(ComparisonAnalysis(comparison, (), (), "unavailable"))
|
|
reasons.append("base_comparison_unavailable" if comparison.base_oid is None else "deadline_exhausted")
|
|
continue
|
|
report = _analyze(
|
|
repository,
|
|
base=comparison.base_oid,
|
|
head=comparison.head_oid,
|
|
artifact=artifact,
|
|
config=config,
|
|
backend=backend,
|
|
max_regions=max_regions,
|
|
cache_dir=cache_dir,
|
|
execution_budget=budget,
|
|
resolved=comparison,
|
|
)
|
|
reports.append(report)
|
|
analyses.extend(report.result.analyses)
|
|
except Exception as error:
|
|
reasons.append("push_input_failed:" + type(error).__name__)
|
|
if time.monotonic() >= budget.deadline_monotonic:
|
|
reasons.append("deadline_exhausted")
|
|
defects: dict[str, ActionableDefect] = {}
|
|
for report in reports:
|
|
for defect in report.admission.defects:
|
|
previous = defects.get(defect.defect_id)
|
|
if previous is None:
|
|
defects[defect.defect_id] = defect
|
|
else:
|
|
fields = ("witnesses", "locations", "comparisons", "refs", "change_evidence", "consequences", "repairs", "evaluation_ids")
|
|
defects[defect.defect_id] = replace(
|
|
previous,
|
|
**cast(dict[str, Any], {key: tuple(dict.fromkeys((*getattr(previous, key), *getattr(defect, key)))) for key in fields}),
|
|
)
|
|
admission = AdmissionResult(
|
|
tuple(defects.values()),
|
|
tuple(d for r in reports for d in r.admission.dispositions),
|
|
tuple(dict.fromkeys((*reasons, *(g for r in reports for g in r.admission.coverage_reasons)))),
|
|
)
|
|
result = admission_push_result(admission, tuple(analyses), config=config)
|
|
if resolution is not None and not resolution.comparisons and not reasons:
|
|
result = PushResult((), "none", "not_applicable", "allow", ())
|
|
provenance = dict(reports[0].provenance) if reports else {key: "not_run" for key in ("engine", "facts", "queries", "policy")}
|
|
# Remote URLs can contain credentials. Retain identity without persisting secrets.
|
|
provenance.update(
|
|
repository=str(repository.absolute()),
|
|
model="disabled",
|
|
remote_name=remote_name or "unnamed",
|
|
remote_url_sha256=hashlib.sha256(remote_url.encode()).hexdigest(),
|
|
)
|
|
timings = {f"comparison_{i}_{k}": v for i, r in enumerate(reports) for k, v in r.timings.items()}
|
|
timings.update(
|
|
total_seconds=time.monotonic() - started,
|
|
deadline_seconds=config.deadline_seconds,
|
|
cancellation_allowance_seconds=config.cancellation_allowance_seconds,
|
|
)
|
|
report = PushReport(
|
|
result,
|
|
admission,
|
|
tuple(s for r in reports for s in r.scans),
|
|
provenance,
|
|
timings,
|
|
resolution=resolution,
|
|
snapshots=tuple(s for r in reports for s in r.snapshots),
|
|
change_contexts=tuple(r.change_context for r in reports if r.change_context is not None),
|
|
)
|
|
from openultrasast.push.assistance import assist
|
|
|
|
report = assist(report, config=model_config, budget=budget)
|
|
return _deliver(repository, report, artifact, budget)
|