From 11059c4cd434509ab786cf7309aa9f73e4f2629e Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:37:44 +0100 Subject: [PATCH] feat(detection-core): migrate benchmark readiness and admission consumers --- .kiro/specs/detection-core/tasks.md | 4 +- benchmarks/ax/batch.py | 111 ++++-- benchmarks/learn/engine_task_entry.py | 155 +++++--- .../2026-10-14-access-control-hunt/analyse.py | 24 +- benchmarks/unseen/calibrate.py | 4 +- benchmarks/unseen/replay.py | 9 +- benchmarks/unseen/score.py | 22 +- benchmarks/valid/admission_run.py | 167 +------- benchmarks/valid/admission_scoring.py | 17 +- benchmarks/valid/arm_regression.py | 162 +++++--- benchmarks/valid/artifact_contract.py | 309 +++++++++++++++ benchmarks/valid/development_runner.py | 6 +- benchmarks/valid/hunt_handoff.py | 306 ++++++++++++++ benchmarks/valid/hunter_arm.py | 75 ++-- benchmarks/valid/judge_pass.py | 114 +++--- benchmarks/valid/metrics.py | 5 +- benchmarks/valid/prover_admission.py | 24 +- benchmarks/valid/prover_arm.py | 138 ++++--- benchmarks/valid/semgrep_arm.py | 23 +- src/openultrasast/hunt/artifact.py | 134 ++++++- src/openultrasast/hunt/report.py | 239 +++++++---- src/openultrasast/hunt/runner.py | 173 +++++--- tests/benchmark_migration_conformance.py | 374 ++++++++++++++++++ tests/benchmark_migration_evidence.py | 358 +++++++++++++++++ tests/core_migration_inventory.py | 54 ++- .../v1/migration_inventory.json | 50 +-- tests/test_admission_run.py | 165 -------- tests/test_admission_scoring.py | 2 +- tests/test_arm_regression.py | 39 +- tests/test_benchmark_artifact_contract.py | 369 +++++++++++++++++ tests/test_core_migration_benchmark.py | 186 +++++++++ tests/test_core_migration_production.py | 6 +- tests/test_core_migration_provenance.py | 6 +- tests/test_hunt_artifact.py | 111 +++++- tests/test_hunt_report.py | 2 +- tests/test_hunt_runner.py | 62 ++- tests/test_hunter_arm.py | 19 +- tests/test_judge_pass_v2.py | 33 ++ tests/test_prover_admission.py | 2 +- tests/test_prover_arm.py | 17 +- tests/test_unseen_replay.py | 9 +- 41 files changed, 3223 insertions(+), 862 deletions(-) create mode 100644 benchmarks/valid/artifact_contract.py create mode 100644 benchmarks/valid/hunt_handoff.py create mode 100644 tests/benchmark_migration_conformance.py create mode 100644 tests/benchmark_migration_evidence.py create mode 100644 tests/test_benchmark_artifact_contract.py create mode 100644 tests/test_core_migration_benchmark.py create mode 100644 tests/test_judge_pass_v2.py diff --git a/.kiro/specs/detection-core/tasks.md b/.kiro/specs/detection-core/tasks.md index 40baccd..d99c7b7 100644 --- a/.kiro/specs/detection-core/tasks.md +++ b/.kiro/specs/detection-core/tasks.md @@ -382,7 +382,7 @@ - _Requirements: 14.1, 14.3, 15.3_ - _Boundary: Core and Adjacent Adapter Migration_ - _Depends: 10.3_ -- [ ] 10.5 Migrate benchmark readiness and admission consumers +- [x] 10.5 Migrate benchmark readiness and admission consumers - Move every registered benchmark reader and writer to v2 while retaining readiness sequencing, persistence, population custody, and admission orchestration in the benchmark boundary. - Update each matching stable inventory entry to `v2_migrated` or `retired` without changing its identity. - Preserve guarded spend, authenticated case release, fixed measurement, and bounded model behavior through the migration. @@ -442,3 +442,5 @@ - The frozen migration inventory is a reviewed worklist with stable identities, not a semantic-use census. It may only close entries as `v2_migrated` or `retired`; it never grows, deletes entries, or proves absence by interpreting Python flow. - Cutover is proved by explicit syntax checks plus deletion of the compatibility and v1 write capability. No task extends the discarded alias/data-flow interpreter. +- Task 10.5 debug established that executable owner probes cannot compensate for blanket source-closure acceptance; resumed hunt attempts also require collision-resistant attempt identity and create-only persistence before handoff validation is authoritative. +- Human review reopened Task 10.5 for one focused remediation cycle covering exact source-closure evidence, collision-resistant create-only attempt persistence, and complexity-cap conformance. diff --git a/benchmarks/ax/batch.py b/benchmarks/ax/batch.py index 220d8ee..e3b5d38 100644 --- a/benchmarks/ax/batch.py +++ b/benchmarks/ax/batch.py @@ -19,11 +19,13 @@ from collections import Counter, deque from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from dataclasses import dataclass, field from datetime import timedelta +from functools import partial from pathlib import Path from urllib.parse import urlsplit import yaml +from benchmarks.valid.artifact_contract import read_benchmark_artifact, write_benchmark_artifact from openultrasast.config import load_dotenv from openultrasast.plane.memory import PRESIGNED_ARCHIVE_SUFFIX, S3Store, open_store from openultrasast.search.task_storage import diagnostic, exception_reason @@ -495,21 +497,21 @@ def schedule(pending, lanes, out, execute, complete, *, fallbacks=(), overflow=T return failures -def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), overflow=True, max_attempts=2): - """Digest checkpoints, one instrument rerun, sandbox retry, priority VM fallback.""" - if not isinstance(max_attempts, int) or not 1 <= max_attempts <= 2: - raise ValueError("batch attempts must be one or two") - out.mkdir(parents=True, exist_ok=True) - items = list(items) - if len({item.id for item in items}) != len(items): - raise ValueError("duplicate dispatch item id") +def _dispatch_path(out, item): + return out / (hashlib.sha256(item.id.encode()).hexdigest() + ".json") + + +def _read_dispatch_record(out, item, record_kind): + path = _dispatch_path(out, item) + if not path.exists(): + return {} + return read_benchmark_artifact(path, record_kind) if record_kind is not None else json.loads(path.read_text()) + + +def _dispatch_queue(items, out, record_kind): pending, fallbacks = [], [] - - def path(item): - return out / (hashlib.sha256(item.id.encode()).hexdigest() + ".json") - for item in items: - previous = json.loads(path(item).read_text()) if path(item).exists() else {} + previous = _read_dispatch_record(out, item, record_kind) if previous.get("input_digest") == item.input_digest: if previous.get("fallback_pending"): fallbacks.append((item, previous)) @@ -517,37 +519,62 @@ def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), o if previous.get("done"): continue pending.append(item) + return pending, fallbacks - def execute(item, lane, previous): - output = out / hashlib.sha256(item.id.encode()).hexdigest() - output.mkdir(exist_ok=True) - attempts = [] - for attempt in range(max_attempts): - attempt_output = output / ("attempt-" + uuid.uuid4().hex) - attempt_output.mkdir() - record = executors[lane](item, attempt_output, attempt) - for artifact in attempt_output.iterdir(): - if artifact.is_file(): - shutil.copyfile(artifact, output / artifact.name) - attempts.append( - { - "lane": lane, - "status": record.get("status"), - "attempt": attempt, - "transport": record.get("ax_attempts", []), - "docker_errors": record.get("docker_errors", []), - } - ) - if record.get("status") not in {"sandbox_failure", "instrument_failure"}: - break - return {**record, "attempts": (previous or {}).get("attempts", []) + attempts, "input_digest": item.input_digest, "done": True} - def complete(item, lane, record, progress): - fallback = lane in {"ax", "kind"} and record.get("status") in {"sandbox_failure", "oom"} and "docker" in lanes - record["fallback_pending"] = fallback - write_json(path(item), record) - progress["last_item"] = hashlib.sha256(item.id.encode()).hexdigest() - return record.get("status") not in {"ok", "unanalysable", "quick_only", "engine_covered"}, record if fallback else None +def _dispatch_attempt(item, lane, output, executors, attempt): + attempt_output = output / ("attempt-" + uuid.uuid4().hex) + attempt_output.mkdir() + record = executors[lane](item, attempt_output, attempt) + for artifact in attempt_output.iterdir(): + if artifact.is_file(): + shutil.copyfile(artifact, output / artifact.name) + summary = { + "lane": lane, + "status": record.get("status"), + "attempt": attempt, + "transport": record.get("ax_attempts", []), + "docker_errors": record.get("docker_errors", []), + } + return record, summary + + +def _execute_dispatch(item, lane, previous, *, out, executors, max_attempts): + output = out / hashlib.sha256(item.id.encode()).hexdigest() + output.mkdir(exist_ok=True) + attempts = [] + for attempt in range(max_attempts): + record, summary = _dispatch_attempt(item, lane, output, executors, attempt) + attempts.append(summary) + if record.get("status") not in {"sandbox_failure", "instrument_failure"}: + break + return {**record, "attempts": (previous or {}).get("attempts", []) + attempts, "input_digest": item.input_digest, "done": True} + + +def _complete_dispatch(item, lane, record, progress, *, out, lanes, record_kind): + fallback = lane in {"ax", "kind"} and record.get("status") in {"sandbox_failure", "oom"} and "docker" in lanes + record["fallback_pending"] = fallback + path = _dispatch_path(out, item) + if record_kind is None: + write_json(path, record) + else: + write_benchmark_artifact(path, record_kind, record, replace=True) + progress["last_item"] = hashlib.sha256(item.id.encode()).hexdigest() + failed = record.get("status") not in {"ok", "unanalysable", "quick_only", "engine_covered"} + return failed, record if fallback else None + + +def dispatch(items, workload, out, executors, *, lanes=("ax", "ax", "docker"), overflow=True, max_attempts=2, record_kind=None): + """Digest checkpoints, one instrument rerun, sandbox retry, priority VM fallback.""" + if not isinstance(max_attempts, int) or not 1 <= max_attempts <= 2: + raise ValueError("batch attempts must be one or two") + out.mkdir(parents=True, exist_ok=True) + items = list(items) + if len({item.id for item in items}) != len(items): + raise ValueError("duplicate dispatch item id") + pending, fallbacks = _dispatch_queue(items, out, record_kind) + execute = partial(_execute_dispatch, out=out, executors=executors, max_attempts=max_attempts) + complete = partial(_complete_dispatch, out=out, lanes=lanes, record_kind=record_kind) return schedule(pending, lanes, out, execute, complete, fallbacks=fallbacks, overflow=overflow) diff --git a/benchmarks/learn/engine_task_entry.py b/benchmarks/learn/engine_task_entry.py index ac4aca8..14d92dc 100644 --- a/benchmarks/learn/engine_task_entry.py +++ b/benchmarks/learn/engine_task_entry.py @@ -14,6 +14,8 @@ import urllib.error import urllib.request from pathlib import Path +from benchmarks.valid.artifact_contract import write_benchmark_artifact + class TaskEntryError(RuntimeError): """A task-entry diagnostic containing only locally controlled text.""" @@ -91,68 +93,94 @@ def failure_record(pin, reason): } -def main(*, work_root=Path("/tmp"), worker_script=None, readiness_budget=60): - pin = {} - failed = False +def _read_worker_result(path, pin): + """Validate the engine worker's raw handoff before benchmark-owned persistence.""" + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as error: + raise TaskEntryError("worker result is unreadable") from error + if not isinstance(value, dict) or {"schema_version", "artifact_kind", "artifact_digest"}.intersection(value): + raise TaskEntryError("worker result has an unsupported raw schema") + units = value.get("units") + expected = [unit.get("unit") for unit in pin.get("units", [])] + if value.get("done") is not True or not isinstance(units, list): + raise TaskEntryError("worker did not write a completed result") + actual = [unit.get("unit") for unit in units if isinstance(unit, dict)] + if len(actual) != len(units) or actual != expected: + raise TaskEntryError("worker result units do not match the prepared pin") + return value + + +def _produce_result(work_root, worker_script, readiness_budget, state): output = work_root / "out" result = output / "result.json" - step = "prepare output" + output.mkdir(parents=True, exist_ok=True) + result.unlink(missing_ok=True) + questions = work_root / "questions.json" + state["step"] = "GET questions" + download(os.environ["QUESTIONS_URL"], questions, readiness_budget=readiness_budget) + state["step"] = "parse questions" + pin = json.loads(questions.read_text()) + if not isinstance(pin, dict): + raise ValueError("questions object must contain a prepared pin") + state["pin"] = pin + source, case = work_root / "source.tar", work_root / "case" + state["step"] = "GET source" + download(os.environ["SOURCE_URL"], source) + state["step"] = "unpack source" + case.mkdir(parents=True, exist_ok=True) + unpack(source, case) + deadline = int(os.environ.get("DEADLINE", "900")) + command = _worker_command(worker_script, questions, case, result, deadline) + state["step"] = "run worker" + completed = subprocess.run(command, timeout=deadline + 5, check=False) + if completed.returncode: + raise TaskEntryError(f"worker exited {completed.returncode}") + state["step"] = "read worker result" + worker_result = _read_worker_result(result, pin) if result.is_file() else {} + if not worker_result: + raise TaskEntryError("worker did not write a completed result") + write_benchmark_artifact(result, "engine-task-result", worker_result, replace=True) + return output, result + + +def _worker_command(worker_script, questions, case, result, deadline): + return [ + sys.executable, + str(worker_script or Path(__file__).with_name("engine_trace_worker.py")), + "--worker", + str(questions), + "--root", + str(case), + "--output", + str(result), + "--heap-profile", + "cluster", + "--deadline", + str(deadline), + "--question-deadline", + os.environ.get("QUESTION_DEADLINE", "120"), + ] + + +def _failure_detail(error): + if isinstance(error, urllib.error.HTTPError): + return f"HTTP {error.code}" + return str(error) if isinstance(error, TaskEntryError) else type(error).__name__ + + +def _persist_failure(work_root, state, error): + output, result = work_root / "out", work_root / "out/result.json" + reason = f"{state['step']}: {_failure_detail(error)}" try: output.mkdir(parents=True, exist_ok=True) - result.unlink(missing_ok=True) - questions = work_root / "questions.json" - step = "GET questions" - download(os.environ["QUESTIONS_URL"], questions, readiness_budget=readiness_budget) - step = "parse questions" - pin = json.loads(questions.read_text()) - if not isinstance(pin, dict): - pin = {} - raise ValueError("questions object must contain a prepared pin") - source = work_root / "source.tar" - step = "GET source" - download(os.environ["SOURCE_URL"], source) - step = "unpack source" - case = work_root / "case" - case.mkdir(parents=True, exist_ok=True) - unpack(source, case) - step = "configure worker" - deadline = int(os.environ.get("DEADLINE", "900")) - command = [ - sys.executable, - str(worker_script or Path(__file__).with_name("engine_trace_worker.py")), - "--worker", - str(questions), - "--root", - str(case), - "--output", - str(result), - "--heap-profile", - "cluster", - "--deadline", - str(deadline), - "--question-deadline", - os.environ.get("QUESTION_DEADLINE", "120"), - ] - step = "run worker" - completed = subprocess.run(command, timeout=deadline + 5, check=False) - if completed.returncode: - raise TaskEntryError(f"worker exited {completed.returncode}") - step = "read worker result" - if not result.is_file() or not json.loads(result.read_text()).get("done"): - raise TaskEntryError("worker did not write a completed result") - except Exception as exc: - failed = True - # Avoid including exception text containing a presigned URL in durable output. - if isinstance(exc, urllib.error.HTTPError): - detail = f"HTTP {exc.code}" - else: - detail = str(exc) if isinstance(exc, TaskEntryError) else type(exc).__name__ - reason = f"{step}: {detail}" - try: - output.mkdir(parents=True, exist_ok=True) - result.write_text(json.dumps(failure_record(pin, reason)) + "\n") - except Exception: - print("task entry could not write failure result", file=sys.stderr) + write_benchmark_artifact(result, "engine-task-failure", failure_record(state["pin"], reason), replace=True) + except Exception: + print("task entry could not write failure result", file=sys.stderr) + return output + + +def _pack_result(work_root, output, readiness_budget, failed): step = "pack result" try: archive = work_root / "result.tar" @@ -166,6 +194,17 @@ def main(*, work_root=Path("/tmp"), worker_script=None, readiness_budget=60): except Exception as exc: detail = f"HTTP {exc.code}" if isinstance(exc, urllib.error.HTTPError) else type(exc).__name__ print(f"{step}: {detail}; dispatcher deadline remains authoritative", file=sys.stderr) + + +def main(*, work_root=Path("/tmp"), worker_script=None, readiness_budget=60): + state = {"pin": {}, "step": "prepare output"} + failed = False + try: + output, _result = _produce_result(work_root, worker_script, readiness_budget, state) + except Exception as error: + failed = True + output = _persist_failure(work_root, state, error) + _pack_result(work_root, output, readiness_budget, failed) return 0 diff --git a/benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py b/benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py index e10111f..c57bed2 100644 --- a/benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py +++ b/benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py @@ -17,6 +17,13 @@ import sys from pathlib import Path from benchmarks.unseen import score +from benchmarks.unseen.replay import load_slice +from benchmarks.valid.artifact_contract import write_benchmark_jsonl +from benchmarks.valid.hunt_handoff import ( + HUNT_RUNNER_PRODUCER, + discover_hunt_runner_context, + persist_hunt_runner_records, +) from openultrasast import stats from openultrasast.model.endpoint import _PRICES as PRICES @@ -130,12 +137,25 @@ def cost(log: Path) -> dict | None: def main(root: Path) -> dict: out: dict = {} all_suspicions = [] + repo = Path(__file__).resolve().parents[3] + manifest = repo / "benchmarks/valid/train/pool-train.toml" + freeze = json.loads((repo / "benchmarks/valid/train/freeze-train.json").read_text()) + changes = load_slice(manifest, "1") for arm, folder in (("dspy", "dspy-1"), ("hunter", "hunter-1")): - records = score.load_records(root / folder) + source = root / folder + destination = root / "benchmark-v2" / arm + context = discover_hunt_runner_context( + source, + manifest, + producer=HUNT_RUNNER_PRODUCER, + freeze_digest=freeze["freeze_digest"], + ) + persist_hunt_runner_records(source, destination, context=context, changes=changes) + records = score.load_records(destination) out[arm], suspicions = arm_summary(records, arm) all_suspicions.extend(suspicions) out["dspy"]["cost"] = cost(root / "dspy-1.log") - (root / "suspicions.jsonl").write_text("".join(json.dumps(s) + "\n" for s in all_suspicions)) + write_benchmark_jsonl(root / "suspicions.jsonl", "access-control-suspicion", all_suspicions, replace=True) return out diff --git a/benchmarks/unseen/calibrate.py b/benchmarks/unseen/calibrate.py index 2bf9d83..d73c060 100644 --- a/benchmarks/unseen/calibrate.py +++ b/benchmarks/unseen/calibrate.py @@ -12,9 +12,9 @@ import statistics from datetime import date from pathlib import Path -from benchmarks.ax.batch import write_json from benchmarks.unseen.replay import canonical, load_slice, run_replays, sample_changes from benchmarks.unseen.score import coverage, load_records +from benchmarks.valid.artifact_contract import write_benchmark_artifact def stage_times(record): @@ -120,7 +120,7 @@ def main(): result.update( image=args.image, deadline_seconds=args.deadline, sample_digest=hashlib.sha256(canonical([c["id"] for c in changes])).hexdigest() ) - write_json(args.record, result) + write_benchmark_artifact(args.record, "deadline-calibration", result, replace=True) return 0 if result["status"] == "ok" else 2 diff --git a/benchmarks/unseen/replay.py b/benchmarks/unseen/replay.py index 570abde..98c8a5b 100644 --- a/benchmarks/unseen/replay.py +++ b/benchmarks/unseen/replay.py @@ -14,6 +14,7 @@ from pathlib import Path from benchmarks.ax.batch import AXLane, DockerLane, Item, Workload, dispatch, write_json from benchmarks.unseen.score import coverage +from benchmarks.valid.artifact_contract import read_benchmark_artifact, write_benchmark_sequence def make_item(change, arm, image, *, deadline_scale=1.0, lane="ax", deadline=None): @@ -180,7 +181,7 @@ def main(): changes = load_slice(args.manifest, args.slice, args.private_manifest) factor = args.deadline_scale if args.calibration: - calibration = json.loads(args.calibration.read_text()) + calibration = read_benchmark_artifact(args.calibration, "deadline-calibration") if calibration.get("status") != "ok" or calibration.get("image") != args.image: parser.error("calibration must be successful and match the replay image") factor = calibration["quick_total"]["factor"] @@ -213,9 +214,11 @@ def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None): if out.resolve().is_relative_to(Path(__file__).resolve().parents[2]): raise ValueError("raw replay output must be outside the repository") items = [make_item(c, args.arm, args.image, deadline_scale=deadline_scale, lane=lane, deadline=deadline) for c in changes] - write_json( + write_benchmark_sequence( out / "inputs.json", + "benchmark-arm-inputs", [{"item_id": item.id, "change": change, "input_digest": item.input_digest} for item, change in zip(items, changes, strict=True)], + replace=True, ) maximum = max( max(json.loads(i.inputs["SPEC_URL"])["deadline_seconds"], json.loads(i.inputs["SPEC_URL"])["base_deadline_seconds"]) for i in items @@ -240,7 +243,7 @@ def run_replays(changes, args, out, lane, *, deadline_scale=1.0, deadline=None): return record executors[name] = execute - return dispatch(items, workload, out, executors, lanes=lanes) + return dispatch(items, workload, out, executors, lanes=lanes, record_kind="benchmark-arm-record") if __name__ == "__main__": diff --git a/benchmarks/unseen/score.py b/benchmarks/unseen/score.py index bc1932c..8c387e2 100644 --- a/benchmarks/unseen/score.py +++ b/benchmarks/unseen/score.py @@ -9,7 +9,13 @@ import math import random from collections import Counter, defaultdict from pathlib import Path +from typing import Any, cast +from benchmarks.valid.artifact_contract import ( + read_benchmark_artifact, + read_benchmark_sequence, + write_benchmark_artifact, +) from openultrasast.model.taxonomy import load_families from openultrasast.push.report import CAPABILITY_VETOES @@ -324,13 +330,17 @@ def gate(a, b, *, slice_id=None, informed=False, binary=True, threshold=None, th } -def load_records(path): +def load_records(path: Path) -> list[dict[str, Any]]: + """Load only canonical benchmark v2 records; producer handoffs are explicit adapters.""" + path = Path(path) if path.is_file(): - return json.loads(path.read_text()) - rows = [] - for entry in json.loads((path / "inputs.json").read_text()): + return [cast(dict[str, Any], row) for row in read_benchmark_sequence(path, "benchmark-arm-records")] + rows: list[dict[str, Any]] = [] + entries = read_benchmark_sequence(path / "inputs.json", "benchmark-arm-inputs") + for raw_entry in entries: + entry = cast(dict[str, Any], raw_entry) result = path / (hashlib.sha256(entry["item_id"].encode()).hexdigest() + ".json") - record = json.loads(result.read_text()) if result.exists() else {} + record: dict[str, Any] = cast(dict[str, Any], read_benchmark_artifact(result, "benchmark-arm-record")) if result.exists() else {} if record.get("input_digest") != entry["input_digest"]: record = {} rows.append({**entry["change"], **record, "id": entry["change"]["id"]}) @@ -356,7 +366,7 @@ def main(): result["unseen"] = gate(load_records(args.against) if args.against else [], current, slice_id=args.slice_id, informed=args.informed) if native and result["flagged"]: result["unseen"] = {"gate": "not_run", "reason": "native_agreement"} - args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n") + write_benchmark_artifact(args.out, "unseen-score", result, replace=True) if __name__ == "__main__": diff --git a/benchmarks/valid/admission_run.py b/benchmarks/valid/admission_run.py index dfeae8e..18989e2 100644 --- a/benchmarks/valid/admission_run.py +++ b/benchmarks/valid/admission_run.py @@ -15,34 +15,22 @@ mutates or re-labels the manifest it consumes. from __future__ import annotations -import hashlib -import json -import tomllib from collections.abc import Callable, Sequence from dataclasses import dataclass -from pathlib import Path from typing import Any -from benchmarks.unseen.replay import canonical -from benchmarks.valid.admission_scoring import AdmissionCase, OrdinaryChange, admission_report, score_v2_admission -from benchmarks.valid.arm_regression import BundleDigests, _input_digest, _prove_one +from benchmarks.valid.admission_scoring import AdmissionCase, OrdinaryChange, score_v2_admission from benchmarks.valid.population_custody import AdmissionSpendAuthority, GuardedPopulationCustodian, ReleaseReview -from benchmarks.valid.prover_admission import AdmissionManifest, validate_admission_manifest from openultrasast.core.admission_case import AdmissionCaseResult from openultrasast.core.admission_report import AdmissionReplay from openultrasast.core.admission_spend import authorize_admission -from openultrasast.preprocess import detect_language -from openultrasast.source import canonical_language, ranges __all__ = [ "admission_ordinary", "admission_pair_cases", - "assemble_admission_report", "coverage_share", "fixed_side_change", - "load_admission_population", "locate_function", - "run_admission", "run_v2_admission", "V2AdmissionRunInput", "site_judgment", @@ -225,156 +213,3 @@ def coverage_share(records: tuple[dict[str, Any], ...]) -> float: return 0.0 covered = sum(1 for total in per_repository.values() if total > 0) return covered / len(per_repository) - - -def load_admission_population( - payload: dict[str, object], pool_path: Path, freeze_path: Path -) -> tuple[tuple[dict[str, Any], ...], tuple[dict[str, Any], ...]]: - """Validate the manifest and the reviewed freeze before opening labels (Req 12.1).""" - manifest = validate_admission_manifest(payload) - pool = tomllib.loads(Path(pool_path).read_text(encoding="utf-8")) - freeze = json.loads(Path(freeze_path).read_text(encoding="utf-8")) - digest = hashlib.sha256(canonical(pool)).hexdigest() - if digest != freeze.get("freeze_digest"): - raise ValueError(f"population: pool digest {digest[:12]} does not match the reviewed freeze") - if freeze.get("guard_result") != 0: - raise ValueError("guard: held-out guard failed for this population") - if manifest.population_digest != digest: - raise ValueError("population: manifest names a different population than the reviewed freeze") - changes = tuple( - {**change, "id": _change_identity(repository, change), "repo": str(repository["id"])} - for repository in pool.get("repository", []) - for change in repository.get("changes", []) - ) - vulnerable = tuple(change for change in changes if change.get("kind") == "vulnerability") - ordinary = tuple(change for change in changes if change.get("kind") == "ordinary") - return vulnerable, ordinary - - -def _change_identity(repository: dict[str, Any], change: dict[str, Any]) -> str: - """The pool's canonical change identity, as ``replay.load_slice`` derives it.""" - payload = [repository["id"], change["kind"], change["base"], change["head"]] - return hashlib.sha256(canonical(payload)).hexdigest() - - -def assemble_admission_report( - manifest: AdmissionManifest, - vulnerable: tuple[dict[str, Any], ...], - fixed: tuple[dict[str, Any], ...], - ordinary: tuple[dict[str, Any], ...], - *, - criteria: dict[str, float] | None, - budget_seconds: float = 120.0, -) -> dict[str, Any]: - """Bridge measured records to the paired admission report (Reqs 12.7, 12.8). - - Every change is budget-gated: pairs by their combined seconds, ordinary - changes individually (design.md:370 "fails any change over 120 seconds"). - """ - failures = sum(1 for record in (*vulnerable, *fixed, *ordinary) if record.get("status") == "instrument_failure") - report = admission_report( - manifest, - admission_pair_cases(vulnerable, fixed), - admission_ordinary(ordinary), - coverage=coverage_share((*vulnerable, *fixed, *ordinary)), - instrument_failures=failures, - budget_seconds=budget_seconds, - criteria=criteria, - ) - overruns = [ - f"budget:{record['id']}:{float(record.get('seconds', 0.0)):g}s>{budget_seconds:g}s" - for record in ordinary - if float(record.get("seconds", 0.0)) > budget_seconds - ] - if overruns: - report["failed_gates"] = [*report["failed_gates"], *overruns] - report["decision"] = "NO-GO" - return report - - -def _resume_or_prove( - bundle: BundleDigests, - population: str, - change: dict[str, Any], - repo_path: Path, - side_dir: Path, -) -> dict[str, Any]: - from benchmarks.valid.prover_arm import _write_json - - side_dir.mkdir(parents=True, exist_ok=True) - path = side_dir / f"{change['id']}.json" - if path.is_file(): - previous: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) - if previous.get("input_digest") == _input_digest(change, ARM_NAME): - return previous - raise ValueError(f"existing record for {change['id']} does not match this arm's inputs") - record = _prove_one(bundle, ARM_NAME, population, change, repo_path) - _write_json(path, record) - return record - - -def run_admission( - *, - payload: dict[str, object], - pool_path: Path, - freeze_path: Path, - repos: dict[str, str], - bundle: BundleDigests, - out_dir: Path, - criteria: dict[str, float] | None, - budget_seconds: float = 120.0, -) -> dict[str, Any]: - """Consume the reviewed manifest, prove the population, return GO/NO-GO.""" - vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path) - manifest = validate_admission_manifest(payload) - out_dir.mkdir(parents=True, exist_ok=True) - vulnerable_records = tuple( - _resume_or_prove(bundle, manifest.population_digest, change, Path(repos[change["repo"]]).expanduser(), out_dir / "vulnerable") - for change in vulnerable - ) - fixed_records = tuple( - _resume_or_prove( - bundle, - manifest.population_digest, - _fixed_side_change(Path(repos[change["repo"]]).expanduser(), change), - Path(repos[change["repo"]]).expanduser(), - out_dir / "fixed", - ) - for change in vulnerable - ) - ordinary_records = tuple( - _resume_or_prove(bundle, manifest.population_digest, change, Path(repos[change["repo"]]).expanduser(), out_dir / "ordinary") - for change in ordinary - ) - return assemble_admission_report( - manifest, vulnerable_records, fixed_records, ordinary_records, criteria=criteria, budget_seconds=budget_seconds - ) - - -def _fixed_side_change(repo_path: Path, change: dict[str, Any]) -> dict[str, Any]: - lines, operation = _fixed_site(repo_path, change) - return fixed_side_change(change, fixed_lines=lines, fixed_operation=operation) - - -def _fixed_site(repo_path: Path, change: dict[str, Any]) -> tuple[tuple[int, int] | None, str]: - """(lines, operation) for the labelled function at the fixed revision. - - A locator miss (the name is present but ``ranges`` could not name it) is - "missed" and can never become a clean judgment (design.md:371). - """ - from benchmarks.valid.catalog import local_git - - git = local_git(repo_path) - text = git.runner( - ["git", "-C", str(repo_path), "show", f"{change['base']}:{change['path']}"], - capture_output=True, - text=True, - check=True, - ).stdout - name = str(change.get("function")) - try: - functions = ranges(str(change["path"]), text, canonical_language(detect_language(repo_path / change["path"]))) - except ValueError: - functions = () - lines = locate_function(functions, name=name) - return lines, classify_fixed_site(lines, name_in_text=name in text) diff --git a/benchmarks/valid/admission_scoring.py b/benchmarks/valid/admission_scoring.py index 5bc1580..e25885b 100644 --- a/benchmarks/valid/admission_scoring.py +++ b/benchmarks/valid/admission_scoring.py @@ -10,12 +10,13 @@ indeterminate and never converted to clean results; ordinary changes whose review found a real missing-authorization defect are refused before scoring. """ +from __future__ import annotations + from collections.abc import Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, Protocol from benchmarks.unseen import score -from benchmarks.valid.prover_admission import AdmissionManifest from openultrasast.core import canonical from openultrasast.core.admission_case import AdmissionCaseRelease, AdmissionCaseResult from openultrasast.core.admission_context import AdmissionValidationContext @@ -31,6 +32,18 @@ __all__ = ["AdmissionCase", "AdmissionScoringError", "OrdinaryChange", "admissio _JUDGMENTS = ("unguarded", "discharged", "indeterminate") +class AdmissionManifest(Protocol): + """Structural shape used only by the parked legacy scoring helpers.""" + + population_role: str + analyzer_revision: str + fact_set_digest: str + acquisition_record_digest: str + configuration_digest: str + population_digest: str + families: tuple[str, ...] + + class AdmissionScoringError(ValueError): """Raised when a population cannot be scored; the message names the reason.""" diff --git a/benchmarks/valid/arm_regression.py b/benchmarks/valid/arm_regression.py index c601a00..8415903 100644 --- a/benchmarks/valid/arm_regression.py +++ b/benchmarks/valid/arm_regression.py @@ -23,11 +23,18 @@ from typing import Any from benchmarks.unseen.extract import InstrumentFailure from benchmarks.unseen.replay import canonical +from benchmarks.valid.artifact_contract import read_benchmark_artifact, write_benchmark_artifact from benchmarks.valid.catalog import local_git from benchmarks.valid.development_runner import AnalyzerBundle, stage_funnel -from benchmarks.valid.prover_arm import _write_json -from openultrasast.core.legacy_v1 import InstrumentFailure as CoreInstrumentFailure -from openultrasast.core.legacy_v1 import check_reads +from openultrasast.core.coverage import ( + InputManifest, + InputReadReceipt, + derive_input_manifest_digest, + validate_instrument, +) +from openultrasast.core.coverage import ( + InstrumentFailure as CoreInstrumentFailure, +) from openultrasast.hunt.producers import Context, _Absent, _safe_path from openultrasast.hunt.prover import prove_change @@ -119,25 +126,8 @@ def _population_digest(changes: tuple[dict[str, Any], ...]) -> str: return hashlib.sha256(joined.encode()).hexdigest()[:12] -def _prove_one( - bundle: BundleDigests, - arm_name: str, - population: str, - change: dict[str, Any], - repo_path: Path, -) -> dict[str, Any]: - started = time.monotonic() - instrument: dict[str, Any] = dict( - hook_exit=0, - hook_bytes_read=0, - changed_bytes={}, - file_bytes={}, - head_verified=False, - base_verified=False, - suspect_fast=False, - failure=None, - ) - record: dict[str, Any] = { +def _arm_record(bundle: BundleDigests, arm_name: str, population: str, change: dict[str, Any], instrument: dict[str, Any]): + return { **change, "arm": arm_name, "analyzer_revision": bundle.analyzer_revision, @@ -157,40 +147,71 @@ def _prove_one( "stage_funnel": {}, "push": {"admission": {"defects": [], "dispositions": []}, "quick_tier": []}, } + + +def _apply_prover_result(record: dict[str, Any], result: Any, change: dict[str, Any], instrument: dict[str, Any]) -> None: + funnel = stage_funnel(result) + record.update( + findings=list(result.findings), + traces=list(result.traces), + coverage=asdict(result.coverage), + degradations=list(result.degradations), + stage_funnel=funnel, + ) + record["complexity"] = { + "regions": len(change.get("files", ())), + "calls": sum(funnel.get("resolution", {}).values()), + "changed_bytes": sum(instrument.get("changed_bytes", {}).values()), + } + record["push"]["admission"]["defects"] = [ + {**finding, "defect_id": finding["id"], "source": "prover", "locations": [[finding["path"], finding["line"]]]} + for finding in result.findings + ] + + +def _prove_checkout(record: dict[str, Any], change: dict[str, Any], repo_path: Path, instrument: dict[str, Any]) -> None: + git = local_git(repo_path) + git.run("cat-file", "-e", change["base"] + "^{commit}") + instrument["base_verified"] = True + with tempfile.TemporaryDirectory(prefix="ousast-arm-") as temporary: + root = Path(temporary) / "head" + git.run("-c", "core.hooksPath=/dev/null", "worktree", "add", "--detach", str(root), change["head"]) + try: + _verify_checkout(root) + ctx = Context(root, change) + ctx.verify_head() + instrument["head_verified"] = True + _instrument_reads(ctx, root, change, instrument) + result = prove_change(root, change, ctx, judgment_mode=True) + instrument["hook_bytes_read"] = sum(instrument["file_bytes"].values()) + _apply_prover_result(record, result, change, instrument) + finally: + git.run("worktree", "remove", "--force", str(root)) + + +def _prove_one( + bundle: BundleDigests, + arm_name: str, + population: str, + change: dict[str, Any], + repo_path: Path, +) -> dict[str, Any]: + started = time.monotonic() + instrument: dict[str, Any] = dict( + hook_exit=0, + hook_bytes_read=0, + changed_bytes={}, + changed_digests={}, + file_bytes={}, + file_digests={}, + head_verified=False, + base_verified=False, + suspect_fast=False, + failure=None, + ) + record = _arm_record(bundle, arm_name, population, change, instrument) try: - git = local_git(repo_path) - git.run("cat-file", "-e", change["base"] + "^{commit}") - instrument["base_verified"] = True - with tempfile.TemporaryDirectory(prefix="ousast-arm-") as temporary: - root = Path(temporary) / "head" - git.run("-c", "core.hooksPath=/dev/null", "worktree", "add", "--detach", str(root), change["head"]) - try: - _verify_checkout(root) - ctx = Context(root, change) - ctx.verify_head() - instrument["head_verified"] = True - _instrument_reads(ctx, root, change, instrument) - result = prove_change(root, change, ctx, judgment_mode=True) - instrument["hook_bytes_read"] = sum(instrument["file_bytes"].values()) - funnel = stage_funnel(result) - record.update( - findings=list(result.findings), - traces=list(result.traces), - coverage=asdict(result.coverage), - degradations=list(result.degradations), - stage_funnel=funnel, - ) - record["complexity"] = { - "regions": len(change.get("files", ())), - "calls": sum(funnel.get("resolution", {}).values()), - "changed_bytes": sum(instrument.get("changed_bytes", {}).values()), - } - record["push"]["admission"]["defects"] = [ - {**finding, "defect_id": finding["id"], "source": "prover", "locations": [[finding["path"], finding["line"]]]} - for finding in result.findings - ] - finally: - git.run("worktree", "remove", "--force", str(root)) + _prove_checkout(record, change, repo_path, instrument) except Exception as error: # noqa: BLE001 - the record must survive any instrument failure instrument.update(hook_exit=2, failure=type(error).__name__) record.update(status="instrument_failure", findings=[]) @@ -223,12 +244,19 @@ def _instrument_reads(ctx: Context, root: Path, change: dict[str, Any], instrume held = local_git(root).runner(["git", "-C", str(root), "cat-file", "-s", f"HEAD:{relative}"], capture_output=True, text=True) if getattr(error, "reason", "") == "empty_source" and held.returncode == 0 and int(held.stdout.strip() or 0) == 0: instrument["empty_files"] = instrument.get("empty_files", 0) + 1 - continue + instrument["file_bytes"][relative] = 0 + instrument["file_digests"][relative] = hashlib.sha256(b"").hexdigest() + if relative in changed: + instrument["changed_bytes"][relative] = 0 + instrument["changed_digests"][relative] = instrument["file_digests"][relative] + raise InstrumentFailure("zero-byte read") from error raise InstrumentFailure("unread tracked file") from error size = len(text.encode("utf-8")) instrument["file_bytes"][relative] = size + instrument["file_digests"][relative] = hashlib.sha256(text.encode("utf-8")).hexdigest() if relative in changed: instrument["changed_bytes"][relative] = size + instrument["changed_digests"][relative] = instrument["file_digests"][relative] if not size: raise InstrumentFailure("zero-byte read") @@ -248,13 +276,13 @@ def run_arm( for change in changes: path = out_dir / f"{change['id']}.json" if path.is_file(): - previous = json.loads(path.read_text(encoding="utf-8")) + previous = read_benchmark_artifact(path, "development-arm-record") if previous.get("input_digest") == _input_digest(change, arm_name): records.append(previous) continue raise ValueError(f"existing record for {change['id']} does not match this arm's inputs") record = _prove_one(bundle, arm_name, population, change, Path(repos[change["repo"]]).expanduser()) - _write_json(path, record) + write_benchmark_artifact(path, "development-arm-record", record) records.append(record) return tuple(records) @@ -263,17 +291,25 @@ def _masked(repo: str) -> str: return hashlib.sha256(repo.encode()).hexdigest()[:12] +def _validate_record_instrument(instrument: dict[str, Any]) -> None: + sizes = {**instrument.get("file_bytes", {}), **instrument.get("changed_bytes", {})} + digests = {**instrument.get("file_digests", {}), **instrument.get("changed_digests", {})} + input_ids = tuple(sorted(sizes)) + try: + manifest = InputManifest(input_ids, derive_input_manifest_digest(input_ids)) + receipts = tuple(InputReadReceipt(path, path, True, int(sizes[path]), digests.get(path)) for path in input_ids) + validate_instrument(manifest, receipts) + except (CoreInstrumentFailure, TypeError, ValueError) as error: + raise InstrumentFailure(str(error)) from error + + def summarize_arm(records: tuple[dict[str, Any], ...]) -> dict[str, Any]: """Aggregate one arm; refuse any zero-byte read, publish budget and masked coverage.""" judgment = {"unguarded": 0, "discharged": 0, "indeterminate": 0} per_repository: dict[str, dict[str, Any]] = {} for record in records: instrument = record.get("instrument", {}) - merged = {**instrument.get("file_bytes", {}), **instrument.get("changed_bytes", {})} - try: - check_reads(merged) # core instrument refusal: a zero-byte read invalidates the run - except CoreInstrumentFailure as error: - raise InstrumentFailure(str(error)) from error + _validate_record_instrument(instrument) funnel = record.get("stage_funnel", {}) for kind, count in funnel.get("judgment", {}).items(): judgment[kind] += int(count) diff --git a/benchmarks/valid/artifact_contract.py b/benchmarks/valid/artifact_contract.py new file mode 100644 index 0000000..b3b8f96 --- /dev/null +++ b/benchmarks/valid/artifact_contract.py @@ -0,0 +1,309 @@ +"""Strict benchmark-owned persistence for canonical v2 artifacts.""" + +from __future__ import annotations + +import fcntl +import json +import os +import tempfile +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from pathlib import Path +from typing import TypeAlias, cast + +from openultrasast.core.canonical import JsonValue, content_digest + +BENCHMARK_ARTIFACT_SCHEMA = "openultrasast.benchmark-artifact.v2" +ArtifactPayload: TypeAlias = Mapping[str, JsonValue] +ArtifactRows: TypeAlias = Sequence[Mapping[str, JsonValue]] +_RESERVED = frozenset({"schema_version", "artifact_kind", "artifact_digest"}) +_SEQUENCE_KEYS = frozenset({"artifact_mode", "sequence_count", "sequence_digest", "rows"}) +_JSONL_HEADER_KEYS = frozenset({"artifact_mode", "sequence_count", "sequence_digest"}) +_JSONL_ROW_KEYS = frozenset({"sequence_index", "sequence_count", "sequence_digest", "artifact"}) + + +def _canonical_value(value: object) -> JsonValue: + if value is None or isinstance(value, (str, bool, int, float)): + return value + if isinstance(value, Mapping): + return {str(key): _canonical_value(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return tuple(_canonical_value(item) for item in value) + raise ValueError(f"unsupported benchmark artifact value: {type(value).__name__}") + + +def _kind(kind: str) -> str: + if not isinstance(kind, str) or not kind: + raise ValueError("benchmark artifact kind must be nonempty text") + return kind + + +def _artifact_digest(kind: str, payload: Mapping[str, JsonValue]) -> str: + return content_digest( + { + "schema_version": BENCHMARK_ARTIFACT_SCHEMA, + "artifact_kind": kind, + "payload": _canonical_value(payload), + } + ) + + +def benchmark_artifact(kind: str, payload: ArtifactPayload) -> dict[str, JsonValue]: + """Create one v2 artifact from an unwrapped payload.""" + kind = _kind(kind) + if not isinstance(payload, Mapping): + raise ValueError("benchmark artifact payload must be a mapping") + if _RESERVED.intersection(payload): + raise ValueError("benchmark artifact payload uses reserved metadata") + value = {str(key): _canonical_value(item) for key, item in payload.items()} + return { + **value, + "schema_version": BENCHMARK_ARTIFACT_SCHEMA, + "artifact_kind": kind, + "artifact_digest": _artifact_digest(kind, value), + } + + +def benchmark_payload(value: object, *, expected_kind: str) -> dict[str, JsonValue]: + """Strictly validate one v2 artifact and return its domain payload.""" + expected_kind = _kind(expected_kind) + if not isinstance(value, dict): + raise ValueError("benchmark artifact metadata is missing") + if _RESERVED.intersection(value) != _RESERVED: + raise ValueError("benchmark artifact metadata is missing or incomplete") + raw = {str(key): item for key, item in value.items() if key not in _RESERVED} + canonical_raw = cast(dict[str, JsonValue], _canonical_value(raw)) + if ( + value["schema_version"] != BENCHMARK_ARTIFACT_SCHEMA + or value["artifact_kind"] != expected_kind + or value["artifact_digest"] != _artifact_digest(expected_kind, canonical_raw) + ): + raise ValueError("benchmark artifact metadata does not match payload") + return cast(dict[str, JsonValue], raw) + + +def encode_benchmark_artifact(kind: str, payload: ArtifactPayload, *, indent: int | None = 2) -> str: + """Encode one canonical v2 artifact.""" + return json.dumps(benchmark_artifact(kind, payload), sort_keys=True, indent=indent, ensure_ascii=False) + + +@contextmanager +def _target_lock(path: Path) -> Iterator[None]: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path.with_name(f".{path.name}.lock"), os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def _write_locked(path: Path, content: bytes, *, replace: bool) -> None: + if path.exists(): + existing = path.read_bytes() + if existing == content: + return + if not replace: + raise ValueError(f"benchmark artifact conflict: {path}") + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False) as temporary: + temporary_name = temporary.name + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_name, path) + temporary_name = None + finally: + if temporary_name is not None: + Path(temporary_name).unlink(missing_ok=True) + + +def _write_bytes(path: Path, content: bytes, *, replace: bool) -> None: + path = Path(path) + with _target_lock(path): + _write_locked(path, content, replace=replace) + + +def write_benchmark_artifact(path: Path, kind: str, payload: ArtifactPayload, *, replace: bool = False) -> None: + """Atomically create one artifact; identical writes are idempotent.""" + _write_bytes(path, (encode_benchmark_artifact(kind, payload) + "\n").encode(), replace=replace) + + +def read_benchmark_artifact(path: Path, expected_kind: str) -> dict[str, JsonValue]: + """Read and strictly validate one artifact from disk.""" + try: + value = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"benchmark artifact unreadable: {path}") from error + return benchmark_payload(value, expected_kind=expected_kind) + + +def _ordered_digest(artifacts: Sequence[Mapping[str, JsonValue]]) -> str: + return content_digest( + tuple( + {"sequence_index": index, "artifact_digest": cast(str, artifact["artifact_digest"])} for index, artifact in enumerate(artifacts) + ) + ) + + +def _sequence_artifact(kind: str, rows: ArtifactRows) -> dict[str, JsonValue]: + artifacts = tuple(benchmark_artifact(f"{kind}.item", row) for row in rows) + members: tuple[JsonValue, ...] = tuple({"sequence_index": index, "artifact": artifact} for index, artifact in enumerate(artifacts)) + return benchmark_artifact( + kind, + { + "artifact_mode": "ordered-sequence", + "sequence_count": len(artifacts), + "sequence_digest": _ordered_digest(artifacts), + "rows": members, + }, + ) + + +def write_benchmark_sequence(path: Path, kind: str, rows: ArtifactRows, *, replace: bool = False) -> None: + """Persist an exact ordered collection with count, indices, and aggregate digest.""" + value = _sequence_artifact(_kind(kind), rows) + content = (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False) + "\n").encode() + _write_bytes(path, content, replace=replace) + + +def _sequence_payload(value: object, kind: str) -> tuple[dict[str, JsonValue], ...]: + payload = benchmark_payload(value, expected_kind=kind) + if set(payload) != _SEQUENCE_KEYS or payload.get("artifact_mode") != "ordered-sequence": + raise ValueError("benchmark sequence metadata does not match contract") + count = payload.get("sequence_count") + sequence_digest = payload.get("sequence_digest") + members = payload.get("rows") + if ( + not isinstance(count, int) + or count < 0 + or not isinstance(sequence_digest, str) + or not isinstance(members, Sequence) + or isinstance(members, (str, bytes)) + ): + raise ValueError("benchmark sequence metadata is invalid") + if len(members) != count: + raise ValueError("benchmark sequence count mismatch") + rows: list[dict[str, JsonValue]] = [] + artifacts: list[dict[str, JsonValue]] = [] + for index, member in enumerate(members): + if not isinstance(member, dict) or set(member) != {"sequence_index", "artifact"} or member.get("sequence_index") != index: + raise ValueError("benchmark sequence index mismatch") + artifact = member.get("artifact") + rows.append(benchmark_payload(artifact, expected_kind=f"{kind}.item")) + artifacts.append(cast(dict[str, JsonValue], artifact)) + digests = tuple(cast(str, artifact["artifact_digest"]) for artifact in artifacts) + if len(set(digests)) != len(digests) or sequence_digest != _ordered_digest(artifacts): + raise ValueError("benchmark sequence digest mismatch or duplicate row") + return tuple(rows) + + +def read_benchmark_sequence(path: Path, expected_kind: str) -> tuple[dict[str, JsonValue], ...]: + """Read and validate a complete ordered collection.""" + try: + value = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"benchmark sequence unreadable: {path}") from error + return _sequence_payload(value, _kind(expected_kind)) + + +def _jsonl_bytes(kind: str, rows: ArtifactRows) -> bytes: + items = tuple(benchmark_artifact(f"{kind}.item", row) for row in rows) + sequence_digest = _ordered_digest(items) + header = benchmark_artifact( + f"{kind}.sequence", + {"artifact_mode": "ordered-jsonl", "sequence_count": len(items), "sequence_digest": sequence_digest}, + ) + encoded = [json.dumps(header, sort_keys=True, ensure_ascii=False)] + for index, artifact in enumerate(items): + encoded.append( + json.dumps( + benchmark_artifact( + f"{kind}.row", + { + "sequence_index": index, + "sequence_count": len(items), + "sequence_digest": sequence_digest, + "artifact": artifact, + }, + ), + sort_keys=True, + ensure_ascii=False, + ) + ) + return ("\n".join(encoded) + "\n").encode() + + +def _jsonl_rows(content: bytes, kind: str) -> tuple[dict[str, JsonValue], ...]: + try: + values = tuple(json.loads(line) for line in content.decode().splitlines()) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("benchmark JSONL is unreadable") from error + if not values: + raise ValueError("benchmark JSONL metadata is missing") + header = benchmark_payload(values[0], expected_kind=f"{kind}.sequence") + if set(header) != _JSONL_HEADER_KEYS or header.get("artifact_mode") != "ordered-jsonl": + raise ValueError("benchmark JSONL metadata does not match contract") + count = header.get("sequence_count") + sequence_digest = header.get("sequence_digest") + if not isinstance(count, int) or count < 0 or not isinstance(sequence_digest, str) or len(values) != count + 1: + raise ValueError("benchmark JSONL count mismatch") + rows: list[dict[str, JsonValue]] = [] + items: list[dict[str, JsonValue]] = [] + for index, value in enumerate(values[1:]): + member = benchmark_payload(value, expected_kind=f"{kind}.row") + if set(member) != _JSONL_ROW_KEYS: + raise ValueError("benchmark JSONL row metadata does not match contract") + if ( + member.get("sequence_index") != index + or member.get("sequence_count") != count + or member.get("sequence_digest") != sequence_digest + ): + raise ValueError("benchmark JSONL sequence binding mismatch") + artifact = member.get("artifact") + rows.append(benchmark_payload(artifact, expected_kind=f"{kind}.item")) + items.append(cast(dict[str, JsonValue], artifact)) + digests = tuple(cast(str, item["artifact_digest"]) for item in items) + if len(set(digests)) != len(digests) or sequence_digest != _ordered_digest(items): + raise ValueError("benchmark JSONL digest mismatch or duplicate row") + return tuple(rows) + + +def write_benchmark_jsonl(path: Path, kind: str, rows: ArtifactRows, *, replace: bool = False) -> None: + """Atomically persist an ordered JSONL manifest and its bound rows.""" + _write_bytes(path, _jsonl_bytes(_kind(kind), rows), replace=replace) + + +def read_benchmark_jsonl(path: Path, expected_kind: str) -> tuple[dict[str, JsonValue], ...]: + """Read and validate a complete ordered JSONL artifact.""" + try: + content = Path(path).read_bytes() + except OSError as error: + raise ValueError(f"benchmark JSONL unreadable: {path}") from error + return _jsonl_rows(content, _kind(expected_kind)) + + +def append_benchmark_jsonl(path: Path, kind: str, row: ArtifactPayload) -> None: + """Append one row under the target lock and rewrite the bound sequence atomically.""" + path = Path(path) + kind = _kind(kind) + with _target_lock(path): + rows = _jsonl_rows(path.read_bytes(), kind) if path.exists() else () + _write_locked(path, _jsonl_bytes(kind, (*rows, row)), replace=True) + + +__all__ = [ + "BENCHMARK_ARTIFACT_SCHEMA", + "append_benchmark_jsonl", + "benchmark_artifact", + "benchmark_payload", + "encode_benchmark_artifact", + "read_benchmark_artifact", + "read_benchmark_jsonl", + "read_benchmark_sequence", + "write_benchmark_artifact", + "write_benchmark_jsonl", + "write_benchmark_sequence", +] diff --git a/benchmarks/valid/development_runner.py b/benchmarks/valid/development_runner.py index b84fb25..d134721 100644 --- a/benchmarks/valid/development_runner.py +++ b/benchmarks/valid/development_runner.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, TypeVar if TYPE_CHECKING: from openultrasast.hunt.prover import ProverResult +from benchmarks.valid.artifact_contract import write_benchmark_artifact from benchmarks.valid.readiness_contract import require_stage_authorized from benchmarks.valid.run_manifests import AnalyzerBundle from openultrasast.core.readiness import AnalysisIdentity, ReadinessResult, ReadinessStage @@ -115,10 +116,7 @@ def run_development( seconds=seconds, ) records.append(record) - (out_dir / f"{record.change_id}.json").write_text( - json.dumps(asdict(record), indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + write_benchmark_artifact(out_dir / f"{record.change_id}.json", "development-observation", asdict(record)) return tuple(records) diff --git a/benchmarks/valid/hunt_handoff.py b/benchmarks/valid/hunt_handoff.py new file mode 100644 index 0000000..d8077c8 --- /dev/null +++ b/benchmarks/valid/hunt_handoff.py @@ -0,0 +1,306 @@ +"""Explicit, non-mutating handoff from the hunt runner to benchmark v2 storage.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, cast + +from benchmarks.unseen.replay import canonical +from benchmarks.valid.artifact_contract import ( + read_benchmark_artifact, + write_benchmark_artifact, + write_benchmark_sequence, +) +from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact +from openultrasast.hunt.runner import input_digest, record_path + +HUNT_RUNNER_PRODUCER = "openultrasast.hunt.runner.v1" +_RESERVED = frozenset({"schema_version", "artifact_kind", "artifact_digest"}) +_RECORD_REQUIRED = frozenset( + {"id", "done", "status", "instrument", "push", "rows", "program_id", "mask", "inputs_present", "seconds", "input_digest", "usd"} +) +_RECORD_OPTIONAL = frozenset({"parse_failed", "loop_parse_failed"}) + + +@dataclass(frozen=True) +class HuntRunContext: + producer: str + program_id: str + mask: tuple[str, ...] + manifest_digest: str + freeze_digest: str + only: str + slice_id: str + artifact_ids: tuple[str, ...] + versions: tuple[tuple[str, str], ...] + + +def _hex_digest(value: object, field: str) -> str: + if not isinstance(value, str) or len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError(f"hunt handoff {field} must be a full lowercase digest") + return value + + +def _validate_context(context: HuntRunContext) -> None: + if context.producer != HUNT_RUNNER_PRODUCER: + raise ValueError(f"unsupported hunt producer: {context.producer}") + _hex_digest(context.program_id, "program identity") + _hex_digest(context.manifest_digest, "manifest identity") + _hex_digest(context.freeze_digest, "freeze identity") + if tuple(sorted(set(context.mask))) != context.mask: + raise ValueError("hunt handoff mask must be sorted and unique") + if not context.only or not context.slice_id: + raise ValueError("hunt handoff runner context is incomplete") + if not context.artifact_ids or tuple(sorted(set(context.artifact_ids))) != context.artifact_ids: + raise ValueError("hunt handoff attempt identities must be nonempty, sorted, and unique") + for artifact_id in context.artifact_ids: + _hex_digest(artifact_id, "attempt identity") + if tuple(sorted(set(context.versions))) != context.versions: + raise ValueError("hunt handoff versions must be sorted and unique") + + +def _read_json(path: Path, label: str) -> object: + try: + data = path.read_bytes() + if not data: + raise ValueError(f"{label} is empty") + return json.loads(data) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{label} is unreadable") from error + + +def _has_reserved(value: object) -> bool: + if isinstance(value, Mapping): + return bool(_RESERVED.intersection(value)) or any(_has_reserved(item) for item in value.values()) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + return any(_has_reserved(item) for item in value) + return False + + +def _expected_entries(changes: Sequence[Mapping[str, Any]], context: HuntRunContext) -> tuple[dict[str, Any], ...]: + if not changes or len({change.get("id") for change in changes}) != len(changes): + raise ValueError("hunt handoff changes must be nonempty with unique identities") + return tuple( + { + "item_id": change["id"], + "change": dict(change), + "input_digest": input_digest(change, context.program_id, context.mask), + } + for change in changes + if context.only == "all" or change.get("kind") == context.only + ) + + +def _raw_entries(source: Path, expected: tuple[dict[str, Any], ...]) -> tuple[dict[str, Any], ...]: + value = _read_json(source / "inputs.json", "hunt input index") + if not isinstance(value, list) or _has_reserved(value): + raise ValueError("hunt input index has reserved metadata or an unsupported schema") + if not all(isinstance(row, dict) and set(row) == {"item_id", "change", "input_digest"} for row in value): + raise ValueError("hunt input index has an unsupported row shape") + rows = cast(tuple[dict[str, Any], ...], tuple(value)) + if rows != expected[: len(rows)]: + raise ValueError("hunt input identity, ordering, or input digest mismatch") + return rows + + +def _artifact_paths(source: Path, record_names: set[str]) -> tuple[Path, ...]: + return tuple(sorted(path for path in source.glob("*.json") if path.name != "inputs.json" and path.name not in record_names)) + + +def _artifact_projection(artifact: RunArtifact) -> tuple[object, ...]: + return ( + artifact.schema_version, + artifact.program_id, + tuple(artifact.mask), + tuple(sorted(artifact.data_digests.items())), + tuple(json.dumps(value, sort_keys=True) for value in artifact.configurations), + tuple(sorted(artifact.versions.items())), + ) + + +def _read_attempts(paths: Sequence[Path]) -> tuple[RunArtifact, ...]: + artifacts: list[RunArtifact] = [] + for path in paths: + try: + artifact = RunArtifact.read(path) + except (OSError, TypeError, ValueError) as error: + raise ValueError("hunt runner context artifact is invalid") from error + if path.stem != artifact.artifact_id: + raise ValueError("hunt runner context artifact filename does not match its identity") + artifacts.append(artifact) + if not artifacts: + raise ValueError("hunt runner context artifact collection is empty") + ordinals = [artifact.attempt_ordinal for artifact in artifacts] + if any(type(ordinal) is not int or not 0 < ordinal <= MAX_ATTEMPT_ORDINAL for ordinal in ordinals) or len(set(ordinals)) != len( + ordinals + ): + raise ValueError("hunt runner attempt ordinals must be positive and unique") + projection = _artifact_projection(artifacts[0]) + if any(_artifact_projection(artifact) != projection for artifact in artifacts[1:]): + raise ValueError("hunt runner attempt invariant mismatch") + return tuple(artifacts) + + +def _validate_run_artifact(source: Path, context: HuntRunContext, expected: tuple[dict[str, Any], ...]) -> None: + record_names = {record_path(source, cast(str, entry["item_id"])).name for entry in expected} + artifacts = _read_attempts(_artifact_paths(source, record_names)) + artifact = artifacts[0] + if tuple(sorted(item.artifact_id for item in artifacts)) != context.artifact_ids: + raise ValueError("hunt runner attempt identities do not match handoff context") + if artifact.schema_version != 1: + raise ValueError("hunt runner schema is unsupported") + if artifact.program_id != context.program_id or tuple(artifact.mask) != context.mask: + raise ValueError("hunt runner program identity does not match handoff context") + if artifact.data_digests != {"manifest": context.manifest_digest}: + raise ValueError("hunt runner manifest identity does not match handoff context") + expected_configuration = [{"only": context.only, "slice": context.slice_id, "changes": len(expected)}] + if artifact.configurations != expected_configuration: + raise ValueError("hunt runner configuration identity does not match handoff context") + if tuple(sorted(artifact.versions.items())) != context.versions: + raise ValueError("hunt runner version identity does not match handoff context") + + +def _raw_record(source: Path, entry: Mapping[str, Any], context: HuntRunContext) -> dict[str, Any] | None: + path = record_path(source, cast(str, entry["item_id"])) + if not path.exists(): + return None + value = _read_json(path, "hunt record") + if not isinstance(value, dict) or _has_reserved(value): + raise ValueError("hunt record uses reserved metadata or an unsupported schema") + keys = frozenset(value) + if not _RECORD_REQUIRED.issubset(keys) or not keys.issubset(_RECORD_REQUIRED | _RECORD_OPTIONAL): + raise ValueError("hunt record has an unsupported shape") + if value["id"] != entry["item_id"]: + raise ValueError("hunt record identity mismatch") + if value["input_digest"] != entry["input_digest"]: + raise ValueError("hunt record input digest mismatch") + if value["program_id"] != context.program_id or tuple(value["mask"]) != context.mask: + raise ValueError("hunt record program or configuration identity mismatch") + if value["done"] is not True or not isinstance(value["instrument"], dict) or not isinstance(value["push"], dict): + raise ValueError("hunt record completion schema is invalid") + return value + + +def read_hunt_runner_records( + source: Path, + *, + context: HuntRunContext, + changes: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, Any], ...]: + """Validate a named raw hunt run and return normalized rows without writing to it.""" + source = Path(source) + _validate_context(context) + expected = _expected_entries(changes, context) + _validate_run_artifact(source, context, expected) + entries = _raw_entries(source, expected) + rows: list[dict[str, Any]] = [] + for entry in entries: + record = _raw_record(source, entry, context) or {} + change = cast(dict[str, Any], entry["change"]) + rows.append({**change, **record, "id": change["id"]}) + return tuple(rows) + + +def _handoff_payload(context: HuntRunContext, entries: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + return { + **asdict(context), + "mask": list(context.mask), + "input_count": len(entries), + "input_sequence_digest": hashlib.sha256(canonical(entries)).hexdigest(), + } + + +def persist_hunt_runner_records( + source: Path, + destination: Path, + *, + context: HuntRunContext, + changes: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, Any], ...]: + """Copy a validated raw run into distinct benchmark-owned canonical v2 files.""" + source, destination = Path(source), Path(destination) + if source.resolve() == destination.resolve(): + raise ValueError("hunt producer and benchmark destination must be distinct") + rows = read_hunt_runner_records(source, context=context, changes=changes) + expected = _expected_entries(changes, context) + entries = _raw_entries(source, expected) + write_benchmark_artifact(destination / "handoff.json", "hunt-runner-handoff", _handoff_payload(context, entries)) + write_benchmark_sequence(destination / "inputs.json", "benchmark-arm-inputs", entries) + for entry in entries: + record = _raw_record(source, entry, context) + if record is not None: + write_benchmark_artifact(record_path(destination, cast(str, entry["item_id"])), "benchmark-arm-record", record) + return rows + + +def read_hunt_handoff_context(path: Path) -> HuntRunContext: + """Read the bound context of an already-normalized benchmark handoff.""" + payload = read_benchmark_artifact(Path(path) / "handoff.json", "hunt-runner-handoff") + context = HuntRunContext( + producer=cast(str, payload["producer"]), + program_id=cast(str, payload["program_id"]), + mask=tuple(cast(Sequence[str], payload["mask"])), + manifest_digest=cast(str, payload["manifest_digest"]), + freeze_digest=cast(str, payload["freeze_digest"]), + only=cast(str, payload["only"]), + slice_id=cast(str, payload["slice_id"]), + artifact_ids=tuple(cast(Sequence[str], payload["artifact_ids"])), + versions=tuple(tuple(cast(Sequence[str], row)) for row in cast(Sequence[Sequence[str]], payload["versions"])), + ) + _validate_context(context) + return context + + +def discover_hunt_runner_context( + source: Path, + manifest: Path, + *, + producer: str, + freeze_digest: str, +) -> HuntRunContext: + """Resolve the content-addressed runner identity for a documented raw-run handoff.""" + if producer != HUNT_RUNNER_PRODUCER: + raise ValueError(f"unsupported hunt producer: {producer}") + manifest_bytes = Path(manifest).read_bytes() + if not manifest_bytes: + raise ValueError("hunt handoff manifest is empty") + manifest_digest = hashlib.sha256(manifest_bytes).hexdigest() + paths: list[Path] = [] + for path in Path(source).glob("*.json"): + value = _read_json(path, "hunt runner candidate") + if isinstance(value, dict) and "artifact_id" in value: + paths.append(path) + artifacts = _read_attempts(tuple(sorted(paths))) + artifact = artifacts[0] + if len(artifact.configurations) != 1: + raise ValueError("hunt runner configuration artifact is ambiguous") + if artifact.data_digests != {"manifest": manifest_digest}: + raise ValueError("hunt runner manifest identity does not match supplied manifest") + configuration = artifact.configurations[0] + context = HuntRunContext( + producer=producer, + program_id=artifact.program_id, + mask=tuple(artifact.mask), + manifest_digest=manifest_digest, + freeze_digest=freeze_digest, + only=cast(str, configuration.get("only")), + slice_id=cast(str, configuration.get("slice")), + artifact_ids=tuple(sorted(item.artifact_id for item in artifacts)), + versions=tuple(sorted(artifact.versions.items())), + ) + _validate_context(context) + return context + + +__all__ = [ + "HUNT_RUNNER_PRODUCER", + "HuntRunContext", + "discover_hunt_runner_context", + "persist_hunt_runner_records", + "read_hunt_handoff_context", + "read_hunt_runner_records", +] diff --git a/benchmarks/valid/hunter_arm.py b/benchmarks/valid/hunter_arm.py index ae2e6ff..8c42c45 100644 --- a/benchmarks/valid/hunter_arm.py +++ b/benchmarks/valid/hunter_arm.py @@ -27,6 +27,12 @@ if __package__ in {None, ""}: from benchmarks.unseen.extract import SHA, InstrumentFailure from benchmarks.unseen.replay import canonical, load_slice from benchmarks.unseen.score import load_records +from benchmarks.valid.artifact_contract import ( + read_benchmark_artifact, + read_benchmark_sequence, + write_benchmark_artifact, + write_benchmark_sequence, +) from benchmarks.valid.catalog import local_git # These names describe findings, never benchmark labels. The taxonomy owns the @@ -154,11 +160,12 @@ def finding_defect(finding, classification=None): } -def _write_json(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(".json.tmp") - temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n") - temporary.replace(path) +def _write_json(path, value, *, kind=None, replace=False): + kind = kind or ("benchmark-arm-inputs" if Path(path).name == "inputs.json" else "benchmark-arm-record") + if isinstance(value, list): + write_benchmark_sequence(path, kind, value, replace=replace) + else: + write_benchmark_artifact(path, kind, value, replace=replace) def _record_path(out, identity): @@ -167,7 +174,7 @@ def _record_path(out, identity): def _inputs(out): path = out / "inputs.json" - entries = json.loads(path.read_text()) if path.exists() else [] + entries = list(read_benchmark_sequence(path, "benchmark-arm-inputs")) if path.exists() else [] for key in (lambda entry: entry["item_id"], lambda entry: entry["change"]["id"]): if len({key(entry) for entry in entries}) != len(entries): raise ValueError(f"duplicate ids in {path}") @@ -277,21 +284,13 @@ def _hunt_change(change, repos, *, client, model, max_steps, no_classify=False): return record -def run( - manifest, repos_path, out, *, slice_id="1", max_steps=tool_hunter.DEFAULT_MAX_STEPS, only="all", client_mode=None, no_classify=False -): - if max_steps < 1: - raise ValueError("max_steps must be positive") - changes = load_slice(manifest, slice_id) - if len({change["id"] for change in changes}) != len(changes): - raise ValueError("duplicate change ids") - changes = [change for change in changes if only == "all" or change["kind"] == only] +def _pending_hunter_entries(changes, out): entries = _inputs(out) indexed = {entry["change"]["id"]: entry for entry in entries} pending = [] for change in changes: path = _record_path(out, change["id"]) - previous = json.loads(path.read_text()) if path.exists() else {} + previous = read_benchmark_artifact(path, "benchmark-arm-record") if path.exists() else {} entry = indexed.get(change["id"]) if entry is None: entry = { @@ -306,31 +305,51 @@ def run( print(f"{change['id']}: skipped (done)", flush=True) continue pending.append(entry) - if not pending: - _write_json(out / "inputs.json", entries) - return 0 + return entries, pending + + +def _hunter_client(client_mode, out): config_path = Path("openultrasast.toml") config = load_config(config_path if config_path.exists() else None, dotenv=client_mode != "scripted") model = resolve_models(config)[0] client = tool_hunter.scripted_hunter_client() if client_mode == "scripted" else tool_hunter.resolve_hunter_client() if client is None: print("No hunter client resolved. Configure a hunter provider or use --client scripted for offline tests.", file=sys.stderr) - return 2 - repos = json.loads(repos_path.read_text()) - _write_json(out / "inputs.json", entries) + return client, model + + +def _run_hunter_entries(pending, repos, out, client, model, client_mode, max_steps, no_classify): failed = False for entry in pending: - # The built-in script is a finite sequence, so each change needs a fresh - # sequence. Live clients remain shared and are resolved only once. - if client_mode == "scripted": - client = tool_hunter.scripted_hunter_client() - record = _hunt_change(entry["change"], repos, client=client, model=model, max_steps=max_steps, no_classify=no_classify) + current_client = tool_hunter.scripted_hunter_client() if client_mode == "scripted" else client + record = _hunt_change(entry["change"], repos, client=current_client, model=model, max_steps=max_steps, no_classify=no_classify) record["input_digest"] = entry["input_digest"] _write_json(_record_path(out, entry["item_id"]), record) failed |= record["status"] == "instrument_failure" return 2 if failed else 0 +def run( + manifest, repos_path, out, *, slice_id="1", max_steps=tool_hunter.DEFAULT_MAX_STEPS, only="all", client_mode=None, no_classify=False +): + if max_steps < 1: + raise ValueError("max_steps must be positive") + changes = load_slice(manifest, slice_id) + if len({change["id"] for change in changes}) != len(changes): + raise ValueError("duplicate change ids") + changes = [change for change in changes if only == "all" or change["kind"] == only] + entries, pending = _pending_hunter_entries(changes, out) + if not pending: + _write_json(out / "inputs.json", entries, kind="benchmark-arm-inputs", replace=True) + return 0 + client, model = _hunter_client(client_mode, out) + if client is None: + return 2 + repos = json.loads(repos_path.read_text()) + _write_json(out / "inputs.json", entries, kind="benchmark-arm-inputs", replace=True) + return _run_hunter_entries(pending, repos, out, client, model, client_mode, max_steps, no_classify) + + def _joined(out): _inputs(out) rows = load_records(out) @@ -393,7 +412,7 @@ def combine(replay_out, hunter_out, out, *, keep_verdicts=None): "b": right.get("status", "missing") if has_b else "missing", } combined.append(merged) - _write_json(out, combined) + _write_json(out, combined, kind="benchmark-arm-records", replace=True) return combined diff --git a/benchmarks/valid/judge_pass.py b/benchmarks/valid/judge_pass.py index c66890d..b8246ce 100644 --- a/benchmarks/valid/judge_pass.py +++ b/benchmarks/valid/judge_pass.py @@ -19,6 +19,14 @@ import json import time from pathlib import Path +from benchmarks.valid.artifact_contract import ( + append_benchmark_jsonl, + encode_benchmark_artifact, + read_benchmark_artifact, + read_benchmark_jsonl, + write_benchmark_artifact, +) + JUDGE_QUESTION = ( "A static analyzer flagged this code path as a possible access-control " "vulnerability: an attacker-reachable entry point reaches a protected " @@ -51,13 +59,13 @@ def _judge_input(repo: str, finding: dict) -> str: "operation": finding.get("operation", {}), "missing": finding.get("missing"), } - return JUDGE_QUESTION + "\n\n" + json.dumps(payload, indent=2, default=str) + return JUDGE_QUESTION + "\n\n" + encode_benchmark_artifact("judge-pass-input", payload) def _iter_findings(prover_out: Path): """Every finding of every completed prover record, with its push context.""" for path in sorted(prover_out.glob("[0-9a-f]*.json")): - record = json.loads(path.read_text()) + record = read_benchmark_artifact(path, "benchmark-arm-record") if not record.get("done"): continue for finding in record.get("findings") or []: @@ -68,20 +76,19 @@ def _load_done(out: Path) -> dict: done = {} judged = out / "judgments.jsonl" if judged.is_file(): - for line in judged.read_text().splitlines(): - if line.strip(): - row = json.loads(line) - done[row["key"]] = row + for row in read_benchmark_jsonl(judged, "judge-pass"): + key = row["key"] + if key in done: + raise ValueError(f"duplicate judge key: {key}") + done[key] = row return done -def run(args) -> int: +def _judge_program(out: Path): import dspy from openultrasast.hunt.program import build_lm - out = Path(args.out) - out.mkdir(parents=True, exist_ok=True) lm = build_lm(cache_dir=out / "dspy-cache") dspy.configure(lm=lm, adapter=dspy.ChatAdapter()) @@ -91,52 +98,59 @@ def run(args) -> int: case: str = dspy.InputField() answer: str = dspy.OutputField() - judge = dspy.Predict(Judge) - done = _load_done(out) - judgments_path = out / "judgments.jsonl" + return dspy, lm, dspy.Predict(Judge) - todo = 0 - failures = [] - started = time.monotonic() - for record, finding in _iter_findings(Path(args.prover_out)): - key = _finding_key(record["id"], finding.get("id") or "") - if key in done: + +def _judgment_payload(record, finding, model): + return { + "key": _finding_key(record["id"], finding.get("id") or ""), + "record_id": record["id"], + "repo": record["repo"], + "kind": record.get("kind"), + "finding_id": finding.get("id"), + "path": finding.get("path"), + "line": finding.get("line"), + "model": model, + } + + +def _apply_judgment(payload, judge, record, finding): + result = judge(case=_judge_input(record["repo"], finding)) + text = (result.answer or "").strip() + body = text.strip("`") + if body.lower().startswith("json"): + body = body[4:] + parsed = json.loads(body[body.index("{") : body.rindex("}") + 1]) + if not isinstance(parsed.get("public"), bool): + raise ValueError("public is not a boolean") + payload.update(public=parsed["public"], confidence=parsed.get("confidence"), reason=parsed.get("reason"), status="ok") + + +def _judge_findings(prover_out, out, lm, judge, done): + todo, failures = 0, [] + for record, finding in _iter_findings(prover_out): + payload = _judgment_payload(record, finding, lm.model) + if payload["key"] in done: continue todo += 1 - payload = { - "key": key, - "record_id": record["id"], - "repo": record["repo"], - "kind": record.get("kind"), - "finding_id": finding.get("id"), - "path": finding.get("path"), - "line": finding.get("line"), - "model": lm.model, - } try: - result = judge(case=_judge_input(record["repo"], finding)) - text = (result.answer or "").strip() - # Tolerate a fenced JSON block, then parse strictly. - body = text.strip("`") - if body.lower().startswith("json"): - body = body[4:] - parsed = json.loads(body[body.index("{") : body.rindex("}") + 1]) - if not isinstance(parsed.get("public"), bool): - raise ValueError("public is not a boolean") - payload["public"] = parsed["public"] - payload["confidence"] = parsed.get("confidence") - payload["reason"] = parsed.get("reason") - payload["status"] = "ok" - except Exception as error: # noqa: BLE001 (recorded, never swallowed silently) - payload["status"] = "judge_error" - payload["error_type"] = type(error).__name__ - payload["error"] = str(error)[:500] + _apply_judgment(payload, judge, record, finding) + except Exception as error: # noqa: BLE001 + payload.update(status="judge_error", error_type=type(error).__name__, error=str(error)[:500]) failures.append(payload) - with judgments_path.open("a") as handle: - handle.write(json.dumps(payload, default=str) + "\n") + append_benchmark_jsonl(out / "judgments.jsonl", "judge-pass", payload) if not todo % 25: print(f"{todo} judged, {len(failures)} failures", flush=True) + return todo, failures + +def run(args) -> int: + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + dspy, lm, judge = _judge_program(out) + done = _load_done(out) + started = time.monotonic() + todo, failures = _judge_findings(Path(args.prover_out), out, lm, judge, done) seconds = time.monotonic() - started meta = { "model": lm.model, @@ -147,7 +161,7 @@ def run(args) -> int: "seconds": round(seconds, 1), "note": "zero-shot: no examples, no thresholds; question fixed before any answer was seen", } - (out / "run.json").write_text(json.dumps(meta, indent=2, default=str) + "\n") + write_benchmark_artifact(out / "run.json", "judge-pass-run", meta, replace=True) print(json.dumps(meta)) return 2 if failures and todo else 0 @@ -203,10 +217,10 @@ def summarize(args) -> int: "headline": ( f"Judge over prover findings: keeps {tp_kept}/{len(vuln)} true flags, " f"drops {fa_dropped}/{fa_total} false flags " - f"(recall {rate(tp_kept, 23)*100:.1f}%, FA {(fa_total - fa_dropped) / 460 * 100:.1f}/100)" + f"(recall {rate(tp_kept, 23) * 100:.1f}%, FA {(fa_total - fa_dropped) / 460 * 100:.1f}/100)" ), } - (out / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n") + write_benchmark_artifact(out / "summary.json", "judge-pass-summary", summary, replace=True) print(json.dumps(summary, indent=2, default=str)) return 0 diff --git a/benchmarks/valid/metrics.py b/benchmarks/valid/metrics.py index 4216ff2..d6bfb8e 100644 --- a/benchmarks/valid/metrics.py +++ b/benchmarks/valid/metrics.py @@ -7,6 +7,7 @@ import json from pathlib import Path from benchmarks.unseen import score +from benchmarks.valid.artifact_contract import read_benchmark_artifact, write_benchmark_artifact def demonstrated(proof: dict) -> bool: @@ -104,10 +105,10 @@ def main(argv=None): args = parser.parse_args(argv) result = summarize( score.load_records(args.records), - scored=json.loads(args.scored.read_text()) if args.scored else None, + scored=read_benchmark_artifact(args.scored, "unseen-score") if args.scored else None, proofs=json.loads(args.proofs.read_text()) if args.proofs else [], ) - args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n") + write_benchmark_artifact(args.out, "valid-benchmark-metrics", result, replace=True) print(result["headline"]) return 0 diff --git a/benchmarks/valid/prover_admission.py b/benchmarks/valid/prover_admission.py index f96e9c5..4b67d72 100644 --- a/benchmarks/valid/prover_admission.py +++ b/benchmarks/valid/prover_admission.py @@ -1,23 +1,7 @@ -"""Admission manifest consumption for the access-control prover arm. +"""Retired benchmark v1 admission seam. -Task 12.5 of the access-control-prover specification (Reqs 12.1, 12.2, 12.3, -12.6, 12.8). The admission runner only CONSUMES manifests that valid-detection-benchmark -produces and reviews; it never produces, mutates or re-labels one. - -Since detection-core task 4.1 this module is a thin compatibility wrapper: the -generalized, family-agnostic contract lives in ``openultrasast.core.admission``. +Live admission is exclusively ``admission_run.run_v2_admission``. Historical +v1 bytes remain readable only through ``openultrasast.core.legacy``. """ -from openultrasast.core.admission import ( - REPORT_SCHEMA_KEYS, - AdmissionManifest, - AdmissionManifestError, - validate_admission_manifest, -) - -__all__ = [ - "REPORT_SCHEMA_KEYS", - "AdmissionManifest", - "AdmissionManifestError", - "validate_admission_manifest", -] +__all__: list[str] = [] diff --git a/benchmarks/valid/prover_arm.py b/benchmarks/valid/prover_arm.py index d75a246..fa66ec5 100644 --- a/benchmarks/valid/prover_arm.py +++ b/benchmarks/valid/prover_arm.py @@ -16,16 +16,23 @@ from benchmarks.unseen import score from benchmarks.unseen.extract import SHA, InstrumentFailure from benchmarks.unseen.replay import canonical, load_slice from benchmarks.valid import metrics +from benchmarks.valid.artifact_contract import ( + read_benchmark_artifact, + read_benchmark_sequence, + write_benchmark_artifact, + write_benchmark_sequence, +) from benchmarks.valid.catalog import local_git from openultrasast.hunt.producers import Context, _Absent, _safe_path from openultrasast.hunt.prover import prove_change -def _write_json(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(".json.tmp") - temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n") - temporary.replace(path) +def _write_json(path, value, *, kind=None, replace=False): + kind = kind or ("benchmark-arm-inputs" if Path(path).name == "inputs.json" else "benchmark-arm-record") + if isinstance(value, list): + write_benchmark_sequence(path, kind, value, replace=replace) + else: + write_benchmark_artifact(path, kind, value, replace=replace) def _record_path(out, identity): @@ -144,7 +151,7 @@ def run(manifest, repos_path, out, *, slice_id="1", only="all", max_depth=3): raise ValueError("duplicate change ids") changes = [change for change in changes if only == "all" or change["kind"] == only] index = out / "inputs.json" - entries = json.loads(index.read_text()) if index.exists() else [] + entries = list(read_benchmark_sequence(index, "benchmark-arm-inputs")) if index.exists() else [] if len({entry["item_id"] for entry in entries}) != len(entries): raise ValueError("duplicate inputs") indexed = {entry["item_id"]: entry for entry in entries} @@ -158,14 +165,14 @@ def run(manifest, repos_path, out, *, slice_id="1", only="all", max_depth=3): if not previous_entry: entries.append(entry) path = _record_path(out, change["id"]) - previous = json.loads(path.read_text()) if path.exists() else {} + previous = read_benchmark_artifact(path, "benchmark-arm-record") if path.exists() else {} if previous and previous.get("input_digest") != digest: raise ValueError("existing prover inputs do not match") if previous.get("done"): print(f"{change['id']}: skipped (done)", flush=True) else: pending.append(entry) - _write_json(index, entries) + _write_json(index, entries, kind="benchmark-arm-inputs", replace=True) repos = json.loads(repos_path.read_text()) failed = False for entry in pending: @@ -176,84 +183,101 @@ def run(manifest, repos_path, out, *, slice_id="1", only="all", max_depth=3): return 2 if failed else 0 -def summarize(out, *, seed=0): - out = Path(out) - records = score.load_records(out) +def _summary_records(out): + records = score.load_records(Path(out)) if not records or len({row["id"] for row in records}) != len(records): raise ValueError("missing or duplicate prover records") for record in records: instrument = record.get("instrument", {}) if any(size == 0 for field in ("changed_bytes", "file_bytes") for size in instrument.get(field, {}).values()): raise ValueError("zero-byte read of a tracked file; summary refused") - result = metrics.summarize(records, seed=seed) - usable = [row for row in records if score.coverage(row) != "instrument_failure"] - locations = [ + return records + + +def _target_locations(usable): + return [ bool(score.catches({**row, **target, "files": [{"path": target["path"], "head_lines": [target["function_lines"]]}]})) for row in usable if row["kind"] == "vulnerability" for target in row.get("targets") or [row] ] - repositories = {} - framework_rows = {} + + +def _coverage_slot(): fields = ("entry_points", "reached", "unreached", "operations_checked", "operations_discharged") + return { + **dict.fromkeys(fields, 0), + "files_by_language": Counter(), + "frameworks_recognised": Counter(), + "unchecked_by_reason": Counter(), + "changes": 0, + } + + +def _add_coverage(target, coverage, fields): + target["changes"] += 1 + for field in fields: + target[field] += coverage.get(field, 0) + for field in ("files_by_language", "frameworks_recognised", "unchecked_by_reason"): + if field in target: + target[field].update(coverage.get(field, {})) + + +def _coverage_tables(records): + fields = ("entry_points", "reached", "unreached", "operations_checked", "operations_discharged") + repositories, framework_rows = {}, {} for row in records: - cov = row.get("coverage", {}) - repository = repositories.setdefault( - row["repo"], - { - **dict.fromkeys(fields, 0), - "files_by_language": Counter(), - "frameworks_recognised": Counter(), - "unchecked_by_reason": Counter(), - "changes": 0, - }, - ) - repository["changes"] += 1 - for field in fields: - repository[field] += cov.get(field, 0) - for field in ("files_by_language", "frameworks_recognised", "unchecked_by_reason"): - repository[field].update(cov.get(field, {})) - for framework, count in cov.get("frameworks_recognised", {}).items(): + coverage = row.get("coverage", {}) + _add_coverage(repositories.setdefault(row["repo"], _coverage_slot()), coverage, fields) + for framework, count in coverage.get("frameworks_recognised", {}).items(): table = framework_rows.setdefault( framework, {"files": 0, "changes": 0, **dict.fromkeys(fields, 0), "unchecked_by_reason": Counter()} ) table["files"] += count - table["changes"] += 1 - for field in fields: - table[field] += cov.get(field, 0) - table["unchecked_by_reason"].update(cov.get("unchecked_by_reason", {})) - uncovered = sorted(repo for repo, cov in repositories.items() if not cov["frameworks_recognised"]) - for repo, cov in repositories.items(): - cov["status"] = "uncovered" if repo in uncovered else "covered" + _add_coverage(table, coverage, fields) + return repositories, framework_rows + + +def _annotate_repositories(repositories, usable): + uncovered = sorted(repo for repo, coverage in repositories.items() if not coverage["frameworks_recognised"]) + for repo, coverage in repositories.items(): + coverage["status"] = "uncovered" if repo in uncovered else "covered" positives = [row for row in usable if row["repo"] == repo and row["kind"] == "vulnerability"] - cov["recall"] = score.rate([bool(score.catches(row)) for row in metrics.scoring_records(positives)]) + coverage["recall"] = score.rate([bool(score.catches(row)) for row in metrics.scoring_records(positives)]) + return uncovered + + +def _scope_recall(usable, uncovered, *, include_uncovered): + values = [ + bool(score.catches(row)) + for row in metrics.scoring_records(usable) + if row["kind"] == "vulnerability" and ((row["repo"] in uncovered) is include_uncovered) + ] + return score.rate(values) + + +def summarize(out, *, seed=0): + out = Path(out) + records = _summary_records(out) + result = metrics.summarize(records, seed=seed) + usable = [row for row in records if score.coverage(row) != "instrument_failure"] + repositories, framework_rows = _coverage_tables(records) + uncovered = _annotate_repositories(repositories, usable) seconds = [row["seconds"] for row in records] result.update( - location_recall=score.rate(locations), + location_recall=score.rate(_target_locations(usable)), per_repository=repositories, per_framework=framework_rows, uncovered_repositories=uncovered, - covered_recall=score.rate( - [ - bool(score.catches(row)) - for row in metrics.scoring_records(usable) - if row["kind"] == "vulnerability" and row["repo"] not in uncovered - ] - ), - uncovered_recall=score.rate( - [ - bool(score.catches(row)) - for row in metrics.scoring_records(usable) - if row["kind"] == "vulnerability" and row["repo"] in uncovered - ] - ), + covered_recall=_scope_recall(usable, uncovered, include_uncovered=False), + uncovered_recall=_scope_recall(usable, uncovered, include_uncovered=True), instrument_failures=sum(score.coverage(row) == "instrument_failure" for row in records), seconds_per_change=dict(n=len(seconds), mean=statistics.mean(seconds), median=statistics.median(seconds), maximum=max(seconds)), changes_over_120_seconds=[{"id": row["id"], "seconds": row["seconds"]} for row in records if row["seconds"] > 120], ) # This is a training pool; the generic metric headline refers to unseen repositories. result["headline"] = result["headline"].replace("On unseen app repos:", "On the training pool:") - _write_json(out / "summary.json", result) + _write_json(out / "summary.json", result, kind="benchmark-arm-summary", replace=True) return result diff --git a/benchmarks/valid/semgrep_arm.py b/benchmarks/valid/semgrep_arm.py index 8775b1e..19404dc 100644 --- a/benchmarks/valid/semgrep_arm.py +++ b/benchmarks/valid/semgrep_arm.py @@ -35,6 +35,12 @@ if __package__ in {None, ""}: from benchmarks.unseen.extract import SHA, InstrumentFailure from benchmarks.unseen.replay import canonical, load_slice +from benchmarks.valid.artifact_contract import ( + read_benchmark_artifact, + read_benchmark_sequence, + write_benchmark_artifact, + write_benchmark_sequence, +) from benchmarks.valid.catalog import local_git # Our language ids to Semgrep's. Groovy has no Semgrep parser: `generic` plus a path filter. @@ -427,11 +433,12 @@ def result_defect(result): } -def _write_json(path, value): - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_suffix(".json.tmp") - temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n") - temporary.replace(path) +def _write_json(path, value, *, kind=None, replace=False): + kind = kind or ("benchmark-arm-inputs" if Path(path).name == "inputs.json" else "benchmark-arm-record") + if isinstance(value, list): + write_benchmark_sequence(path, kind, value, replace=replace) + else: + write_benchmark_artifact(path, kind, value, replace=replace) def _record_path(out, identity): @@ -440,7 +447,7 @@ def _record_path(out, identity): def _inputs(out): path = out / "inputs.json" - entries = json.loads(path.read_text()) if path.exists() else [] + entries = list(read_benchmark_sequence(path, "benchmark-arm-inputs")) if path.exists() else [] for key in (lambda entry: entry["item_id"], lambda entry: entry["change"]["id"]): if len({key(entry) for entry in entries}) != len(entries): raise ValueError(f"duplicate ids in {path}") @@ -568,7 +575,7 @@ def run(manifest, repos_path, rules, out, *, slice_id="1", only="all", semgrep=N pending = [] for change in changes: path = _record_path(out, change["id"]) - previous = json.loads(path.read_text()) if path.exists() else {} + previous = read_benchmark_artifact(path, "benchmark-arm-record") if path.exists() else {} entry = indexed.get(change["id"]) if entry is None: entry = { @@ -583,7 +590,7 @@ def run(manifest, repos_path, rules, out, *, slice_id="1", only="all", semgrep=N print(f"{change['id']}: skipped (done)", flush=True) continue pending.append(entry) - _write_json(out / "inputs.json", entries) + _write_json(out / "inputs.json", entries, kind="benchmark-arm-inputs", replace=True) if not pending: return 0 repos = json.loads(Path(repos_path).expanduser().read_text()) diff --git a/src/openultrasast/hunt/artifact.py b/src/openultrasast/hunt/artifact.py index f25a1ed..d2d2695 100644 --- a/src/openultrasast/hunt/artifact.py +++ b/src/openultrasast/hunt/artifact.py @@ -2,16 +2,113 @@ from __future__ import annotations +import errno import hashlib import json import math -from collections.abc import Mapping +import os +import tempfile +from collections.abc import Iterator, Mapping +from contextlib import contextmanager from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any from .table import canonical_json +# Persisted attempt sequences use the portable signed 64-bit integer domain. +MAX_ATTEMPT_ORDINAL = (1 << 63) - 1 +_UNSUPPORTED_DIRECTORY_SYNC = frozenset( + error for name in ("EINVAL", "ENOTSUP", "EOPNOTSUPP") if (error := getattr(errno, name, None)) is not None +) + + +@contextmanager +def _artifact_lock(directory: Path) -> Iterator[None]: + directory.mkdir(parents=True, exist_ok=True) + descriptor = os.open(directory / ".run-artifact.lock", os.O_CREAT | os.O_RDWR, 0o600) + try: + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def _sync_directory(directory: Path) -> None: + """Durably publish a directory entry where the platform supports it.""" + if os.name == "nt": + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + try: + descriptor = os.open(directory, flags) + except OSError as error: + if error.errno in _UNSUPPORTED_DIRECTORY_SYNC: + return + raise + try: + os.fsync(descriptor) + except OSError as error: + if error.errno not in _UNSUPPORTED_DIRECTORY_SYNC: + raise + finally: + os.close(descriptor) + + +def _persist_sequence(directory: Path, ordinal: int) -> None: + descriptor, temporary = tempfile.mkstemp(dir=directory, prefix=".attempt-sequence.", text=True) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(f"{ordinal}\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, directory / ".attempt-sequence") + _sync_directory(directory) + finally: + Path(temporary).unlink(missing_ok=True) + + +def _reserve_attempt_ordinal(directory: Path) -> int: + sequence = directory / ".attempt-sequence" + try: + content = sequence.read_text() + except FileNotFoundError: + if _valid_attempt_artifact_exists(directory): + raise ValueError("run attempt sequence is missing for existing artifacts; explicit recovery is required") from None + previous = 0 + except OSError as error: + raise ValueError("run attempt sequence is unreadable") from error + else: + try: + previous = int(content.strip()) + except ValueError as error: + raise ValueError("run attempt sequence is unreadable") from error + if previous < 0: + raise ValueError("run attempt sequence is invalid") + if previous >= MAX_ATTEMPT_ORDINAL: + raise ValueError("run attempt sequence is exhausted") + ordinal = previous + 1 + _persist_sequence(directory, ordinal) + return ordinal + + +def _create_only(path: Path, content: bytes) -> None: + if path.exists(): + if path.read_bytes() == content: + return + raise ValueError(f"run artifact conflict: {path}") + try: + with path.open("xb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + _sync_directory(path.parent) + except FileExistsError: + if path.read_bytes() != content: + raise ValueError(f"run artifact conflict: {path}") from None + def _exhausted(usage: Mapping[str, float], budget: Mapping[str, float]) -> bool: for name in ("usd", "calls"): @@ -34,23 +131,36 @@ class RunArtifact: usage: dict[str, float] versions: dict[str, str] created: str + attempt_ordinal: int = 0 @property def artifact_id(self) -> str: - return hashlib.sha256(canonical_json(asdict(self)).encode("utf-8")).hexdigest() + payload = asdict(self) + if not self.attempt_ordinal: + payload.pop("attempt_ordinal") + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() def finish(self, budget: Mapping[str, float]) -> RunArtifact | None: """Return no completed artifact when either registered budget is exhausted.""" return None if _exhausted(self.usage, budget) else self - def write(self, directory: Path | str, *, budget: Mapping[str, float] | None = None) -> Path | None: + def write( + self, + directory: Path | str, + *, + budget: Mapping[str, float] | None = None, + reserve_attempt: bool = False, + ) -> Path | None: """Write metadata under its content id, after an optional final budget check.""" if budget is not None and self.finish(budget) is None: return None directory = Path(directory) - directory.mkdir(parents=True, exist_ok=True) - path = directory / f"{self.artifact_id}.json" - path.write_text(canonical_json({**asdict(self), "artifact_id": self.artifact_id}) + "\n", encoding="utf-8") + with _artifact_lock(directory): + if reserve_attempt and not self.attempt_ordinal: + self.attempt_ordinal = _reserve_attempt_ordinal(directory) + path = directory / f"{self.artifact_id}.json" + content = (canonical_json({**asdict(self), "artifact_id": self.artifact_id}) + "\n").encode() + _create_only(path, content) return path @classmethod @@ -58,12 +168,24 @@ class RunArtifact: payload = json.loads(Path(path).read_text(encoding="utf-8")) identifier = payload.pop("artifact_id", None) payload["mask"] = tuple(payload["mask"]) + payload.setdefault("attempt_ordinal", 0) artifact = cls(**payload) if identifier != artifact.artifact_id: raise ValueError("run artifact id does not match its contents") return artifact +def _valid_attempt_artifact_exists(directory: Path) -> bool: + for path in directory.glob("*.json"): + try: + artifact = RunArtifact.read(path) + except (OSError, TypeError, ValueError): + continue + if path.stem == artifact.artifact_id: + return True + return False + + @dataclass class UsageLedger: """Count paid calls, replay hits, spend and elapsed call time separately.""" diff --git a/src/openultrasast/hunt/report.py b/src/openultrasast/hunt/report.py index bc1819a..c0ece47 100644 --- a/src/openultrasast/hunt/report.py +++ b/src/openultrasast/hunt/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import fcntl +import hashlib import json import math import os @@ -40,11 +41,21 @@ def _reserve(root: Path, program_id: str) -> Path: return directory -def _records(directory: Path, changes: Sequence[Mapping]) -> list[dict]: +def _records( + directory: Path, + destination: Path, + changes: Sequence[Mapping], + *, + context: Any, +) -> list[dict]: """Load benchmark records and retain every expected change, even after a budget stop.""" from benchmarks.unseen import score + from benchmarks.valid.hunt_handoff import persist_hunt_runner_records - records = score.load_records(Path(directory)) if (Path(directory) / "inputs.json").exists() else [] + records = [] + if (Path(directory) / "inputs.json").exists(): + persist_hunt_runner_records(Path(directory), Path(destination), context=context, changes=changes) + records = score.load_records(Path(destination)) indexed = {r["id"]: r for r in records} if len(indexed) != len(records) or set(indexed) - {c["id"] for c in changes}: raise ValueError("records do not match the pilot population") @@ -57,6 +68,24 @@ def _records(directory: Path, changes: Sequence[Mapping]) -> list[dict]: return result +def _hunt_context(source: Path, manifest: Path, program_id: str, mask: Sequence[str]) -> Any: + from benchmarks.valid.hunt_handoff import HUNT_RUNNER_PRODUCER, discover_hunt_runner_context + + content = Path(manifest).read_bytes() + if not content: + raise ValueError("hunt manifest is empty") + manifest_digest = hashlib.sha256(content).hexdigest() + freeze = Path(manifest).with_name("freeze-" + Path(manifest).stem.removeprefix("pool-") + ".json") + freeze_digest = manifest_digest + if freeze.is_file(): + payload = json.loads(freeze.read_text()) + freeze_digest = str(payload.get("freeze_digest") or "") + context = discover_hunt_runner_context(source, manifest, producer=HUNT_RUNNER_PRODUCER, freeze_digest=freeze_digest) + if context.program_id != program_id or context.mask != tuple(sorted(set(mask))) or context.manifest_digest != manifest_digest: + raise ValueError("hunt report context does not match the requested program") + return context + + def _instrument(records: Sequence[Mapping]) -> dict: """Record changed-file bytes and verified revisions; unreadable and absent changes fail.""" from benchmarks.unseen import score @@ -368,6 +397,137 @@ def _notes(analysis: dict) -> str: return "\n".join(lines) +def _pilot_inputs( + candidate: Candidate, + decision: accept.Decision, + training_runs: Sequence[Path], + pilot_metrics: Mapping[str, Path], + pilot_manifest: Path, + load_changes: Any, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + from benchmarks.unseen.replay import load_slice + + if not decision.accepted or decision.comparison.get("candidate_id") != candidate.program_id: + raise ValueError("pilot gain requires the accepted decision for this candidate") + normalized_runs = [Path(path) for path in training_runs] + if contamination.clean_runs(normalized_runs) != normalized_runs: + raise ValueError("contaminated training runs cannot report pilot gain") + changes = list((load_changes or load_slice)(Path(pilot_manifest), "1")) + if not changes or len({change["id"] for change in changes}) != len(changes): + raise ValueError("pilot changes must be nonempty and have unique IDs") + return _before(pilot_metrics), changes + + +def _run_pilot_candidate( + candidate: Candidate, + pilot_manifest: Path, + repos: Mapping[str, str], + directory: Path, + lm: Any, + budget: Mapping[str, float], + tools_for: Any, + changes: list[dict[str, Any]], +) -> runner.RunArtifact | None: + import dspy + + from .program import repository_tools + + program = _candidate_program(candidate, tools_for) + with dspy.context(lm=lm, adapter=dspy.ChatAdapter()): + return runner.run( + Path(pilot_manifest), + repos, + directory / "records", + program=program, + program_id=candidate.program_id, + mask=candidate.mask, + pilot_report=True, + only="all", + budget=budget, + tools_for=tools_for or repository_tools, + load_changes=lambda _path, _slice: changes, + versions={"dspy": dspy.__version__}, + ) + + +def _pilot_baselines( + candidate: Candidate, + pilot_manifest: Path, + baseline_runs: Sequence[Path], + directory: Path, + changes: list[dict[str, Any]], +) -> list[list[dict[str, Any]]]: + return [ + _clean( + _records( + Path(path), + directory / "benchmark-baselines" / str(index), + changes, + context=_hunt_context(Path(path), Path(pilot_manifest), candidate.base_program_id, ()), + ) + ) + for index, path in enumerate(baseline_runs) + ] + + +def _successful_pilot_analysis( + analysis: dict[str, Any], + records: list[dict[str, Any]], + before: dict[str, Any], + baseline_records: list[list[dict[str, Any]]], +) -> tuple[list[dict[str, Any]], list[str]]: + from benchmarks.unseen import score + + records = _clean(records) + score.score(records) + families = _families([dict(change=record, after=_outcome(record, record, 10)) for record in records], ("after",)) + for baseline in baseline_records: + before_units = [dict(change=record, before=_outcome(record, record, 10)) for record in baseline] + families = sorted(set(families) | set(_families(before_units, ("before",)))) + curve, distance = _curve(records) + counts = [search_outcome(record, _findings(record["push"]["admission"]["defects"])).suspicions for record in records] + analysis.update( + before=before, + after=_summaries(records, families), + curve=curve, + gate_distance=distance, + workload={ + "suspicions_per_100_pushes": _rate(counts, [record["repo"] for record in records], scale=100), + "flagged_pushes_per_100_pushes": _rate( + [int(count > 0) for count in counts], [record["repo"] for record in records], binary=True, scale=100 + ), + }, + ) + return records, families + + +def _add_pilot_baselines( + analysis: dict[str, Any], + baseline_runs: Sequence[Path], + baseline_records: list[list[dict[str, Any]]], + records: list[dict[str, Any]], + families: list[str], +) -> None: + if not baseline_records: + return + analysis["baseline"] = [] + for path, baseline in zip(baseline_runs, baseline_records, strict=True): + instrument = _instrument(baseline) + entry = {"directory": str(path), "valid": instrument["valid"], "instrument": instrument} + if entry["valid"]: + entry["summary"] = _summaries(baseline, families) + analysis["baseline"].append(entry) + if all(row["valid"] for row in analysis["baseline"]): + analysis["paired"] = _baseline_comparison(baseline_records, records, families) + + +def _persist_pilot_analysis(directory: Path, analysis: dict[str, Any]) -> dict[str, Any]: + normalized: dict[str, Any] = json.loads(json.dumps(analysis)) + _write(directory / "analysis.json", normalized) + (directory / "NOTES.md").write_text(_notes(normalized), encoding="utf-8") + return normalized + + def pilot_gain( candidate: Candidate, decision: accept.Decision, @@ -390,40 +550,11 @@ def pilot_gain( line scorers or real model clients are constructed here. Budget exhaustion or more than five percent instrument failures leaves an invalid measurement without performance numbers. """ - import dspy - - from benchmarks.unseen import score - from benchmarks.unseen.replay import load_slice - - from .program import repository_tools - - if not decision.accepted or decision.comparison.get("candidate_id") != candidate.program_id: - raise ValueError("pilot gain requires the accepted decision for this candidate") - training_runs = [Path(p) for p in training_runs] - if contamination.clean_runs(training_runs) != training_runs: - raise ValueError("contaminated training runs cannot report pilot gain") - before = _before(pilot_metrics) - program = _candidate_program(candidate, tools_for) - changes = list((load_changes or load_slice)(Path(pilot_manifest), "1")) - if not changes or len({c["id"] for c in changes}) != len(changes): - raise ValueError("pilot changes must be nonempty and have unique IDs") + before, changes = _pilot_inputs(candidate, decision, training_runs, pilot_metrics, pilot_manifest, load_changes) directory = _reserve(Path(out_root), candidate.program_id) - with dspy.context(lm=lm, adapter=dspy.ChatAdapter()): - artifact = runner.run( - Path(pilot_manifest), - repos, - directory / "records", - program=program, - program_id=candidate.program_id, - mask=candidate.mask, - pilot_report=True, - only="all", - budget=budget, - tools_for=tools_for or repository_tools, - load_changes=lambda _path, _slice: changes, - versions={"dspy": dspy.__version__}, - ) - records = _records(directory / "records", changes) + artifact = _run_pilot_candidate(candidate, pilot_manifest, repos, directory, lm, budget, tools_for, changes) + context = _hunt_context(directory / "records", Path(pilot_manifest), candidate.program_id, candidate.mask) + records = _records(directory / "records", directory / "benchmark-records", changes, context=context) proof = _instrument(records) _write(directory / "instrument.json", proof) analysis: dict[str, Any] = dict( @@ -436,37 +567,7 @@ def pilot_gain( if not analysis["valid"]: analysis["reason"] = "more than 5% instrument failures" if not proof["valid"] else "runner budget exhausted" else: - records = _clean(records) - # Run the same overall instrument gate and scorer as the pilot before quoting any scores. - score.score(records) - families = _families([dict(change=r, after=_outcome(r, r, 10)) for r in records], ("after",)) - baseline_records = [_clean(_records(Path(path), changes)) for path in baseline_runs] - for baseline in baseline_records: - families = sorted(set(families) | set(_families([dict(change=r, before=_outcome(r, r, 10)) for r in baseline], ("before",)))) - curve, distance = _curve(records) - counts = [search_outcome(r, _findings(r["push"]["admission"]["defects"])).suspicions for r in records] - analysis.update( - before=before, - after=_summaries(records, families), - curve=curve, - gate_distance=distance, - workload=dict( - suspicions_per_100_pushes=_rate(counts, [r["repo"] for r in records], scale=100), - flagged_pushes_per_100_pushes=_rate([int(n > 0) for n in counts], [r["repo"] for r in records], binary=True, scale=100), - ), - ) - if baseline_records: - analysis["baseline"] = [] - for path, baseline in zip(baseline_runs, baseline_records, strict=True): - instrument = _instrument(baseline) - entry = dict(directory=str(path), valid=instrument["valid"], instrument=instrument) - if entry["valid"]: - entry["summary"] = _summaries(baseline, families) - analysis["baseline"].append(entry) - if all(row["valid"] for row in analysis["baseline"]): - analysis["paired"] = _baseline_comparison(baseline_records, records, families) - # Return the same JSON types that a reader of analysis.json receives. - analysis = json.loads(json.dumps(analysis)) - _write(directory / "analysis.json", analysis) - (directory / "NOTES.md").write_text(_notes(analysis), encoding="utf-8") - return analysis + baseline_records = _pilot_baselines(candidate, pilot_manifest, baseline_runs, directory, changes) + records, families = _successful_pilot_analysis(analysis, records, before, baseline_records) + _add_pilot_baselines(analysis, baseline_runs, baseline_records, records, families) + return _persist_pilot_analysis(directory, analysis) diff --git a/src/openultrasast/hunt/runner.py b/src/openultrasast/hunt/runner.py index 014dcf3..cb9abbe 100644 --- a/src/openultrasast/hunt/runner.py +++ b/src/openultrasast/hunt/runner.py @@ -14,7 +14,7 @@ import subprocess import sys import tempfile import time -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -219,6 +219,119 @@ def _function_at(root: Path, finding: Mapping[str, Any]) -> str | None: return None +def _run_entries(out: Path) -> tuple[Path, list[dict[str, Any]], dict[str, dict[str, Any]]]: + inputs_path = out / "inputs.json" + entries = json.loads(inputs_path.read_text()) if inputs_path.is_file() else [] + return inputs_path, entries, {entry["item_id"]: entry for entry in entries} + + +def _change_entry( + change: Mapping[str, Any], + program_id: str, + mask: tuple[str, ...], + entries: list[dict[str, Any]], + indexed: dict[str, dict[str, Any]], +) -> str: + digest = input_digest(change, program_id, mask) + entry = indexed.get(change["id"]) + if entry is None: + entry = {"item_id": change["id"], "change": change, "input_digest": digest} + entries.append(entry) + indexed[change["id"]] = entry + elif entry["input_digest"] != digest: + entry.update(change=change, input_digest=digest) + return digest + + +def _record_is_complete(out: Path, change_id: str, digest: str) -> bool: + path = record_path(out, change_id) + if not path.is_file(): + return False + previous = json.loads(path.read_text()) + return bool(previous.get("done") and previous.get("input_digest") == digest) + + +def _run_change( + change: Mapping[str, Any], + repos: Mapping[str, str], + out: Path, + program: Any, + program_id: str, + mask: tuple[str, ...], + render: Any, + tools_for: Any, + ledger: UsageLedger, +) -> None: + history = _lm_history() + first_call = len(history) if history is not None else 0 + record = hunt_change( + change, + Path(repos[change["repo"]]).expanduser(), + program=program, + program_id=program_id, + mask=mask, + render=render, + tools_for=tools_for, + ) + record["input_digest"] = input_digest(change, program_id, mask) + record["usd"] = round(spent_since(history, first_call), 6) + ledger.add({"calls": 1, "usd": record["usd"]}) + _write_json(record_path(out, change["id"]), record) + + +def _process_changes( + changes: Sequence[Mapping[str, Any]], + repos: Mapping[str, str], + out: Path, + program: Any, + program_id: str, + mask: tuple[str, ...], + render: Any, + tools_for: Any, + budget: Mapping[str, float] | None, + entries: list[dict[str, Any]], + indexed: dict[str, dict[str, Any]], + inputs_path: Path, +) -> UsageLedger: + ledger = UsageLedger() + for change in changes: + digest = _change_entry(change, program_id, mask, entries, indexed) + if _record_is_complete(out, change["id"], digest): + continue + if budget and ledger.exhausted(budget): + print("budget exhausted; stopping before the next change", flush=True) + break + _run_change(change, repos, out, program, program_id, mask, render, tools_for, ledger) + _write_json(inputs_path, entries) + _write_json(inputs_path, entries) + return ledger + + +def _run_artifact( + manifest: Path, + program_id: str, + mask: tuple[str, ...], + only: str, + slice_id: str, + changes: Sequence[Mapping[str, Any]], + ledger: UsageLedger, + versions: Mapping[str, str] | None, + started: float, +) -> RunArtifact: + usage = dict(ledger.usage) + usage["wall_seconds"] = round(time.monotonic() - started, 1) + return RunArtifact( + program_id=program_id, + schema_version=1, + mask=list(mask), + data_digests={"manifest": hashlib.sha256(manifest.read_bytes()).hexdigest()}, + configurations=[{"only": only, "slice": slice_id, "changes": len(changes)}], + usage=usage, + versions=dict(versions or {}), + created=datetime.now(UTC).isoformat(timespec="seconds"), + ) + + def run( manifest: Path, repos: Mapping[str, str], @@ -245,56 +358,10 @@ def run( changes = load_changes(manifest, slice_id) changes = [change for change in changes if only == "all" or change["kind"] == only] out.mkdir(parents=True, exist_ok=True) - inputs_path = out / "inputs.json" - entries = json.loads(inputs_path.read_text()) if inputs_path.is_file() else [] - indexed = {entry["item_id"]: entry for entry in entries} - ledger = UsageLedger() + inputs_path, entries, indexed = _run_entries(out) started = time.monotonic() - for change in changes: - digest = input_digest(change, program_id, mask) - entry = indexed.get(change["id"]) - if entry is None: - entry = {"item_id": change["id"], "change": change, "input_digest": digest} - entries.append(entry) - indexed[change["id"]] = entry - elif entry["input_digest"] != digest: - entry.update(change=change, input_digest=digest) - path = record_path(out, change["id"]) - if path.is_file(): - previous = json.loads(path.read_text()) - if previous.get("done") and previous.get("input_digest") == digest: - continue - if budget and ledger.exhausted(budget): - print("budget exhausted; stopping before the next change", flush=True) - break - history = _lm_history() - first_call = len(history) if history is not None else 0 - record = hunt_change( - change, - Path(repos[change["repo"]]).expanduser(), - program=program, - program_id=program_id, - mask=mask, - render=render, - tools_for=tools_for, - ) - record["input_digest"] = digest - record["usd"] = round(spent_since(history, first_call), 6) - ledger.add({"calls": 1, "usd": record["usd"]}) - _write_json(path, record) - _write_json(inputs_path, entries) - _write_json(inputs_path, entries) - usage = dict(ledger.usage) - usage["wall_seconds"] = round(time.monotonic() - started, 1) - artifact = RunArtifact( - program_id=program_id, - schema_version=1, - mask=list(mask), - data_digests={"manifest": hashlib.sha256(manifest.read_bytes()).hexdigest()}, - configurations=[{"only": only, "slice": slice_id, "changes": len(changes)}], - usage=usage, - versions=dict(versions or {}), - created=datetime.now(UTC).isoformat(timespec="seconds"), - ) - written = artifact.write(out, budget=budget) if budget else artifact.write(out) - return artifact if written is not None else None + ledger = _process_changes(changes, repos, out, program, program_id, mask, render, tools_for, budget, entries, indexed, inputs_path) + artifact = _run_artifact(manifest, program_id, mask, only, slice_id, changes, ledger, versions, started) + completed: RunArtifact | None = artifact.finish(budget) if budget else artifact + artifact.write(out, reserve_attempt=True) + return completed diff --git a/tests/benchmark_migration_conformance.py b/tests/benchmark_migration_conformance.py new file mode 100644 index 0000000..d35f917 --- /dev/null +++ b/tests/benchmark_migration_conformance.py @@ -0,0 +1,374 @@ +"""Executable owner conformance for the stable benchmark migration worklist.""" + +from __future__ import annotations + +import importlib +import json +import os +import subprocess +import sys +from collections.abc import Callable +from contextlib import ExitStack +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, migrated_owner_targets +from core_migration_inventory import LegacyInventoryEntry, LegacyMigrationInventory, MigrationDisposition + +from benchmarks.valid.artifact_contract import ( + benchmark_payload, + read_benchmark_artifact, + read_benchmark_jsonl, + read_benchmark_sequence, + write_benchmark_artifact, + write_benchmark_jsonl, + write_benchmark_sequence, +) + + +class BenchmarkMigrationConformanceError(ValueError): + """A terminal worklist entry lacks its exact executable owner.""" + + +OWNER_TARGETS = migrated_owner_targets() + + +def _change() -> dict[str, Any]: + return {"id": "case", "repo": "fixture/repo", "kind": "ordinary", "base": "a" * 40, "head": "b" * 40, "files": []} + + +def _record() -> dict[str, Any]: + return { + **_change(), + "done": True, + "status": "quick_only", + "seconds": 0.1, + "instrument": { + "hook_exit": 0, + "hook_bytes_read": 1, + "head_verified": True, + "base_verified": True, + "changed_bytes": {"app.py": 1}, + "file_bytes": {"app.py": 1}, + }, + "push": {"admission": {"defects": []}, "quick_tier": []}, + "coverage": {}, + } + + +def _owner(entry: LegacyInventoryEntry) -> tuple[Any, Callable[..., Any]]: + module_name, symbol = OWNER_TARGETS[entry.entry_id] + module = importlib.import_module(module_name) + owner = getattr(module, symbol, None) + if not callable(owner): + raise BenchmarkMigrationConformanceError(f"registered benchmark owner is absent: {entry.entry_id}") + return module, owner + + +def _probe_engine(module: Any, owner: Callable[..., Any], root: Path) -> None: + pin = {"pin": "fixture", "units": [], "questions": {}} + + def download(_url: str, path: Path, **_kwargs: Any) -> None: + path.write_text(json.dumps(pin)) if path.name == "questions.json" else path.write_bytes(b"source") + + def run(command: list[str], **_kwargs: Any) -> SimpleNamespace: + Path(command[command.index("--output") + 1]).write_text(json.dumps({"done": True, "units": []})) + return SimpleNamespace(returncode=0) + + with ExitStack() as stack: + stack.enter_context(patch.object(module, "download", download)) + stack.enter_context(patch.object(module, "unpack", lambda *_args: None)) + stack.enter_context(patch.object(module.subprocess, "run", run)) + stack.enter_context(patch.object(module, "transfer", lambda *_args, **_kwargs: None)) + stack.enter_context(patch.dict(os.environ, {"QUESTIONS_URL": "q", "SOURCE_URL": "s", "RESULT_URL": "r"})) + owner(work_root=root) + read_benchmark_artifact(root / "out/result.json", "engine-task-result") + + +def _probe_measurement(module: Any, owner: Callable[..., Any], root: Path) -> None: + with ExitStack() as stack: + stack.enter_context(patch.object(module, "discover_hunt_runner_context", lambda *_args, **_kwargs: object())) + stack.enter_context(patch.object(module, "persist_hunt_runner_records", lambda *_args, **_kwargs: ())) + stack.enter_context(patch.object(module.score, "load_records", lambda _path: [])) + stack.enter_context(patch.object(module, "arm_summary", lambda _records, arm: ({}, [{"id": arm}]))) + stack.enter_context(patch.object(module, "cost", lambda _path: None)) + stack.enter_context(patch.object(module, "load_slice", lambda *_args: ())) + owner(root) + read_benchmark_jsonl(root / "suspicions.jsonl", "access-control-suspicion") + + +def _probe_calibrate(module: Any, owner: Callable[..., Any], root: Path) -> None: + argv = [ + "calibrate", + "--manifest", + str(root / "pool.toml"), + "--image", + "image", + "--docker-image-bytes", + "1", + "--out", + str(root / "runs"), + "--record", + str(root / "record.json"), + ] + with ExitStack() as stack: + stack.enter_context(patch.object(sys, "argv", argv)) + stack.enter_context(patch.object(module, "load_slice", lambda *_args: (_change(),))) + stack.enter_context(patch.object(module, "run_replays", lambda *_args, **_kwargs: False)) + stack.enter_context(patch.object(module, "load_records", lambda _path: [_record()])) + stack.enter_context(patch.object(module, "calibration_record", lambda *_args: {"status": "ok"})) + owner() + read_benchmark_artifact(root / "record.json", "deadline-calibration") + + +def _probe_replay(module: Any, owner: Callable[..., Any], root: Path) -> None: + class Transport: + def __init__(self, *_args: Any): + self.store = SimpleNamespace(_put=lambda *_values: None) + + def __call__(self, item: Any, _output: Path, _attempt: int) -> dict[str, Any]: + return {"done": True, "status": "quick_only", "input_digest": item.input_digest} + + args = SimpleNamespace(arm="default", image="image", docker_image_bytes=1) + with patch.object(module, "DockerLane", Transport): + owner((_change(),), args, root, "docker") + read_benchmark_sequence(root / "inputs.json", "benchmark-arm-inputs") + result = next(path for path in root.glob("*.json") if '"artifact_kind": "benchmark-arm-record"' in path.read_text()) + read_benchmark_artifact(result, "benchmark-arm-record") + + +def _probe_score(module: Any, owner: Callable[..., Any], root: Path) -> None: + records, output = root / "records.json", root / "score.json" + write_benchmark_sequence(records, "benchmark-arm-records", (_record(),)) + with patch.object(sys, "argv", ["score", str(records), "--out", str(output)]): + owner() + read_benchmark_artifact(output, "unseen-score") + + +def _probe_instrument(module: Any, owner: Callable[..., Any], *, zero: bool) -> None: + from openultrasast.core import coverage + + if zero: + if owner is not coverage.validate_instrument or module.validate_instrument is not coverage.validate_instrument: + raise ValueError("validate_instrument alias is not canonical") + manifest = coverage.InputManifest(("input",), coverage.derive_input_manifest_digest(("input",))) + owner(manifest, (coverage.InputReadReceipt("input", "input", True, 1, "a" * 64),)) + try: + owner(manifest, (coverage.InputReadReceipt("input", "input", True, 0, "a" * 64),)) + except coverage.InstrumentFailure: + return + raise ValueError("canonical instrument accepted a zero-byte read") + if module.CoreInstrumentFailure is not coverage.InstrumentFailure: + raise ValueError("instrument failure alias is not canonical") + owner({"file_bytes": {"input": 1}, "file_digests": {"input": "a" * 64}}) + + +def _probe_development(module: Any, owner: Callable[..., Any], root: Path) -> None: + bundle = module.AnalyzerBundle("a", "b", "c", "d") + funnel = {"entry_recognition": {}, "resolution": {}, "judgment": {}} + with ( + patch.object(module, "prove_change", lambda *_args, **_kwargs: object()), + patch.object(module, "stage_funnel", lambda _result: funnel), + ): + owner(bundle, root, (_change(),), root / "out") + read_benchmark_artifact(root / "out/case.json", "development-observation") + + +def _probe_run_arm(module: Any, owner: Callable[..., Any], root: Path) -> None: + bundle = module.BundleDigests("a", "b", "c", "d") + with patch.object(module, "_prove_one", lambda *_args, **_kwargs: _record()): + owner(arm_name="arm", bundle=bundle, changes=(_change(),), repos={"fixture/repo": str(root)}, out_dir=root / "out") + read_benchmark_artifact(root / "out/case.json", "development-arm-record") + + +def _arm_inputs(path: Path, record: dict[str, Any] | None = None) -> None: + change = _change() + entry = {"item_id": "case", "change": change, "input_digest": "d" * 64} + write_benchmark_sequence(path / "inputs.json", "benchmark-arm-inputs", (entry,)) + if record is not None: + from openultrasast.hunt.runner import record_path + + write_benchmark_artifact(record_path(path, "case"), "benchmark-arm-record", {**record, "input_digest": entry["input_digest"]}) + + +def _probe_combine(module: Any, owner: Callable[..., Any], root: Path) -> None: + left, right, output = root / "left", root / "right", root / "combined.json" + left.mkdir() + right.mkdir() + _arm_inputs(left, _record()) + _arm_inputs(right, _record()) + owner(left, right, output) + read_benchmark_sequence(output, "benchmark-arm-records") + + +def _probe_hunter(module: Any, owner: Callable[..., Any], root: Path) -> None: + repos = root / "repos.json" + repos.write_text(json.dumps({"fixture/repo": str(root)})) + with ExitStack() as stack: + stack.enter_context(patch.object(module, "load_slice", lambda *_args: [_change()])) + stack.enter_context(patch.object(module, "load_config", lambda *_args, **_kwargs: object())) + stack.enter_context(patch.object(module, "resolve_models", lambda _config: ("model",))) + stack.enter_context(patch.object(module.tool_hunter, "resolve_hunter_client", lambda: object())) + stack.enter_context(patch.object(module, "_hunt_change", lambda *_args, **_kwargs: _record())) + owner(root / "pool", repos, root / "out") + read_benchmark_sequence(root / "out/inputs.json", "benchmark-arm-inputs") + read_benchmark_artifact(next(path for path in (root / "out").glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record") + + +def _probe_judge_input(owner: Callable[..., Any]) -> None: + result = owner("repo", {"trace": {}, "operation": {}, "missing": "guard"}) + start = result.rindex('{\n "artifact_digest"') + encoded = result[start:] + benchmark_payload(json.loads(encoded), expected_kind="judge-pass-input") + + +def _probe_judge_run(module: Any, owner: Callable[..., Any], root: Path) -> None: + fake_lm = SimpleNamespace(model="offline") + program = importlib.import_module("openultrasast.hunt.program") + with patch.object(program, "build_lm", lambda **_kwargs: fake_lm), patch.object(module, "_iter_findings", lambda _path: ()): + owner(SimpleNamespace(out=root, prover_out=root / "prover")) + read_benchmark_artifact(root / "run.json", "judge-pass-run") + + +def _probe_judge_summary(module: Any, owner: Callable[..., Any], root: Path) -> None: + row = {"key": "one", "record_id": "case", "repo": "repo", "kind": "ordinary", "public": False, "status": "ok", "model": "offline"} + write_benchmark_jsonl(root / "judgments.jsonl", "judge-pass", (row,)) + owner(SimpleNamespace(out=root)) + read_benchmark_artifact(root / "summary.json", "judge-pass-summary") + + +def _probe_metrics(module: Any, owner: Callable[..., Any], root: Path) -> None: + records, output = root / "records.json", root / "metrics.json" + write_benchmark_sequence(records, "benchmark-arm-records", (_record(),)) + owner([str(records), "--out", str(output)]) + read_benchmark_artifact(output, "valid-benchmark-metrics") + + +def _probe_prover(module: Any, owner: Callable[..., Any], root: Path, *, summary: bool) -> None: + out = root / "out" + if summary: + out.mkdir() + _arm_inputs(out, _record()) + base = { + "headline": "On unseen app repos: fixture", + "recall": {}, + "precision": {}, + "false_alarms_per_100_pushes": {}, + "exploit_verified": {}, + } + with patch.object(module.metrics, "summarize", lambda *_args, **_kwargs: base): + owner(out) + read_benchmark_artifact(out / "summary.json", "benchmark-arm-summary") + return + repos = root / "repos.json" + repos.write_text("{}") + with ( + patch.object(module, "load_slice", lambda *_args: [_change()]), + patch.object(module, "_prove", lambda *_args, **_kwargs: _record()), + ): + owner(root / "pool", repos, out) + read_benchmark_artifact(next(path for path in out.glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record") + + +def _git_fixture(root: Path) -> tuple[str, str]: + root.mkdir() + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Fixture"], cwd=root, check=True) + (root / "app.py").write_text("print('base')\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=root, check=True) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + (root / "app.py").write_text("print('head')\n") + subprocess.run(["git", "commit", "-qam", "head"], cwd=root, check=True) + head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + return base, head + + +def _probe_semgrep(module: Any, owner: Callable[..., Any], root: Path) -> None: + repo = root / "repo" + base, head = _git_fixture(repo) + change = {**_change(), "base": base, "head": head, "files": [{"path": "app.py"}]} + repos, rules, binary = root / "repos.json", root / "rules", root / "fake-semgrep" + repos.write_text(json.dumps({"fixture/repo": str(repo)})) + rules.mkdir() + binary.write_text('#!/bin/sh\nprintf \'%s\\n\' \'{"results":[],"errors":[],"paths":{"scanned":["app.py"]}}\'\n') + binary.chmod(0o755) + with patch.object(module, "load_slice", lambda *_args: [change]), patch.object(module, "MIN_PLAUSIBLE_SECONDS", 0.0): + owner(root / "pool", repos, rules, root / "out", semgrep=str(binary)) + read_benchmark_artifact(next(path for path in (root / "out").glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record") + + +def probe_benchmark_entry(entry: LegacyInventoryEntry, *, output_root: Path) -> None: + """Invoke the exact registered owner and strict-read only its produced output.""" + module, owner = _owner(entry) + root = output_root / entry.entry_id + root.mkdir(parents=True, exist_ok=True) + probes: dict[str, Callable[[], None]] = { + "40465a45": lambda: _probe_engine(module, owner, root), + "f0fd6f38": lambda: _probe_measurement(module, owner, root), + "fd21a8b7": lambda: _probe_measurement(module, owner, root), + "61d6e4cf": lambda: _probe_calibrate(module, owner, root), + "c01d6865": lambda: _probe_replay(module, owner, root), + "039988ff": lambda: _probe_score(module, owner, root), + "ae8dd586": lambda: _probe_instrument(module, owner, zero=False), + "f3ba25f4": lambda: _probe_instrument(module, owner, zero=True), + "b2bee433": lambda: _probe_run_arm(module, owner, root), + "2acb6086": lambda: _probe_development(module, owner, root), + "c4fe7770": lambda: _probe_combine(module, owner, root), + "880bbb68": lambda: _probe_hunter(module, owner, root), + "6a962869": lambda: _probe_judge_input(owner), + "dea8fd3d": lambda: _probe_judge_run(module, owner, root), + "8bf0cd9c": lambda: _probe_judge_summary(module, owner, root), + "c9906d43": lambda: _probe_metrics(module, owner, root), + "bfeaa25d": lambda: _probe_prover(module, owner, root, summary=False), + "c3e95086": lambda: _probe_prover(module, owner, root, summary=True), + "8db722b9": lambda: _probe_semgrep(module, owner, root), + } + try: + probes[entry.entry_id[:8]]() + except Exception as error: + raise BenchmarkMigrationConformanceError(f"benchmark owner probe failed: {entry.entry_id}: {error}") from error + + +def _validate_retired(entry: LegacyInventoryEntry) -> None: + module = importlib.import_module(entry.path.removesuffix(".py").replace("/", ".")) + names = ( + (entry.symbol,) + if entry.symbol != "" + else tuple(anchor.rsplit(":", 1)[-1] for anchor in entry.capability_anchors if anchor.startswith("import_from:")) + ) + if any(hasattr(module, name) for name in names): + raise BenchmarkMigrationConformanceError(f"retired benchmark entry remains live: {entry.entry_id}") + + +def _stable_fields(entry: LegacyInventoryEntry) -> tuple[str, str, str, str, tuple[str, ...], str]: + return ( + entry.path, + entry.symbol, + entry.dependency_kind.value, + entry.artifact_family, + entry.capability_anchors, + entry.owner_boundary, + ) + + +def validate_benchmark_terminal_conformance(inventory: LegacyMigrationInventory, *, output_root: Path) -> None: + entries = tuple(entry for entry in inventory.entries if entry.path.startswith("benchmarks/")) + migrated = {entry.entry_id for entry in entries if entry.disposition is MigrationDisposition.V2_MIGRATED} + if not entries or {entry.entry_id for entry in entries} != set(BENCHMARK_TERMINAL_EVIDENCE) or migrated != set(OWNER_TARGETS): + raise BenchmarkMigrationConformanceError("benchmark probe registry and terminal evidence do not exactly match inventory entries") + for entry in entries: + evidence = BENCHMARK_TERMINAL_EVIDENCE[entry.entry_id] + if evidence.disposition != entry.disposition.value or evidence.stable_fields != _stable_fields(entry): + raise BenchmarkMigrationConformanceError(f"benchmark terminal evidence mismatch: {entry.entry_id}") + if entry.disposition is MigrationDisposition.OPEN: + raise BenchmarkMigrationConformanceError(f"benchmark entry remains open: {entry.entry_id}") + _validate_retired(entry) if entry.disposition is MigrationDisposition.RETIRED else probe_benchmark_entry( + entry, output_root=output_root + ) + + +__all__ = ["BenchmarkMigrationConformanceError", "OWNER_TARGETS", "probe_benchmark_entry", "validate_benchmark_terminal_conformance"] diff --git a/tests/benchmark_migration_evidence.py b/tests/benchmark_migration_evidence.py new file mode 100644 index 0000000..2348714 --- /dev/null +++ b/tests/benchmark_migration_evidence.py @@ -0,0 +1,358 @@ +"""Immutable reviewed closure evidence for detection-core benchmark migration entries.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BenchmarkTerminalEvidence: + disposition: str + path: str + symbol: str + dependency_kind: str + artifact_family: str + capability_anchors: tuple[str, ...] + owner_boundary: str + successor_module: str | None + successor_symbol: str | None + + @property + def stable_fields(self) -> tuple[str, str, str, str, tuple[str, ...], str]: + return ( + self.path, + self.symbol, + self.dependency_kind, + self.artifact_family, + self.capability_anchors, + self.owner_boundary, + ) + + +BENCHMARK_TERMINAL_EVIDENCE = { + "40465a45eb2ef682fecad973566a0bd94956bfb804d1fc6ea1d43075533232b1": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/learn/engine_task_entry.py", + "main", + "writer", + "v1-artifact", + ("definition:main", "constant:done", "call:dumps", "call:write_text"), + "benchmark", + "benchmarks.learn.engine_task_entry", + "main", + ), + "f0fd6f3806e7f50b826c6b3fad99d50c4ac02aeedb5cc3afca6c140f3bebbcf7": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py", + "", + "writer", + "v1-artifact", + ( + "constant:family", + "constant:id", + "constant:instrument", + "constant:instrument_failures", + "constant:precision", + "constant:push", + "call:dumps", + "call:write_text", + ), + "benchmark", + "benchmarks.measurements.2026-10-14-access-control-hunt.analyse", + "main", + ), + "fd21a8b7f0759a69e9f51ad3d2bae51e435ebb88cabea3bc44b8fec534fae32f": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py", + "main", + "writer", + "v1-artifact", + ("definition:main", "call:dumps", "call:write_text"), + "benchmark", + "benchmarks.measurements.2026-10-14-access-control-hunt.analyse", + "main", + ), + "61d6e4cf01bb735ec13ea6dcd9aeb3e804abaccd26e0c7ad18819f2b28a7db0c": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/unseen/calibrate.py", + "main", + "writer", + "v1-artifact", + ("definition:main", "constant:id", "constant:status", "call:canonical"), + "benchmark", + "benchmarks.unseen.calibrate", + "main", + ), + "c01d686518e8c0d17fa60092007604923b133200328a975bbf39e4727d75d900": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/unseen/replay.py", + "run_replays", + "writer", + "v1-artifact", + ("definition:run_replays", "constant:change", "constant:input_digest", "constant:item_id"), + "benchmark", + "benchmarks.unseen.replay", + "run_replays", + ), + "039988ff20d8f58a169cc7cd183e25a88b2bb2638fa969becb3553d8d7bc927b": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/unseen/score.py", + "main", + "writer", + "v1-artifact", + ("definition:main", "call:dumps", "call:write_text"), + "benchmark", + "benchmarks.unseen.score", + "main", + ), + "fe616a80de18d275283ad26ccd9027c4fbf151c017d02c14205e9cfc374a8b53": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/admission_run.py", + "_resume_or_prove", + "writer", + "v1-artifact", + ("definition:_resume_or_prove", "constant:id", "constant:input_digest", "call:_write_json"), + "benchmark", + None, + None, + ), + "41eade0e409ea4726173e851fcad5c9c81076bd7e094fc4c2ad830ac40f958b4": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/admission_run.py", + "assemble_admission_report", + "writer", + "v1-artifact", + ("definition:assemble_admission_report", "constant:id", "constant:status", "name:admission_report"), + "benchmark", + None, + None, + ), + "ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/arm_regression.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.coverage:InstrumentFailure",), + "benchmark", + "benchmarks.valid.arm_regression", + "_validate_record_instrument", + ), + "f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/arm_regression.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.coverage:check_reads",), + "benchmark", + "benchmarks.valid.arm_regression", + "validate_instrument", + ), + "b2bee4333d7374d21d2f4d0a8cdf49bb7c2f398daba5f63d7c3c2f7349f9be72": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/arm_regression.py", + "run_arm", + "writer", + "v1-artifact", + ("definition:run_arm", "constant:id", "constant:input_digest", "call:_write_json"), + "benchmark", + "benchmarks.valid.arm_regression", + "run_arm", + ), + "2acb6086c4eba2ee4a6bd6f6bacc86219207dd71548586c7966bb38153de9150": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/development_runner.py", + "run_development", + "writer", + "v1-artifact", + ("definition:run_development", "constant:id", "call:dumps", "call:write_text", "name:DevelopmentRecord"), + "benchmark", + "benchmarks.valid.development_runner", + "run_development", + ), + "c4fe7770e44fb6c37a97cc2b5c686927d116522cdd62229fe7203aa58405477b": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/hunter_arm.py", + "combine", + "writer", + "v1-artifact", + ("definition:combine", "constant:missing", "constant:push", "constant:status", "call:_write_json"), + "benchmark", + "benchmarks.valid.hunter_arm", + "combine", + ), + "880bbb68575bcb453127df5930a3d73d5dc67b478096bc504fafa878d91e534c": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/hunter_arm.py", + "run", + "writer", + "v1-artifact", + ( + "definition:run", + "constant:change", + "constant:done", + "constant:id", + "constant:input_digest", + "constant:item_id", + "constant:status", + "call:_write_json", + "call:canonical", + ), + "benchmark", + "benchmarks.valid.hunter_arm", + "run", + ), + "6a96286909032b9997570a1f42afd97ffb92258a7d6ad0ebd077f4cdcb65adef": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/judge_pass.py", + "_judge_input", + "writer", + "v1-artifact", + ("definition:_judge_input", "constant:missing", "constant:operation", "constant:trace", "call:dumps"), + "benchmark", + "benchmarks.valid.judge_pass", + "_judge_input", + ), + "dea8fd3d112e1cbb4abb54b5d0e77e83c0a6e1e75e5ba2171dbd6f7ba436b75b": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/judge_pass.py", + "run", + "writer", + "v1-artifact", + ("definition:run", "constant:id", "constant:status", "call:dumps", "call:write", "call:write_text"), + "benchmark", + "benchmarks.valid.judge_pass", + "run", + ), + "8bf0cd9c6a28264ad9ff1492f203d384b33432ddf0d9027381497cc8352de48c": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/judge_pass.py", + "summarize", + "writer", + "v1-artifact", + ("definition:summarize", "constant:status", "call:dumps", "call:write_text"), + "benchmark", + "benchmarks.valid.judge_pass", + "summarize", + ), + "c9906d437a27969e465c4d3139b1e865de0ac82761d0a980cf41efce20a0c178": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/metrics.py", + "main", + "writer", + "v1-artifact", + ("definition:main", "call:dumps", "call:write_text"), + "benchmark", + "benchmarks.valid.metrics", + "main", + ), + "ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/prover_admission.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.admission:AdmissionManifest",), + "benchmark", + None, + None, + ), + "f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/prover_admission.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.admission:AdmissionManifestError",), + "benchmark", + None, + None, + ), + "b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/prover_admission.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.admission:REPORT_SCHEMA_KEYS",), + "benchmark", + None, + None, + ), + "cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429": BenchmarkTerminalEvidence( + "retired", + "benchmarks/valid/prover_admission.py", + "", + "reader", + "v1-contract", + ("import_from:openultrasast.core.admission:validate_admission_manifest",), + "benchmark", + None, + None, + ), + "bfeaa25d41ff876c1211b69f0bc497e246cabb4bcdd0569f27ae2ae15e84d3f5": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/prover_arm.py", + "run", + "writer", + "v1-artifact", + ( + "definition:run", + "constant:change", + "constant:done", + "constant:id", + "constant:input_digest", + "constant:item_id", + "constant:status", + "call:_write_json", + "call:canonical", + ), + "benchmark", + "benchmarks.valid.prover_arm", + "run", + ), + "c3e95086ce661de7f589d787c324b4dd08977481fb427452837e8482d24210bf": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/prover_arm.py", + "summarize", + "writer", + "v1-artifact", + ("definition:summarize", "constant:coverage", "constant:id", "constant:instrument", "constant:status", "call:_write_json"), + "benchmark", + "benchmarks.valid.prover_arm", + "summarize", + ), + "8db722b97cb4411fd2a12391f23d5b79b46832fcd7d62d08905aa667871c6c6a": BenchmarkTerminalEvidence( + "v2_migrated", + "benchmarks/valid/semgrep_arm.py", + "run", + "writer", + "v1-artifact", + ( + "definition:run", + "constant:change", + "constant:done", + "constant:id", + "constant:input_digest", + "constant:item_id", + "constant:status", + "call:_write_json", + "call:canonical", + ), + "benchmark", + "benchmarks.valid.semgrep_arm", + "run", + ), +} + + +def migrated_owner_targets() -> dict[str, tuple[str, str]]: + return { + entry_id: (evidence.successor_module, evidence.successor_symbol) + for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items() + if evidence.disposition == "v2_migrated" and evidence.successor_module is not None and evidence.successor_symbol is not None + } + + +__all__ = ["BENCHMARK_TERMINAL_EVIDENCE", "BenchmarkTerminalEvidence", "migrated_owner_targets"] diff --git a/tests/core_migration_inventory.py b/tests/core_migration_inventory.py index 8728ae2..498751c 100644 --- a/tests/core_migration_inventory.py +++ b/tests/core_migration_inventory.py @@ -9,6 +9,8 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path +from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, BenchmarkTerminalEvidence + from openultrasast.core.legacy import LegacyArtifact, decode_legacy_artifact REPO = Path(__file__).resolve().parents[1] @@ -55,6 +57,12 @@ _APPROVED_TERMINAL_SOURCE_CLOSURES = { "9f2097b8a673c3079a8a61cc28e9099c671e48e6c29a3e95a9e3aac3d53ccb00": MigrationDisposition.RETIRED, "eb963401f43fca11e38c50e5e064f58b33233b9564a3a85e5b629a6fb2820d7d": MigrationDisposition.V2_MIGRATED, "f8a5c42661f6a79b0abbe668e8993fd0ec26d1c9dad5df4e141472e6de62f586": MigrationDisposition.V2_MIGRATED, + "ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36": MigrationDisposition.V2_MIGRATED, + "f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f": MigrationDisposition.V2_MIGRATED, + "b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d": MigrationDisposition.RETIRED, + "cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429": MigrationDisposition.RETIRED, + "ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522": MigrationDisposition.RETIRED, + "f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c": MigrationDisposition.RETIRED, } @@ -322,7 +330,7 @@ def _dotted(node: ast.AST) -> str: def _top_level_reads(tree: ast.Module) -> set[str]: - aliases = set() + aliases: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): aliases.update(alias.asname for alias in node.names if alias.name == "openultrasast.core" and alias.asname) @@ -448,8 +456,46 @@ def _moved_mixed_v1_owner(entry: LegacyInventoryEntry, *, root: Path) -> bool: return set(destination_symbols) <= owned -def _approved_terminal_source_closure(entry: LegacyInventoryEntry) -> bool: - return _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition and entry.entry_id == entry_identity(entry) +def _successor_path(root: Path, evidence: BenchmarkTerminalEvidence) -> Path: + assert evidence.successor_module is not None + relative = Path(*evidence.successor_module.split(".")).with_suffix(".py") + direct = root / relative + return direct if direct.is_file() else root / "src" / relative + + +def _validate_retired_absence(entry: LegacyInventoryEntry, tree: ast.Module) -> None: + if entry.symbol != "" and entry.symbol in _defined_symbols(tree): + raise LegacyInventoryError(f"retired owner remains live: {entry.path}:{entry.symbol}") + if entry.symbol == "" and not _missing_anchors(entry, tree): + raise LegacyInventoryError(f"retired capability remains live: {entry.path}:{entry.capability_anchors}") + + +def _benchmark_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool: + if not entry.path.startswith("benchmarks/") or entry.disposition is MigrationDisposition.OPEN: + return False + evidence = BENCHMARK_TERMINAL_EVIDENCE.get(entry.entry_id) + if ( + evidence is None + or evidence.disposition != entry.disposition.value + or evidence.stable_fields != _stable_entry_fields(entry) + or entry.entry_id != entry_identity(entry) + ): + raise LegacyInventoryError(f"benchmark terminal evidence is absent or mismatched: {entry.entry_id}") + if entry.disposition is MigrationDisposition.RETIRED: + _validate_retired_absence(entry, tree) + return True + successor = _read_owner_module(_successor_path(root, evidence), "migrated benchmark successor") + assert evidence.successor_symbol is not None + if evidence.successor_symbol not in _defined_symbols(successor): + raise LegacyInventoryError(f"migrated owner is absent: {evidence.successor_module}:{evidence.successor_symbol}") + return True + + +def _approved_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool: + if _benchmark_terminal_source_closure(entry, root=root, tree=tree): + return True + approved = _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition + return approved and entry.entry_id == entry_identity(entry) def validate_inventory_sources(inventory: LegacyMigrationInventory, *, root: Path) -> None: @@ -468,7 +514,7 @@ def validate_inventory_sources(inventory: LegacyMigrationInventory, *, root: Pat raise LegacyInventoryError(f"inventory source is not parseable: {entry.path}") from exc is_shim = _explicit_legacy_shim(tree, entry.symbol) moved_owner = _moved_mixed_v1_owner(entry, root=root) - terminal_closure = _approved_terminal_source_closure(entry) + terminal_closure = _approved_terminal_source_closure(entry, root=root, tree=tree) if entry.symbol not in _defined_symbols(tree) and not is_shim and not moved_owner and not terminal_closure: raise LegacyInventoryError(f"inventory target is absent: {entry.path}:{entry.symbol}") missing = _missing_anchors(entry, tree) diff --git a/tests/fixtures/detection_core/v1/migration_inventory.json b/tests/fixtures/detection_core/v1/migration_inventory.json index d967c7a..b7d38d1 100644 --- a/tests/fixtures/detection_core/v1/migration_inventory.json +++ b/tests/fixtures/detection_core/v1/migration_inventory.json @@ -82,7 +82,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "40465a45eb2ef682fecad973566a0bd94956bfb804d1fc6ea1d43075533232b1", "owner_boundary": "benchmark", "path": "benchmarks/learn/engine_task_entry.py", @@ -101,7 +101,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "f0fd6f3806e7f50b826c6b3fad99d50c4ac02aeedb5cc3afca6c140f3bebbcf7", "owner_boundary": "benchmark", "path": "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py", @@ -115,7 +115,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "fd21a8b7f0759a69e9f51ad3d2bae51e435ebb88cabea3bc44b8fec534fae32f", "owner_boundary": "benchmark", "path": "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py", @@ -130,7 +130,7 @@ "call:canonical" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "61d6e4cf01bb735ec13ea6dcd9aeb3e804abaccd26e0c7ad18819f2b28a7db0c", "owner_boundary": "benchmark", "path": "benchmarks/unseen/calibrate.py", @@ -145,7 +145,7 @@ "constant:item_id" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "c01d686518e8c0d17fa60092007604923b133200328a975bbf39e4727d75d900", "owner_boundary": "benchmark", "path": "benchmarks/unseen/replay.py", @@ -159,7 +159,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "039988ff20d8f58a169cc7cd183e25a88b2bb2638fa969becb3553d8d7bc927b", "owner_boundary": "benchmark", "path": "benchmarks/unseen/score.py", @@ -174,7 +174,7 @@ "call:_write_json" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "retired", "entry_id": "fe616a80de18d275283ad26ccd9027c4fbf151c017d02c14205e9cfc374a8b53", "owner_boundary": "benchmark", "path": "benchmarks/valid/admission_run.py", @@ -189,7 +189,7 @@ "name:admission_report" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "retired", "entry_id": "41eade0e409ea4726173e851fcad5c9c81076bd7e094fc4c2ad830ac40f958b4", "owner_boundary": "benchmark", "path": "benchmarks/valid/admission_run.py", @@ -201,7 +201,7 @@ "import_from:openultrasast.core.coverage:InstrumentFailure" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36", "owner_boundary": "benchmark", "path": "benchmarks/valid/arm_regression.py", @@ -213,7 +213,7 @@ "import_from:openultrasast.core.coverage:check_reads" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f", "owner_boundary": "benchmark", "path": "benchmarks/valid/arm_regression.py", @@ -228,7 +228,7 @@ "call:_write_json" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "b2bee4333d7374d21d2f4d0a8cdf49bb7c2f398daba5f63d7c3c2f7349f9be72", "owner_boundary": "benchmark", "path": "benchmarks/valid/arm_regression.py", @@ -244,7 +244,7 @@ "name:DevelopmentRecord" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "2acb6086c4eba2ee4a6bd6f6bacc86219207dd71548586c7966bb38153de9150", "owner_boundary": "benchmark", "path": "benchmarks/valid/development_runner.py", @@ -260,7 +260,7 @@ "call:_write_json" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "c4fe7770e44fb6c37a97cc2b5c686927d116522cdd62229fe7203aa58405477b", "owner_boundary": "benchmark", "path": "benchmarks/valid/hunter_arm.py", @@ -280,7 +280,7 @@ "call:canonical" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "880bbb68575bcb453127df5930a3d73d5dc67b478096bc504fafa878d91e534c", "owner_boundary": "benchmark", "path": "benchmarks/valid/hunter_arm.py", @@ -296,7 +296,7 @@ "call:dumps" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "6a96286909032b9997570a1f42afd97ffb92258a7d6ad0ebd077f4cdcb65adef", "owner_boundary": "benchmark", "path": "benchmarks/valid/judge_pass.py", @@ -313,7 +313,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "dea8fd3d112e1cbb4abb54b5d0e77e83c0a6e1e75e5ba2171dbd6f7ba436b75b", "owner_boundary": "benchmark", "path": "benchmarks/valid/judge_pass.py", @@ -328,7 +328,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "8bf0cd9c6a28264ad9ff1492f203d384b33432ddf0d9027381497cc8352de48c", "owner_boundary": "benchmark", "path": "benchmarks/valid/judge_pass.py", @@ -342,7 +342,7 @@ "call:write_text" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "c9906d437a27969e465c4d3139b1e865de0ac82761d0a980cf41efce20a0c178", "owner_boundary": "benchmark", "path": "benchmarks/valid/metrics.py", @@ -354,7 +354,7 @@ "import_from:openultrasast.core.admission:AdmissionManifest" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "retired", "entry_id": "ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_admission.py", @@ -366,7 +366,7 @@ "import_from:openultrasast.core.admission:AdmissionManifestError" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "retired", "entry_id": "f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_admission.py", @@ -378,7 +378,7 @@ "import_from:openultrasast.core.admission:REPORT_SCHEMA_KEYS" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "retired", "entry_id": "b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_admission.py", @@ -390,7 +390,7 @@ "import_from:openultrasast.core.admission:validate_admission_manifest" ], "dependency_kind": "reader", - "disposition": "open", + "disposition": "retired", "entry_id": "cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_admission.py", @@ -410,7 +410,7 @@ "call:canonical" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "bfeaa25d41ff876c1211b69f0bc497e246cabb4bcdd0569f27ae2ae15e84d3f5", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_arm.py", @@ -427,7 +427,7 @@ "call:_write_json" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "c3e95086ce661de7f589d787c324b4dd08977481fb427452837e8482d24210bf", "owner_boundary": "benchmark", "path": "benchmarks/valid/prover_arm.py", @@ -447,7 +447,7 @@ "call:canonical" ], "dependency_kind": "writer", - "disposition": "open", + "disposition": "v2_migrated", "entry_id": "8db722b97cb4411fd2a12391f23d5b79b46832fcd7d62d08905aa667871c6c6a", "owner_boundary": "benchmark", "path": "benchmarks/valid/semgrep_arm.py", diff --git a/tests/test_admission_run.py b/tests/test_admission_run.py index 93e4151..66a0321 100644 --- a/tests/test_admission_run.py +++ b/tests/test_admission_run.py @@ -6,30 +6,21 @@ reviewed ordinary change, maps per-site prover evidence onto the three-way judgment vocabulary and returns the paired admission report. """ -import hashlib -import json import sys -import tempfile -import tomllib from pathlib import Path from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from benchmarks.unseen.replay import canonical # noqa: E402 from benchmarks.valid.admission_run import ( # noqa: E402 admission_ordinary, admission_pair_cases, - assemble_admission_report, classify_fixed_site, coverage_share, fixed_side_change, - load_admission_population, locate_function, site_judgment, ) -from benchmarks.valid.admission_scoring import AdmissionScoringError # noqa: E402 -from benchmarks.valid.prover_admission import AdmissionManifest, AdmissionManifestError # noqa: E402 SITE = ("src/app.py", 30, 40) @@ -194,159 +185,3 @@ def test_coverage_share_over_repositories() -> None: bare = _record("b", repo="repo-b", stage_funnel={"entry_recognition": {"frameworks_recognised": {}}}) other = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 2}}}) assert coverage_share((covered, bare, other)) == 0.5 - - -def _manifest_payload(**over: object) -> dict[str, object]: - payload = { - "produced_by": "valid-detection-benchmark", - "reviewed": True, - "label_isolated": True, - "spent": False, - "population_role": "admission", - "analyzer_revision": "d063167", - "fact_set_digest": "3eb5bf699434", - "acquisition_record_digest": "none", - "configuration_digest": "be6d0f688598", - "population_digest": "0" * 64, - "labels": [], - } - payload.update(over) - return payload - - -POOL_TOML = ( - 'status = "frozen"\n' - 'pool = "valid"\n' - "seed = 0\n" - "\n" - "[[repository]]\n" - 'id = "example/project"\n' - 'repository = "example/project"\n' - 'url = "https://github.com/example/project.git"\n' - "slice = 1\n" - "changes = [\n" - ' {base = "fixcommit", head = "vulnparent", case_id = "example-project-vuln-1", ' - 'id = "example-project-vuln-1", repo = "example/project", kind = "vulnerability", ' - 'path = "src/app.py", function = "handler", function_lines = [30, 40], ' - 'files = [{path = "src/app.py", head_lines = [[30, 40]]}], family = "access_control"},\n' - ' {base = "parent1", head = "ordinary1", case_id = "example-project-ordinary-1", ' - 'id = "example-project-ordinary-1", repo = "example/project", kind = "ordinary", ' - 'files = [{path = "src/lib.py", head_lines = [[1, 5]]}]},\n' - "]\n" -) - - -def _population_files(root: Path, *, freeze_digest: str | None = None, guard_result: int = 0) -> tuple[Path, Path, str]: - root.mkdir(parents=True, exist_ok=True) - pool_path = root / "pool.toml" - pool_path.write_text(POOL_TOML, encoding="utf-8") - pool = tomllib.loads(pool_path.read_text(encoding="utf-8")) - digest = freeze_digest if freeze_digest is not None else hashlib.sha256(canonical(pool)).hexdigest() - freeze_path = root / "freeze.json" - freeze_path.write_text(json.dumps({"freeze_digest": digest, "guard_result": guard_result}), encoding="utf-8") - return pool_path, freeze_path, digest - - -def test_load_population_verifies_freeze_digest_and_guard() -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - pool_path, freeze_path, digest = _population_files(root) - payload = _manifest_payload(population_digest=digest) - vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path) - assert [change["kind"] for change in (*vulnerable, *ordinary)] == ["vulnerability", "ordinary"] - assert all(len(change["id"]) == 64 for change in (*vulnerable, *ordinary)) - tampered = _manifest_payload(population_digest="f" * 64) - try: - load_admission_population(tampered, pool_path, freeze_path) - raise AssertionError("population digest mismatch must be refused") - except ValueError as error: - assert "population" in str(error) - guarded = _population_files(root / "g", guard_result=1) - payload_guarded = _manifest_payload(population_digest=guarded[2]) - try: - load_admission_population(payload_guarded, guarded[0], guarded[1]) - raise AssertionError("held-out guard failure must be refused") - except ValueError as error: - assert "guard" in str(error) - - -def test_load_population_validates_manifest_before_labels() -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - pool_path, freeze_path, _ = _population_files(root) - spent = _manifest_payload(spent=True) - try: - load_admission_population(spent, pool_path, freeze_path) - raise AssertionError("spent population must be refused") - except AdmissionManifestError: - pass - - -def _admission_manifest(digest: str = "0" * 64) -> AdmissionManifest: - from benchmarks.valid.prover_admission import validate_admission_manifest - - return validate_admission_manifest(_manifest_payload(population_digest=digest)) - - -def test_assemble_report_returns_go_without_failed_gates() -> None: - vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], family="access_control") - fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")]) - ordinary = _record("ord-1", kind="ordinary") - manifest = _admission_manifest() - report = assemble_admission_report( - manifest, - (vulnerable,), - (fixed,), - (ordinary,), - criteria={"min_fix_recall": 1.0, "max_flagged_ordinary_per_100": 0.0}, - ) - assert report["decision"] == "GO" - assert report["fix_recall"]["count"] == 1 - assert report["pairs"] == {"n": 1, "correct": 1} - assert report["flagged_ordinary_per_100"]["count"] == 0 - - -def test_assemble_report_no_go_on_ordinary_overrun() -> None: - vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)]) - fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")]) - slow = _record("ord-1", kind="ordinary", seconds=300.0) - report = assemble_admission_report(_admission_manifest(), (vulnerable,), (fixed,), (slow,), criteria=None, budget_seconds=120.0) - assert report["decision"] == "NO-GO" - assert any(gate.startswith("budget:ord-1:") for gate in report["failed_gates"]) - - -def test_assemble_report_no_go_on_budget_and_criteria() -> None: - vulnerable = _record("case-1", seconds=200.0) - fixed = _record("case-1", fixed_lines=(31, 42), seconds=1.0) - ordinary = _record("ord-1", kind="ordinary", findings=[_finding("src/x.py", 3)]) - manifest = _admission_manifest() - report = assemble_admission_report( - manifest, - (vulnerable,), - (fixed,), - (ordinary,), - criteria={"min_fix_recall": 0.9, "max_flagged_ordinary_per_100": 0.0}, - ) - assert report["decision"] == "NO-GO" - assert any(gate.startswith("budget:case-1:") for gate in report["failed_gates"]) - assert any("min_fix_recall" in gate for gate in report["failed_gates"]) - assert any("max_flagged_ordinary_per_100" in gate for gate in report["failed_gates"]) - - -def test_assemble_report_counts_instrument_failures() -> None: - vulnerable = _record("case-1", status="instrument_failure") - fixed = _record("case-1", fixed_lines=(31, 42), status="instrument_failure") - manifest = _admission_manifest() - report = assemble_admission_report(manifest, (vulnerable,), (fixed,), (), criteria=None) - assert report["instrument_failures"] == 2 - assert report["decision"] == "NO-GO" - - -def test_assemble_report_refuses_development_role() -> None: - manifest = _admission_manifest() - object.__setattr__(manifest, "population_role", "development") - try: - assemble_admission_report(manifest, (), (), (), criteria=None) - raise AssertionError("development population must be refused for admission") - except AdmissionScoringError: - pass diff --git a/tests/test_admission_scoring.py b/tests/test_admission_scoring.py index 5ef68c7..8fd0986 100644 --- a/tests/test_admission_scoring.py +++ b/tests/test_admission_scoring.py @@ -14,7 +14,7 @@ from benchmarks.valid.admission_scoring import ( OrdinaryChange, admission_report, ) -from benchmarks.valid.prover_admission import AdmissionManifest, validate_admission_manifest +from openultrasast.core.legacy_v1 import AdmissionManifest, validate_admission_manifest def admission_manifest(role: str = "admission") -> AdmissionManifest: diff --git a/tests/test_arm_regression.py b/tests/test_arm_regression.py index 5a71ca2..cd72494 100644 --- a/tests/test_arm_regression.py +++ b/tests/test_arm_regression.py @@ -141,7 +141,33 @@ def test_run_arm_writes_resumable_record_with_digests(sample_repo: Path, tmp_pat assert (out / "sample-repo-ordinary-abc.json").stat().st_mtime_ns == first_mtime -def test_run_arm_allows_legitimately_empty_tracked_file(sample_repo: Path, tmp_path: Path) -> None: +def test_run_arm_resume_refuses_stripped_v2_metadata(sample_repo: Path, tmp_path: Path) -> None: + change = _change(sample_repo) + out = tmp_path / "arm" + arm_regression.run_arm( + arm_name="candidate-analyzer/candidate-facts", + bundle=_bundle(), + changes=(change,), + repos={"sample/repo": str(sample_repo)}, + out_dir=out, + ) + path = out / "sample-repo-ordinary-abc.json" + value = json.loads(path.read_text()) + for field in ("schema_version", "artifact_kind", "artifact_digest"): + value.pop(field) + path.write_text(json.dumps(value)) + + with pytest.raises(ValueError, match="metadata"): + arm_regression.run_arm( + arm_name="candidate-analyzer/candidate-facts", + bundle=_bundle(), + changes=(change,), + repos={"sample/repo": str(sample_repo)}, + out_dir=out, + ) + + +def test_run_arm_refuses_empty_tracked_file_through_v2_instrument(sample_repo: Path, tmp_path: Path) -> None: (sample_repo / "empty.py").write_text("") (sample_repo / "app.py").write_text("def handler(event):\n return 2\n") _git(sample_repo, "add", ".") @@ -158,9 +184,12 @@ def test_run_arm_allows_legitimately_empty_tracked_file(sample_repo: Path, tmp_p repos={"sample/repo": str(sample_repo)}, out_dir=tmp_path / "arm", ) - assert records[0]["status"] == "prover_covered" - assert records[0]["instrument"]["empty_files"] == 1 + assert records[0]["status"] == "instrument_failure" + assert records[0]["instrument"]["failure"] == "InstrumentFailure" + assert records[0]["instrument"]["file_bytes"]["empty.py"] == 0 assert records[0]["instrument"]["file_bytes"]["app.py"] > 0 + with pytest.raises(arm_regression.InstrumentFailure, match="zero-byte"): + arm_regression.summarize_arm(records) def test_summarize_arm_refuses_zero_byte_summary(tmp_path: Path) -> None: @@ -186,7 +215,7 @@ def test_summarize_arm_publishes_budget_result_and_masks_repositories() -> None: "repo": "private/name", "kind": "ordinary", "status": "prover_covered", - "instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10}, + "instrument": {"file_bytes": {"app.py": 10}, "file_digests": {"app.py": "a" * 64}, "hook_bytes_read": 10}, "stage_funnel": { "entry_recognition": {"entry_points": 1, "frameworks_recognised": {"web": 1}}, "resolution": {"resolved": 1}, @@ -244,7 +273,7 @@ def test_summarize_arm_reports_uncovered_repository_separately() -> None: "repo": "hidden/other", "kind": "vulnerability", "status": "prover_covered", - "instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10}, + "instrument": {"file_bytes": {"app.py": 10}, "file_digests": {"app.py": "a" * 64}, "hook_bytes_read": 10}, "stage_funnel": { "entry_recognition": {"entry_points": 0, "frameworks_recognised": {}}, "resolution": {}, diff --git a/tests/test_benchmark_artifact_contract.py b/tests/test_benchmark_artifact_contract.py new file mode 100644 index 0000000..b6e201b --- /dev/null +++ b/tests/test_benchmark_artifact_contract.py @@ -0,0 +1,369 @@ +"""Strict v2 persistence, ordered collections, and atomic collision behavior.""" + +from __future__ import annotations + +import json +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from benchmarks.unseen import score # noqa: E402 +from benchmarks.valid import ( + artifact_contract, # noqa: E402 + hunt_handoff, # noqa: E402 +) +from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact # noqa: E402 +from openultrasast.hunt.runner import input_digest, record_path # noqa: E402 + + +def _rewrite(path: Path, value: object) -> None: + path.write_text(json.dumps(value, sort_keys=True) + "\n") + + +def test_payload_requires_complete_metadata_expected_kind_schema_and_digest() -> None: + artifact = artifact_contract.benchmark_artifact("record", {"id": "one"}) + assert artifact_contract.benchmark_payload(artifact, expected_kind="record") == {"id": "one"} + + with pytest.raises(ValueError, match="metadata"): + artifact_contract.benchmark_payload({"id": "one"}, expected_kind="record") + for field in ("schema_version", "artifact_kind", "artifact_digest"): + stripped = dict(artifact) + stripped.pop(field) + with pytest.raises(ValueError, match="metadata"): + artifact_contract.benchmark_payload(stripped, expected_kind="record") + for field, value in ( + ("schema_version", "v1"), + ("artifact_kind", "other"), + ("artifact_digest", "0" * 64), + ): + tampered = dict(artifact) + tampered[field] = value + with pytest.raises(ValueError, match="metadata"): + artifact_contract.benchmark_payload(tampered, expected_kind="record") + + +def test_ordered_sequence_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None: + path = tmp_path / "inputs.json" + rows = ({"id": "one"}, {"id": "two"}) + artifact_contract.write_benchmark_sequence(path, "inputs", rows) + assert artifact_contract.read_benchmark_sequence(path, "inputs") == rows + original = json.loads(path.read_text()) + + mutations = [] + reordered = json.loads(json.dumps(original)) + reordered["rows"].reverse() + mutations.append(reordered) + truncated = json.loads(json.dumps(original)) + truncated["rows"].pop() + mutations.append(truncated) + duplicated = json.loads(json.dumps(original)) + duplicated["rows"][1] = duplicated["rows"][0] + mutations.append(duplicated) + extra = json.loads(json.dumps(original)) + extra["rows"].append(extra["rows"][0]) + mutations.append(extra) + tampered = json.loads(json.dumps(original)) + tampered["rows"][0]["artifact"]["id"] = "changed" + mutations.append(tampered) + + for mutation in mutations: + _rewrite(path, mutation) + with pytest.raises(ValueError): + artifact_contract.read_benchmark_sequence(path, "inputs") + + +def test_jsonl_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None: + path = tmp_path / "judgments.jsonl" + rows = ({"id": "one"}, {"id": "two"}) + artifact_contract.write_benchmark_jsonl(path, "judgment", rows) + assert artifact_contract.read_benchmark_jsonl(path, "judgment") == rows + original = path.read_text().splitlines() + mutations = ( + list(reversed(original)), + original[:-1], + [original[0], original[0]], + [*original, original[0]], + [original[0].replace('"one"', '"changed"'), original[1]], + ) + for lines in mutations: + path.write_text("\n".join(lines) + "\n") + with pytest.raises(ValueError): + artifact_contract.read_benchmark_jsonl(path, "judgment") + + +def test_atomic_writer_is_idempotent_and_refuses_conflicting_overwrite(tmp_path: Path) -> None: + path = tmp_path / "record.json" + artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"}) + before = (path.read_bytes(), path.stat().st_mtime_ns) + artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"}) + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + with pytest.raises(ValueError, match="conflict"): + artifact_contract.write_benchmark_artifact(path, "record", {"id": "two"}) + with pytest.raises(ValueError, match="conflict"): + artifact_contract.write_benchmark_artifact(path, "other", {"id": "one"}) + assert not list(tmp_path.glob("*.tmp")) + + +def test_concurrent_conflicting_writers_have_one_winner(tmp_path: Path) -> None: + path = tmp_path / "record.json" + + def write(identity: str) -> str: + artifact_contract.write_benchmark_artifact(path, "record", {"id": identity}) + return identity + + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(write, identity) for identity in ("one", "two")] + winners = [future.result() for future in futures if future.exception() is None] + failures = [future.exception() for future in futures if future.exception() is not None] + assert len(winners) == 1 + assert len(failures) == 1 and isinstance(failures[0], ValueError) + assert artifact_contract.read_benchmark_artifact(path, "record") == {"id": winners[0]} + + +def test_concurrent_jsonl_appends_preserve_both_rows(tmp_path: Path) -> None: + path = tmp_path / "judgments.jsonl" + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(artifact_contract.append_benchmark_jsonl, path, "judgment", {"id": identity}) for identity in ("one", "two")] + assert all(future.exception() is None for future in futures) + rows = artifact_contract.read_benchmark_jsonl(path, "judgment") + assert len(rows) == 2 + assert {row["id"] for row in rows} == {"one", "two"} + + +def test_score_reader_refuses_stripped_v2_metadata(tmp_path: Path) -> None: + path = tmp_path / "score.json" + _rewrite(path, [{"id": "one"}]) + with pytest.raises(ValueError, match="metadata"): + score.load_records(path) + + +def _hunt_change() -> dict[str, object]: + return { + "id": "change-1", + "repo": "fixture/repo", + "base": "a" * 40, + "head": "b" * 40, + "kind": "ordinary", + "files": [{"path": "app.py", "head_lines": [[1, 1]]}], + } + + +def _hunt_record(change: dict[str, object], program_id: str, mask: tuple[str, ...], digest: str) -> dict[str, object]: + return { + "id": change["id"], + "done": True, + "status": "hunt_covered", + "instrument": {"head_verified": True, "base_verified": True, "changed_bytes": {"app.py": 10}}, + "push": {"admission": {"defects": [], "dispositions": []}, "quick_tier": []}, + "rows": 1, + "program_id": program_id, + "mask": list(mask), + "inputs_present": ["function_text"], + "seconds": 0.1, + "input_digest": digest, + "usd": 0.0, + } + + +def _hunt_artifact(program_id: str, mask: tuple[str, ...]) -> RunArtifact: + return RunArtifact( + program_id=program_id, + schema_version=1, + mask=mask, + data_digests={"manifest": "d" * 64}, + configurations=[{"only": "all", "slice": "1", "changes": 1}], + usage={"usd": 0.0, "calls": 1.0, "replay_hits": 0.0, "wall_seconds": 0.1}, + versions={}, + created="2026-11-02T00:00:00+00:00", + ) + + +def _raw_hunt_run(root: Path) -> tuple[hunt_handoff.HuntRunContext, tuple[dict[str, object], ...]]: + change = _hunt_change() + program_id = "c" * 64 + mask = ("obligations",) + digest = input_digest(change, program_id, mask) + root.mkdir() + _rewrite(root / "inputs.json", [{"item_id": change["id"], "change": change, "input_digest": digest}]) + _rewrite(record_path(root, str(change["id"])), _hunt_record(change, program_id, mask, digest)) + artifact = _hunt_artifact(program_id, mask) + artifact.write(root, reserve_attempt=True) + context = hunt_handoff.HuntRunContext( + producer="openultrasast.hunt.runner.v1", + program_id=program_id, + mask=mask, + manifest_digest="d" * 64, + freeze_digest="e" * 64, + only="all", + slice_id="1", + artifact_ids=(artifact.artifact_id,), + versions=(), + ) + return context, (change,) + + +def test_hunt_handoff_is_non_mutating_repeatable_and_persists_only_to_benchmark_destination(tmp_path: Path) -> None: + source = tmp_path / "producer" + destination = tmp_path / "benchmark-v2" + context, changes = _raw_hunt_run(source) + before = {path.name: path.read_bytes() for path in source.iterdir()} + + first = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes) + second = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes) + assert first == second and first[0]["id"] == "change-1" + assert {path.name: path.read_bytes() for path in source.iterdir()} == before + + hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes) + persisted = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} + hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes) + assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} == persisted + assert score.load_records(destination) == list(first) + assert {path.name: path.read_bytes() for path in source.iterdir()} == before + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + ("producer", "producer"), + ("program", "program"), + ("manifest", "manifest"), + ("input", "input digest"), + ("record", "record identity"), + ("reserved", "reserved metadata"), + ), +) +def test_hunt_handoff_refuses_unsupported_producer_schema_and_identity(tmp_path: Path, mutation: str, message: str) -> None: + source = tmp_path / "producer" + context, changes = _raw_hunt_run(source) + if mutation == "producer": + context = hunt_handoff.HuntRunContext(**{**context.__dict__, "producer": "arbitrary-json"}) + elif mutation == "program": + context = hunt_handoff.HuntRunContext(**{**context.__dict__, "program_id": "f" * 64}) + elif mutation == "manifest": + context = hunt_handoff.HuntRunContext(**{**context.__dict__, "manifest_digest": "f" * 64}) + elif mutation == "input": + entries = json.loads((source / "inputs.json").read_text()) + entries[0]["input_digest"] = "0" * 64 + _rewrite(source / "inputs.json", entries) + else: + path = record_path(source, "change-1") + record = json.loads(path.read_text()) + if mutation == "record": + record["id"] = "other" + else: + record["schema_version"] = "forged" + _rewrite(path, record) + with pytest.raises(ValueError, match=message): + hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes) + + +def test_hunt_handoff_refuses_conflicting_benchmark_destination(tmp_path: Path) -> None: + source = tmp_path / "producer" + destination = tmp_path / "benchmark-v2" + context, changes = _raw_hunt_run(source) + hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes) + artifact_contract.write_benchmark_artifact(destination / "handoff.json", "hunt-runner-handoff", {"producer": "conflict"}, replace=True) + with pytest.raises(ValueError, match="conflict"): + hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes) + + +@pytest.mark.parametrize("mutation", ("program", "manifest", "configuration", "versions")) +def test_hunt_handoff_refuses_mixed_attempt_invariants(tmp_path: Path, mutation: str) -> None: + source = tmp_path / "producer" + manifest = tmp_path / "pool-fixture.toml" + manifest.write_text("status = 'fixture'\n") + _context, _changes = _raw_hunt_run(source) + path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text()) + original = RunArtifact.read(path) + values = { + "program_id": original.program_id, + "schema_version": original.schema_version, + "mask": tuple(original.mask), + "data_digests": dict(original.data_digests), + "configurations": list(original.configurations), + "usage": {**original.usage, "calls": 2.0}, + "versions": dict(original.versions), + "created": "2026-11-02T00:00:01+00:00", + } + if mutation == "program": + values["program_id"] = "f" * 64 + elif mutation == "manifest": + values["data_digests"] = {"manifest": "f" * 64} + elif mutation == "configuration": + values["configurations"] = [{"only": "ordinary", "slice": "1", "changes": 1}] + else: + values["versions"] = {"dspy": "different"} + RunArtifact(**values).write(source, reserve_attempt=True) + + with pytest.raises(ValueError, match="invariant"): + hunt_handoff.discover_hunt_runner_context( + source, + manifest, + producer=hunt_handoff.HUNT_RUNNER_PRODUCER, + freeze_digest="e" * 64, + ) + + +@pytest.mark.parametrize("attempt_ordinal", (-1, 0, 1, True, 1.5)) +def test_hunt_handoff_refuses_nonpositive_or_duplicate_attempt_ordinals(tmp_path: Path, attempt_ordinal: int) -> None: + source = tmp_path / "producer" + context, changes = _raw_hunt_run(source) + path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text()) + original = RunArtifact.read(path) + duplicate = RunArtifact( + program_id=original.program_id, + schema_version=original.schema_version, + mask=tuple(original.mask), + data_digests=dict(original.data_digests), + configurations=list(original.configurations), + usage={**original.usage, "calls": 2.0}, + versions=dict(original.versions), + created="2026-11-02T00:00:01+00:00", + attempt_ordinal=attempt_ordinal, + ) + duplicate.write(source) + artifact_ids = tuple(sorted((original.artifact_id, duplicate.artifact_id))) + + with pytest.raises(ValueError, match="attempt ordinals"): + hunt_handoff.read_hunt_runner_records(source, context=replace(context, artifact_ids=artifact_ids), changes=changes) + + +@pytest.mark.parametrize(("attempt_ordinal", "accepted"), ((MAX_ATTEMPT_ORDINAL, True), (MAX_ATTEMPT_ORDINAL + 1, False))) +def test_hunt_handoff_enforces_maximum_attempt_ordinal(tmp_path: Path, attempt_ordinal: int, accepted: bool) -> None: + source = tmp_path / "producer" + context, changes = _raw_hunt_run(source) + path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text()) + original = RunArtifact.read(path) + additional = RunArtifact( + program_id=original.program_id, + schema_version=original.schema_version, + mask=tuple(original.mask), + data_digests=dict(original.data_digests), + configurations=list(original.configurations), + usage={**original.usage, "calls": 2.0}, + versions=dict(original.versions), + created="2026-11-02T00:00:01+00:00", + attempt_ordinal=attempt_ordinal, + ) + additional.write(source) + artifact_ids = tuple(sorted((original.artifact_id, additional.artifact_id))) + bound_context = replace(context, artifact_ids=artifact_ids) + + if accepted: + assert hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes) + else: + with pytest.raises(ValueError, match="attempt ordinals"): + hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes) + + +def test_score_reader_has_no_external_adoption_escape_hatch(tmp_path: Path) -> None: + path = tmp_path / "raw.json" + _rewrite(path, [{"id": "one"}]) + with pytest.raises(TypeError, match="adopt_external"): + score.load_records(path, adopt_external=True) diff --git a/tests/test_core_migration_benchmark.py b/tests/test_core_migration_benchmark.py new file mode 100644 index 0000000..9802d3d --- /dev/null +++ b/tests/test_core_migration_benchmark.py @@ -0,0 +1,186 @@ +"""Benchmark-source closure for the detection-core v1 migration.""" + +from __future__ import annotations + +import sys +from dataclasses import replace +from pathlib import Path + +import pytest +from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE +from core_migration_boundary import inspect_legacy_boundary +from core_migration_inventory import ( + REPO, + LegacyInventoryError, + MigrationDisposition, + entry_identity, + load_migration_inventory, + validate_historical_artifacts, + validate_inventory_sources, + validate_inventory_transition, +) +from core_migration_scope import boundary_python + +INVENTORY = REPO / "tests/fixtures/detection_core/v1/migration_inventory.json" +sys.path.insert(0, str(REPO)) + +_V2_MIGRATED = frozenset(entry_id for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated") +_RETIRED = frozenset(entry_id for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items() if evidence.disposition == "retired") +_BENCHMARK_ENTRY_IDS = _V2_MIGRATED | _RETIRED + + +def _benchmark_program_paths(root: Path = REPO) -> tuple[Path, ...]: + """Owned benchmark programs, excluding harvested fixtures and historical corpora.""" + return tuple( + path + for path in boundary_python(root) + if (relative := path.relative_to(root).as_posix()).startswith("benchmarks/") and not relative.startswith("benchmarks/measurements/") + ) + + +def test_registered_benchmark_entries_close_without_rekeying_other_consumers() -> None: + candidate = load_migration_inventory(INVENTORY) + benchmark = {entry.entry_id: entry for entry in candidate.entries if entry.path.startswith("benchmarks/")} + + assert set(benchmark) == _BENCHMARK_ENTRY_IDS + assert all(benchmark[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED) + assert all(benchmark[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED) + frozen = replace( + candidate, + entries=tuple( + replace(entry, disposition=MigrationDisposition.OPEN) if entry.entry_id in _BENCHMARK_ENTRY_IDS else entry + for entry in candidate.entries + ), + ) + validate_inventory_transition(frozen, candidate) + validate_inventory_sources(candidate, root=REPO) + + +def test_source_validation_refuses_unregistered_terminal_entry() -> None: + inventory = load_migration_inventory(INVENTORY) + original = next(entry for entry in inventory.entries if entry.entry_id in _V2_MIGRATED) + forged = replace(original, artifact_family="unregistered-terminal", entry_id="") + forged = replace(forged, entry_id=entry_identity(forged)) + + with pytest.raises(LegacyInventoryError, match="terminal evidence"): + validate_inventory_sources(replace(inventory, entries=(forged,)), root=REPO) + + +def test_source_validation_refuses_removed_migrated_owner(tmp_path: Path) -> None: + inventory = load_migration_inventory(INVENTORY) + original = next(entry for entry in inventory.entries if entry.entry_id.startswith("40465a45")) + target = tmp_path / original.path + target.parent.mkdir(parents=True) + target.write_text((REPO / original.path).read_text().replace("def main(", "def removed_main(", 1)) + + with pytest.raises(LegacyInventoryError, match="migrated owner is absent"): + validate_inventory_sources(replace(inventory, entries=(original,)), root=tmp_path) + + +def test_test_and_operational_entries_remain_open_for_task_10_6() -> None: + candidate = load_migration_inventory(INVENTORY) + deferred = { + entry.disposition for entry in candidate.entries if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"} + } + + assert deferred == {MigrationDisposition.OPEN} + + +def test_old_benchmark_admission_contract_and_runner_are_not_live() -> None: + from benchmarks.valid import admission_run, prover_admission + + assert not hasattr(admission_run, "run_admission") + assert not hasattr(admission_run, "assemble_admission_report") + assert not hasattr(prover_admission, "AdmissionManifest") + assert not hasattr(prover_admission, "validate_admission_manifest") + + +def test_registered_live_writers_execute_strict_v2_artifact_conformance(tmp_path: Path) -> None: + from benchmark_migration_conformance import validate_benchmark_terminal_conformance + + inventory = load_migration_inventory(INVENTORY) + validate_benchmark_terminal_conformance(inventory, output_root=tmp_path) + + +@pytest.mark.parametrize("entry_id", sorted(_V2_MIGRATED)) +def test_each_registered_owner_is_required_for_its_exact_probe(tmp_path: Path, monkeypatch, entry_id: str) -> None: + from benchmark_migration_conformance import ( + OWNER_TARGETS, + BenchmarkMigrationConformanceError, + probe_benchmark_entry, + ) + + inventory = load_migration_inventory(INVENTORY) + entry = next(candidate for candidate in inventory.entries if candidate.entry_id == entry_id) + module_name, symbol = OWNER_TARGETS[entry_id] + module = __import__(module_name, fromlist=[symbol]) + + def removed_owner(*_args, **_kwargs): + raise RuntimeError("registered owner removed") + + monkeypatch.setattr(module, symbol, removed_owner) + with pytest.raises(BenchmarkMigrationConformanceError, match=entry_id): + probe_benchmark_entry(entry, output_root=tmp_path) + + +@pytest.mark.parametrize( + "forged", + ["if 0", "if None", "while False", "local def", "local import", "parameter", "assignment", "comprehension"], +) +def test_dead_or_shadowed_syntax_cannot_close_an_unregistered_entry(tmp_path: Path, forged: str) -> None: + from benchmark_migration_conformance import BenchmarkMigrationConformanceError, validate_benchmark_terminal_conformance + + inventory = load_migration_inventory(INVENTORY) + original = next(entry for entry in inventory.entries if entry.entry_id in _V2_MIGRATED) + unregistered = replace(original, entry_id="f" * 64) + candidate = replace(inventory, entries=(unregistered,)) + (tmp_path / "forged.py").write_text(f"# {forged}\n") + with pytest.raises(BenchmarkMigrationConformanceError, match="probe registry"): + validate_benchmark_terminal_conformance(candidate, output_root=tmp_path / "out") + + +def test_v2_benchmark_artifact_binds_its_kind_and_payload() -> None: + from benchmarks.valid.artifact_contract import benchmark_artifact, benchmark_payload + + artifact = benchmark_artifact("readiness-result", {"result_digest": "a" * 64}) + assert benchmark_payload(artifact, expected_kind="readiness-result") == {"result_digest": "a" * 64} + + artifact["artifact_kind"] = "admission-report" + with pytest.raises(ValueError, match="metadata does not match"): + benchmark_payload(artifact, expected_kind="readiness-result") + + +def test_v2_benchmark_jsonl_preserves_ordered_row_persistence(tmp_path: Path) -> None: + from benchmarks.valid.artifact_contract import read_benchmark_jsonl, write_benchmark_jsonl + + path = tmp_path / "suspicions.jsonl" + write_benchmark_jsonl(path, "suspicion", ({"id": "first"}, {"id": "second"})) + + assert read_benchmark_jsonl(path, "suspicion") == ({"id": "first"}, {"id": "second"}) + + +def test_verified_nonzero_benchmark_program_scope_has_exact_zero_legacy_references() -> None: + inventory = load_migration_inventory(INVENTORY) + paths = _benchmark_program_paths() + report = inspect_legacy_boundary( + paths, + frozen_inventory=inventory, + candidate_inventory=inventory, + expected_scope=paths, + root=REPO, + ) + + assert report.files_read == len(paths) > 0 + assert report.bytes_read > 0 + assert report.scanned_scope_paths == report.expected_scope_paths + assert report.consumer_legacy_import_sites == () + assert report.consumer_legacy_writer_sites == () + assert report.forbidden_dynamic_sites == () + assert report.compatibility_implementation_sites == () + + +def test_benchmark_closure_preserves_frozen_historical_bytes() -> None: + inventory = load_migration_inventory(INVENTORY) + artifacts = validate_historical_artifacts(inventory, root=REPO) + + assert len(artifacts) == len(inventory.historical_artifacts) > 0 diff --git a/tests/test_core_migration_production.py b/tests/test_core_migration_production.py index d8dd76e..0fe8230 100644 --- a/tests/test_core_migration_production.py +++ b/tests/test_core_migration_production.py @@ -66,7 +66,11 @@ def test_registered_production_entries_close_without_rekeying_other_consumers() assert set(production) == _PRODUCTION_ENTRY_IDS assert all(production[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED) assert all(production[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED) - assert all(entry.disposition is MigrationDisposition.OPEN for entry in candidate.entries if entry.entry_id not in production) + assert all( + entry.disposition is MigrationDisposition.OPEN + for entry in candidate.entries + if entry.entry_id not in production and not entry.path.startswith("benchmarks/") + ) frozen = replace( candidate, diff --git a/tests/test_core_migration_provenance.py b/tests/test_core_migration_provenance.py index feb11e8..184a7ee 100644 --- a/tests/test_core_migration_provenance.py +++ b/tests/test_core_migration_provenance.py @@ -81,9 +81,9 @@ def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None: inventory = load_migration_inventory(INVENTORY) assert inventory.entries - assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 180 - assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 6 - assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 12 + assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 155 + assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 25 + assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 18 assert all(entry.entry_id == entry_identity(entry) for entry in inventory.entries) validate_inventory_sources(inventory, root=REPO) diff --git a/tests/test_hunt_artifact.py b/tests/test_hunt_artifact.py index bcf31de..c04c169 100644 --- a/tests/test_hunt_artifact.py +++ b/tests/test_hunt_artifact.py @@ -1,10 +1,15 @@ """Checks for stable run records and budget accounting.""" +import errno import json +import os +import stat +from concurrent.futures import ThreadPoolExecutor import pytest -from openultrasast.hunt.artifact import RunArtifact, UsageLedger +from openultrasast.hunt import artifact as artifact_module +from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact, UsageLedger def artifact(): @@ -32,6 +37,110 @@ def test_artifact_round_trip_and_tamper(tmp_path): RunArtifact.read(path) +def test_artifact_persistence_is_create_only_and_idempotent(tmp_path): + run = artifact() + path = run.write(tmp_path) + before = (path.read_bytes(), path.stat().st_mtime_ns) + assert run.write(tmp_path) == path + assert (path.read_bytes(), path.stat().st_mtime_ns) == before + + path.write_text("conflicting bytes\n") + with pytest.raises(ValueError, match="conflict"): + run.write(tmp_path) + + +def test_missing_attempt_sequence_fails_closed_when_artifacts_exist(tmp_path): + first = artifact() + first_path = first.write(tmp_path, reserve_attempt=True) + before = (first_path.read_bytes(), first_path.stat().st_mtime_ns) + (tmp_path / ".attempt-sequence").unlink() + resumed = artifact() + resumed.created = "2026-11-02T00:00:01+00:00" + + with pytest.raises(ValueError, match="attempt sequence is missing"): + resumed.write(tmp_path, reserve_attempt=True) + assert (first_path.read_bytes(), first_path.stat().st_mtime_ns) == before + assert not (tmp_path / ".attempt-sequence").exists() + + +def test_attempt_ordinals_are_unique_under_concurrency_and_resume(tmp_path): + def persist(index): + run = artifact() + run.created = f"2026-11-02T00:00:{index:02d}+00:00" + run.write(tmp_path, reserve_attempt=True) + return run.attempt_ordinal + + with ThreadPoolExecutor(max_workers=4) as pool: + ordinals = list(pool.map(persist, range(4))) + resumed = artifact() + resumed.created = "2026-11-02T00:01:00+00:00" + resumed.write(tmp_path, reserve_attempt=True) + + assert sorted(ordinals) == [1, 2, 3, 4] + assert resumed.attempt_ordinal == 5 + assert (tmp_path / ".attempt-sequence").read_text() == "5\n" + + +@pytest.mark.parametrize(("state", "message"), (("corrupt\n", "unreadable"), ("-1\n", "invalid"))) +def test_corrupt_attempt_sequence_fails_closed(tmp_path, state, message): + (tmp_path / ".attempt-sequence").write_text(state) + + with pytest.raises(ValueError, match=message): + artifact().write(tmp_path, reserve_attempt=True) + + +def test_attempt_sequence_allocates_the_signed_64_bit_maximum(tmp_path): + (tmp_path / ".attempt-sequence").write_text(f"{MAX_ATTEMPT_ORDINAL - 1}\n") + run = artifact() + + run.write(tmp_path, reserve_attempt=True) + + assert run.attempt_ordinal == MAX_ATTEMPT_ORDINAL + assert (tmp_path / ".attempt-sequence").read_text() == f"{MAX_ATTEMPT_ORDINAL}\n" + + +def test_attempt_sequence_refuses_overflow_without_mutation(tmp_path): + sequence = tmp_path / ".attempt-sequence" + sequence.write_text(f"{MAX_ATTEMPT_ORDINAL}\n") + before = (sequence.read_bytes(), sequence.stat().st_mtime_ns) + + with pytest.raises(ValueError, match="exhausted"): + artifact().write(tmp_path, reserve_attempt=True) + + assert (sequence.read_bytes(), sequence.stat().st_mtime_ns) == before + assert not list(tmp_path.glob("*.json")) + + +def test_attempt_sequence_and_artifact_creation_fsync_the_directory(tmp_path, monkeypatch): + real_fsync = os.fsync + directory_syncs = [] + + def recording_fsync(descriptor): + if stat.S_ISDIR(os.fstat(descriptor).st_mode): + directory_syncs.append(descriptor) + real_fsync(descriptor) + + monkeypatch.setattr(artifact_module.os, "fsync", recording_fsync) + artifact().write(tmp_path, reserve_attempt=True) + + assert len(directory_syncs) == 2 + + +def test_directory_fsync_ignores_only_unsupported_platform_errors(tmp_path, monkeypatch): + def unsupported(_descriptor): + raise OSError(errno.EOPNOTSUPP, "unsupported") + + monkeypatch.setattr(artifact_module.os, "fsync", unsupported) + artifact_module._sync_directory(tmp_path) + + def io_failure(_descriptor): + raise OSError(errno.EIO, "I/O failure") + + monkeypatch.setattr(artifact_module.os, "fsync", io_failure) + with pytest.raises(OSError, match="I/O failure"): + artifact_module._sync_directory(tmp_path) + + def test_ledger_and_budget_prevent_artifact(tmp_path): run = artifact() ledger = UsageLedger(artifact=run) diff --git a/tests/test_hunt_report.py b/tests/test_hunt_report.py index 2c502d8..7015340 100644 --- a/tests/test_hunt_report.py +++ b/tests/test_hunt_report.py @@ -84,7 +84,7 @@ def test_valid_report_before_after_workload_curve_and_ledger(pilot_case, populat assert row["changed_bytes"]["app.py"] > 0 assert row["base_verified"] and row["head_verified"] and row["base"] and row["head"] assert analysis["before"]["raw"]["metrics"] == json.loads(kwargs["pilot_metrics"]["raw"].read_text()) - records = score.load_records(directory / "records") + records = score.load_records(directory / "benchmark-records") assert analysis["after"]["pooled"]["push"] == metrics.summarize(records, scored=score.score(metrics.scoring_records(records))) assert analysis["after"]["injection"]["search"]["recall"]["rate"] == 1 assert analysis["after"]["pooled"]["search"]["suspicions_per_push"]["rate"] == 0 diff --git a/tests/test_hunt_runner.py b/tests/test_hunt_runner.py index 6ce4da2..76266f3 100644 --- a/tests/test_hunt_runner.py +++ b/tests/test_hunt_runner.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import subprocess +from datetime import UTC, datetime import pytest @@ -11,6 +12,12 @@ dspy = pytest.importorskip("dspy") from dspy.utils import DummyLM # noqa: E402 from benchmarks.unseen import score # noqa: E402 +from benchmarks.valid.hunt_handoff import ( # noqa: E402 + HUNT_RUNNER_PRODUCER, + discover_hunt_runner_context, + persist_hunt_runner_records, + read_hunt_runner_records, +) from openultrasast.hunt import program as hp # noqa: E402 from openultrasast.hunt import runner # noqa: E402 @@ -73,7 +80,8 @@ def test_runner_writes_scorable_records_and_resumes(repository, tmp_path): manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes ) assert artifact is not None and artifact.program_id == pid - rows = score.load_records(out) + context = discover_hunt_runner_context(out, manifest, producer=HUNT_RUNNER_PRODUCER, freeze_digest="e" * 64) + rows = list(read_hunt_runner_records(out, context=context, changes=changes)) assert {r["id"]: r["status"] for r in rows} == {"pos": "hunt_covered", "ord": "hunt_covered"} pos = next(r for r in rows if r["id"] == "pos") assert pos["instrument"]["head_verified"] and pos["instrument"]["hook_bytes_read"] > 0 @@ -91,6 +99,58 @@ def test_runner_writes_scorable_records_and_resumes(repository, tmp_path): assert {p.name: p.read_bytes() for p in out.glob("*.json") if p.name in before} == before +def _budget_attempt(manifest, root, source, program, pid, changes): + return runner.run( + manifest, + {"fixture/repo": str(root)}, + source, + program=program, + program_id=pid, + mask=program.mask, + load_changes=lambda _manifest, _slice: changes, + budget={"calls": 1}, + versions={"dspy": dspy.__version__}, + ) + + +def test_budget_stop_resume_handoff_binds_all_attempt_artifacts(repository, tmp_path, monkeypatch): + root, manifest, changes = repository + program = _scripted_program() + pid = hp.program_id(program, mask=program.mask) + source = tmp_path / "attempts" + destination = tmp_path / "benchmark-v2" + instant = datetime(2026, 11, 2, tzinfo=UTC) + instants = iter((instant, instant)) + + class AttemptClock: + @classmethod + def now(cls, _timezone): + return next(instants) + + monkeypatch.setattr(runner, "datetime", AttemptClock) + monkeypatch.setattr(runner.time, "monotonic", lambda: 100.0) + + first = _budget_attempt(manifest, root, source, program, pid, changes) + (first_path,) = [path for path in source.glob("*.json") if "artifact_id" in path.read_text()] + first_before = (first_path.read_bytes(), first_path.stat().st_mtime_ns) + second = _budget_attempt(manifest, root, source, program, pid, changes) + artifacts = [path for path in source.glob("*.json") if "artifact_id" in path.read_text()] + assert first is None and second is None + assert len(artifacts) == 2 + assert all(path.stat().st_size > 0 for path in artifacts) + assert (first_path.read_bytes(), first_path.stat().st_mtime_ns) == first_before + source_before = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in source.iterdir()} + + context = discover_hunt_runner_context(source, manifest, producer=HUNT_RUNNER_PRODUCER, freeze_digest="e" * 64) + assert context.artifact_ids == tuple(sorted(path.stem for path in artifacts)) + assert {row["id"] for row in read_hunt_runner_records(source, context=context, changes=changes)} == {"pos", "ord"} + persist_hunt_runner_records(source, destination, context=context, changes=changes) + destination_before = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} + persist_hunt_runner_records(source, destination, context=context, changes=changes) + assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in source.iterdir()} == source_before + assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} == destination_before + + def test_runner_refuses_the_pilot_unless_it_is_a_report_run(tmp_path): manifest = tmp_path / "pool-valid.toml" manifest.write_text("status = 'frozen'\n") diff --git a/tests/test_hunter_arm.py b/tests/test_hunter_arm.py index 4275381..79e7b29 100644 --- a/tests/test_hunter_arm.py +++ b/tests/test_hunter_arm.py @@ -22,6 +22,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) hunter_arm = importlib.import_module("benchmarks.valid.hunter_arm") catalog = importlib.import_module("benchmarks.valid.catalog") score = importlib.import_module("benchmarks.unseen.score") +artifact_contract = importlib.import_module("benchmarks.valid.artifact_contract") @pytest.fixture(autouse=True) @@ -136,7 +137,7 @@ def test_positive_is_scoreable_and_checkout_is_removed(fixture, monkeypatch, cap assert row["status"] == "hunter_covered" and row["done"] assert row["instrument"]["hook_bytes_read"] == len(git.blob(change["head"], "app.py").encode()) assert score.coverage(row) != "instrument_failure" - assert row["input_digest"] == json.loads((out / "inputs.json").read_text())[0]["input_digest"] + assert row["input_digest"] == artifact_contract.read_benchmark_sequence(out / "inputs.json", "benchmark-arm-inputs")[0]["input_digest"] assert client.calls[0]["model"] == resolve_models(load_config(dotenv=False))[0] assert "fixture-positive app.py:" in capsys.readouterr().out assert git.run("worktree", "list", "--porcelain").count("worktree ") == 1 @@ -204,9 +205,11 @@ def test_resume_skips_done_without_client_or_checkout(fixture, monkeypatch, with _, change, manifest, repos, out = fixture out.mkdir() existing = {"id": change["id"], "done": True, "input_digest": "existing-digest", "status": "hunter_covered"} - record_path(out, change["id"]).write_text(json.dumps(existing)) + artifact_contract.write_benchmark_artifact(record_path(out, change["id"]), "benchmark-arm-record", existing) if with_inputs: - (out / "inputs.json").write_text(json.dumps([{"item_id": change["id"], "change": change, "input_digest": "existing-digest"}])) + artifact_contract.write_benchmark_sequence( + out / "inputs.json", "benchmark-arm-inputs", [{"item_id": change["id"], "change": change, "input_digest": "existing-digest"}] + ) def forbidden(*args, **kwargs): pytest.fail("completed change was executed again") @@ -214,7 +217,7 @@ def test_resume_skips_done_without_client_or_checkout(fixture, monkeypatch, with monkeypatch.setattr(tool_hunter, "resolve_hunter_client", forbidden) monkeypatch.setattr(hunter_arm, "local_git", forbidden) assert hunter_arm.main(["run", str(manifest), "--repos", str(repos), "--out", str(out)]) == 0 - assert json.loads(record_path(out, change["id"]).read_text()) == existing + assert artifact_contract.read_benchmark_artifact(record_path(out, change["id"]), "benchmark-arm-record") == existing assert score.load_records(out)[0]["input_digest"] == "existing-digest" @@ -418,9 +421,9 @@ def test_classify_family_reuses_parser_and_retry(tmp_path, monkeypatch, failure) def write_arm(out, entries, records): out.mkdir() - (out / "inputs.json").write_text(json.dumps(entries)) + artifact_contract.write_benchmark_sequence(out / "inputs.json", "benchmark-arm-inputs", entries) for item_id, record in records.items(): - record_path(out, item_id).write_text(json.dumps(record)) + artifact_contract.write_benchmark_artifact(record_path(out, item_id), "benchmark-arm-record", record) @pytest.fixture @@ -489,7 +492,7 @@ def test_combine_missing_classifier_counts_as_unsure(combine_arms, classificatio replay_out, hunter_out, combined, entry, a, b = combine_arms defect = {"defect_id": "unclassified", "source": "hunter", "family": "path", "locations": [["app.py", 6]], **classification} b["push"]["admission"]["defects"] = [defect] - record_path(hunter_out, entry["item_id"]).write_text(json.dumps(b)) + artifact_contract.write_benchmark_artifact(record_path(hunter_out, entry["item_id"]), "benchmark-arm-record", b, replace=True) assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", keep_verdicts]) == 0 (row,) = score.load_records(combined) kept = int("unsure" in keep_verdicts.split(",")) @@ -546,7 +549,7 @@ def test_combine_preserves_a_and_scores_b(fixture, tmp_path): } assert b_only["arms"] == {"a": "missing", "b": "hunter_covered"} a["push"]["admission"]["defects"] = [{**defect, "defect_id": "a-defect", "source": "engine"}] - record_path(replay_out, "replay-both").write_text(json.dumps(a)) + artifact_contract.write_benchmark_artifact(record_path(replay_out, "replay-both"), "benchmark-arm-record", a, replace=True) hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined)]) defects = score.load_records(combined)[0]["push"]["admission"]["defects"] assert len(defects) == 1 and defects[0]["source"] == "engine" diff --git a/tests/test_judge_pass_v2.py b/tests/test_judge_pass_v2.py new file mode 100644 index 0000000..652e73a --- /dev/null +++ b/tests/test_judge_pass_v2.py @@ -0,0 +1,33 @@ +"""The judge resume boundary accepts only complete ordered v2 artifacts.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from benchmarks.valid import judge_pass # noqa: E402 + + +def test_load_done_refuses_stripped_metadata_and_duplicate_ids(tmp_path: Path) -> None: + out = tmp_path / "judge" + out.mkdir() + path = out / "judgments.jsonl" + path.write_text(json.dumps({"id": "one", "judgment": "keep"}) + "\n") + with pytest.raises(ValueError, match="metadata"): + judge_pass._load_done(out) + + from benchmarks.valid.artifact_contract import write_benchmark_jsonl + + write_benchmark_jsonl( + path, + "judge-pass", + ({"key": "one", "judgment": "keep"}, {"key": "one", "judgment": "drop"}), + replace=True, + ) + with pytest.raises(ValueError, match="duplicate"): + judge_pass._load_done(out) diff --git a/tests/test_prover_admission.py b/tests/test_prover_admission.py index 22b1324..292ba0a 100644 --- a/tests/test_prover_admission.py +++ b/tests/test_prover_admission.py @@ -14,7 +14,7 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from benchmarks.valid import prover_admission # noqa: E402 +from openultrasast.core import legacy_v1 as prover_admission # noqa: E402 def _manifest() -> dict[str, object]: diff --git a/tests/test_prover_arm.py b/tests/test_prover_arm.py index a995cd0..afb613d 100644 --- a/tests/test_prover_arm.py +++ b/tests/test_prover_arm.py @@ -14,6 +14,7 @@ from test_hunt_prover import repository # noqa: E402 from benchmarks.unseen import score # noqa: E402 from benchmarks.valid import prover_arm # noqa: E402 +from benchmarks.valid.artifact_contract import benchmark_payload # noqa: E402 @pytest.fixture @@ -99,13 +100,25 @@ def test_run_resume_summary_and_locations(pool, monkeypatch, capsys): ) +def test_resume_refuses_stripped_v2_metadata(pool): + _, manifest, repos, out = pool + assert prover_arm.run(manifest, repos, out) == 0 + path = out / (hashlib.sha256(b"positive").hexdigest() + ".json") + value = json.loads(path.read_text()) + for field in ("schema_version", "artifact_kind", "artifact_digest"): + value.pop(field) + path.write_text(json.dumps(value)) + with pytest.raises(ValueError, match="metadata"): + prover_arm.run(manifest, repos, out) + + def test_zero_byte_record_refuses(pool): _, manifest, repos, out = pool assert prover_arm.run(manifest, repos, out) == 0 path = out / (hashlib.sha256(b"positive").hexdigest() + ".json") - record = json.loads(path.read_text()) + record = benchmark_payload(json.loads(path.read_text()), expected_kind="benchmark-arm-record") record["instrument"]["changed_bytes"]["Fixture.java"] = 0 - path.write_text(json.dumps(record)) + prover_arm._write_json(path, record, replace=True) with pytest.raises(ValueError, match="zero-byte"): prover_arm.summarize(out) diff --git a/tests/test_unseen_replay.py b/tests/test_unseen_replay.py index 37a22a2..f34c499 100644 --- a/tests/test_unseen_replay.py +++ b/tests/test_unseen_replay.py @@ -48,6 +48,7 @@ def test_frozen_toml_slice_and_scoring_join(tmp_path): import hashlib from benchmarks.unseen.score import load_records + from benchmarks.valid.artifact_contract import write_benchmark_artifact, write_benchmark_sequence pool = tmp_path / "pool-p1.toml" pool.write_text("""pool="p1" @@ -68,11 +69,15 @@ changes=[{kind="ordinary",base="ccc",head="ddd",files=[]}] (tmp_path / "freeze-p1.json").write_text(json.dumps({"freeze_digest": freeze})) changes = r.load_slice(pool, 2) assert len(changes) == 1 and changes[0]["repo"] == "opaque-b" - (tmp_path / "inputs.json").write_text(json.dumps([{"item_id": "replay-opaque", "change": changes[0], "input_digest": "digest"}])) + write_benchmark_sequence( + tmp_path / "inputs.json", + "benchmark-arm-inputs", + [{"item_id": "replay-opaque", "change": changes[0], "input_digest": "digest"}], + ) assert load_records(tmp_path)[0]["kind"] == "ordinary" assert "instrument" not in load_records(tmp_path)[0] # Missing result remains a measured failure. result = tmp_path / (hashlib.sha256(b"replay-opaque").hexdigest() + ".json") - result.write_text(json.dumps({"input_digest": "digest", "instrument": {"head_verified": True}})) + write_benchmark_artifact(result, "benchmark-arm-record", {"input_digest": "digest", "instrument": {"head_verified": True}}) assert load_records(tmp_path)[0]["instrument"]["head_verified"] pool.write_text(pool.read_text() + "changed=true\n") with pytest.raises(ValueError, match="digest"):