mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
feat(detection-core): migrate benchmark readiness and admission consumers
This commit is contained in:
Marius Ciepluch
@@ -0,0 +1,374 @@
|
||||
"""Executable owner conformance for the stable benchmark migration worklist."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, migrated_owner_targets
|
||||
from core_migration_inventory import LegacyInventoryEntry, LegacyMigrationInventory, MigrationDisposition
|
||||
|
||||
from benchmarks.valid.artifact_contract import (
|
||||
benchmark_payload,
|
||||
read_benchmark_artifact,
|
||||
read_benchmark_jsonl,
|
||||
read_benchmark_sequence,
|
||||
write_benchmark_artifact,
|
||||
write_benchmark_jsonl,
|
||||
write_benchmark_sequence,
|
||||
)
|
||||
|
||||
|
||||
class BenchmarkMigrationConformanceError(ValueError):
|
||||
"""A terminal worklist entry lacks its exact executable owner."""
|
||||
|
||||
|
||||
OWNER_TARGETS = migrated_owner_targets()
|
||||
|
||||
|
||||
def _change() -> dict[str, Any]:
|
||||
return {"id": "case", "repo": "fixture/repo", "kind": "ordinary", "base": "a" * 40, "head": "b" * 40, "files": []}
|
||||
|
||||
|
||||
def _record() -> dict[str, Any]:
|
||||
return {
|
||||
**_change(),
|
||||
"done": True,
|
||||
"status": "quick_only",
|
||||
"seconds": 0.1,
|
||||
"instrument": {
|
||||
"hook_exit": 0,
|
||||
"hook_bytes_read": 1,
|
||||
"head_verified": True,
|
||||
"base_verified": True,
|
||||
"changed_bytes": {"app.py": 1},
|
||||
"file_bytes": {"app.py": 1},
|
||||
},
|
||||
"push": {"admission": {"defects": []}, "quick_tier": []},
|
||||
"coverage": {},
|
||||
}
|
||||
|
||||
|
||||
def _owner(entry: LegacyInventoryEntry) -> tuple[Any, Callable[..., Any]]:
|
||||
module_name, symbol = OWNER_TARGETS[entry.entry_id]
|
||||
module = importlib.import_module(module_name)
|
||||
owner = getattr(module, symbol, None)
|
||||
if not callable(owner):
|
||||
raise BenchmarkMigrationConformanceError(f"registered benchmark owner is absent: {entry.entry_id}")
|
||||
return module, owner
|
||||
|
||||
|
||||
def _probe_engine(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
pin = {"pin": "fixture", "units": [], "questions": {}}
|
||||
|
||||
def download(_url: str, path: Path, **_kwargs: Any) -> None:
|
||||
path.write_text(json.dumps(pin)) if path.name == "questions.json" else path.write_bytes(b"source")
|
||||
|
||||
def run(command: list[str], **_kwargs: Any) -> SimpleNamespace:
|
||||
Path(command[command.index("--output") + 1]).write_text(json.dumps({"done": True, "units": []}))
|
||||
return SimpleNamespace(returncode=0)
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(module, "download", download))
|
||||
stack.enter_context(patch.object(module, "unpack", lambda *_args: None))
|
||||
stack.enter_context(patch.object(module.subprocess, "run", run))
|
||||
stack.enter_context(patch.object(module, "transfer", lambda *_args, **_kwargs: None))
|
||||
stack.enter_context(patch.dict(os.environ, {"QUESTIONS_URL": "q", "SOURCE_URL": "s", "RESULT_URL": "r"}))
|
||||
owner(work_root=root)
|
||||
read_benchmark_artifact(root / "out/result.json", "engine-task-result")
|
||||
|
||||
|
||||
def _probe_measurement(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(module, "discover_hunt_runner_context", lambda *_args, **_kwargs: object()))
|
||||
stack.enter_context(patch.object(module, "persist_hunt_runner_records", lambda *_args, **_kwargs: ()))
|
||||
stack.enter_context(patch.object(module.score, "load_records", lambda _path: []))
|
||||
stack.enter_context(patch.object(module, "arm_summary", lambda _records, arm: ({}, [{"id": arm}])))
|
||||
stack.enter_context(patch.object(module, "cost", lambda _path: None))
|
||||
stack.enter_context(patch.object(module, "load_slice", lambda *_args: ()))
|
||||
owner(root)
|
||||
read_benchmark_jsonl(root / "suspicions.jsonl", "access-control-suspicion")
|
||||
|
||||
|
||||
def _probe_calibrate(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
argv = [
|
||||
"calibrate",
|
||||
"--manifest",
|
||||
str(root / "pool.toml"),
|
||||
"--image",
|
||||
"image",
|
||||
"--docker-image-bytes",
|
||||
"1",
|
||||
"--out",
|
||||
str(root / "runs"),
|
||||
"--record",
|
||||
str(root / "record.json"),
|
||||
]
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(sys, "argv", argv))
|
||||
stack.enter_context(patch.object(module, "load_slice", lambda *_args: (_change(),)))
|
||||
stack.enter_context(patch.object(module, "run_replays", lambda *_args, **_kwargs: False))
|
||||
stack.enter_context(patch.object(module, "load_records", lambda _path: [_record()]))
|
||||
stack.enter_context(patch.object(module, "calibration_record", lambda *_args: {"status": "ok"}))
|
||||
owner()
|
||||
read_benchmark_artifact(root / "record.json", "deadline-calibration")
|
||||
|
||||
|
||||
def _probe_replay(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
class Transport:
|
||||
def __init__(self, *_args: Any):
|
||||
self.store = SimpleNamespace(_put=lambda *_values: None)
|
||||
|
||||
def __call__(self, item: Any, _output: Path, _attempt: int) -> dict[str, Any]:
|
||||
return {"done": True, "status": "quick_only", "input_digest": item.input_digest}
|
||||
|
||||
args = SimpleNamespace(arm="default", image="image", docker_image_bytes=1)
|
||||
with patch.object(module, "DockerLane", Transport):
|
||||
owner((_change(),), args, root, "docker")
|
||||
read_benchmark_sequence(root / "inputs.json", "benchmark-arm-inputs")
|
||||
result = next(path for path in root.glob("*.json") if '"artifact_kind": "benchmark-arm-record"' in path.read_text())
|
||||
read_benchmark_artifact(result, "benchmark-arm-record")
|
||||
|
||||
|
||||
def _probe_score(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
records, output = root / "records.json", root / "score.json"
|
||||
write_benchmark_sequence(records, "benchmark-arm-records", (_record(),))
|
||||
with patch.object(sys, "argv", ["score", str(records), "--out", str(output)]):
|
||||
owner()
|
||||
read_benchmark_artifact(output, "unseen-score")
|
||||
|
||||
|
||||
def _probe_instrument(module: Any, owner: Callable[..., Any], *, zero: bool) -> None:
|
||||
from openultrasast.core import coverage
|
||||
|
||||
if zero:
|
||||
if owner is not coverage.validate_instrument or module.validate_instrument is not coverage.validate_instrument:
|
||||
raise ValueError("validate_instrument alias is not canonical")
|
||||
manifest = coverage.InputManifest(("input",), coverage.derive_input_manifest_digest(("input",)))
|
||||
owner(manifest, (coverage.InputReadReceipt("input", "input", True, 1, "a" * 64),))
|
||||
try:
|
||||
owner(manifest, (coverage.InputReadReceipt("input", "input", True, 0, "a" * 64),))
|
||||
except coverage.InstrumentFailure:
|
||||
return
|
||||
raise ValueError("canonical instrument accepted a zero-byte read")
|
||||
if module.CoreInstrumentFailure is not coverage.InstrumentFailure:
|
||||
raise ValueError("instrument failure alias is not canonical")
|
||||
owner({"file_bytes": {"input": 1}, "file_digests": {"input": "a" * 64}})
|
||||
|
||||
|
||||
def _probe_development(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
bundle = module.AnalyzerBundle("a", "b", "c", "d")
|
||||
funnel = {"entry_recognition": {}, "resolution": {}, "judgment": {}}
|
||||
with (
|
||||
patch.object(module, "prove_change", lambda *_args, **_kwargs: object()),
|
||||
patch.object(module, "stage_funnel", lambda _result: funnel),
|
||||
):
|
||||
owner(bundle, root, (_change(),), root / "out")
|
||||
read_benchmark_artifact(root / "out/case.json", "development-observation")
|
||||
|
||||
|
||||
def _probe_run_arm(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
bundle = module.BundleDigests("a", "b", "c", "d")
|
||||
with patch.object(module, "_prove_one", lambda *_args, **_kwargs: _record()):
|
||||
owner(arm_name="arm", bundle=bundle, changes=(_change(),), repos={"fixture/repo": str(root)}, out_dir=root / "out")
|
||||
read_benchmark_artifact(root / "out/case.json", "development-arm-record")
|
||||
|
||||
|
||||
def _arm_inputs(path: Path, record: dict[str, Any] | None = None) -> None:
|
||||
change = _change()
|
||||
entry = {"item_id": "case", "change": change, "input_digest": "d" * 64}
|
||||
write_benchmark_sequence(path / "inputs.json", "benchmark-arm-inputs", (entry,))
|
||||
if record is not None:
|
||||
from openultrasast.hunt.runner import record_path
|
||||
|
||||
write_benchmark_artifact(record_path(path, "case"), "benchmark-arm-record", {**record, "input_digest": entry["input_digest"]})
|
||||
|
||||
|
||||
def _probe_combine(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
left, right, output = root / "left", root / "right", root / "combined.json"
|
||||
left.mkdir()
|
||||
right.mkdir()
|
||||
_arm_inputs(left, _record())
|
||||
_arm_inputs(right, _record())
|
||||
owner(left, right, output)
|
||||
read_benchmark_sequence(output, "benchmark-arm-records")
|
||||
|
||||
|
||||
def _probe_hunter(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
repos = root / "repos.json"
|
||||
repos.write_text(json.dumps({"fixture/repo": str(root)}))
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(module, "load_slice", lambda *_args: [_change()]))
|
||||
stack.enter_context(patch.object(module, "load_config", lambda *_args, **_kwargs: object()))
|
||||
stack.enter_context(patch.object(module, "resolve_models", lambda _config: ("model",)))
|
||||
stack.enter_context(patch.object(module.tool_hunter, "resolve_hunter_client", lambda: object()))
|
||||
stack.enter_context(patch.object(module, "_hunt_change", lambda *_args, **_kwargs: _record()))
|
||||
owner(root / "pool", repos, root / "out")
|
||||
read_benchmark_sequence(root / "out/inputs.json", "benchmark-arm-inputs")
|
||||
read_benchmark_artifact(next(path for path in (root / "out").glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record")
|
||||
|
||||
|
||||
def _probe_judge_input(owner: Callable[..., Any]) -> None:
|
||||
result = owner("repo", {"trace": {}, "operation": {}, "missing": "guard"})
|
||||
start = result.rindex('{\n "artifact_digest"')
|
||||
encoded = result[start:]
|
||||
benchmark_payload(json.loads(encoded), expected_kind="judge-pass-input")
|
||||
|
||||
|
||||
def _probe_judge_run(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
fake_lm = SimpleNamespace(model="offline")
|
||||
program = importlib.import_module("openultrasast.hunt.program")
|
||||
with patch.object(program, "build_lm", lambda **_kwargs: fake_lm), patch.object(module, "_iter_findings", lambda _path: ()):
|
||||
owner(SimpleNamespace(out=root, prover_out=root / "prover"))
|
||||
read_benchmark_artifact(root / "run.json", "judge-pass-run")
|
||||
|
||||
|
||||
def _probe_judge_summary(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
row = {"key": "one", "record_id": "case", "repo": "repo", "kind": "ordinary", "public": False, "status": "ok", "model": "offline"}
|
||||
write_benchmark_jsonl(root / "judgments.jsonl", "judge-pass", (row,))
|
||||
owner(SimpleNamespace(out=root))
|
||||
read_benchmark_artifact(root / "summary.json", "judge-pass-summary")
|
||||
|
||||
|
||||
def _probe_metrics(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
records, output = root / "records.json", root / "metrics.json"
|
||||
write_benchmark_sequence(records, "benchmark-arm-records", (_record(),))
|
||||
owner([str(records), "--out", str(output)])
|
||||
read_benchmark_artifact(output, "valid-benchmark-metrics")
|
||||
|
||||
|
||||
def _probe_prover(module: Any, owner: Callable[..., Any], root: Path, *, summary: bool) -> None:
|
||||
out = root / "out"
|
||||
if summary:
|
||||
out.mkdir()
|
||||
_arm_inputs(out, _record())
|
||||
base = {
|
||||
"headline": "On unseen app repos: fixture",
|
||||
"recall": {},
|
||||
"precision": {},
|
||||
"false_alarms_per_100_pushes": {},
|
||||
"exploit_verified": {},
|
||||
}
|
||||
with patch.object(module.metrics, "summarize", lambda *_args, **_kwargs: base):
|
||||
owner(out)
|
||||
read_benchmark_artifact(out / "summary.json", "benchmark-arm-summary")
|
||||
return
|
||||
repos = root / "repos.json"
|
||||
repos.write_text("{}")
|
||||
with (
|
||||
patch.object(module, "load_slice", lambda *_args: [_change()]),
|
||||
patch.object(module, "_prove", lambda *_args, **_kwargs: _record()),
|
||||
):
|
||||
owner(root / "pool", repos, out)
|
||||
read_benchmark_artifact(next(path for path in out.glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record")
|
||||
|
||||
|
||||
def _git_fixture(root: Path) -> tuple[str, str]:
|
||||
root.mkdir()
|
||||
subprocess.run(["git", "init", "-q"], cwd=root, check=True)
|
||||
subprocess.run(["git", "config", "user.email", "[email protected]"], cwd=root, check=True)
|
||||
subprocess.run(["git", "config", "user.name", "Fixture"], cwd=root, check=True)
|
||||
(root / "app.py").write_text("print('base')\n")
|
||||
subprocess.run(["git", "add", "."], cwd=root, check=True)
|
||||
subprocess.run(["git", "commit", "-qm", "base"], cwd=root, check=True)
|
||||
base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
||||
(root / "app.py").write_text("print('head')\n")
|
||||
subprocess.run(["git", "commit", "-qam", "head"], cwd=root, check=True)
|
||||
head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
|
||||
return base, head
|
||||
|
||||
|
||||
def _probe_semgrep(module: Any, owner: Callable[..., Any], root: Path) -> None:
|
||||
repo = root / "repo"
|
||||
base, head = _git_fixture(repo)
|
||||
change = {**_change(), "base": base, "head": head, "files": [{"path": "app.py"}]}
|
||||
repos, rules, binary = root / "repos.json", root / "rules", root / "fake-semgrep"
|
||||
repos.write_text(json.dumps({"fixture/repo": str(repo)}))
|
||||
rules.mkdir()
|
||||
binary.write_text('#!/bin/sh\nprintf \'%s\\n\' \'{"results":[],"errors":[],"paths":{"scanned":["app.py"]}}\'\n')
|
||||
binary.chmod(0o755)
|
||||
with patch.object(module, "load_slice", lambda *_args: [change]), patch.object(module, "MIN_PLAUSIBLE_SECONDS", 0.0):
|
||||
owner(root / "pool", repos, rules, root / "out", semgrep=str(binary))
|
||||
read_benchmark_artifact(next(path for path in (root / "out").glob("*.json") if path.name != "inputs.json"), "benchmark-arm-record")
|
||||
|
||||
|
||||
def probe_benchmark_entry(entry: LegacyInventoryEntry, *, output_root: Path) -> None:
|
||||
"""Invoke the exact registered owner and strict-read only its produced output."""
|
||||
module, owner = _owner(entry)
|
||||
root = output_root / entry.entry_id
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
probes: dict[str, Callable[[], None]] = {
|
||||
"40465a45": lambda: _probe_engine(module, owner, root),
|
||||
"f0fd6f38": lambda: _probe_measurement(module, owner, root),
|
||||
"fd21a8b7": lambda: _probe_measurement(module, owner, root),
|
||||
"61d6e4cf": lambda: _probe_calibrate(module, owner, root),
|
||||
"c01d6865": lambda: _probe_replay(module, owner, root),
|
||||
"039988ff": lambda: _probe_score(module, owner, root),
|
||||
"ae8dd586": lambda: _probe_instrument(module, owner, zero=False),
|
||||
"f3ba25f4": lambda: _probe_instrument(module, owner, zero=True),
|
||||
"b2bee433": lambda: _probe_run_arm(module, owner, root),
|
||||
"2acb6086": lambda: _probe_development(module, owner, root),
|
||||
"c4fe7770": lambda: _probe_combine(module, owner, root),
|
||||
"880bbb68": lambda: _probe_hunter(module, owner, root),
|
||||
"6a962869": lambda: _probe_judge_input(owner),
|
||||
"dea8fd3d": lambda: _probe_judge_run(module, owner, root),
|
||||
"8bf0cd9c": lambda: _probe_judge_summary(module, owner, root),
|
||||
"c9906d43": lambda: _probe_metrics(module, owner, root),
|
||||
"bfeaa25d": lambda: _probe_prover(module, owner, root, summary=False),
|
||||
"c3e95086": lambda: _probe_prover(module, owner, root, summary=True),
|
||||
"8db722b9": lambda: _probe_semgrep(module, owner, root),
|
||||
}
|
||||
try:
|
||||
probes[entry.entry_id[:8]]()
|
||||
except Exception as error:
|
||||
raise BenchmarkMigrationConformanceError(f"benchmark owner probe failed: {entry.entry_id}: {error}") from error
|
||||
|
||||
|
||||
def _validate_retired(entry: LegacyInventoryEntry) -> None:
|
||||
module = importlib.import_module(entry.path.removesuffix(".py").replace("/", "."))
|
||||
names = (
|
||||
(entry.symbol,)
|
||||
if entry.symbol != "<module>"
|
||||
else tuple(anchor.rsplit(":", 1)[-1] for anchor in entry.capability_anchors if anchor.startswith("import_from:"))
|
||||
)
|
||||
if any(hasattr(module, name) for name in names):
|
||||
raise BenchmarkMigrationConformanceError(f"retired benchmark entry remains live: {entry.entry_id}")
|
||||
|
||||
|
||||
def _stable_fields(entry: LegacyInventoryEntry) -> tuple[str, str, str, str, tuple[str, ...], str]:
|
||||
return (
|
||||
entry.path,
|
||||
entry.symbol,
|
||||
entry.dependency_kind.value,
|
||||
entry.artifact_family,
|
||||
entry.capability_anchors,
|
||||
entry.owner_boundary,
|
||||
)
|
||||
|
||||
|
||||
def validate_benchmark_terminal_conformance(inventory: LegacyMigrationInventory, *, output_root: Path) -> None:
|
||||
entries = tuple(entry for entry in inventory.entries if entry.path.startswith("benchmarks/"))
|
||||
migrated = {entry.entry_id for entry in entries if entry.disposition is MigrationDisposition.V2_MIGRATED}
|
||||
if not entries or {entry.entry_id for entry in entries} != set(BENCHMARK_TERMINAL_EVIDENCE) or migrated != set(OWNER_TARGETS):
|
||||
raise BenchmarkMigrationConformanceError("benchmark probe registry and terminal evidence do not exactly match inventory entries")
|
||||
for entry in entries:
|
||||
evidence = BENCHMARK_TERMINAL_EVIDENCE[entry.entry_id]
|
||||
if evidence.disposition != entry.disposition.value or evidence.stable_fields != _stable_fields(entry):
|
||||
raise BenchmarkMigrationConformanceError(f"benchmark terminal evidence mismatch: {entry.entry_id}")
|
||||
if entry.disposition is MigrationDisposition.OPEN:
|
||||
raise BenchmarkMigrationConformanceError(f"benchmark entry remains open: {entry.entry_id}")
|
||||
_validate_retired(entry) if entry.disposition is MigrationDisposition.RETIRED else probe_benchmark_entry(
|
||||
entry, output_root=output_root
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BenchmarkMigrationConformanceError", "OWNER_TARGETS", "probe_benchmark_entry", "validate_benchmark_terminal_conformance"]
|
||||
@@ -0,0 +1,358 @@
|
||||
"""Immutable reviewed closure evidence for detection-core benchmark migration entries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchmarkTerminalEvidence:
|
||||
disposition: str
|
||||
path: str
|
||||
symbol: str
|
||||
dependency_kind: str
|
||||
artifact_family: str
|
||||
capability_anchors: tuple[str, ...]
|
||||
owner_boundary: str
|
||||
successor_module: str | None
|
||||
successor_symbol: str | None
|
||||
|
||||
@property
|
||||
def stable_fields(self) -> tuple[str, str, str, str, tuple[str, ...], str]:
|
||||
return (
|
||||
self.path,
|
||||
self.symbol,
|
||||
self.dependency_kind,
|
||||
self.artifact_family,
|
||||
self.capability_anchors,
|
||||
self.owner_boundary,
|
||||
)
|
||||
|
||||
|
||||
BENCHMARK_TERMINAL_EVIDENCE = {
|
||||
"40465a45eb2ef682fecad973566a0bd94956bfb804d1fc6ea1d43075533232b1": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/learn/engine_task_entry.py",
|
||||
"main",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:main", "constant:done", "call:dumps", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.learn.engine_task_entry",
|
||||
"main",
|
||||
),
|
||||
"f0fd6f3806e7f50b826c6b3fad99d50c4ac02aeedb5cc3afca6c140f3bebbcf7": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py",
|
||||
"<module>",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
(
|
||||
"constant:family",
|
||||
"constant:id",
|
||||
"constant:instrument",
|
||||
"constant:instrument_failures",
|
||||
"constant:precision",
|
||||
"constant:push",
|
||||
"call:dumps",
|
||||
"call:write_text",
|
||||
),
|
||||
"benchmark",
|
||||
"benchmarks.measurements.2026-10-14-access-control-hunt.analyse",
|
||||
"main",
|
||||
),
|
||||
"fd21a8b7f0759a69e9f51ad3d2bae51e435ebb88cabea3bc44b8fec534fae32f": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py",
|
||||
"main",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:main", "call:dumps", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.measurements.2026-10-14-access-control-hunt.analyse",
|
||||
"main",
|
||||
),
|
||||
"61d6e4cf01bb735ec13ea6dcd9aeb3e804abaccd26e0c7ad18819f2b28a7db0c": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/unseen/calibrate.py",
|
||||
"main",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:main", "constant:id", "constant:status", "call:canonical"),
|
||||
"benchmark",
|
||||
"benchmarks.unseen.calibrate",
|
||||
"main",
|
||||
),
|
||||
"c01d686518e8c0d17fa60092007604923b133200328a975bbf39e4727d75d900": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/unseen/replay.py",
|
||||
"run_replays",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:run_replays", "constant:change", "constant:input_digest", "constant:item_id"),
|
||||
"benchmark",
|
||||
"benchmarks.unseen.replay",
|
||||
"run_replays",
|
||||
),
|
||||
"039988ff20d8f58a169cc7cd183e25a88b2bb2638fa969becb3553d8d7bc927b": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/unseen/score.py",
|
||||
"main",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:main", "call:dumps", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.unseen.score",
|
||||
"main",
|
||||
),
|
||||
"fe616a80de18d275283ad26ccd9027c4fbf151c017d02c14205e9cfc374a8b53": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/admission_run.py",
|
||||
"_resume_or_prove",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:_resume_or_prove", "constant:id", "constant:input_digest", "call:_write_json"),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"41eade0e409ea4726173e851fcad5c9c81076bd7e094fc4c2ad830ac40f958b4": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/admission_run.py",
|
||||
"assemble_admission_report",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:assemble_admission_report", "constant:id", "constant:status", "name:admission_report"),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/arm_regression.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.coverage:InstrumentFailure",),
|
||||
"benchmark",
|
||||
"benchmarks.valid.arm_regression",
|
||||
"_validate_record_instrument",
|
||||
),
|
||||
"f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/arm_regression.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.coverage:check_reads",),
|
||||
"benchmark",
|
||||
"benchmarks.valid.arm_regression",
|
||||
"validate_instrument",
|
||||
),
|
||||
"b2bee4333d7374d21d2f4d0a8cdf49bb7c2f398daba5f63d7c3c2f7349f9be72": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/arm_regression.py",
|
||||
"run_arm",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:run_arm", "constant:id", "constant:input_digest", "call:_write_json"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.arm_regression",
|
||||
"run_arm",
|
||||
),
|
||||
"2acb6086c4eba2ee4a6bd6f6bacc86219207dd71548586c7966bb38153de9150": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/development_runner.py",
|
||||
"run_development",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:run_development", "constant:id", "call:dumps", "call:write_text", "name:DevelopmentRecord"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.development_runner",
|
||||
"run_development",
|
||||
),
|
||||
"c4fe7770e44fb6c37a97cc2b5c686927d116522cdd62229fe7203aa58405477b": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/hunter_arm.py",
|
||||
"combine",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:combine", "constant:missing", "constant:push", "constant:status", "call:_write_json"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.hunter_arm",
|
||||
"combine",
|
||||
),
|
||||
"880bbb68575bcb453127df5930a3d73d5dc67b478096bc504fafa878d91e534c": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/hunter_arm.py",
|
||||
"run",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
(
|
||||
"definition:run",
|
||||
"constant:change",
|
||||
"constant:done",
|
||||
"constant:id",
|
||||
"constant:input_digest",
|
||||
"constant:item_id",
|
||||
"constant:status",
|
||||
"call:_write_json",
|
||||
"call:canonical",
|
||||
),
|
||||
"benchmark",
|
||||
"benchmarks.valid.hunter_arm",
|
||||
"run",
|
||||
),
|
||||
"6a96286909032b9997570a1f42afd97ffb92258a7d6ad0ebd077f4cdcb65adef": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/judge_pass.py",
|
||||
"_judge_input",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:_judge_input", "constant:missing", "constant:operation", "constant:trace", "call:dumps"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.judge_pass",
|
||||
"_judge_input",
|
||||
),
|
||||
"dea8fd3d112e1cbb4abb54b5d0e77e83c0a6e1e75e5ba2171dbd6f7ba436b75b": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/judge_pass.py",
|
||||
"run",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:run", "constant:id", "constant:status", "call:dumps", "call:write", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.judge_pass",
|
||||
"run",
|
||||
),
|
||||
"8bf0cd9c6a28264ad9ff1492f203d384b33432ddf0d9027381497cc8352de48c": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/judge_pass.py",
|
||||
"summarize",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:summarize", "constant:status", "call:dumps", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.judge_pass",
|
||||
"summarize",
|
||||
),
|
||||
"c9906d437a27969e465c4d3139b1e865de0ac82761d0a980cf41efce20a0c178": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/metrics.py",
|
||||
"main",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:main", "call:dumps", "call:write_text"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.metrics",
|
||||
"main",
|
||||
),
|
||||
"ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/prover_admission.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.admission:AdmissionManifest",),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/prover_admission.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.admission:AdmissionManifestError",),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/prover_admission.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.admission:REPORT_SCHEMA_KEYS",),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429": BenchmarkTerminalEvidence(
|
||||
"retired",
|
||||
"benchmarks/valid/prover_admission.py",
|
||||
"<module>",
|
||||
"reader",
|
||||
"v1-contract",
|
||||
("import_from:openultrasast.core.admission:validate_admission_manifest",),
|
||||
"benchmark",
|
||||
None,
|
||||
None,
|
||||
),
|
||||
"bfeaa25d41ff876c1211b69f0bc497e246cabb4bcdd0569f27ae2ae15e84d3f5": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/prover_arm.py",
|
||||
"run",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
(
|
||||
"definition:run",
|
||||
"constant:change",
|
||||
"constant:done",
|
||||
"constant:id",
|
||||
"constant:input_digest",
|
||||
"constant:item_id",
|
||||
"constant:status",
|
||||
"call:_write_json",
|
||||
"call:canonical",
|
||||
),
|
||||
"benchmark",
|
||||
"benchmarks.valid.prover_arm",
|
||||
"run",
|
||||
),
|
||||
"c3e95086ce661de7f589d787c324b4dd08977481fb427452837e8482d24210bf": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/prover_arm.py",
|
||||
"summarize",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
("definition:summarize", "constant:coverage", "constant:id", "constant:instrument", "constant:status", "call:_write_json"),
|
||||
"benchmark",
|
||||
"benchmarks.valid.prover_arm",
|
||||
"summarize",
|
||||
),
|
||||
"8db722b97cb4411fd2a12391f23d5b79b46832fcd7d62d08905aa667871c6c6a": BenchmarkTerminalEvidence(
|
||||
"v2_migrated",
|
||||
"benchmarks/valid/semgrep_arm.py",
|
||||
"run",
|
||||
"writer",
|
||||
"v1-artifact",
|
||||
(
|
||||
"definition:run",
|
||||
"constant:change",
|
||||
"constant:done",
|
||||
"constant:id",
|
||||
"constant:input_digest",
|
||||
"constant:item_id",
|
||||
"constant:status",
|
||||
"call:_write_json",
|
||||
"call:canonical",
|
||||
),
|
||||
"benchmark",
|
||||
"benchmarks.valid.semgrep_arm",
|
||||
"run",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def migrated_owner_targets() -> dict[str, tuple[str, str]]:
|
||||
return {
|
||||
entry_id: (evidence.successor_module, evidence.successor_symbol)
|
||||
for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items()
|
||||
if evidence.disposition == "v2_migrated" and evidence.successor_module is not None and evidence.successor_symbol is not None
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["BENCHMARK_TERMINAL_EVIDENCE", "BenchmarkTerminalEvidence", "migrated_owner_targets"]
|
||||
@@ -9,6 +9,8 @@ from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE, BenchmarkTerminalEvidence
|
||||
|
||||
from openultrasast.core.legacy import LegacyArtifact, decode_legacy_artifact
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
@@ -55,6 +57,12 @@ _APPROVED_TERMINAL_SOURCE_CLOSURES = {
|
||||
"9f2097b8a673c3079a8a61cc28e9099c671e48e6c29a3e95a9e3aac3d53ccb00": MigrationDisposition.RETIRED,
|
||||
"eb963401f43fca11e38c50e5e064f58b33233b9564a3a85e5b629a6fb2820d7d": MigrationDisposition.V2_MIGRATED,
|
||||
"f8a5c42661f6a79b0abbe668e8993fd0ec26d1c9dad5df4e141472e6de62f586": MigrationDisposition.V2_MIGRATED,
|
||||
"ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36": MigrationDisposition.V2_MIGRATED,
|
||||
"f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f": MigrationDisposition.V2_MIGRATED,
|
||||
"b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d": MigrationDisposition.RETIRED,
|
||||
"cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429": MigrationDisposition.RETIRED,
|
||||
"ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522": MigrationDisposition.RETIRED,
|
||||
"f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c": MigrationDisposition.RETIRED,
|
||||
}
|
||||
|
||||
|
||||
@@ -322,7 +330,7 @@ def _dotted(node: ast.AST) -> str:
|
||||
|
||||
|
||||
def _top_level_reads(tree: ast.Module) -> set[str]:
|
||||
aliases = set()
|
||||
aliases: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
aliases.update(alias.asname for alias in node.names if alias.name == "openultrasast.core" and alias.asname)
|
||||
@@ -448,8 +456,46 @@ def _moved_mixed_v1_owner(entry: LegacyInventoryEntry, *, root: Path) -> bool:
|
||||
return set(destination_symbols) <= owned
|
||||
|
||||
|
||||
def _approved_terminal_source_closure(entry: LegacyInventoryEntry) -> bool:
|
||||
return _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition and entry.entry_id == entry_identity(entry)
|
||||
def _successor_path(root: Path, evidence: BenchmarkTerminalEvidence) -> Path:
|
||||
assert evidence.successor_module is not None
|
||||
relative = Path(*evidence.successor_module.split(".")).with_suffix(".py")
|
||||
direct = root / relative
|
||||
return direct if direct.is_file() else root / "src" / relative
|
||||
|
||||
|
||||
def _validate_retired_absence(entry: LegacyInventoryEntry, tree: ast.Module) -> None:
|
||||
if entry.symbol != "<module>" and entry.symbol in _defined_symbols(tree):
|
||||
raise LegacyInventoryError(f"retired owner remains live: {entry.path}:{entry.symbol}")
|
||||
if entry.symbol == "<module>" and not _missing_anchors(entry, tree):
|
||||
raise LegacyInventoryError(f"retired capability remains live: {entry.path}:{entry.capability_anchors}")
|
||||
|
||||
|
||||
def _benchmark_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
|
||||
if not entry.path.startswith("benchmarks/") or entry.disposition is MigrationDisposition.OPEN:
|
||||
return False
|
||||
evidence = BENCHMARK_TERMINAL_EVIDENCE.get(entry.entry_id)
|
||||
if (
|
||||
evidence is None
|
||||
or evidence.disposition != entry.disposition.value
|
||||
or evidence.stable_fields != _stable_entry_fields(entry)
|
||||
or entry.entry_id != entry_identity(entry)
|
||||
):
|
||||
raise LegacyInventoryError(f"benchmark terminal evidence is absent or mismatched: {entry.entry_id}")
|
||||
if entry.disposition is MigrationDisposition.RETIRED:
|
||||
_validate_retired_absence(entry, tree)
|
||||
return True
|
||||
successor = _read_owner_module(_successor_path(root, evidence), "migrated benchmark successor")
|
||||
assert evidence.successor_symbol is not None
|
||||
if evidence.successor_symbol not in _defined_symbols(successor):
|
||||
raise LegacyInventoryError(f"migrated owner is absent: {evidence.successor_module}:{evidence.successor_symbol}")
|
||||
return True
|
||||
|
||||
|
||||
def _approved_terminal_source_closure(entry: LegacyInventoryEntry, *, root: Path, tree: ast.Module) -> bool:
|
||||
if _benchmark_terminal_source_closure(entry, root=root, tree=tree):
|
||||
return True
|
||||
approved = _APPROVED_TERMINAL_SOURCE_CLOSURES.get(entry.entry_id) is entry.disposition
|
||||
return approved and entry.entry_id == entry_identity(entry)
|
||||
|
||||
|
||||
def validate_inventory_sources(inventory: LegacyMigrationInventory, *, root: Path) -> None:
|
||||
@@ -468,7 +514,7 @@ def validate_inventory_sources(inventory: LegacyMigrationInventory, *, root: Pat
|
||||
raise LegacyInventoryError(f"inventory source is not parseable: {entry.path}") from exc
|
||||
is_shim = _explicit_legacy_shim(tree, entry.symbol)
|
||||
moved_owner = _moved_mixed_v1_owner(entry, root=root)
|
||||
terminal_closure = _approved_terminal_source_closure(entry)
|
||||
terminal_closure = _approved_terminal_source_closure(entry, root=root, tree=tree)
|
||||
if entry.symbol not in _defined_symbols(tree) and not is_shim and not moved_owner and not terminal_closure:
|
||||
raise LegacyInventoryError(f"inventory target is absent: {entry.path}:{entry.symbol}")
|
||||
missing = _missing_anchors(entry, tree)
|
||||
|
||||
+25
-25
@@ -82,7 +82,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "40465a45eb2ef682fecad973566a0bd94956bfb804d1fc6ea1d43075533232b1",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/learn/engine_task_entry.py",
|
||||
@@ -101,7 +101,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "f0fd6f3806e7f50b826c6b3fad99d50c4ac02aeedb5cc3afca6c140f3bebbcf7",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py",
|
||||
@@ -115,7 +115,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "fd21a8b7f0759a69e9f51ad3d2bae51e435ebb88cabea3bc44b8fec534fae32f",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/measurements/2026-10-14-access-control-hunt/analyse.py",
|
||||
@@ -130,7 +130,7 @@
|
||||
"call:canonical"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "61d6e4cf01bb735ec13ea6dcd9aeb3e804abaccd26e0c7ad18819f2b28a7db0c",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/unseen/calibrate.py",
|
||||
@@ -145,7 +145,7 @@
|
||||
"constant:item_id"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "c01d686518e8c0d17fa60092007604923b133200328a975bbf39e4727d75d900",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/unseen/replay.py",
|
||||
@@ -159,7 +159,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "039988ff20d8f58a169cc7cd183e25a88b2bb2638fa969becb3553d8d7bc927b",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/unseen/score.py",
|
||||
@@ -174,7 +174,7 @@
|
||||
"call:_write_json"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "fe616a80de18d275283ad26ccd9027c4fbf151c017d02c14205e9cfc374a8b53",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/admission_run.py",
|
||||
@@ -189,7 +189,7 @@
|
||||
"name:admission_report"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "41eade0e409ea4726173e851fcad5c9c81076bd7e094fc4c2ad830ac40f958b4",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/admission_run.py",
|
||||
@@ -201,7 +201,7 @@
|
||||
"import_from:openultrasast.core.coverage:InstrumentFailure"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "ae8dd5862ff8cb6fb4cdfb83f99eb0ac881fed4dd6d0e01d19f7e4bad25f1a36",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/arm_regression.py",
|
||||
@@ -213,7 +213,7 @@
|
||||
"import_from:openultrasast.core.coverage:check_reads"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "f3ba25f48c0da317242323061f27a45213a459b93a12aeeb16389bd0f3e5525f",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/arm_regression.py",
|
||||
@@ -228,7 +228,7 @@
|
||||
"call:_write_json"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "b2bee4333d7374d21d2f4d0a8cdf49bb7c2f398daba5f63d7c3c2f7349f9be72",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/arm_regression.py",
|
||||
@@ -244,7 +244,7 @@
|
||||
"name:DevelopmentRecord"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "2acb6086c4eba2ee4a6bd6f6bacc86219207dd71548586c7966bb38153de9150",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/development_runner.py",
|
||||
@@ -260,7 +260,7 @@
|
||||
"call:_write_json"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "c4fe7770e44fb6c37a97cc2b5c686927d116522cdd62229fe7203aa58405477b",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/hunter_arm.py",
|
||||
@@ -280,7 +280,7 @@
|
||||
"call:canonical"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "880bbb68575bcb453127df5930a3d73d5dc67b478096bc504fafa878d91e534c",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/hunter_arm.py",
|
||||
@@ -296,7 +296,7 @@
|
||||
"call:dumps"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "6a96286909032b9997570a1f42afd97ffb92258a7d6ad0ebd077f4cdcb65adef",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/judge_pass.py",
|
||||
@@ -313,7 +313,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "dea8fd3d112e1cbb4abb54b5d0e77e83c0a6e1e75e5ba2171dbd6f7ba436b75b",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/judge_pass.py",
|
||||
@@ -328,7 +328,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "8bf0cd9c6a28264ad9ff1492f203d384b33432ddf0d9027381497cc8352de48c",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/judge_pass.py",
|
||||
@@ -342,7 +342,7 @@
|
||||
"call:write_text"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "c9906d437a27969e465c4d3139b1e865de0ac82761d0a980cf41efce20a0c178",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/metrics.py",
|
||||
@@ -354,7 +354,7 @@
|
||||
"import_from:openultrasast.core.admission:AdmissionManifest"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "ce683bd3299f95f9972af9e760939fc0c465a4ce0f682a0eb6e1de3e29538522",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_admission.py",
|
||||
@@ -366,7 +366,7 @@
|
||||
"import_from:openultrasast.core.admission:AdmissionManifestError"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "f206dbf8024c582e401dc14fe1dd336f0d7abfac016f6ab0c3e3980236eb6b2c",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_admission.py",
|
||||
@@ -378,7 +378,7 @@
|
||||
"import_from:openultrasast.core.admission:REPORT_SCHEMA_KEYS"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "b7de07df33d29a2cba9543b53505f9789d7b51b4890797964b571c20f69fa26d",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_admission.py",
|
||||
@@ -390,7 +390,7 @@
|
||||
"import_from:openultrasast.core.admission:validate_admission_manifest"
|
||||
],
|
||||
"dependency_kind": "reader",
|
||||
"disposition": "open",
|
||||
"disposition": "retired",
|
||||
"entry_id": "cb86f9a346e08246f627daa0ad1ba71b4e203d467dd08c983772b73d0926a429",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_admission.py",
|
||||
@@ -410,7 +410,7 @@
|
||||
"call:canonical"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "bfeaa25d41ff876c1211b69f0bc497e246cabb4bcdd0569f27ae2ae15e84d3f5",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_arm.py",
|
||||
@@ -427,7 +427,7 @@
|
||||
"call:_write_json"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "c3e95086ce661de7f589d787c324b4dd08977481fb427452837e8482d24210bf",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/prover_arm.py",
|
||||
@@ -447,7 +447,7 @@
|
||||
"call:canonical"
|
||||
],
|
||||
"dependency_kind": "writer",
|
||||
"disposition": "open",
|
||||
"disposition": "v2_migrated",
|
||||
"entry_id": "8db722b97cb4411fd2a12391f23d5b79b46832fcd7d62d08905aa667871c6c6a",
|
||||
"owner_boundary": "benchmark",
|
||||
"path": "benchmarks/valid/semgrep_arm.py",
|
||||
|
||||
@@ -6,30 +6,21 @@ reviewed ordinary change, maps per-site prover evidence onto the three-way
|
||||
judgment vocabulary and returns the paired admission report.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.unseen.replay import canonical # noqa: E402
|
||||
from benchmarks.valid.admission_run import ( # noqa: E402
|
||||
admission_ordinary,
|
||||
admission_pair_cases,
|
||||
assemble_admission_report,
|
||||
classify_fixed_site,
|
||||
coverage_share,
|
||||
fixed_side_change,
|
||||
load_admission_population,
|
||||
locate_function,
|
||||
site_judgment,
|
||||
)
|
||||
from benchmarks.valid.admission_scoring import AdmissionScoringError # noqa: E402
|
||||
from benchmarks.valid.prover_admission import AdmissionManifest, AdmissionManifestError # noqa: E402
|
||||
|
||||
SITE = ("src/app.py", 30, 40)
|
||||
|
||||
@@ -194,159 +185,3 @@ def test_coverage_share_over_repositories() -> None:
|
||||
bare = _record("b", repo="repo-b", stage_funnel={"entry_recognition": {"frameworks_recognised": {}}})
|
||||
other = _record("a", repo="repo-a", stage_funnel={"entry_recognition": {"frameworks_recognised": {"jenkins": 2}}})
|
||||
assert coverage_share((covered, bare, other)) == 0.5
|
||||
|
||||
|
||||
def _manifest_payload(**over: object) -> dict[str, object]:
|
||||
payload = {
|
||||
"produced_by": "valid-detection-benchmark",
|
||||
"reviewed": True,
|
||||
"label_isolated": True,
|
||||
"spent": False,
|
||||
"population_role": "admission",
|
||||
"analyzer_revision": "d063167",
|
||||
"fact_set_digest": "3eb5bf699434",
|
||||
"acquisition_record_digest": "none",
|
||||
"configuration_digest": "be6d0f688598",
|
||||
"population_digest": "0" * 64,
|
||||
"labels": [],
|
||||
}
|
||||
payload.update(over)
|
||||
return payload
|
||||
|
||||
|
||||
POOL_TOML = (
|
||||
'status = "frozen"\n'
|
||||
'pool = "valid"\n'
|
||||
"seed = 0\n"
|
||||
"\n"
|
||||
"[[repository]]\n"
|
||||
'id = "example/project"\n'
|
||||
'repository = "example/project"\n'
|
||||
'url = "https://github.com/example/project.git"\n'
|
||||
"slice = 1\n"
|
||||
"changes = [\n"
|
||||
' {base = "fixcommit", head = "vulnparent", case_id = "example-project-vuln-1", '
|
||||
'id = "example-project-vuln-1", repo = "example/project", kind = "vulnerability", '
|
||||
'path = "src/app.py", function = "handler", function_lines = [30, 40], '
|
||||
'files = [{path = "src/app.py", head_lines = [[30, 40]]}], family = "access_control"},\n'
|
||||
' {base = "parent1", head = "ordinary1", case_id = "example-project-ordinary-1", '
|
||||
'id = "example-project-ordinary-1", repo = "example/project", kind = "ordinary", '
|
||||
'files = [{path = "src/lib.py", head_lines = [[1, 5]]}]},\n'
|
||||
"]\n"
|
||||
)
|
||||
|
||||
|
||||
def _population_files(root: Path, *, freeze_digest: str | None = None, guard_result: int = 0) -> tuple[Path, Path, str]:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
pool_path = root / "pool.toml"
|
||||
pool_path.write_text(POOL_TOML, encoding="utf-8")
|
||||
pool = tomllib.loads(pool_path.read_text(encoding="utf-8"))
|
||||
digest = freeze_digest if freeze_digest is not None else hashlib.sha256(canonical(pool)).hexdigest()
|
||||
freeze_path = root / "freeze.json"
|
||||
freeze_path.write_text(json.dumps({"freeze_digest": digest, "guard_result": guard_result}), encoding="utf-8")
|
||||
return pool_path, freeze_path, digest
|
||||
|
||||
|
||||
def test_load_population_verifies_freeze_digest_and_guard() -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
pool_path, freeze_path, digest = _population_files(root)
|
||||
payload = _manifest_payload(population_digest=digest)
|
||||
vulnerable, ordinary = load_admission_population(payload, pool_path, freeze_path)
|
||||
assert [change["kind"] for change in (*vulnerable, *ordinary)] == ["vulnerability", "ordinary"]
|
||||
assert all(len(change["id"]) == 64 for change in (*vulnerable, *ordinary))
|
||||
tampered = _manifest_payload(population_digest="f" * 64)
|
||||
try:
|
||||
load_admission_population(tampered, pool_path, freeze_path)
|
||||
raise AssertionError("population digest mismatch must be refused")
|
||||
except ValueError as error:
|
||||
assert "population" in str(error)
|
||||
guarded = _population_files(root / "g", guard_result=1)
|
||||
payload_guarded = _manifest_payload(population_digest=guarded[2])
|
||||
try:
|
||||
load_admission_population(payload_guarded, guarded[0], guarded[1])
|
||||
raise AssertionError("held-out guard failure must be refused")
|
||||
except ValueError as error:
|
||||
assert "guard" in str(error)
|
||||
|
||||
|
||||
def test_load_population_validates_manifest_before_labels() -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
pool_path, freeze_path, _ = _population_files(root)
|
||||
spent = _manifest_payload(spent=True)
|
||||
try:
|
||||
load_admission_population(spent, pool_path, freeze_path)
|
||||
raise AssertionError("spent population must be refused")
|
||||
except AdmissionManifestError:
|
||||
pass
|
||||
|
||||
|
||||
def _admission_manifest(digest: str = "0" * 64) -> AdmissionManifest:
|
||||
from benchmarks.valid.prover_admission import validate_admission_manifest
|
||||
|
||||
return validate_admission_manifest(_manifest_payload(population_digest=digest))
|
||||
|
||||
|
||||
def test_assemble_report_returns_go_without_failed_gates() -> None:
|
||||
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)], family="access_control")
|
||||
fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")])
|
||||
ordinary = _record("ord-1", kind="ordinary")
|
||||
manifest = _admission_manifest()
|
||||
report = assemble_admission_report(
|
||||
manifest,
|
||||
(vulnerable,),
|
||||
(fixed,),
|
||||
(ordinary,),
|
||||
criteria={"min_fix_recall": 1.0, "max_flagged_ordinary_per_100": 0.0},
|
||||
)
|
||||
assert report["decision"] == "GO"
|
||||
assert report["fix_recall"]["count"] == 1
|
||||
assert report["pairs"] == {"n": 1, "correct": 1}
|
||||
assert report["flagged_ordinary_per_100"]["count"] == 0
|
||||
|
||||
|
||||
def test_assemble_report_no_go_on_ordinary_overrun() -> None:
|
||||
vulnerable = _record("case-1", findings=[_finding("src/app.py", 35)])
|
||||
fixed = _record("case-1", fixed_lines=(31, 42), traces=[_trace("src/app.py", 33, "discharged")])
|
||||
slow = _record("ord-1", kind="ordinary", seconds=300.0)
|
||||
report = assemble_admission_report(_admission_manifest(), (vulnerable,), (fixed,), (slow,), criteria=None, budget_seconds=120.0)
|
||||
assert report["decision"] == "NO-GO"
|
||||
assert any(gate.startswith("budget:ord-1:") for gate in report["failed_gates"])
|
||||
|
||||
|
||||
def test_assemble_report_no_go_on_budget_and_criteria() -> None:
|
||||
vulnerable = _record("case-1", seconds=200.0)
|
||||
fixed = _record("case-1", fixed_lines=(31, 42), seconds=1.0)
|
||||
ordinary = _record("ord-1", kind="ordinary", findings=[_finding("src/x.py", 3)])
|
||||
manifest = _admission_manifest()
|
||||
report = assemble_admission_report(
|
||||
manifest,
|
||||
(vulnerable,),
|
||||
(fixed,),
|
||||
(ordinary,),
|
||||
criteria={"min_fix_recall": 0.9, "max_flagged_ordinary_per_100": 0.0},
|
||||
)
|
||||
assert report["decision"] == "NO-GO"
|
||||
assert any(gate.startswith("budget:case-1:") for gate in report["failed_gates"])
|
||||
assert any("min_fix_recall" in gate for gate in report["failed_gates"])
|
||||
assert any("max_flagged_ordinary_per_100" in gate for gate in report["failed_gates"])
|
||||
|
||||
|
||||
def test_assemble_report_counts_instrument_failures() -> None:
|
||||
vulnerable = _record("case-1", status="instrument_failure")
|
||||
fixed = _record("case-1", fixed_lines=(31, 42), status="instrument_failure")
|
||||
manifest = _admission_manifest()
|
||||
report = assemble_admission_report(manifest, (vulnerable,), (fixed,), (), criteria=None)
|
||||
assert report["instrument_failures"] == 2
|
||||
assert report["decision"] == "NO-GO"
|
||||
|
||||
|
||||
def test_assemble_report_refuses_development_role() -> None:
|
||||
manifest = _admission_manifest()
|
||||
object.__setattr__(manifest, "population_role", "development")
|
||||
try:
|
||||
assemble_admission_report(manifest, (), (), (), criteria=None)
|
||||
raise AssertionError("development population must be refused for admission")
|
||||
except AdmissionScoringError:
|
||||
pass
|
||||
|
||||
@@ -14,7 +14,7 @@ from benchmarks.valid.admission_scoring import (
|
||||
OrdinaryChange,
|
||||
admission_report,
|
||||
)
|
||||
from benchmarks.valid.prover_admission import AdmissionManifest, validate_admission_manifest
|
||||
from openultrasast.core.legacy_v1 import AdmissionManifest, validate_admission_manifest
|
||||
|
||||
|
||||
def admission_manifest(role: str = "admission") -> AdmissionManifest:
|
||||
|
||||
@@ -141,7 +141,33 @@ def test_run_arm_writes_resumable_record_with_digests(sample_repo: Path, tmp_pat
|
||||
assert (out / "sample-repo-ordinary-abc.json").stat().st_mtime_ns == first_mtime
|
||||
|
||||
|
||||
def test_run_arm_allows_legitimately_empty_tracked_file(sample_repo: Path, tmp_path: Path) -> None:
|
||||
def test_run_arm_resume_refuses_stripped_v2_metadata(sample_repo: Path, tmp_path: Path) -> None:
|
||||
change = _change(sample_repo)
|
||||
out = tmp_path / "arm"
|
||||
arm_regression.run_arm(
|
||||
arm_name="candidate-analyzer/candidate-facts",
|
||||
bundle=_bundle(),
|
||||
changes=(change,),
|
||||
repos={"sample/repo": str(sample_repo)},
|
||||
out_dir=out,
|
||||
)
|
||||
path = out / "sample-repo-ordinary-abc.json"
|
||||
value = json.loads(path.read_text())
|
||||
for field in ("schema_version", "artifact_kind", "artifact_digest"):
|
||||
value.pop(field)
|
||||
path.write_text(json.dumps(value))
|
||||
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
arm_regression.run_arm(
|
||||
arm_name="candidate-analyzer/candidate-facts",
|
||||
bundle=_bundle(),
|
||||
changes=(change,),
|
||||
repos={"sample/repo": str(sample_repo)},
|
||||
out_dir=out,
|
||||
)
|
||||
|
||||
|
||||
def test_run_arm_refuses_empty_tracked_file_through_v2_instrument(sample_repo: Path, tmp_path: Path) -> None:
|
||||
(sample_repo / "empty.py").write_text("")
|
||||
(sample_repo / "app.py").write_text("def handler(event):\n return 2\n")
|
||||
_git(sample_repo, "add", ".")
|
||||
@@ -158,9 +184,12 @@ def test_run_arm_allows_legitimately_empty_tracked_file(sample_repo: Path, tmp_p
|
||||
repos={"sample/repo": str(sample_repo)},
|
||||
out_dir=tmp_path / "arm",
|
||||
)
|
||||
assert records[0]["status"] == "prover_covered"
|
||||
assert records[0]["instrument"]["empty_files"] == 1
|
||||
assert records[0]["status"] == "instrument_failure"
|
||||
assert records[0]["instrument"]["failure"] == "InstrumentFailure"
|
||||
assert records[0]["instrument"]["file_bytes"]["empty.py"] == 0
|
||||
assert records[0]["instrument"]["file_bytes"]["app.py"] > 0
|
||||
with pytest.raises(arm_regression.InstrumentFailure, match="zero-byte"):
|
||||
arm_regression.summarize_arm(records)
|
||||
|
||||
|
||||
def test_summarize_arm_refuses_zero_byte_summary(tmp_path: Path) -> None:
|
||||
@@ -186,7 +215,7 @@ def test_summarize_arm_publishes_budget_result_and_masks_repositories() -> None:
|
||||
"repo": "private/name",
|
||||
"kind": "ordinary",
|
||||
"status": "prover_covered",
|
||||
"instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10},
|
||||
"instrument": {"file_bytes": {"app.py": 10}, "file_digests": {"app.py": "a" * 64}, "hook_bytes_read": 10},
|
||||
"stage_funnel": {
|
||||
"entry_recognition": {"entry_points": 1, "frameworks_recognised": {"web": 1}},
|
||||
"resolution": {"resolved": 1},
|
||||
@@ -244,7 +273,7 @@ def test_summarize_arm_reports_uncovered_repository_separately() -> None:
|
||||
"repo": "hidden/other",
|
||||
"kind": "vulnerability",
|
||||
"status": "prover_covered",
|
||||
"instrument": {"file_bytes": {"app.py": 10}, "hook_bytes_read": 10},
|
||||
"instrument": {"file_bytes": {"app.py": 10}, "file_digests": {"app.py": "a" * 64}, "hook_bytes_read": 10},
|
||||
"stage_funnel": {
|
||||
"entry_recognition": {"entry_points": 0, "frameworks_recognised": {}},
|
||||
"resolution": {},
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
"""Strict v2 persistence, ordered collections, and atomic collision behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.unseen import score # noqa: E402
|
||||
from benchmarks.valid import (
|
||||
artifact_contract, # noqa: E402
|
||||
hunt_handoff, # noqa: E402
|
||||
)
|
||||
from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact # noqa: E402
|
||||
from openultrasast.hunt.runner import input_digest, record_path # noqa: E402
|
||||
|
||||
|
||||
def _rewrite(path: Path, value: object) -> None:
|
||||
path.write_text(json.dumps(value, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def test_payload_requires_complete_metadata_expected_kind_schema_and_digest() -> None:
|
||||
artifact = artifact_contract.benchmark_artifact("record", {"id": "one"})
|
||||
assert artifact_contract.benchmark_payload(artifact, expected_kind="record") == {"id": "one"}
|
||||
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
artifact_contract.benchmark_payload({"id": "one"}, expected_kind="record")
|
||||
for field in ("schema_version", "artifact_kind", "artifact_digest"):
|
||||
stripped = dict(artifact)
|
||||
stripped.pop(field)
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
artifact_contract.benchmark_payload(stripped, expected_kind="record")
|
||||
for field, value in (
|
||||
("schema_version", "v1"),
|
||||
("artifact_kind", "other"),
|
||||
("artifact_digest", "0" * 64),
|
||||
):
|
||||
tampered = dict(artifact)
|
||||
tampered[field] = value
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
artifact_contract.benchmark_payload(tampered, expected_kind="record")
|
||||
|
||||
|
||||
def test_ordered_sequence_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None:
|
||||
path = tmp_path / "inputs.json"
|
||||
rows = ({"id": "one"}, {"id": "two"})
|
||||
artifact_contract.write_benchmark_sequence(path, "inputs", rows)
|
||||
assert artifact_contract.read_benchmark_sequence(path, "inputs") == rows
|
||||
original = json.loads(path.read_text())
|
||||
|
||||
mutations = []
|
||||
reordered = json.loads(json.dumps(original))
|
||||
reordered["rows"].reverse()
|
||||
mutations.append(reordered)
|
||||
truncated = json.loads(json.dumps(original))
|
||||
truncated["rows"].pop()
|
||||
mutations.append(truncated)
|
||||
duplicated = json.loads(json.dumps(original))
|
||||
duplicated["rows"][1] = duplicated["rows"][0]
|
||||
mutations.append(duplicated)
|
||||
extra = json.loads(json.dumps(original))
|
||||
extra["rows"].append(extra["rows"][0])
|
||||
mutations.append(extra)
|
||||
tampered = json.loads(json.dumps(original))
|
||||
tampered["rows"][0]["artifact"]["id"] = "changed"
|
||||
mutations.append(tampered)
|
||||
|
||||
for mutation in mutations:
|
||||
_rewrite(path, mutation)
|
||||
with pytest.raises(ValueError):
|
||||
artifact_contract.read_benchmark_sequence(path, "inputs")
|
||||
|
||||
|
||||
def test_jsonl_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None:
|
||||
path = tmp_path / "judgments.jsonl"
|
||||
rows = ({"id": "one"}, {"id": "two"})
|
||||
artifact_contract.write_benchmark_jsonl(path, "judgment", rows)
|
||||
assert artifact_contract.read_benchmark_jsonl(path, "judgment") == rows
|
||||
original = path.read_text().splitlines()
|
||||
mutations = (
|
||||
list(reversed(original)),
|
||||
original[:-1],
|
||||
[original[0], original[0]],
|
||||
[*original, original[0]],
|
||||
[original[0].replace('"one"', '"changed"'), original[1]],
|
||||
)
|
||||
for lines in mutations:
|
||||
path.write_text("\n".join(lines) + "\n")
|
||||
with pytest.raises(ValueError):
|
||||
artifact_contract.read_benchmark_jsonl(path, "judgment")
|
||||
|
||||
|
||||
def test_atomic_writer_is_idempotent_and_refuses_conflicting_overwrite(tmp_path: Path) -> None:
|
||||
path = tmp_path / "record.json"
|
||||
artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"})
|
||||
before = (path.read_bytes(), path.stat().st_mtime_ns)
|
||||
artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"})
|
||||
assert (path.read_bytes(), path.stat().st_mtime_ns) == before
|
||||
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
artifact_contract.write_benchmark_artifact(path, "record", {"id": "two"})
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
artifact_contract.write_benchmark_artifact(path, "other", {"id": "one"})
|
||||
assert not list(tmp_path.glob("*.tmp"))
|
||||
|
||||
|
||||
def test_concurrent_conflicting_writers_have_one_winner(tmp_path: Path) -> None:
|
||||
path = tmp_path / "record.json"
|
||||
|
||||
def write(identity: str) -> str:
|
||||
artifact_contract.write_benchmark_artifact(path, "record", {"id": identity})
|
||||
return identity
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(write, identity) for identity in ("one", "two")]
|
||||
winners = [future.result() for future in futures if future.exception() is None]
|
||||
failures = [future.exception() for future in futures if future.exception() is not None]
|
||||
assert len(winners) == 1
|
||||
assert len(failures) == 1 and isinstance(failures[0], ValueError)
|
||||
assert artifact_contract.read_benchmark_artifact(path, "record") == {"id": winners[0]}
|
||||
|
||||
|
||||
def test_concurrent_jsonl_appends_preserve_both_rows(tmp_path: Path) -> None:
|
||||
path = tmp_path / "judgments.jsonl"
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(artifact_contract.append_benchmark_jsonl, path, "judgment", {"id": identity}) for identity in ("one", "two")]
|
||||
assert all(future.exception() is None for future in futures)
|
||||
rows = artifact_contract.read_benchmark_jsonl(path, "judgment")
|
||||
assert len(rows) == 2
|
||||
assert {row["id"] for row in rows} == {"one", "two"}
|
||||
|
||||
|
||||
def test_score_reader_refuses_stripped_v2_metadata(tmp_path: Path) -> None:
|
||||
path = tmp_path / "score.json"
|
||||
_rewrite(path, [{"id": "one"}])
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
score.load_records(path)
|
||||
|
||||
|
||||
def _hunt_change() -> dict[str, object]:
|
||||
return {
|
||||
"id": "change-1",
|
||||
"repo": "fixture/repo",
|
||||
"base": "a" * 40,
|
||||
"head": "b" * 40,
|
||||
"kind": "ordinary",
|
||||
"files": [{"path": "app.py", "head_lines": [[1, 1]]}],
|
||||
}
|
||||
|
||||
|
||||
def _hunt_record(change: dict[str, object], program_id: str, mask: tuple[str, ...], digest: str) -> dict[str, object]:
|
||||
return {
|
||||
"id": change["id"],
|
||||
"done": True,
|
||||
"status": "hunt_covered",
|
||||
"instrument": {"head_verified": True, "base_verified": True, "changed_bytes": {"app.py": 10}},
|
||||
"push": {"admission": {"defects": [], "dispositions": []}, "quick_tier": []},
|
||||
"rows": 1,
|
||||
"program_id": program_id,
|
||||
"mask": list(mask),
|
||||
"inputs_present": ["function_text"],
|
||||
"seconds": 0.1,
|
||||
"input_digest": digest,
|
||||
"usd": 0.0,
|
||||
}
|
||||
|
||||
|
||||
def _hunt_artifact(program_id: str, mask: tuple[str, ...]) -> RunArtifact:
|
||||
return RunArtifact(
|
||||
program_id=program_id,
|
||||
schema_version=1,
|
||||
mask=mask,
|
||||
data_digests={"manifest": "d" * 64},
|
||||
configurations=[{"only": "all", "slice": "1", "changes": 1}],
|
||||
usage={"usd": 0.0, "calls": 1.0, "replay_hits": 0.0, "wall_seconds": 0.1},
|
||||
versions={},
|
||||
created="2026-11-02T00:00:00+00:00",
|
||||
)
|
||||
|
||||
|
||||
def _raw_hunt_run(root: Path) -> tuple[hunt_handoff.HuntRunContext, tuple[dict[str, object], ...]]:
|
||||
change = _hunt_change()
|
||||
program_id = "c" * 64
|
||||
mask = ("obligations",)
|
||||
digest = input_digest(change, program_id, mask)
|
||||
root.mkdir()
|
||||
_rewrite(root / "inputs.json", [{"item_id": change["id"], "change": change, "input_digest": digest}])
|
||||
_rewrite(record_path(root, str(change["id"])), _hunt_record(change, program_id, mask, digest))
|
||||
artifact = _hunt_artifact(program_id, mask)
|
||||
artifact.write(root, reserve_attempt=True)
|
||||
context = hunt_handoff.HuntRunContext(
|
||||
producer="openultrasast.hunt.runner.v1",
|
||||
program_id=program_id,
|
||||
mask=mask,
|
||||
manifest_digest="d" * 64,
|
||||
freeze_digest="e" * 64,
|
||||
only="all",
|
||||
slice_id="1",
|
||||
artifact_ids=(artifact.artifact_id,),
|
||||
versions=(),
|
||||
)
|
||||
return context, (change,)
|
||||
|
||||
|
||||
def test_hunt_handoff_is_non_mutating_repeatable_and_persists_only_to_benchmark_destination(tmp_path: Path) -> None:
|
||||
source = tmp_path / "producer"
|
||||
destination = tmp_path / "benchmark-v2"
|
||||
context, changes = _raw_hunt_run(source)
|
||||
before = {path.name: path.read_bytes() for path in source.iterdir()}
|
||||
|
||||
first = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
||||
second = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
||||
assert first == second and first[0]["id"] == "change-1"
|
||||
assert {path.name: path.read_bytes() for path in source.iterdir()} == before
|
||||
|
||||
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
persisted = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()}
|
||||
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} == persisted
|
||||
assert score.load_records(destination) == list(first)
|
||||
assert {path.name: path.read_bytes() for path in source.iterdir()} == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "message"),
|
||||
(
|
||||
("producer", "producer"),
|
||||
("program", "program"),
|
||||
("manifest", "manifest"),
|
||||
("input", "input digest"),
|
||||
("record", "record identity"),
|
||||
("reserved", "reserved metadata"),
|
||||
),
|
||||
)
|
||||
def test_hunt_handoff_refuses_unsupported_producer_schema_and_identity(tmp_path: Path, mutation: str, message: str) -> None:
|
||||
source = tmp_path / "producer"
|
||||
context, changes = _raw_hunt_run(source)
|
||||
if mutation == "producer":
|
||||
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "producer": "arbitrary-json"})
|
||||
elif mutation == "program":
|
||||
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "program_id": "f" * 64})
|
||||
elif mutation == "manifest":
|
||||
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "manifest_digest": "f" * 64})
|
||||
elif mutation == "input":
|
||||
entries = json.loads((source / "inputs.json").read_text())
|
||||
entries[0]["input_digest"] = "0" * 64
|
||||
_rewrite(source / "inputs.json", entries)
|
||||
else:
|
||||
path = record_path(source, "change-1")
|
||||
record = json.loads(path.read_text())
|
||||
if mutation == "record":
|
||||
record["id"] = "other"
|
||||
else:
|
||||
record["schema_version"] = "forged"
|
||||
_rewrite(path, record)
|
||||
with pytest.raises(ValueError, match=message):
|
||||
hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
||||
|
||||
|
||||
def test_hunt_handoff_refuses_conflicting_benchmark_destination(tmp_path: Path) -> None:
|
||||
source = tmp_path / "producer"
|
||||
destination = tmp_path / "benchmark-v2"
|
||||
context, changes = _raw_hunt_run(source)
|
||||
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
artifact_contract.write_benchmark_artifact(destination / "handoff.json", "hunt-runner-handoff", {"producer": "conflict"}, replace=True)
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ("program", "manifest", "configuration", "versions"))
|
||||
def test_hunt_handoff_refuses_mixed_attempt_invariants(tmp_path: Path, mutation: str) -> None:
|
||||
source = tmp_path / "producer"
|
||||
manifest = tmp_path / "pool-fixture.toml"
|
||||
manifest.write_text("status = 'fixture'\n")
|
||||
_context, _changes = _raw_hunt_run(source)
|
||||
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
||||
original = RunArtifact.read(path)
|
||||
values = {
|
||||
"program_id": original.program_id,
|
||||
"schema_version": original.schema_version,
|
||||
"mask": tuple(original.mask),
|
||||
"data_digests": dict(original.data_digests),
|
||||
"configurations": list(original.configurations),
|
||||
"usage": {**original.usage, "calls": 2.0},
|
||||
"versions": dict(original.versions),
|
||||
"created": "2026-11-02T00:00:01+00:00",
|
||||
}
|
||||
if mutation == "program":
|
||||
values["program_id"] = "f" * 64
|
||||
elif mutation == "manifest":
|
||||
values["data_digests"] = {"manifest": "f" * 64}
|
||||
elif mutation == "configuration":
|
||||
values["configurations"] = [{"only": "ordinary", "slice": "1", "changes": 1}]
|
||||
else:
|
||||
values["versions"] = {"dspy": "different"}
|
||||
RunArtifact(**values).write(source, reserve_attempt=True)
|
||||
|
||||
with pytest.raises(ValueError, match="invariant"):
|
||||
hunt_handoff.discover_hunt_runner_context(
|
||||
source,
|
||||
manifest,
|
||||
producer=hunt_handoff.HUNT_RUNNER_PRODUCER,
|
||||
freeze_digest="e" * 64,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attempt_ordinal", (-1, 0, 1, True, 1.5))
|
||||
def test_hunt_handoff_refuses_nonpositive_or_duplicate_attempt_ordinals(tmp_path: Path, attempt_ordinal: int) -> None:
|
||||
source = tmp_path / "producer"
|
||||
context, changes = _raw_hunt_run(source)
|
||||
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
||||
original = RunArtifact.read(path)
|
||||
duplicate = RunArtifact(
|
||||
program_id=original.program_id,
|
||||
schema_version=original.schema_version,
|
||||
mask=tuple(original.mask),
|
||||
data_digests=dict(original.data_digests),
|
||||
configurations=list(original.configurations),
|
||||
usage={**original.usage, "calls": 2.0},
|
||||
versions=dict(original.versions),
|
||||
created="2026-11-02T00:00:01+00:00",
|
||||
attempt_ordinal=attempt_ordinal,
|
||||
)
|
||||
duplicate.write(source)
|
||||
artifact_ids = tuple(sorted((original.artifact_id, duplicate.artifact_id)))
|
||||
|
||||
with pytest.raises(ValueError, match="attempt ordinals"):
|
||||
hunt_handoff.read_hunt_runner_records(source, context=replace(context, artifact_ids=artifact_ids), changes=changes)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("attempt_ordinal", "accepted"), ((MAX_ATTEMPT_ORDINAL, True), (MAX_ATTEMPT_ORDINAL + 1, False)))
|
||||
def test_hunt_handoff_enforces_maximum_attempt_ordinal(tmp_path: Path, attempt_ordinal: int, accepted: bool) -> None:
|
||||
source = tmp_path / "producer"
|
||||
context, changes = _raw_hunt_run(source)
|
||||
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
||||
original = RunArtifact.read(path)
|
||||
additional = RunArtifact(
|
||||
program_id=original.program_id,
|
||||
schema_version=original.schema_version,
|
||||
mask=tuple(original.mask),
|
||||
data_digests=dict(original.data_digests),
|
||||
configurations=list(original.configurations),
|
||||
usage={**original.usage, "calls": 2.0},
|
||||
versions=dict(original.versions),
|
||||
created="2026-11-02T00:00:01+00:00",
|
||||
attempt_ordinal=attempt_ordinal,
|
||||
)
|
||||
additional.write(source)
|
||||
artifact_ids = tuple(sorted((original.artifact_id, additional.artifact_id)))
|
||||
bound_context = replace(context, artifact_ids=artifact_ids)
|
||||
|
||||
if accepted:
|
||||
assert hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="attempt ordinals"):
|
||||
hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes)
|
||||
|
||||
|
||||
def test_score_reader_has_no_external_adoption_escape_hatch(tmp_path: Path) -> None:
|
||||
path = tmp_path / "raw.json"
|
||||
_rewrite(path, [{"id": "one"}])
|
||||
with pytest.raises(TypeError, match="adopt_external"):
|
||||
score.load_records(path, adopt_external=True)
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Benchmark-source closure for the detection-core v1 migration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from benchmark_migration_evidence import BENCHMARK_TERMINAL_EVIDENCE
|
||||
from core_migration_boundary import inspect_legacy_boundary
|
||||
from core_migration_inventory import (
|
||||
REPO,
|
||||
LegacyInventoryError,
|
||||
MigrationDisposition,
|
||||
entry_identity,
|
||||
load_migration_inventory,
|
||||
validate_historical_artifacts,
|
||||
validate_inventory_sources,
|
||||
validate_inventory_transition,
|
||||
)
|
||||
from core_migration_scope import boundary_python
|
||||
|
||||
INVENTORY = REPO / "tests/fixtures/detection_core/v1/migration_inventory.json"
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
_V2_MIGRATED = frozenset(entry_id for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items() if evidence.disposition == "v2_migrated")
|
||||
_RETIRED = frozenset(entry_id for entry_id, evidence in BENCHMARK_TERMINAL_EVIDENCE.items() if evidence.disposition == "retired")
|
||||
_BENCHMARK_ENTRY_IDS = _V2_MIGRATED | _RETIRED
|
||||
|
||||
|
||||
def _benchmark_program_paths(root: Path = REPO) -> tuple[Path, ...]:
|
||||
"""Owned benchmark programs, excluding harvested fixtures and historical corpora."""
|
||||
return tuple(
|
||||
path
|
||||
for path in boundary_python(root)
|
||||
if (relative := path.relative_to(root).as_posix()).startswith("benchmarks/") and not relative.startswith("benchmarks/measurements/")
|
||||
)
|
||||
|
||||
|
||||
def test_registered_benchmark_entries_close_without_rekeying_other_consumers() -> None:
|
||||
candidate = load_migration_inventory(INVENTORY)
|
||||
benchmark = {entry.entry_id: entry for entry in candidate.entries if entry.path.startswith("benchmarks/")}
|
||||
|
||||
assert set(benchmark) == _BENCHMARK_ENTRY_IDS
|
||||
assert all(benchmark[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED)
|
||||
assert all(benchmark[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED)
|
||||
frozen = replace(
|
||||
candidate,
|
||||
entries=tuple(
|
||||
replace(entry, disposition=MigrationDisposition.OPEN) if entry.entry_id in _BENCHMARK_ENTRY_IDS else entry
|
||||
for entry in candidate.entries
|
||||
),
|
||||
)
|
||||
validate_inventory_transition(frozen, candidate)
|
||||
validate_inventory_sources(candidate, root=REPO)
|
||||
|
||||
|
||||
def test_source_validation_refuses_unregistered_terminal_entry() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
original = next(entry for entry in inventory.entries if entry.entry_id in _V2_MIGRATED)
|
||||
forged = replace(original, artifact_family="unregistered-terminal", entry_id="")
|
||||
forged = replace(forged, entry_id=entry_identity(forged))
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="terminal evidence"):
|
||||
validate_inventory_sources(replace(inventory, entries=(forged,)), root=REPO)
|
||||
|
||||
|
||||
def test_source_validation_refuses_removed_migrated_owner(tmp_path: Path) -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
original = next(entry for entry in inventory.entries if entry.entry_id.startswith("40465a45"))
|
||||
target = tmp_path / original.path
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_text((REPO / original.path).read_text().replace("def main(", "def removed_main(", 1))
|
||||
|
||||
with pytest.raises(LegacyInventoryError, match="migrated owner is absent"):
|
||||
validate_inventory_sources(replace(inventory, entries=(original,)), root=tmp_path)
|
||||
|
||||
|
||||
def test_test_and_operational_entries_remain_open_for_task_10_6() -> None:
|
||||
candidate = load_migration_inventory(INVENTORY)
|
||||
deferred = {
|
||||
entry.disposition for entry in candidate.entries if entry.owner_boundary in {"tests", "operational-tooling", "repository-tooling"}
|
||||
}
|
||||
|
||||
assert deferred == {MigrationDisposition.OPEN}
|
||||
|
||||
|
||||
def test_old_benchmark_admission_contract_and_runner_are_not_live() -> None:
|
||||
from benchmarks.valid import admission_run, prover_admission
|
||||
|
||||
assert not hasattr(admission_run, "run_admission")
|
||||
assert not hasattr(admission_run, "assemble_admission_report")
|
||||
assert not hasattr(prover_admission, "AdmissionManifest")
|
||||
assert not hasattr(prover_admission, "validate_admission_manifest")
|
||||
|
||||
|
||||
def test_registered_live_writers_execute_strict_v2_artifact_conformance(tmp_path: Path) -> None:
|
||||
from benchmark_migration_conformance import validate_benchmark_terminal_conformance
|
||||
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
validate_benchmark_terminal_conformance(inventory, output_root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry_id", sorted(_V2_MIGRATED))
|
||||
def test_each_registered_owner_is_required_for_its_exact_probe(tmp_path: Path, monkeypatch, entry_id: str) -> None:
|
||||
from benchmark_migration_conformance import (
|
||||
OWNER_TARGETS,
|
||||
BenchmarkMigrationConformanceError,
|
||||
probe_benchmark_entry,
|
||||
)
|
||||
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
entry = next(candidate for candidate in inventory.entries if candidate.entry_id == entry_id)
|
||||
module_name, symbol = OWNER_TARGETS[entry_id]
|
||||
module = __import__(module_name, fromlist=[symbol])
|
||||
|
||||
def removed_owner(*_args, **_kwargs):
|
||||
raise RuntimeError("registered owner removed")
|
||||
|
||||
monkeypatch.setattr(module, symbol, removed_owner)
|
||||
with pytest.raises(BenchmarkMigrationConformanceError, match=entry_id):
|
||||
probe_benchmark_entry(entry, output_root=tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"forged",
|
||||
["if 0", "if None", "while False", "local def", "local import", "parameter", "assignment", "comprehension"],
|
||||
)
|
||||
def test_dead_or_shadowed_syntax_cannot_close_an_unregistered_entry(tmp_path: Path, forged: str) -> None:
|
||||
from benchmark_migration_conformance import BenchmarkMigrationConformanceError, validate_benchmark_terminal_conformance
|
||||
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
original = next(entry for entry in inventory.entries if entry.entry_id in _V2_MIGRATED)
|
||||
unregistered = replace(original, entry_id="f" * 64)
|
||||
candidate = replace(inventory, entries=(unregistered,))
|
||||
(tmp_path / "forged.py").write_text(f"# {forged}\n")
|
||||
with pytest.raises(BenchmarkMigrationConformanceError, match="probe registry"):
|
||||
validate_benchmark_terminal_conformance(candidate, output_root=tmp_path / "out")
|
||||
|
||||
|
||||
def test_v2_benchmark_artifact_binds_its_kind_and_payload() -> None:
|
||||
from benchmarks.valid.artifact_contract import benchmark_artifact, benchmark_payload
|
||||
|
||||
artifact = benchmark_artifact("readiness-result", {"result_digest": "a" * 64})
|
||||
assert benchmark_payload(artifact, expected_kind="readiness-result") == {"result_digest": "a" * 64}
|
||||
|
||||
artifact["artifact_kind"] = "admission-report"
|
||||
with pytest.raises(ValueError, match="metadata does not match"):
|
||||
benchmark_payload(artifact, expected_kind="readiness-result")
|
||||
|
||||
|
||||
def test_v2_benchmark_jsonl_preserves_ordered_row_persistence(tmp_path: Path) -> None:
|
||||
from benchmarks.valid.artifact_contract import read_benchmark_jsonl, write_benchmark_jsonl
|
||||
|
||||
path = tmp_path / "suspicions.jsonl"
|
||||
write_benchmark_jsonl(path, "suspicion", ({"id": "first"}, {"id": "second"}))
|
||||
|
||||
assert read_benchmark_jsonl(path, "suspicion") == ({"id": "first"}, {"id": "second"})
|
||||
|
||||
|
||||
def test_verified_nonzero_benchmark_program_scope_has_exact_zero_legacy_references() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
paths = _benchmark_program_paths()
|
||||
report = inspect_legacy_boundary(
|
||||
paths,
|
||||
frozen_inventory=inventory,
|
||||
candidate_inventory=inventory,
|
||||
expected_scope=paths,
|
||||
root=REPO,
|
||||
)
|
||||
|
||||
assert report.files_read == len(paths) > 0
|
||||
assert report.bytes_read > 0
|
||||
assert report.scanned_scope_paths == report.expected_scope_paths
|
||||
assert report.consumer_legacy_import_sites == ()
|
||||
assert report.consumer_legacy_writer_sites == ()
|
||||
assert report.forbidden_dynamic_sites == ()
|
||||
assert report.compatibility_implementation_sites == ()
|
||||
|
||||
|
||||
def test_benchmark_closure_preserves_frozen_historical_bytes() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
artifacts = validate_historical_artifacts(inventory, root=REPO)
|
||||
|
||||
assert len(artifacts) == len(inventory.historical_artifacts) > 0
|
||||
@@ -66,7 +66,11 @@ def test_registered_production_entries_close_without_rekeying_other_consumers()
|
||||
assert set(production) == _PRODUCTION_ENTRY_IDS
|
||||
assert all(production[entry_id].disposition is MigrationDisposition.V2_MIGRATED for entry_id in _V2_MIGRATED)
|
||||
assert all(production[entry_id].disposition is MigrationDisposition.RETIRED for entry_id in _RETIRED)
|
||||
assert all(entry.disposition is MigrationDisposition.OPEN for entry in candidate.entries if entry.entry_id not in production)
|
||||
assert all(
|
||||
entry.disposition is MigrationDisposition.OPEN
|
||||
for entry in candidate.entries
|
||||
if entry.entry_id not in production and not entry.path.startswith("benchmarks/")
|
||||
)
|
||||
|
||||
frozen = replace(
|
||||
candidate,
|
||||
|
||||
@@ -81,9 +81,9 @@ def test_frozen_inventory_resolves_nonzero_sources_and_targets() -> None:
|
||||
inventory = load_migration_inventory(INVENTORY)
|
||||
|
||||
assert inventory.entries
|
||||
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 180
|
||||
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 6
|
||||
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 12
|
||||
assert sum(entry.disposition is MigrationDisposition.OPEN for entry in inventory.entries) == 155
|
||||
assert sum(entry.disposition is MigrationDisposition.V2_MIGRATED for entry in inventory.entries) == 25
|
||||
assert sum(entry.disposition is MigrationDisposition.RETIRED for entry in inventory.entries) == 18
|
||||
assert all(entry.entry_id == entry_identity(entry) for entry in inventory.entries)
|
||||
validate_inventory_sources(inventory, root=REPO)
|
||||
|
||||
|
||||
+110
-1
@@ -1,10 +1,15 @@
|
||||
"""Checks for stable run records and budget accounting."""
|
||||
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
|
||||
from openultrasast.hunt.artifact import RunArtifact, UsageLedger
|
||||
from openultrasast.hunt import artifact as artifact_module
|
||||
from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact, UsageLedger
|
||||
|
||||
|
||||
def artifact():
|
||||
@@ -32,6 +37,110 @@ def test_artifact_round_trip_and_tamper(tmp_path):
|
||||
RunArtifact.read(path)
|
||||
|
||||
|
||||
def test_artifact_persistence_is_create_only_and_idempotent(tmp_path):
|
||||
run = artifact()
|
||||
path = run.write(tmp_path)
|
||||
before = (path.read_bytes(), path.stat().st_mtime_ns)
|
||||
assert run.write(tmp_path) == path
|
||||
assert (path.read_bytes(), path.stat().st_mtime_ns) == before
|
||||
|
||||
path.write_text("conflicting bytes\n")
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
run.write(tmp_path)
|
||||
|
||||
|
||||
def test_missing_attempt_sequence_fails_closed_when_artifacts_exist(tmp_path):
|
||||
first = artifact()
|
||||
first_path = first.write(tmp_path, reserve_attempt=True)
|
||||
before = (first_path.read_bytes(), first_path.stat().st_mtime_ns)
|
||||
(tmp_path / ".attempt-sequence").unlink()
|
||||
resumed = artifact()
|
||||
resumed.created = "2026-11-02T00:00:01+00:00"
|
||||
|
||||
with pytest.raises(ValueError, match="attempt sequence is missing"):
|
||||
resumed.write(tmp_path, reserve_attempt=True)
|
||||
assert (first_path.read_bytes(), first_path.stat().st_mtime_ns) == before
|
||||
assert not (tmp_path / ".attempt-sequence").exists()
|
||||
|
||||
|
||||
def test_attempt_ordinals_are_unique_under_concurrency_and_resume(tmp_path):
|
||||
def persist(index):
|
||||
run = artifact()
|
||||
run.created = f"2026-11-02T00:00:{index:02d}+00:00"
|
||||
run.write(tmp_path, reserve_attempt=True)
|
||||
return run.attempt_ordinal
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
ordinals = list(pool.map(persist, range(4)))
|
||||
resumed = artifact()
|
||||
resumed.created = "2026-11-02T00:01:00+00:00"
|
||||
resumed.write(tmp_path, reserve_attempt=True)
|
||||
|
||||
assert sorted(ordinals) == [1, 2, 3, 4]
|
||||
assert resumed.attempt_ordinal == 5
|
||||
assert (tmp_path / ".attempt-sequence").read_text() == "5\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("state", "message"), (("corrupt\n", "unreadable"), ("-1\n", "invalid")))
|
||||
def test_corrupt_attempt_sequence_fails_closed(tmp_path, state, message):
|
||||
(tmp_path / ".attempt-sequence").write_text(state)
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
artifact().write(tmp_path, reserve_attempt=True)
|
||||
|
||||
|
||||
def test_attempt_sequence_allocates_the_signed_64_bit_maximum(tmp_path):
|
||||
(tmp_path / ".attempt-sequence").write_text(f"{MAX_ATTEMPT_ORDINAL - 1}\n")
|
||||
run = artifact()
|
||||
|
||||
run.write(tmp_path, reserve_attempt=True)
|
||||
|
||||
assert run.attempt_ordinal == MAX_ATTEMPT_ORDINAL
|
||||
assert (tmp_path / ".attempt-sequence").read_text() == f"{MAX_ATTEMPT_ORDINAL}\n"
|
||||
|
||||
|
||||
def test_attempt_sequence_refuses_overflow_without_mutation(tmp_path):
|
||||
sequence = tmp_path / ".attempt-sequence"
|
||||
sequence.write_text(f"{MAX_ATTEMPT_ORDINAL}\n")
|
||||
before = (sequence.read_bytes(), sequence.stat().st_mtime_ns)
|
||||
|
||||
with pytest.raises(ValueError, match="exhausted"):
|
||||
artifact().write(tmp_path, reserve_attempt=True)
|
||||
|
||||
assert (sequence.read_bytes(), sequence.stat().st_mtime_ns) == before
|
||||
assert not list(tmp_path.glob("*.json"))
|
||||
|
||||
|
||||
def test_attempt_sequence_and_artifact_creation_fsync_the_directory(tmp_path, monkeypatch):
|
||||
real_fsync = os.fsync
|
||||
directory_syncs = []
|
||||
|
||||
def recording_fsync(descriptor):
|
||||
if stat.S_ISDIR(os.fstat(descriptor).st_mode):
|
||||
directory_syncs.append(descriptor)
|
||||
real_fsync(descriptor)
|
||||
|
||||
monkeypatch.setattr(artifact_module.os, "fsync", recording_fsync)
|
||||
artifact().write(tmp_path, reserve_attempt=True)
|
||||
|
||||
assert len(directory_syncs) == 2
|
||||
|
||||
|
||||
def test_directory_fsync_ignores_only_unsupported_platform_errors(tmp_path, monkeypatch):
|
||||
def unsupported(_descriptor):
|
||||
raise OSError(errno.EOPNOTSUPP, "unsupported")
|
||||
|
||||
monkeypatch.setattr(artifact_module.os, "fsync", unsupported)
|
||||
artifact_module._sync_directory(tmp_path)
|
||||
|
||||
def io_failure(_descriptor):
|
||||
raise OSError(errno.EIO, "I/O failure")
|
||||
|
||||
monkeypatch.setattr(artifact_module.os, "fsync", io_failure)
|
||||
with pytest.raises(OSError, match="I/O failure"):
|
||||
artifact_module._sync_directory(tmp_path)
|
||||
|
||||
|
||||
def test_ledger_and_budget_prevent_artifact(tmp_path):
|
||||
run = artifact()
|
||||
ledger = UsageLedger(artifact=run)
|
||||
|
||||
@@ -84,7 +84,7 @@ def test_valid_report_before_after_workload_curve_and_ledger(pilot_case, populat
|
||||
assert row["changed_bytes"]["app.py"] > 0
|
||||
assert row["base_verified"] and row["head_verified"] and row["base"] and row["head"]
|
||||
assert analysis["before"]["raw"]["metrics"] == json.loads(kwargs["pilot_metrics"]["raw"].read_text())
|
||||
records = score.load_records(directory / "records")
|
||||
records = score.load_records(directory / "benchmark-records")
|
||||
assert analysis["after"]["pooled"]["push"] == metrics.summarize(records, scored=score.score(metrics.scoring_records(records)))
|
||||
assert analysis["after"]["injection"]["search"]["recall"]["rate"] == 1
|
||||
assert analysis["after"]["pooled"]["search"]["suspicions_per_push"]["rate"] == 0
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -11,6 +12,12 @@ dspy = pytest.importorskip("dspy")
|
||||
from dspy.utils import DummyLM # noqa: E402
|
||||
|
||||
from benchmarks.unseen import score # noqa: E402
|
||||
from benchmarks.valid.hunt_handoff import ( # noqa: E402
|
||||
HUNT_RUNNER_PRODUCER,
|
||||
discover_hunt_runner_context,
|
||||
persist_hunt_runner_records,
|
||||
read_hunt_runner_records,
|
||||
)
|
||||
from openultrasast.hunt import program as hp # noqa: E402
|
||||
from openultrasast.hunt import runner # noqa: E402
|
||||
|
||||
@@ -73,7 +80,8 @@ def test_runner_writes_scorable_records_and_resumes(repository, tmp_path):
|
||||
manifest, {"fixture/repo": str(root)}, out, program=program, program_id=pid, mask=program.mask, load_changes=lambda m, s: changes
|
||||
)
|
||||
assert artifact is not None and artifact.program_id == pid
|
||||
rows = score.load_records(out)
|
||||
context = discover_hunt_runner_context(out, manifest, producer=HUNT_RUNNER_PRODUCER, freeze_digest="e" * 64)
|
||||
rows = list(read_hunt_runner_records(out, context=context, changes=changes))
|
||||
assert {r["id"]: r["status"] for r in rows} == {"pos": "hunt_covered", "ord": "hunt_covered"}
|
||||
pos = next(r for r in rows if r["id"] == "pos")
|
||||
assert pos["instrument"]["head_verified"] and pos["instrument"]["hook_bytes_read"] > 0
|
||||
@@ -91,6 +99,58 @@ def test_runner_writes_scorable_records_and_resumes(repository, tmp_path):
|
||||
assert {p.name: p.read_bytes() for p in out.glob("*.json") if p.name in before} == before
|
||||
|
||||
|
||||
def _budget_attempt(manifest, root, source, program, pid, changes):
|
||||
return runner.run(
|
||||
manifest,
|
||||
{"fixture/repo": str(root)},
|
||||
source,
|
||||
program=program,
|
||||
program_id=pid,
|
||||
mask=program.mask,
|
||||
load_changes=lambda _manifest, _slice: changes,
|
||||
budget={"calls": 1},
|
||||
versions={"dspy": dspy.__version__},
|
||||
)
|
||||
|
||||
|
||||
def test_budget_stop_resume_handoff_binds_all_attempt_artifacts(repository, tmp_path, monkeypatch):
|
||||
root, manifest, changes = repository
|
||||
program = _scripted_program()
|
||||
pid = hp.program_id(program, mask=program.mask)
|
||||
source = tmp_path / "attempts"
|
||||
destination = tmp_path / "benchmark-v2"
|
||||
instant = datetime(2026, 11, 2, tzinfo=UTC)
|
||||
instants = iter((instant, instant))
|
||||
|
||||
class AttemptClock:
|
||||
@classmethod
|
||||
def now(cls, _timezone):
|
||||
return next(instants)
|
||||
|
||||
monkeypatch.setattr(runner, "datetime", AttemptClock)
|
||||
monkeypatch.setattr(runner.time, "monotonic", lambda: 100.0)
|
||||
|
||||
first = _budget_attempt(manifest, root, source, program, pid, changes)
|
||||
(first_path,) = [path for path in source.glob("*.json") if "artifact_id" in path.read_text()]
|
||||
first_before = (first_path.read_bytes(), first_path.stat().st_mtime_ns)
|
||||
second = _budget_attempt(manifest, root, source, program, pid, changes)
|
||||
artifacts = [path for path in source.glob("*.json") if "artifact_id" in path.read_text()]
|
||||
assert first is None and second is None
|
||||
assert len(artifacts) == 2
|
||||
assert all(path.stat().st_size > 0 for path in artifacts)
|
||||
assert (first_path.read_bytes(), first_path.stat().st_mtime_ns) == first_before
|
||||
source_before = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in source.iterdir()}
|
||||
|
||||
context = discover_hunt_runner_context(source, manifest, producer=HUNT_RUNNER_PRODUCER, freeze_digest="e" * 64)
|
||||
assert context.artifact_ids == tuple(sorted(path.stem for path in artifacts))
|
||||
assert {row["id"] for row in read_hunt_runner_records(source, context=context, changes=changes)} == {"pos", "ord"}
|
||||
persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
destination_before = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()}
|
||||
persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
||||
assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in source.iterdir()} == source_before
|
||||
assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} == destination_before
|
||||
|
||||
|
||||
def test_runner_refuses_the_pilot_unless_it_is_a_report_run(tmp_path):
|
||||
manifest = tmp_path / "pool-valid.toml"
|
||||
manifest.write_text("status = 'frozen'\n")
|
||||
|
||||
@@ -22,6 +22,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
hunter_arm = importlib.import_module("benchmarks.valid.hunter_arm")
|
||||
catalog = importlib.import_module("benchmarks.valid.catalog")
|
||||
score = importlib.import_module("benchmarks.unseen.score")
|
||||
artifact_contract = importlib.import_module("benchmarks.valid.artifact_contract")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -136,7 +137,7 @@ def test_positive_is_scoreable_and_checkout_is_removed(fixture, monkeypatch, cap
|
||||
assert row["status"] == "hunter_covered" and row["done"]
|
||||
assert row["instrument"]["hook_bytes_read"] == len(git.blob(change["head"], "app.py").encode())
|
||||
assert score.coverage(row) != "instrument_failure"
|
||||
assert row["input_digest"] == json.loads((out / "inputs.json").read_text())[0]["input_digest"]
|
||||
assert row["input_digest"] == artifact_contract.read_benchmark_sequence(out / "inputs.json", "benchmark-arm-inputs")[0]["input_digest"]
|
||||
assert client.calls[0]["model"] == resolve_models(load_config(dotenv=False))[0]
|
||||
assert "fixture-positive app.py:" in capsys.readouterr().out
|
||||
assert git.run("worktree", "list", "--porcelain").count("worktree ") == 1
|
||||
@@ -204,9 +205,11 @@ def test_resume_skips_done_without_client_or_checkout(fixture, monkeypatch, with
|
||||
_, change, manifest, repos, out = fixture
|
||||
out.mkdir()
|
||||
existing = {"id": change["id"], "done": True, "input_digest": "existing-digest", "status": "hunter_covered"}
|
||||
record_path(out, change["id"]).write_text(json.dumps(existing))
|
||||
artifact_contract.write_benchmark_artifact(record_path(out, change["id"]), "benchmark-arm-record", existing)
|
||||
if with_inputs:
|
||||
(out / "inputs.json").write_text(json.dumps([{"item_id": change["id"], "change": change, "input_digest": "existing-digest"}]))
|
||||
artifact_contract.write_benchmark_sequence(
|
||||
out / "inputs.json", "benchmark-arm-inputs", [{"item_id": change["id"], "change": change, "input_digest": "existing-digest"}]
|
||||
)
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
pytest.fail("completed change was executed again")
|
||||
@@ -214,7 +217,7 @@ def test_resume_skips_done_without_client_or_checkout(fixture, monkeypatch, with
|
||||
monkeypatch.setattr(tool_hunter, "resolve_hunter_client", forbidden)
|
||||
monkeypatch.setattr(hunter_arm, "local_git", forbidden)
|
||||
assert hunter_arm.main(["run", str(manifest), "--repos", str(repos), "--out", str(out)]) == 0
|
||||
assert json.loads(record_path(out, change["id"]).read_text()) == existing
|
||||
assert artifact_contract.read_benchmark_artifact(record_path(out, change["id"]), "benchmark-arm-record") == existing
|
||||
assert score.load_records(out)[0]["input_digest"] == "existing-digest"
|
||||
|
||||
|
||||
@@ -418,9 +421,9 @@ def test_classify_family_reuses_parser_and_retry(tmp_path, monkeypatch, failure)
|
||||
|
||||
def write_arm(out, entries, records):
|
||||
out.mkdir()
|
||||
(out / "inputs.json").write_text(json.dumps(entries))
|
||||
artifact_contract.write_benchmark_sequence(out / "inputs.json", "benchmark-arm-inputs", entries)
|
||||
for item_id, record in records.items():
|
||||
record_path(out, item_id).write_text(json.dumps(record))
|
||||
artifact_contract.write_benchmark_artifact(record_path(out, item_id), "benchmark-arm-record", record)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -489,7 +492,7 @@ def test_combine_missing_classifier_counts_as_unsure(combine_arms, classificatio
|
||||
replay_out, hunter_out, combined, entry, a, b = combine_arms
|
||||
defect = {"defect_id": "unclassified", "source": "hunter", "family": "path", "locations": [["app.py", 6]], **classification}
|
||||
b["push"]["admission"]["defects"] = [defect]
|
||||
record_path(hunter_out, entry["item_id"]).write_text(json.dumps(b))
|
||||
artifact_contract.write_benchmark_artifact(record_path(hunter_out, entry["item_id"]), "benchmark-arm-record", b, replace=True)
|
||||
assert hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined), "--keep-verdicts", keep_verdicts]) == 0
|
||||
(row,) = score.load_records(combined)
|
||||
kept = int("unsure" in keep_verdicts.split(","))
|
||||
@@ -546,7 +549,7 @@ def test_combine_preserves_a_and_scores_b(fixture, tmp_path):
|
||||
}
|
||||
assert b_only["arms"] == {"a": "missing", "b": "hunter_covered"}
|
||||
a["push"]["admission"]["defects"] = [{**defect, "defect_id": "a-defect", "source": "engine"}]
|
||||
record_path(replay_out, "replay-both").write_text(json.dumps(a))
|
||||
artifact_contract.write_benchmark_artifact(record_path(replay_out, "replay-both"), "benchmark-arm-record", a, replace=True)
|
||||
hunter_arm.main(["combine", str(replay_out), str(hunter_out), "--out", str(combined)])
|
||||
defects = score.load_records(combined)[0]["push"]["admission"]["defects"]
|
||||
assert len(defects) == 1 and defects[0]["source"] == "engine"
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""The judge resume boundary accepts only complete ordered v2 artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.valid import judge_pass # noqa: E402
|
||||
|
||||
|
||||
def test_load_done_refuses_stripped_metadata_and_duplicate_ids(tmp_path: Path) -> None:
|
||||
out = tmp_path / "judge"
|
||||
out.mkdir()
|
||||
path = out / "judgments.jsonl"
|
||||
path.write_text(json.dumps({"id": "one", "judgment": "keep"}) + "\n")
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
judge_pass._load_done(out)
|
||||
|
||||
from benchmarks.valid.artifact_contract import write_benchmark_jsonl
|
||||
|
||||
write_benchmark_jsonl(
|
||||
path,
|
||||
"judge-pass",
|
||||
({"key": "one", "judgment": "keep"}, {"key": "one", "judgment": "drop"}),
|
||||
replace=True,
|
||||
)
|
||||
with pytest.raises(ValueError, match="duplicate"):
|
||||
judge_pass._load_done(out)
|
||||
@@ -14,7 +14,7 @@ import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from benchmarks.valid import prover_admission # noqa: E402
|
||||
from openultrasast.core import legacy_v1 as prover_admission # noqa: E402
|
||||
|
||||
|
||||
def _manifest() -> dict[str, object]:
|
||||
|
||||
@@ -14,6 +14,7 @@ from test_hunt_prover import repository # noqa: E402
|
||||
|
||||
from benchmarks.unseen import score # noqa: E402
|
||||
from benchmarks.valid import prover_arm # noqa: E402
|
||||
from benchmarks.valid.artifact_contract import benchmark_payload # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -99,13 +100,25 @@ def test_run_resume_summary_and_locations(pool, monkeypatch, capsys):
|
||||
)
|
||||
|
||||
|
||||
def test_resume_refuses_stripped_v2_metadata(pool):
|
||||
_, manifest, repos, out = pool
|
||||
assert prover_arm.run(manifest, repos, out) == 0
|
||||
path = out / (hashlib.sha256(b"positive").hexdigest() + ".json")
|
||||
value = json.loads(path.read_text())
|
||||
for field in ("schema_version", "artifact_kind", "artifact_digest"):
|
||||
value.pop(field)
|
||||
path.write_text(json.dumps(value))
|
||||
with pytest.raises(ValueError, match="metadata"):
|
||||
prover_arm.run(manifest, repos, out)
|
||||
|
||||
|
||||
def test_zero_byte_record_refuses(pool):
|
||||
_, manifest, repos, out = pool
|
||||
assert prover_arm.run(manifest, repos, out) == 0
|
||||
path = out / (hashlib.sha256(b"positive").hexdigest() + ".json")
|
||||
record = json.loads(path.read_text())
|
||||
record = benchmark_payload(json.loads(path.read_text()), expected_kind="benchmark-arm-record")
|
||||
record["instrument"]["changed_bytes"]["Fixture.java"] = 0
|
||||
path.write_text(json.dumps(record))
|
||||
prover_arm._write_json(path, record, replace=True)
|
||||
with pytest.raises(ValueError, match="zero-byte"):
|
||||
prover_arm.summarize(out)
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ def test_frozen_toml_slice_and_scoring_join(tmp_path):
|
||||
import hashlib
|
||||
|
||||
from benchmarks.unseen.score import load_records
|
||||
from benchmarks.valid.artifact_contract import write_benchmark_artifact, write_benchmark_sequence
|
||||
|
||||
pool = tmp_path / "pool-p1.toml"
|
||||
pool.write_text("""pool="p1"
|
||||
@@ -68,11 +69,15 @@ changes=[{kind="ordinary",base="ccc",head="ddd",files=[]}]
|
||||
(tmp_path / "freeze-p1.json").write_text(json.dumps({"freeze_digest": freeze}))
|
||||
changes = r.load_slice(pool, 2)
|
||||
assert len(changes) == 1 and changes[0]["repo"] == "opaque-b"
|
||||
(tmp_path / "inputs.json").write_text(json.dumps([{"item_id": "replay-opaque", "change": changes[0], "input_digest": "digest"}]))
|
||||
write_benchmark_sequence(
|
||||
tmp_path / "inputs.json",
|
||||
"benchmark-arm-inputs",
|
||||
[{"item_id": "replay-opaque", "change": changes[0], "input_digest": "digest"}],
|
||||
)
|
||||
assert load_records(tmp_path)[0]["kind"] == "ordinary"
|
||||
assert "instrument" not in load_records(tmp_path)[0] # Missing result remains a measured failure.
|
||||
result = tmp_path / (hashlib.sha256(b"replay-opaque").hexdigest() + ".json")
|
||||
result.write_text(json.dumps({"input_digest": "digest", "instrument": {"head_verified": True}}))
|
||||
write_benchmark_artifact(result, "benchmark-arm-record", {"input_digest": "digest", "instrument": {"head_verified": True}})
|
||||
assert load_records(tmp_path)[0]["instrument"]["head_verified"]
|
||||
pool.write_text(pool.read_text() + "changed=true\n")
|
||||
with pytest.raises(ValueError, match="digest"):
|
||||
|
||||
Reference in New Issue
Block a user