mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-11-13 01:48:17 +00:00
370 lines
16 KiB
Python
370 lines
16 KiB
Python
"""Strict v2 persistence, ordered collections, and atomic collision behavior."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from benchmarks.unseen import score # noqa: E402
|
|
from benchmarks.valid import (
|
|
artifact_contract, # noqa: E402
|
|
hunt_handoff, # noqa: E402
|
|
)
|
|
from openultrasast.hunt.artifact import MAX_ATTEMPT_ORDINAL, RunArtifact # noqa: E402
|
|
from openultrasast.hunt.runner import input_digest, record_path # noqa: E402
|
|
|
|
|
|
def _rewrite(path: Path, value: object) -> None:
|
|
path.write_text(json.dumps(value, sort_keys=True) + "\n")
|
|
|
|
|
|
def test_payload_requires_complete_metadata_expected_kind_schema_and_digest() -> None:
|
|
artifact = artifact_contract.benchmark_artifact("record", {"id": "one"})
|
|
assert artifact_contract.benchmark_payload(artifact, expected_kind="record") == {"id": "one"}
|
|
|
|
with pytest.raises(ValueError, match="metadata"):
|
|
artifact_contract.benchmark_payload({"id": "one"}, expected_kind="record")
|
|
for field in ("schema_version", "artifact_kind", "artifact_digest"):
|
|
stripped = dict(artifact)
|
|
stripped.pop(field)
|
|
with pytest.raises(ValueError, match="metadata"):
|
|
artifact_contract.benchmark_payload(stripped, expected_kind="record")
|
|
for field, value in (
|
|
("schema_version", "v1"),
|
|
("artifact_kind", "other"),
|
|
("artifact_digest", "0" * 64),
|
|
):
|
|
tampered = dict(artifact)
|
|
tampered[field] = value
|
|
with pytest.raises(ValueError, match="metadata"):
|
|
artifact_contract.benchmark_payload(tampered, expected_kind="record")
|
|
|
|
|
|
def test_ordered_sequence_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None:
|
|
path = tmp_path / "inputs.json"
|
|
rows = ({"id": "one"}, {"id": "two"})
|
|
artifact_contract.write_benchmark_sequence(path, "inputs", rows)
|
|
assert artifact_contract.read_benchmark_sequence(path, "inputs") == rows
|
|
original = json.loads(path.read_text())
|
|
|
|
mutations = []
|
|
reordered = json.loads(json.dumps(original))
|
|
reordered["rows"].reverse()
|
|
mutations.append(reordered)
|
|
truncated = json.loads(json.dumps(original))
|
|
truncated["rows"].pop()
|
|
mutations.append(truncated)
|
|
duplicated = json.loads(json.dumps(original))
|
|
duplicated["rows"][1] = duplicated["rows"][0]
|
|
mutations.append(duplicated)
|
|
extra = json.loads(json.dumps(original))
|
|
extra["rows"].append(extra["rows"][0])
|
|
mutations.append(extra)
|
|
tampered = json.loads(json.dumps(original))
|
|
tampered["rows"][0]["artifact"]["id"] = "changed"
|
|
mutations.append(tampered)
|
|
|
|
for mutation in mutations:
|
|
_rewrite(path, mutation)
|
|
with pytest.raises(ValueError):
|
|
artifact_contract.read_benchmark_sequence(path, "inputs")
|
|
|
|
|
|
def test_jsonl_rejects_reorder_truncate_duplicate_extra_and_row_tamper(tmp_path: Path) -> None:
|
|
path = tmp_path / "judgments.jsonl"
|
|
rows = ({"id": "one"}, {"id": "two"})
|
|
artifact_contract.write_benchmark_jsonl(path, "judgment", rows)
|
|
assert artifact_contract.read_benchmark_jsonl(path, "judgment") == rows
|
|
original = path.read_text().splitlines()
|
|
mutations = (
|
|
list(reversed(original)),
|
|
original[:-1],
|
|
[original[0], original[0]],
|
|
[*original, original[0]],
|
|
[original[0].replace('"one"', '"changed"'), original[1]],
|
|
)
|
|
for lines in mutations:
|
|
path.write_text("\n".join(lines) + "\n")
|
|
with pytest.raises(ValueError):
|
|
artifact_contract.read_benchmark_jsonl(path, "judgment")
|
|
|
|
|
|
def test_atomic_writer_is_idempotent_and_refuses_conflicting_overwrite(tmp_path: Path) -> None:
|
|
path = tmp_path / "record.json"
|
|
artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"})
|
|
before = (path.read_bytes(), path.stat().st_mtime_ns)
|
|
artifact_contract.write_benchmark_artifact(path, "record", {"id": "one"})
|
|
assert (path.read_bytes(), path.stat().st_mtime_ns) == before
|
|
|
|
with pytest.raises(ValueError, match="conflict"):
|
|
artifact_contract.write_benchmark_artifact(path, "record", {"id": "two"})
|
|
with pytest.raises(ValueError, match="conflict"):
|
|
artifact_contract.write_benchmark_artifact(path, "other", {"id": "one"})
|
|
assert not list(tmp_path.glob("*.tmp"))
|
|
|
|
|
|
def test_concurrent_conflicting_writers_have_one_winner(tmp_path: Path) -> None:
|
|
path = tmp_path / "record.json"
|
|
|
|
def write(identity: str) -> str:
|
|
artifact_contract.write_benchmark_artifact(path, "record", {"id": identity})
|
|
return identity
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [pool.submit(write, identity) for identity in ("one", "two")]
|
|
winners = [future.result() for future in futures if future.exception() is None]
|
|
failures = [future.exception() for future in futures if future.exception() is not None]
|
|
assert len(winners) == 1
|
|
assert len(failures) == 1 and isinstance(failures[0], ValueError)
|
|
assert artifact_contract.read_benchmark_artifact(path, "record") == {"id": winners[0]}
|
|
|
|
|
|
def test_concurrent_jsonl_appends_preserve_both_rows(tmp_path: Path) -> None:
|
|
path = tmp_path / "judgments.jsonl"
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [pool.submit(artifact_contract.append_benchmark_jsonl, path, "judgment", {"id": identity}) for identity in ("one", "two")]
|
|
assert all(future.exception() is None for future in futures)
|
|
rows = artifact_contract.read_benchmark_jsonl(path, "judgment")
|
|
assert len(rows) == 2
|
|
assert {row["id"] for row in rows} == {"one", "two"}
|
|
|
|
|
|
def test_score_reader_refuses_stripped_v2_metadata(tmp_path: Path) -> None:
|
|
path = tmp_path / "score.json"
|
|
_rewrite(path, [{"id": "one"}])
|
|
with pytest.raises(ValueError, match="metadata"):
|
|
score.load_records(path)
|
|
|
|
|
|
def _hunt_change() -> dict[str, object]:
|
|
return {
|
|
"id": "change-1",
|
|
"repo": "fixture/repo",
|
|
"base": "a" * 40,
|
|
"head": "b" * 40,
|
|
"kind": "ordinary",
|
|
"files": [{"path": "app.py", "head_lines": [[1, 1]]}],
|
|
}
|
|
|
|
|
|
def _hunt_record(change: dict[str, object], program_id: str, mask: tuple[str, ...], digest: str) -> dict[str, object]:
|
|
return {
|
|
"id": change["id"],
|
|
"done": True,
|
|
"status": "hunt_covered",
|
|
"instrument": {"head_verified": True, "base_verified": True, "changed_bytes": {"app.py": 10}},
|
|
"push": {"admission": {"defects": [], "dispositions": []}, "quick_tier": []},
|
|
"rows": 1,
|
|
"program_id": program_id,
|
|
"mask": list(mask),
|
|
"inputs_present": ["function_text"],
|
|
"seconds": 0.1,
|
|
"input_digest": digest,
|
|
"usd": 0.0,
|
|
}
|
|
|
|
|
|
def _hunt_artifact(program_id: str, mask: tuple[str, ...]) -> RunArtifact:
|
|
return RunArtifact(
|
|
program_id=program_id,
|
|
schema_version=1,
|
|
mask=mask,
|
|
data_digests={"manifest": "d" * 64},
|
|
configurations=[{"only": "all", "slice": "1", "changes": 1}],
|
|
usage={"usd": 0.0, "calls": 1.0, "replay_hits": 0.0, "wall_seconds": 0.1},
|
|
versions={},
|
|
created="2026-11-02T00:00:00+00:00",
|
|
)
|
|
|
|
|
|
def _raw_hunt_run(root: Path) -> tuple[hunt_handoff.HuntRunContext, tuple[dict[str, object], ...]]:
|
|
change = _hunt_change()
|
|
program_id = "c" * 64
|
|
mask = ("obligations",)
|
|
digest = input_digest(change, program_id, mask)
|
|
root.mkdir()
|
|
_rewrite(root / "inputs.json", [{"item_id": change["id"], "change": change, "input_digest": digest}])
|
|
_rewrite(record_path(root, str(change["id"])), _hunt_record(change, program_id, mask, digest))
|
|
artifact = _hunt_artifact(program_id, mask)
|
|
artifact.write(root, reserve_attempt=True)
|
|
context = hunt_handoff.HuntRunContext(
|
|
producer="openultrasast.hunt.runner.v1",
|
|
program_id=program_id,
|
|
mask=mask,
|
|
manifest_digest="d" * 64,
|
|
freeze_digest="e" * 64,
|
|
only="all",
|
|
slice_id="1",
|
|
artifact_ids=(artifact.artifact_id,),
|
|
versions=(),
|
|
)
|
|
return context, (change,)
|
|
|
|
|
|
def test_hunt_handoff_is_non_mutating_repeatable_and_persists_only_to_benchmark_destination(tmp_path: Path) -> None:
|
|
source = tmp_path / "producer"
|
|
destination = tmp_path / "benchmark-v2"
|
|
context, changes = _raw_hunt_run(source)
|
|
before = {path.name: path.read_bytes() for path in source.iterdir()}
|
|
|
|
first = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
|
second = hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
|
assert first == second and first[0]["id"] == "change-1"
|
|
assert {path.name: path.read_bytes() for path in source.iterdir()} == before
|
|
|
|
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
|
persisted = {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()}
|
|
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
|
assert {path.name: (path.read_bytes(), path.stat().st_mtime_ns) for path in destination.iterdir()} == persisted
|
|
assert score.load_records(destination) == list(first)
|
|
assert {path.name: path.read_bytes() for path in source.iterdir()} == before
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mutation", "message"),
|
|
(
|
|
("producer", "producer"),
|
|
("program", "program"),
|
|
("manifest", "manifest"),
|
|
("input", "input digest"),
|
|
("record", "record identity"),
|
|
("reserved", "reserved metadata"),
|
|
),
|
|
)
|
|
def test_hunt_handoff_refuses_unsupported_producer_schema_and_identity(tmp_path: Path, mutation: str, message: str) -> None:
|
|
source = tmp_path / "producer"
|
|
context, changes = _raw_hunt_run(source)
|
|
if mutation == "producer":
|
|
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "producer": "arbitrary-json"})
|
|
elif mutation == "program":
|
|
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "program_id": "f" * 64})
|
|
elif mutation == "manifest":
|
|
context = hunt_handoff.HuntRunContext(**{**context.__dict__, "manifest_digest": "f" * 64})
|
|
elif mutation == "input":
|
|
entries = json.loads((source / "inputs.json").read_text())
|
|
entries[0]["input_digest"] = "0" * 64
|
|
_rewrite(source / "inputs.json", entries)
|
|
else:
|
|
path = record_path(source, "change-1")
|
|
record = json.loads(path.read_text())
|
|
if mutation == "record":
|
|
record["id"] = "other"
|
|
else:
|
|
record["schema_version"] = "forged"
|
|
_rewrite(path, record)
|
|
with pytest.raises(ValueError, match=message):
|
|
hunt_handoff.read_hunt_runner_records(source, context=context, changes=changes)
|
|
|
|
|
|
def test_hunt_handoff_refuses_conflicting_benchmark_destination(tmp_path: Path) -> None:
|
|
source = tmp_path / "producer"
|
|
destination = tmp_path / "benchmark-v2"
|
|
context, changes = _raw_hunt_run(source)
|
|
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
|
artifact_contract.write_benchmark_artifact(destination / "handoff.json", "hunt-runner-handoff", {"producer": "conflict"}, replace=True)
|
|
with pytest.raises(ValueError, match="conflict"):
|
|
hunt_handoff.persist_hunt_runner_records(source, destination, context=context, changes=changes)
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ("program", "manifest", "configuration", "versions"))
|
|
def test_hunt_handoff_refuses_mixed_attempt_invariants(tmp_path: Path, mutation: str) -> None:
|
|
source = tmp_path / "producer"
|
|
manifest = tmp_path / "pool-fixture.toml"
|
|
manifest.write_text("status = 'fixture'\n")
|
|
_context, _changes = _raw_hunt_run(source)
|
|
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
|
original = RunArtifact.read(path)
|
|
values = {
|
|
"program_id": original.program_id,
|
|
"schema_version": original.schema_version,
|
|
"mask": tuple(original.mask),
|
|
"data_digests": dict(original.data_digests),
|
|
"configurations": list(original.configurations),
|
|
"usage": {**original.usage, "calls": 2.0},
|
|
"versions": dict(original.versions),
|
|
"created": "2026-11-02T00:00:01+00:00",
|
|
}
|
|
if mutation == "program":
|
|
values["program_id"] = "f" * 64
|
|
elif mutation == "manifest":
|
|
values["data_digests"] = {"manifest": "f" * 64}
|
|
elif mutation == "configuration":
|
|
values["configurations"] = [{"only": "ordinary", "slice": "1", "changes": 1}]
|
|
else:
|
|
values["versions"] = {"dspy": "different"}
|
|
RunArtifact(**values).write(source, reserve_attempt=True)
|
|
|
|
with pytest.raises(ValueError, match="invariant"):
|
|
hunt_handoff.discover_hunt_runner_context(
|
|
source,
|
|
manifest,
|
|
producer=hunt_handoff.HUNT_RUNNER_PRODUCER,
|
|
freeze_digest="e" * 64,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("attempt_ordinal", (-1, 0, 1, True, 1.5))
|
|
def test_hunt_handoff_refuses_nonpositive_or_duplicate_attempt_ordinals(tmp_path: Path, attempt_ordinal: int) -> None:
|
|
source = tmp_path / "producer"
|
|
context, changes = _raw_hunt_run(source)
|
|
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
|
original = RunArtifact.read(path)
|
|
duplicate = RunArtifact(
|
|
program_id=original.program_id,
|
|
schema_version=original.schema_version,
|
|
mask=tuple(original.mask),
|
|
data_digests=dict(original.data_digests),
|
|
configurations=list(original.configurations),
|
|
usage={**original.usage, "calls": 2.0},
|
|
versions=dict(original.versions),
|
|
created="2026-11-02T00:00:01+00:00",
|
|
attempt_ordinal=attempt_ordinal,
|
|
)
|
|
duplicate.write(source)
|
|
artifact_ids = tuple(sorted((original.artifact_id, duplicate.artifact_id)))
|
|
|
|
with pytest.raises(ValueError, match="attempt ordinals"):
|
|
hunt_handoff.read_hunt_runner_records(source, context=replace(context, artifact_ids=artifact_ids), changes=changes)
|
|
|
|
|
|
@pytest.mark.parametrize(("attempt_ordinal", "accepted"), ((MAX_ATTEMPT_ORDINAL, True), (MAX_ATTEMPT_ORDINAL + 1, False)))
|
|
def test_hunt_handoff_enforces_maximum_attempt_ordinal(tmp_path: Path, attempt_ordinal: int, accepted: bool) -> None:
|
|
source = tmp_path / "producer"
|
|
context, changes = _raw_hunt_run(source)
|
|
path = next(candidate for candidate in source.glob("*.json") if "artifact_id" in candidate.read_text())
|
|
original = RunArtifact.read(path)
|
|
additional = RunArtifact(
|
|
program_id=original.program_id,
|
|
schema_version=original.schema_version,
|
|
mask=tuple(original.mask),
|
|
data_digests=dict(original.data_digests),
|
|
configurations=list(original.configurations),
|
|
usage={**original.usage, "calls": 2.0},
|
|
versions=dict(original.versions),
|
|
created="2026-11-02T00:00:01+00:00",
|
|
attempt_ordinal=attempt_ordinal,
|
|
)
|
|
additional.write(source)
|
|
artifact_ids = tuple(sorted((original.artifact_id, additional.artifact_id)))
|
|
bound_context = replace(context, artifact_ids=artifact_ids)
|
|
|
|
if accepted:
|
|
assert hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes)
|
|
else:
|
|
with pytest.raises(ValueError, match="attempt ordinals"):
|
|
hunt_handoff.read_hunt_runner_records(source, context=bound_context, changes=changes)
|
|
|
|
|
|
def test_score_reader_has_no_external_adoption_escape_hatch(tmp_path: Path) -> None:
|
|
path = tmp_path / "raw.json"
|
|
_rewrite(path, [{"id": "one"}])
|
|
with pytest.raises(TypeError, match="adopt_external"):
|
|
score.load_records(path, adopt_external=True)
|