mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
learn: registered() returns the frozen registration, not the first row
A pending-then-frozen registration left two rows and rows[0] could be the stale pending one, so fill and run refused a properly frozen experiment. Implemented with Codex. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8aeca972c0
commit
90a2642019
@@ -35,6 +35,7 @@ import math
|
|||||||
import random
|
import random
|
||||||
import subprocess
|
import subprocess
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||||
|
from contextlib import suppress
|
||||||
from dataclasses import asdict, dataclass, field, fields, replace
|
from dataclasses import asdict, dataclass, field, fields, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -340,8 +341,22 @@ def registration_id(experiment_id: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None:
|
def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None:
|
||||||
|
"""Return the authoritative registration, preferring a set units digest over pending rows.
|
||||||
|
|
||||||
|
Within that group, use the greatest ``created`` value when all values are nonempty and mutually
|
||||||
|
comparable (strings compare lexically). Otherwise, or on a tie, use the lexically greatest ``pin``,
|
||||||
|
then the last row in store order (key/id order for MemoryStore, insertion order for an append-only store).
|
||||||
|
"""
|
||||||
rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK]
|
rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK]
|
||||||
return rows[0] if rows else None
|
if not rows:
|
||||||
|
return None
|
||||||
|
rows = [r for r in rows if r.get("units_digest")] or rows
|
||||||
|
rows = sorted(rows, key=lambda r: str(r.get("pin", "")))
|
||||||
|
if all(r.get("created") for r in rows):
|
||||||
|
# Incomparable timestamps retain the deterministic pin/store ordering.
|
||||||
|
with suppress(TypeError):
|
||||||
|
rows = sorted(rows, key=lambda r: r["created"])
|
||||||
|
return rows[-1]
|
||||||
|
|
||||||
|
|
||||||
def register(
|
def register(
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
"""A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and
|
"""A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and
|
||||||
pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule,
|
pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule,
|
||||||
arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores
|
arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores
|
||||||
are ``FileStore`` under ``tmp_path``."""
|
are in-memory fakes or ``FileStore`` under ``tmp_path``."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from learn_fixtures import ScriptedChat, corpus, x_record
|
from learn_fixtures import ScriptedChat, corpus, x_record
|
||||||
@@ -18,7 +19,7 @@ from openultrasast.learn.compile import canonical, program_id
|
|||||||
from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec
|
from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec
|
||||||
from openultrasast.learn.retrieve import Target
|
from openultrasast.learn.retrieve import Target
|
||||||
from openultrasast.plane.budget import MeteredClient
|
from openultrasast.plane.budget import MeteredClient
|
||||||
from openultrasast.plane.memory import FileStore
|
from openultrasast.plane.memory import FileStore, MemoryStore, Record
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
|
PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
|
||||||
@@ -170,6 +171,91 @@ def test_register_refuses_uncommitted_and_modified_manifests(tmp_path: Path) ->
|
|||||||
assert len(store.rows(kind="experiment")) == 1
|
assert len(store.rows(kind="experiment")) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def registration_store() -> Mock:
|
||||||
|
records: list[Record] = []
|
||||||
|
store = Mock(spec=MemoryStore)
|
||||||
|
store.put_row.side_effect = lambda row: records.append(Record(dict(row), "memory", None))
|
||||||
|
store.rows.side_effect = lambda *, repo, kind: [r for r in records if r.row["repo"] == repo and r.row["kind"] == kind]
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_prefers_frozen_over_pending(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
pending = ex.register(registration_store, manifest, commit_of=committed, created="2026-10-03")
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=lambda _: PIN, created="2026-10-02")
|
||||||
|
assert pending["units_status"] == "pending"
|
||||||
|
assert len(registration_store.rows(repo=ex.experiment_repo(manifest.id), kind=ex.EXPERIMENT_KIND)) == 2
|
||||||
|
assert ex.registered(registration_store, manifest.id) == frozen
|
||||||
|
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_frozen_only(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
assert ex.registered(registration_store, manifest.id) == frozen
|
||||||
|
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_pending_only_refuses_units(tmp_path: Path, registration_store: Mock) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
pending = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
assert ex.registered(registration_store, manifest.id) == pending
|
||||||
|
with pytest.raises(ex.ExperimentError, match=r"the units file was not registered \(pending\)"):
|
||||||
|
ex.check_registered(registration_store, manifest, need_units=True)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("freeze", [False, True])
|
||||||
|
def test_register_still_refuses_different_manifest_digest(tmp_path: Path, registration_store: Mock, freeze: bool) -> None:
|
||||||
|
path = write_manifest(tmp_path, "0" * 64)
|
||||||
|
manifest = ex.load_manifest(path)
|
||||||
|
ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
if freeze:
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
ex.register(registration_store, manifest, commit_of=lambda _: PIN)
|
||||||
|
path.write_text(path.read_text().replace("mde: 0.1", "mde: 0.2"))
|
||||||
|
with pytest.raises(ex.ExperimentError, match="A changed manifest is a new experiment"):
|
||||||
|
ex.register(registration_store, ex.load_manifest(path), commit_of=committed)
|
||||||
|
assert registration_store.put_row.call_count == 1 + freeze
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("first_created", "second_created", "first_pin", "second_pin", "expected"),
|
||||||
|
[
|
||||||
|
("2026-10-03", "2026-10-02", PIN, COMMIT, 0),
|
||||||
|
("2026-10-02", "2026-10-03", COMMIT, PIN, 1),
|
||||||
|
("2026-10-02", "2026-10-02", PIN, COMMIT, 1),
|
||||||
|
("", "2026-10-02", COMMIT, PIN, 0),
|
||||||
|
(None, None, PIN, COMMIT, 1),
|
||||||
|
("2026-10-03", 123, PIN, COMMIT, 1),
|
||||||
|
("", "", PIN, PIN, 1),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_registered_orders_frozen_rows(
|
||||||
|
tmp_path: Path,
|
||||||
|
registration_store: Mock,
|
||||||
|
first_created: object,
|
||||||
|
second_created: object,
|
||||||
|
first_pin: str,
|
||||||
|
second_pin: str,
|
||||||
|
expected: int,
|
||||||
|
) -> None:
|
||||||
|
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
|
||||||
|
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
|
||||||
|
frozen = ex.register(registration_store, manifest, commit_of=committed)
|
||||||
|
rows = [dict(frozen, pin=pin, created=created) for pin, created in ((first_pin, first_created), (second_pin, second_created))]
|
||||||
|
for row in rows:
|
||||||
|
if row["created"] is None:
|
||||||
|
del row["created"]
|
||||||
|
# A run row and another experiment must not participate in registration selection.
|
||||||
|
ignored = [dict(frozen, task=ex.RUN_TASK), dict(frozen, repo="experiments/other")]
|
||||||
|
registration_store.rows.side_effect = lambda *, repo, kind: [Record(r, "memory", None) for r in rows + ignored if r["repo"] == repo]
|
||||||
|
assert ex.registered(registration_store, manifest.id) is rows[expected]
|
||||||
|
|
||||||
|
|
||||||
def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None:
|
def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None:
|
||||||
store = FileStore(tmp_path / "memory")
|
store = FileStore(tmp_path / "memory")
|
||||||
store.put_rows(example_rows())
|
store.put_rows(example_rows())
|
||||||
|
|||||||
Reference in New Issue
Block a user