learn: registered() returns the frozen registration, not the first row

A pending-then-frozen registration left two rows and rows[0] could be the stale
pending one, so fill and run refused a properly frozen experiment. Implemented
with Codex.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Marius Ciepluch
2026-10-03 12:42:39 +02:00
co-authored by Claude Opus 5.5
parent 8aeca972c0
commit 90a2642019
2 changed files with 104 additions and 3 deletions
+16 -1
View File
@@ -35,6 +35,7 @@ import math
import random import random
import subprocess import subprocess
from collections.abc import Callable, Iterable, Mapping, Sequence from collections.abc import Callable, Iterable, Mapping, Sequence
from contextlib import suppress
from dataclasses import asdict, dataclass, field, fields, replace from dataclasses import asdict, dataclass, field, fields, replace
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -340,8 +341,22 @@ def registration_id(experiment_id: str) -> str:
def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None: def registered(store: MemoryStore, experiment_id: str) -> dict[str, Any] | None:
"""Return the authoritative registration, preferring a set units digest over pending rows.
Within that group, use the greatest ``created`` value when all values are nonempty and mutually
comparable (strings compare lexically). Otherwise, or on a tie, use the lexically greatest ``pin``,
then the last row in store order (key/id order for MemoryStore, insertion order for an append-only store).
"""
rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK] rows = [r.row for r in store.rows(repo=experiment_repo(experiment_id), kind=EXPERIMENT_KIND) if r.row.get("task") == REGISTER_TASK]
return rows[0] if rows else None if not rows:
return None
rows = [r for r in rows if r.get("units_digest")] or rows
rows = sorted(rows, key=lambda r: str(r.get("pin", "")))
if all(r.get("created") for r in rows):
# Incomparable timestamps retain the deterministic pin/store ordering.
with suppress(TypeError):
rows = sorted(rows, key=lambda r: r["created"])
return rows[-1]
def register( def register(
+88 -2
View File
@@ -1,13 +1,14 @@
"""A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and """A/B experiments (learned-decision-engine design section 6, task 7): manifest digest refusal, frozen units and
pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule, pairing, the paired repository bootstrap, exact McNemar on a known table, the two-look boundary, the adoption rule,
arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores arm A replaying at zero client calls, and the retrieval-ensemble sampling variant. Every client is scripted; stores
are ``FileStore`` under ``tmp_path``.""" are in-memory fakes or ``FileStore`` under ``tmp_path``."""
from __future__ import annotations from __future__ import annotations
import json import json
import math import math
from pathlib import Path from pathlib import Path
from unittest.mock import Mock
import pytest import pytest
from learn_fixtures import ScriptedChat, corpus, x_record from learn_fixtures import ScriptedChat, corpus, x_record
@@ -18,7 +19,7 @@ from openultrasast.learn.compile import canonical, program_id
from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec from openultrasast.learn.program import TEMPERATURES, Caller, Candidate, Program, ProgramSpec
from openultrasast.learn.retrieve import Target from openultrasast.learn.retrieve import Target
from openultrasast.plane.budget import MeteredClient from openultrasast.plane.budget import MeteredClient
from openultrasast.plane.memory import FileStore from openultrasast.plane.memory import FileStore, MemoryStore, Record
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32} PRICES = {"cache_hit_per_m": 0.014, "input_per_m": 0.44, "output_per_m": 1.32}
@@ -170,6 +171,91 @@ def test_register_refuses_uncommitted_and_modified_manifests(tmp_path: Path) ->
assert len(store.rows(kind="experiment")) == 1 assert len(store.rows(kind="experiment")) == 1
@pytest.fixture
def registration_store() -> Mock:
records: list[Record] = []
store = Mock(spec=MemoryStore)
store.put_row.side_effect = lambda row: records.append(Record(dict(row), "memory", None))
store.rows.side_effect = lambda *, repo, kind: [r for r in records if r.row["repo"] == repo and r.row["kind"] == kind]
return store
def test_registered_prefers_frozen_over_pending(tmp_path: Path, registration_store: Mock) -> None:
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
pending = ex.register(registration_store, manifest, commit_of=committed, created="2026-10-03")
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
frozen = ex.register(registration_store, manifest, commit_of=lambda _: PIN, created="2026-10-02")
assert pending["units_status"] == "pending"
assert len(registration_store.rows(repo=ex.experiment_repo(manifest.id), kind=ex.EXPERIMENT_KIND)) == 2
assert ex.registered(registration_store, manifest.id) == frozen
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
def test_registered_frozen_only(tmp_path: Path, registration_store: Mock) -> None:
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
frozen = ex.register(registration_store, manifest, commit_of=committed)
assert ex.registered(registration_store, manifest.id) == frozen
assert ex.check_registered(registration_store, manifest, need_units=True) == frozen
def test_registered_pending_only_refuses_units(tmp_path: Path, registration_store: Mock) -> None:
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
pending = ex.register(registration_store, manifest, commit_of=committed)
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
assert ex.registered(registration_store, manifest.id) == pending
with pytest.raises(ex.ExperimentError, match=r"the units file was not registered \(pending\)"):
ex.check_registered(registration_store, manifest, need_units=True)
@pytest.mark.parametrize("freeze", [False, True])
def test_register_still_refuses_different_manifest_digest(tmp_path: Path, registration_store: Mock, freeze: bool) -> None:
path = write_manifest(tmp_path, "0" * 64)
manifest = ex.load_manifest(path)
ex.register(registration_store, manifest, commit_of=committed)
if freeze:
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
ex.register(registration_store, manifest, commit_of=lambda _: PIN)
path.write_text(path.read_text().replace("mde: 0.1", "mde: 0.2"))
with pytest.raises(ex.ExperimentError, match="A changed manifest is a new experiment"):
ex.register(registration_store, ex.load_manifest(path), commit_of=committed)
assert registration_store.put_row.call_count == 1 + freeze
@pytest.mark.parametrize(
("first_created", "second_created", "first_pin", "second_pin", "expected"),
[
("2026-10-03", "2026-10-02", PIN, COMMIT, 0),
("2026-10-02", "2026-10-03", COMMIT, PIN, 1),
("2026-10-02", "2026-10-02", PIN, COMMIT, 1),
("", "2026-10-02", COMMIT, PIN, 0),
(None, None, PIN, COMMIT, 1),
("2026-10-03", 123, PIN, COMMIT, 1),
("", "", PIN, PIN, 1),
],
)
def test_registered_orders_frozen_rows(
tmp_path: Path,
registration_store: Mock,
first_created: object,
second_created: object,
first_pin: str,
second_pin: str,
expected: int,
) -> None:
manifest = ex.load_manifest(write_manifest(tmp_path, "0" * 64))
ex.write_units(manifest.units_file, [unit("u", "g", 1)])
frozen = ex.register(registration_store, manifest, commit_of=committed)
rows = [dict(frozen, pin=pin, created=created) for pin, created in ((first_pin, first_created), (second_pin, second_created))]
for row in rows:
if row["created"] is None:
del row["created"]
# A run row and another experiment must not participate in registration selection.
ignored = [dict(frozen, task=ex.RUN_TASK), dict(frozen, repo="experiments/other")]
registration_store.rows.side_effect = lambda *, repo, kind: [Record(r, "memory", None) for r in rows + ignored if r["repo"] == repo]
assert ex.registered(registration_store, manifest.id) is rows[expected]
def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None: def test_frozen_units_cover_the_evaluated_groups_and_pair_the_sides(tmp_path: Path) -> None:
store = FileStore(tmp_path / "memory") store = FileStore(tmp_path / "memory")
store.put_rows(example_rows()) store.put_rows(example_rows())