unseen-eval group 1: used set (1,029 repositories, every source read), extended guard, usage ledger; record 01

Implemented with Codex; task 1.5 run by the coordinator against the real
sources (S3 memory read-only).

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/test_independent_population.py

@@ -8,6 +8,8 @@ anywhere else, which is how an evaluation set quietly becomes a development one.
 
 from __future__ import annotations
 
+import hashlib
+import json
 import re
 import tomllib
 from pathlib import Path
@@ -64,17 +66,131 @@ def test_a_multi_language_repository_is_reserved(path: Path) -> None:
     assert members and all(by_id[m]["language"] == "php" and "javascript" in by_id[m].get("also", []) for m in members)
 
 
-def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None:
+def _pool_manifests() -> list[Path]:
+    directory = ROOT / "benchmarks/unseen"
+    return sorted([*directory.glob("*.toml"), *directory.glob("private/*.toml")])
+
+
+def _pool_names() -> set[str]:
+    names = set()
+    for path in _pool_manifests():
+        text = path.read_text().lower()
+        names.update(m.rstrip("/").removesuffix(".git") for m in re.findall(r"github\.com/([\w.-]+/[\w.-]+)", text))
+    return names
+
+
+def _population_names() -> set[str]:
     reserved = {case["repo"].rstrip("/").lower() for path in POPULATIONS for case in _population(path)["case"]}
-    names = {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved}
-    offenders = []
-    for top in ("benchmarks", "src", "tests"):
+    return {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved}
+
+
+def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None:
+    reservations = [(ROOT / "benchmarks/independent", _population_names()), (ROOT / "benchmarks/unseen", _pool_names())]
+    offenders = 0
+    counts = {}
+    for top in ("benchmarks", "src", "tests", "plane"):
         for path in (ROOT / top).rglob("*"):
-            if not path.is_file() or POPULATION.parent in path.parents or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}:
+            if not path.is_file() or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}:
+                continue
+            # Preserve the population guard's existing scope; plane is newly reserved for pools.
+            active = [
+                (directory, names)
+                for directory, names in reservations
+                if directory not in path.parents and names and not (top == "plane" and directory == ROOT / "benchmarks/independent")
+            ]
+            if not active:
                 continue
             try:
                 text = path.read_text(errors="ignore").lower()
             except OSError:
                 continue
-            offenders += [f"{path.relative_to(ROOT)}: {name}" for name in names if f"github.com/{name}" in text]
-    assert not offenders, "a reserved repository is referenced outside the reservation: " + "; ".join(offenders[:10])
+            count = sum(1 for _, names in active for name in names if f"github.com/{name}" in text)
+            offenders += count
+            counts[top] = counts.get(top, 0) + count
+    assert not offenders, f"reserved repository referenced outside its reservation: count={offenders}, sources={counts}"
+
+
+def test_pool_and_population_names_are_disjoint() -> None:
+    count = len(_population_names() & _pool_names())
+    assert not count, f"overlapping reservations: count={count}"
+
+
+def manifest_digest(path: Path) -> str:
+    """Canonical parsed TOML, UTF-8 JSON with sorted keys and compact separators.
+
+    Private entries are covered by their opaque pointers' digests in the public manifest.
+    The freeze writer must use the same encoding (task 3.2).
+    """
+    raw = json.dumps(tomllib.loads(path.read_text()), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
+    return hashlib.sha256(raw).hexdigest()
+
+
+def check_pool_digest(path: Path) -> None:
+    record = path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json")
+    expected = json.loads(record.read_text())["freeze_digest"]
+    assert manifest_digest(path) == expected, "frozen pool digest mismatch"
+
+
+def test_frozen_pool_manifest_digest() -> None:
+    pools = sorted((ROOT / "benchmarks/unseen").glob("pool-p*.toml"))
+    if not pools:
+        pytest.skip("no pool frozen")
+    for path in pools:
+        check_pool_digest(path)
+
+
+def test_pool_guard_controls(tmp_path, monkeypatch) -> None:
+    """A synthetic reservation proves green, then red outside its own directory."""
+    population = tmp_path / "benchmarks/independent/population-v1.toml"
+    pool = tmp_path / "benchmarks/unseen/pool-p1.toml"
+    population.parent.mkdir(parents=True)
+    pool.parent.mkdir(parents=True)
+    prefix = "https://" + "github.com/"
+    population.write_text('[[case]]\nrepo = "' + prefix + 'synthetic/population"\n')
+    pool.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/pool"\n')
+    monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path)
+    monkeypatch.setattr(__import__(__name__), "POPULATIONS", [population])
+    test_no_reserved_repository_is_referenced_by_the_development_tree()
+    leak = tmp_path / "benchmarks/measurements/fixture.txt"
+    leak.parent.mkdir(parents=True)
+    leak.write_text(prefix + "synthetic/pool")
+    with pytest.raises(AssertionError, match="count=1"):
+        test_no_reserved_repository_is_referenced_by_the_development_tree()
+    leak.unlink()
+    plane = tmp_path / "plane/fixture.txt"
+    plane.parent.mkdir()
+    plane.write_text(prefix + "synthetic/pool")
+    with pytest.raises(AssertionError, match="count=1"):
+        test_no_reserved_repository_is_referenced_by_the_development_tree()
+    plane.write_text(prefix + "synthetic/population")
+    test_no_reserved_repository_is_referenced_by_the_development_tree()
+    plane.unlink()
+    private = pool.parent / "private/draft-p1.toml"
+    private.parent.mkdir()
+    private.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/population"\n')
+    with pytest.raises(AssertionError, match="count=2"):
+        test_no_reserved_repository_is_referenced_by_the_development_tree()
+
+
+def test_pool_digest_control(tmp_path) -> None:
+    pool = tmp_path / "pool-p1.toml"
+    pool.write_text('status = "frozen"\n[[repository]]\nid = "opaque"\n')
+    record = tmp_path / "freeze-p1.json"
+    record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)}))
+    check_pool_digest(pool)
+    pool.write_text(pool.read_text() + "slice = 2\n")
+    with pytest.raises(AssertionError, match="digest"):
+        check_pool_digest(pool)
+
+
+def test_frozen_pool_discovery_cannot_be_disabled(tmp_path, monkeypatch) -> None:
+    pool = tmp_path / "benchmarks/unseen/pool-p1.toml"
+    pool.parent.mkdir(parents=True)
+    pool.write_text('status = "frozen"\n')
+    record = pool.with_name("freeze-p1.json")
+    record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)}))
+    monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path)
+    test_frozen_pool_manifest_digest()
+    pool.write_text('status = "draft"\n')
+    with pytest.raises(AssertionError, match="digest"):
+        test_frozen_pool_manifest_digest()

tests/test_unseen_eligibility.py

@@ -0,0 +1,293 @@
+"""Offline eligibility fixtures; no real repository identities are needed."""
+
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.plane.memory import FileStore
+
+with pytest.MonkeyPatch.context() as patch:
+    patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen"))
+    import eligibility as e
+
+
+def plant(root, path, text):
+    target = root / path
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(text)
+    return target
+
+
+def test_sources_and_reserved_input_never_opened(tmp_path, monkeypatch, capsys):
+    sources = {
+        "benchmarks/pairs/a/catalog.toml": ("pairs", "pair"),
+        "benchmarks/pairs/a/recipes.toml": ("pairs", "recipe"),
+        "benchmarks/pairs/a/dataset.json": ("pairs", "dataset"),
+        "benchmarks/pairs/advisory-fixes-b/catalog.toml": ("harvest", "harvest"),
+        "benchmarks/independent/population-v1.toml": ("populations", "first"),
+        "benchmarks/independent/population-v2.toml": ("populations", "second"),
+        "plane/experiments/a.units.jsonl": ("units", "unit"),
+        "benchmarks/measurements/a/record.json": ("measurements", "measurement"),
+        "benchmarks/experiments/a.json": ("measurements", "experiment"),
+        "src/example.txt": ("sweep", "sweep"),
+    }
+    for path, (_, name) in sources.items():
+        text = f'repo = "https://github.com/fixture/{name}"' if path.endswith(".toml") else json.dumps({"repo": f"fixture/{name}"})
+        if name == "sweep":
+            text = "https://github.com/fixture/sweep"
+        plant(tmp_path, path, text)
+    pointer = plant(tmp_path, "local/pointers.toml", '[[pair]]\nrepo = "fixture/pointer"\nfix_repo = "fixture/fix"\nvendored = false')
+    plant(tmp_path, "benchmarks/independent/population-v3.toml", "do not read")
+    plant(tmp_path, "benchmarks/independent/results-v3.json", "do not read")
+    original = io.open
+
+    def guarded(file, *args, **kwargs):
+        if "v3" in str(file):
+            pytest.fail("reserved input opened")
+        return original(file, *args, **kwargs)
+
+    monkeypatch.setattr(io, "open", guarded)
+    store = FileStore(tmp_path / "memory")
+    plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"id": "1", "kind": "example", "repo": "fixture/memory"}) + "\n")
+    used = e.build_used(tmp_path, stores={"local": store}, pointer_manifests=[pointer])
+    for label, name in sources.values():
+        assert f"fixture/{name}" in used.sources[label]
+    assert {"fixture/pointer", "fixture/fix"} <= used.sources["pointers"]
+    assert "fixture/memory" in used.sources["memory_local"]
+    report = used.report()
+    assert report["sources"]["memory_local"]["bytes"] > 0
+    assert all(item["files"] > 0 and item["bytes"] > 0 for item in report["sources"].values())
+    assert "fixture/" not in json.dumps(report)
+    assert capsys.readouterr().out == ""
+    assert e.normalize("HTTPS://GitHub.com/Fixture/Pair.GIT/") == "fixture/pair"
+    assert len({e.normalize(s) for s in ["fixture/pair", "Fixture/Pair.git/", "github.com/fixture/pair"]}) == 1
+
+
+def test_empty_sources_fail_unless_declared(tmp_path):
+    used = e.build_used(tmp_path)
+    with pytest.raises(ValueError, match="empty_source"):
+        used.report()
+    assert used.report(known_empty=set(used.sources))["count"] == 0
+
+
+class FakeAPI:
+    def __init__(self, records):
+        self.records = records
+
+    def get_repo(self, name):
+        return self.records[name]
+
+
+def test_resolution_rejects_used_renames_and_forks():
+    used = e.UsedSet()
+    used.add("pairs", {"fixture/used"}, 10)
+    api = FakeAPI(
+        {
+            "fixture/used": {"full_name": "fixture/used"},
+            "fixture/old": {"full_name": "fixture/new", "former_names": ["fixture/used"]},
+            "fixture/fork": {"full_name": "fixture/fork", "parent": {"full_name": "fixture/used"}, "source": {"full_name": "fixture/used"}},
+            "fixture/a": {"full_name": "fixture/a", "source": {"full_name": "fixture/root"}},
+            "fixture/b": {"full_name": "fixture/b", "source": {"full_name": "fixture/root"}},
+        }
+    )
+    selected = set()
+    for name in ("used", "old", "fork"):
+        resolved = e.resolve("fixture/" + name, api)
+        assert e.rejection(resolved, used, selected) == "pairs"
+    assert e.rejection(e.resolve("fixture/a", api), used, selected) is None
+    assert e.rejection(e.resolve("fixture/b", api), used, selected) == "fork_network"
+
+
+def test_registered_units_and_read_only_store(tmp_path):
+    units = plant(tmp_path, "local/units.jsonl", json.dumps({"repo": "fixture/registered"}))
+    store = FileStore(tmp_path / "memory")
+    plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"kind": "experiment", "units_file": str(units)}))
+    used = e.build_used(tmp_path, stores={"s3": store})
+    assert "fixture/registered" in used.sources["units"]
+
+
+def test_cli_injected_store_and_credentials(tmp_path, monkeypatch, capsys):
+    from openultrasast import config
+    from openultrasast.plane import memory
+
+    calls = []
+    monkeypatch.setattr(config, "load_dotenv", lambda path: calls.append("dotenv"))
+    monkeypatch.setenv("GH_TOKEN", "test-token")
+    store = FileStore(tmp_path / "store")
+    monkeypatch.setattr(memory, "open_store", lambda spec, **kwargs: calls.append(spec) or store)
+    monkeypatch.setattr(e, "GitHub", lambda token: calls.append(token) or FakeAPI({}))
+    args = ["--root", str(tmp_path), "--s3-store", "s3://", "--local-store", "file:///unused"]
+    for label in (*e.SOURCES, "memory_local", "memory_s3"):
+        args += ["--known-empty", label]
+    assert e.main(args) == 0
+    assert calls == ["dotenv", "file:///unused", "s3://", "test-token"]
+    output = capsys.readouterr().out
+    assert json.loads(output)["count"] == 0
+    assert "test-token" not in output
+
+
+def test_s3_open_is_read_only(tmp_path, monkeypatch):
+    from openultrasast.plane import memory
+
+    class Objects:
+        def keys(self, prefix):
+            return ["repos/opaque/pin.jsonl"]
+
+        def get(self, key):
+            return json.dumps({"kind": "example", "repo": "fixture/stored"}).encode(), "1"
+
+        def __getattr__(self, name):
+            raise AssertionError("unexpected_store_operation")
+
+    monkeypatch.setattr(memory, "S3Client", lambda **kwargs: Objects())
+    monkeypatch.setattr(memory, "s3_settings", lambda env: {})
+    store = memory.open_store("s3://fixture-bucket", read_only=True)
+    used = e.build_used(tmp_path, stores={"s3": store})
+    assert used.sources["memory_s3"] == {"fixture/stored"}
+    with pytest.raises(memory.MemoryStoreError, match="read.only"):
+        store._put("key", b"value")
+    with pytest.raises(memory.MemoryStoreError, match="read.only"):
+        store._delete("key")
+
+
+def test_non_repository_provenance_and_missing_used_identity(tmp_path):
+    plant(tmp_path, "benchmarks/pairs/recipes.toml", 'repo = "provenance-label"')
+    assert not e.build_used(tmp_path).sources["pairs"]
+    used = e.UsedSet()
+    used.add("pairs", {"fixture/old", "fixture/nonexistent"}, 1)
+
+    class API:
+        def get_repo(self, name):
+            if name.endswith("nonexistent"):
+                raise e.RepositoryNotFound
+            return {"full_name": "fixture/new"}
+
+    e.canonicalize_used(used, API())
+    assert "fixture/nonexistent" in used.sources["pairs"]
+    assert e.rejection(e.resolve("fixture/new", API()), used, set()) == "pairs"
+
+
+def test_binary_measurement_and_local_pointer_discovery(tmp_path):
+    target = tmp_path / "benchmarks/measurements/a/image.bin"
+    target.parent.mkdir(parents=True)
+    target.write_bytes(b"\xff\x00")
+    plant(tmp_path, "benchmarks/pairs/a/recipes.toml", '[[pair]]\nrepo = "fixture/pointer"\nvendored = false')
+    used = e.build_used(tmp_path)
+    assert used.instrument["measurements"]["bytes"] == 2
+    assert used.sources["pointers"] == {"fixture/pointer"}
+
+
+def test_coordinator_verification_is_black_box(tmp_path, monkeypatch):
+    import subprocess
+    from types import SimpleNamespace
+
+    def run(command, **kwargs):
+        assert kwargs["stdout"] == kwargs["stderr"] == kwargs["stdin"] == subprocess.DEVNULL
+        path = Path(next(part.split("=", 1)[1] for part in command if part.startswith("--junitxml=")))
+        path.write_text('<testsuites><testsuite tests="25" failures="0" errors="0" skipped="1"/></testsuites>')
+        return SimpleNamespace(returncode=0)
+
+    monkeypatch.setattr(subprocess, "run", run)
+    assert e.verify_tests(tmp_path)["tests"]["tests"] == 25
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: SimpleNamespace(returncode=2))
+    with pytest.raises(ValueError, match="guard_or_tests_failed"):
+        e.verify_tests(tmp_path)
+
+
+def test_registered_unit_group_is_repository(tmp_path):
+    plant(tmp_path, "plane/experiments/a.units.jsonl", json.dumps({"unit": "opaque", "group": "fixture/unit", "label": 1}))
+    assert "fixture/unit" in e.build_used(tmp_path).sources["units"]
+
+
+def test_multidocument_measurement_and_missing_registered_units(tmp_path):
+    plant(tmp_path, "benchmarks/measurements/a/run.yaml", "repo: fixture/first\n---\nrepo: fixture/second\n")
+    assert e.build_used(tmp_path).sources["measurements"] == {"fixture/first", "fixture/second"}
+    store = FileStore(tmp_path / "memory")
+    plant(
+        tmp_path,
+        "memory/repos/opaque/pin.jsonl",
+        json.dumps(
+            {
+                "kind": "experiment",
+                "units_file": "absent.units.jsonl",
+                "units_digest": "a" * 64,
+            }
+        ),
+    )
+    with pytest.raises(FileNotFoundError):
+        e.build_used(tmp_path, stores={"local": store})
+
+
+def test_catalog_skips_fixture_and_preserves_host_identity(tmp_path, capsys):
+    plant(
+        tmp_path,
+        "benchmarks/pairs/catalog.toml",
+        '[[pair]]\norigin = "in-tree-fixture"\nvendored = false\n[[pair]]\nrepo = "https://GitLab.com/Fixture/Private.git"\n',
+    )
+    used = e.build_used(tmp_path)
+    assert used.sources["pairs"] == {"gitlab.com/fixture/private"}
+    assert used.instrument["pairs"]["skipped_values"] == 1
+    assert used.instrument["pointers"]["skipped_values"] == 1
+    assert used.sources["sweep"] == set()
+    e.canonicalize_used(used, FakeAPI({}))  # Non-GitHub identities never reach this API.
+    assert e.rejection(e.resolve("https://gitlab.com/fixture/private", FakeAPI({})), used, set()) == "pairs"
+    report = used.report(set(e.SOURCES) - {"pairs"})
+    assert report["sources"]["pairs"]["skipped_values"] == 1
+    assert "fixture" not in json.dumps(report)
+    assert capsys.readouterr() == ("", "")
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("https://bitbucket.org/Fixture/Repo.git", "bitbucket.org/fixture/repo"),
+        ("https://git.example.org/Fixture/Repo", "git.example.org/fixture/repo"),
+        ("https://github.com/Fixture/Repo/tree/main", "fixture/repo"),
+        ("Fixture/Repo.git/", "fixture/repo"),
+    ],
+)
+def test_repository_hosts(value, expected):
+    assert e.normalize(value) == expected
+    assert e.normalize(expected) == expected
+
+
+def test_memory_values_are_skipped_and_counted(tmp_path, capsys):
+    rows = [
+        {"kind": "example", "repo": value}
+        for value in ["local-slice", "free text https://github.com/fixture/prose", None, "https://bitbucket.org/fixture/repo"]
+    ]
+    plant(tmp_path, "memory/repos/opaque/pin.jsonl", "\n".join(json.dumps(row) for row in rows))
+    used = e.build_used(tmp_path, stores={"local": FileStore(tmp_path / "memory")})
+    assert used.sources["memory_local"] == {"bitbucket.org/fixture/repo"}
+    assert used.instrument["memory_local"]["skipped_values"] == 3
+    assert capsys.readouterr() == ("", "")
+
+
+@pytest.mark.parametrize("step", ["open_stores", "build_used", "report", "canonicalize", "verify_tests"])
+def test_cli_failure_reports_only_step_and_exception_type(tmp_path, monkeypatch, capsys, step):
+    from openultrasast import config
+    from openultrasast.plane import memory
+
+    def fail(*args, **kwargs):
+        raise RuntimeError("sensitive/identity")
+
+    monkeypatch.setattr(config, "load_dotenv", lambda path: None)
+    monkeypatch.setattr(memory, "open_store", lambda *args, **kwargs: object())
+    monkeypatch.setattr(e, "build_used", lambda *args, **kwargs: e.UsedSet())
+    monkeypatch.setattr(e.UsedSet, "report", lambda *args, **kwargs: {})
+    monkeypatch.setattr(e, "GitHub", lambda token: object())
+    monkeypatch.setattr(e, "canonicalize_used", lambda *args: None)
+    monkeypatch.setattr(e, "verify_tests", lambda *args: {})
+    if step == "open_stores":
+        monkeypatch.setattr(memory, "open_store", fail)
+    elif step == "report":
+        monkeypatch.setattr(e.UsedSet, "report", fail)
+    else:
+        monkeypatch.setattr(e, "canonicalize_used" if step == "canonicalize" else step, fail)
+    assert e.main(["--root", str(tmp_path), "--verify-tests"]) == 1
+    output = capsys.readouterr()
+    assert output.err == ""
+    assert json.loads(output.out) == {"status": "instrument_failure", "step": step, "type": "RuntimeError"}
+    assert "sensitive" not in output.out

tests/test_unseen_ledger.py

@@ -0,0 +1,63 @@
+"""Append-only use and tamper detection, with opaque fixture identifiers."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+with pytest.MonkeyPatch.context() as patch:
+    patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen"))
+    import ledger
+
+
+def row(**extra):
+    return {
+        "slice": 1,
+        "decision": "decision-001",
+        "purpose": "baseline",
+        "date": "2026-10-04",
+        "result_digest": "a" * 64,
+        "freeze_digest": "b" * 64,
+        **extra,
+    }
+
+
+def test_append_and_refuse_rewrite(tmp_path):
+    path = tmp_path / "usage-p1.jsonl"
+    ledger.append(path, row())
+    original = path.read_bytes()
+    ledger.append(path, row(purpose="exploratory"))
+    assert path.read_bytes().startswith(original)
+    assert len(ledger.read(path)) == 2
+    with pytest.raises(ValueError, match="duplicate"):
+        ledger.append(path, row())
+
+
+def test_informed_slice_cannot_qualify_same_decision(tmp_path):
+    path = tmp_path / "usage-p1.jsonl"
+    ledger.append(path, row(purpose="informed"))
+    before = path.read_bytes()
+    with pytest.raises(ValueError, match="informed"):
+        ledger.append(path, row(purpose="qualifies"))
+    assert path.read_bytes() == before
+    ledger.append(path, row(purpose="qualifies", slice=2))
+    ledger.append(path, row(purpose="qualifies", decision="decision-002"))
+
+
+def test_rewritten_row_breaks_digest_chain(tmp_path):
+    path = tmp_path / "usage-p1.jsonl"
+    ledger.append(path, row())
+    ledger.append(path, row(purpose="exploratory"))
+    rows = [json.loads(line) for line in path.read_text().splitlines()]
+    rows[0]["purpose"] = "informed"
+    path.write_text("".join(json.dumps(r) + "\n" for r in rows))
+    with pytest.raises(ValueError, match="chain"):
+        ledger.read(path)
+    with pytest.raises(ValueError, match="chain"):
+        ledger.append(path, row(slice=2))
+
+
+@pytest.mark.parametrize("extra", [{"decision": "owner/name"}, {"purpose": "other"}, {"freeze_digest": "bad"}, {"slice": 0}])
+def test_invalid_rows_refused(tmp_path, extra):
+    with pytest.raises(ValueError):
+        ledger.append(tmp_path / "usage-p1.jsonl", row(**extra))