From 8213d3baad7b885db1fa7120b759ac927488809f Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:17:35 +0200 Subject: [PATCH] unseen-eval group 1: used set (1,029 repositories, every source read), extended guard, usage ledger; record 01 Implemented with Codex; task 1.5 run by the coordinator against the real sources (S3 memory read-only). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + .kiro/specs/unseen-repo-evaluation/tasks.md | 27 +- .../2026-10-04-unseen-01-used-set/record.json | 86 ++++ benchmarks/unseen/README.md | 62 +++ benchmarks/unseen/eligibility.py | 379 ++++++++++++++++++ benchmarks/unseen/ledger.py | 88 ++++ src/openultrasast/plane/memory.py | 22 +- tests/test_independent_population.py | 130 +++++- tests/test_unseen_eligibility.py | 293 ++++++++++++++ tests/test_unseen_ledger.py | 63 +++ 10 files changed, 1134 insertions(+), 19 deletions(-) create mode 100644 benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json create mode 100644 benchmarks/unseen/README.md create mode 100644 benchmarks/unseen/eligibility.py create mode 100644 benchmarks/unseen/ledger.py create mode 100644 tests/test_unseen_eligibility.py create mode 100644 tests/test_unseen_ledger.py diff --git a/.gitignore b/.gitignore index 9dd8031..82df916 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ workspace/ # MkDocs build output (mkdocs build) site/ + +# Unseen evaluation pointers for non-permissive repositories. +benchmarks/unseen/private/ diff --git a/.kiro/specs/unseen-repo-evaluation/tasks.md b/.kiro/specs/unseen-repo-evaluation/tasks.md index 124060e..0ad2cb4 100644 --- a/.kiro/specs/unseen-repo-evaluation/tasks.md +++ b/.kiro/specs/unseen-repo-evaluation/tasks.md @@ -29,8 +29,8 @@ Execution rules: - **Instrument first** (AGENTS.md). A zero, an empty draft or a fast run is treated as an instrument failure until bytes read and wall time prove otherwise. -- [ ] 1. Used set and guard (design 1.2, 6; increment 1) -- [ ] 1.1 Used-set builder +- [x] 1. Used set and guard (design 1.2, 6; increment 1) +- [x] 1.1 Used-set builder - `benchmarks/unseen/eligibility.py`: normalises names (lower case, no `.git`, no trailing slash) and builds the used set from every source of design table 1.2: pair catalogs, recipes and datasets under `benchmarks/pairs/` plus the local pointer-pair manifests as `benchmarks/pairs/catalog_gen.py` reads them; `population-v1.toml` @@ -43,14 +43,14 @@ Execution rules: and trailing-slash variants collapse; stdout and the output JSON contain no name; a v3-shaped file planted in the synthetic tree is not opened (open is patched to fail on it). - _Requirements: 1.1_ -- [ ] 1.2 Canonical resolution, renames and forks +- [x] 1.2 Canonical resolution, renames and forks - `eligibility.py`: `resolve(candidate, api)` with an injected GitHub client returns the canonical `full_name`, former names (redirects) and the fork network (`parent`, `source`). A candidate is ineligible when any of them is in the used set; one candidate per fork network is kept. - Tests with a fake client: a planted used repository, a renamed one and a fork of a used one are each rejected; two forks of one network keep one; the rejection reason is a source label, not a name. - _Requirements: 1.1_ -- [ ] 1.3 Guard extended to pools +- [x] 1.3 Guard extended to pools - `tests/test_independent_population.py`: the sweep becomes a list of (reservation directory, names): `benchmarks/independent/` for populations, `benchmarks/unseen/` for pools (public manifests, plus `benchmarks/unseen/private/*.toml` when present). The sweep also covers `plane/`. New tests: pool and @@ -62,13 +62,13 @@ Execution rules: - Evidence: the guard is green with an empty `benchmarks/unseen/`, and red when a test fixture names a pool repository in `benchmarks/measurements/` (shown once in the record). - _Requirements: 1.2, 1.3, 1.4_ -- [ ] 1.4 Usage ledger +- [x] 1.4 Usage ledger - `benchmarks/unseen/ledger.py`: append-only `usage-pN.jsonl`, one row per use (slice, decision or experiment id, purpose `baseline | exploratory | informed | qualifies`, date, result digest, freeze digest). It refuses a `qualifies` row for a change whose slice already has an `informed` row for it, and refuses to rewrite rows. - `tests/test_unseen_ledger.py`: append, the refusal, a rewritten row detected by a digest chain. - _Requirements: 5.2_ -- [ ] 1.5 Used-set measurement and record 01 **(coordinator, network)** +- [x] 1.5 Used-set measurement and record 01 **(coordinator, network)** - Run `eligibility.py` against the real sources, the S3 memory store included (read only). - Instrument: each source reports files read and bytes; a source with zero rows fails the run unless it is known empty. @@ -319,3 +319,18 @@ Execution rules: false-alarm rates, coverage per state, spend; `exploratory` rows in `usage-p1.jsonl`. - Observable: one committed table of arms on slice 1; G1's table exists. - _Requirements: 3.4, 5.2, 5.3_ + +## Implementation Notes + +- 2026-10-04: Tasks 1.1–1.4 implemented offline and independently reviewed. Task 1.5 remains for the + coordinator; no networked measurement or commit was performed. Command and API contracts: + `benchmarks/unseen/README.md`. Offline evidence: + `benchmarks/measurements/2026-10-04-unseen-01-offline/record.json`. +- Population guard scope is preserved under the maintainer's hard rule: benchmarks/src/tests for + populations, plus plane for pools. Each directory exemption is reservation-specific, including private + pool manifests. Extending population scope to plane exposed 30 existing references; no historical + records were changed or exempted inside the existing population scope. +- S3 inventory opens through `open_store(read_only=True)` because ordinary store construction writes a + Select capability probe. Default store behavior is unchanged; inventory uses LIST/GET only. +- The freeze writer must match the canonical TOML digest encoding documented in the pool README and + exercised by `test_independent_population.py`; final pool discovery does not trust mutable status. diff --git a/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json b/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json new file mode 100644 index 0000000..2554207 --- /dev/null +++ b/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json @@ -0,0 +1,86 @@ +{ + "count": 1029, + "guard": "green", + "sha256": "5a644a1e93c68a0a35c794e653286d0532b7eb8d757123ccce5f08b11bf9a4a2", + "sources": { + "harvest": { + "bytes": 501031, + "count": 467, + "files": 3, + "known_empty": false, + "rows": 446, + "skipped_values": 446 + }, + "measurements": { + "bytes": 21169903, + "count": 278, + "files": 221, + "known_empty": false, + "rows": 376, + "skipped_values": 681 + }, + "memory_local": { + "bytes": 8084523, + "count": 192, + "files": 776, + "known_empty": false, + "rows": 759, + "skipped_values": 2 + }, + "memory_s3": { + "bytes": 15928676, + "count": 572, + "files": 1499, + "known_empty": false, + "rows": 1490, + "skipped_values": 2 + }, + "pairs": { + "bytes": 1680480, + "count": 728, + "files": 28, + "known_empty": false, + "rows": 900, + "skipped_values": 1260 + }, + "pointers": { + "bytes": 892448, + "count": 537, + "files": 7, + "known_empty": false, + "rows": 586, + "skipped_values": 666 + }, + "populations": { + "bytes": 32431, + "count": 28, + "files": 2, + "known_empty": false, + "rows": 28, + "skipped_values": 0 + }, + "sweep": { + "bytes": 34006439, + "count": 1029, + "files": 1619, + "known_empty": false, + "rows": 2155, + "skipped_values": 0 + }, + "units": { + "bytes": 319641, + "count": 229, + "files": 9, + "known_empty": false, + "rows": 642, + "skipped_values": 0 + } + }, + "test_exit_code": 0, + "tests": { + "errors": 0, + "failures": 0, + "skipped": 1, + "tests": 41 + } +} diff --git a/benchmarks/unseen/README.md b/benchmarks/unseen/README.md new file mode 100644 index 0000000..fdc1ff8 --- /dev/null +++ b/benchmarks/unseen/README.md @@ -0,0 +1,62 @@ +# Unseen evaluation, group 1 + +The coordinator runs task 1.5 from the repository root: + +```sh +export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH +export PYTHONPATH=$PWD/src +python -m benchmarks.unseen.eligibility --s3-store s3:// --verify-tests --output benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json +``` + +This is the networked command; implementation tests use fakes. It loads `.env` with +`openultrasast.config.load_dotenv` (exported values win), opens the local FileStore and +S3 MemoryStore through `openultrasast.plane.memory.open_store(..., read_only=True)`, +and uses `GH_TOKEN` for GitHub GET resolution. `s3://` selects `S3_BUCKET`; pass +`--s3-store s3://BUCKET/PREFIX` for a prefixed store and `--local-store file:///PATH` +for a different local store. Read-only S3 opening skips the normal write/Select probe; +collection uses LIST/GET only. Existing normal store opening is unchanged. + +Pair catalogs/recipes include local pointer rows (`vendored = false`, including +`fix_repo`). Additional external manifests can be supplied with repeatable +`--pointer-manifest PATH`. Registered units in memory and experiment YAML are followed; +missing units with a registered digest fail. Plans without frozen units are not used data. +Experiment units identify repositories in `group`. Binary measurement artifacts are +read and counted; structured JSON, TOML and multi-document YAML are parsed strictly. + +Each source records files/objects read, bytes read, repository-bearing rows (distinct +names per input), and distinct repository counts. The digest is SHA-256 of sorted, +normalized names joined with newlines. Source counts overlap. An empty source fails; +only after establishing it is empty may the coordinator pass `--known-empty LABEL`. +The output records that declaration. Missing/unreadable inputs still fail. + +GitHub redirects add the requested and canonical names, plus parent/source fork +identities. Used names are also resolved so an old used name excludes a renamed +candidate. GitHub cannot enumerate all historical names: injected clients may supply +`former_names`; actual historical aliases come from used references and redirects. +A 404 for a historical/synthetic used identity retains its literal identity; other +API failures stop collection. Candidate resolution always fails closed on a 404. +`resolve(candidate, api)` and `rejection(resolution, used, selected_networks)` are +available to the later sourcing task. Names stay in memory; output contains only +counts, labels and SHA-256 digests. Output files are created exclusively, never overwritten. + +`--verify-tests` runs the three group-1 test modules with both output streams discarded, +including the existing population guard. Only aggregate test counts and the exit code +enter the record; a failed test prevents the record. V3 paths are excluded before +eligibility reads; only the existing guard handles v3 inside its black-box test process. + +Population reservations retain their existing sweep over `benchmarks/`, `src/`, and +`tests/`. Pool reservations also cover `plane/`. Each reservation exempts only its own +directory; population checks include all of `benchmarks/unseen/private/`. This preserves +the existing population behavior while adding pools (the hard rule for this increment). + +All `pool-pN.toml` files require a matching `freeze-pN.json`; drafts use `draft-pN.toml`. +The freeze digest is SHA-256 of parsed public TOML encoded as UTF-8 JSON with sorted +keys, compact separators and `ensure_ascii=False`. Private entries must be represented +by opaque digests in that public manifest, as task 3.2 requires. + +The ledger exposes `append(path, row)` and `read(path)`. Rows contain `slice`, `decision` +(an opaque decision or experiment label), `purpose`, ISO `date`, `result_digest`, and +`freeze_digest`. Appends lock and validate the existing chain, reject duplicates or a +changed freeze, and reject qualification of an informed slice for the same decision. +Keep the published final digest to authenticate history: the chain detects row edits, +but cannot independently prove that an entire history was not replaced or truncated. diff --git a/benchmarks/unseen/eligibility.py b/benchmarks/unseen/eligibility.py new file mode 100644 index 0000000..ec1186f --- /dev/null +++ b/benchmarks/unseen/eligibility.py @@ -0,0 +1,379 @@ +"""Read-only used-set collection. Only counts, source labels and digests leave this module's CLI. + +Run from the repository root: python -m benchmarks.unseen.eligibility --help. +Population v3 is exclusively the existing guard's responsibility. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tomllib +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import urlsplit + +import yaml + +from openultrasast.plane.memory import MemoryStore + +URL = re.compile(r"github\.com/([\w.-]+/[\w.-]+)", re.I) +NAME = re.compile(r"[\w.-]+/[\w.-]+") +SOURCES = ("pairs", "pointers", "populations", "harvest", "units", "measurements", "sweep") + + +def normalize(value: str) -> str: + text = value.strip().lower().rstrip("/") + if NAME.fullmatch(text): + name = text.removesuffix(".git") + else: + # Structured fields must be identities, not prose containing a URL. + if any(char.isspace() for char in text): + raise ValueError("invalid_repository") + url = urlsplit(text if "://" in text else "https://" + text) + if url.scheme not in {"http", "https", "ssh", "git"} or not url.hostname or "." not in url.hostname: + raise ValueError("invalid_repository") + parts = url.path.strip("/").split("/") + if len(parts) < 2: + raise ValueError("invalid_repository") + name = "/".join(parts[:2]).removesuffix(".git") + if not NAME.fullmatch(name): + raise ValueError("invalid_repository") + if url.hostname != "github.com": + return url.hostname + "/" + name + if not NAME.fullmatch(name) or any(part in {".", ".."} for part in name.split("/")): + raise ValueError("invalid_repository") + return name + + +def excluded(path: Path) -> bool: + return any("v3" in part.lower() or part in {".git", "__pycache__", ".venv"} for part in path.parts) + + +def repositories(value: object, counts: dict[str, int] | None = None) -> set[str]: + """Structured repo fields include bare owner/name pointers; URLs also occur in prose.""" + found = set() + if isinstance(value, dict): + for key, item in value.items(): + if key in {"repo", "fix_repo", "repository", "full_name", "origin"} or (key == "group" and "unit" in value): + try: + if not isinstance(item, str): + raise ValueError("invalid_repository") + found.add(normalize(item)) + except ValueError: + if counts is not None: + counts["skipped_values"] = counts.get("skipped_values", 0) + 1 + else: + found.update(repositories(item, counts)) + elif isinstance(value, list): + for item in value: + found.update(repositories(item, counts)) + elif isinstance(value, str): + found.update(normalize(m[1]) for m in URL.finditer(value)) + return found + + +def pointer_repositories(value: object, counts: dict[str, int] | None = None) -> set[str]: + if isinstance(value, dict): + if value.get("vendored") is False: + return repositories(value, counts) + return set().union(*(pointer_repositories(v, counts) for v in value.values())) + if isinstance(value, list): + return set().union(*(pointer_repositories(v, counts) for v in value)) + return set() + + +def parse(path: Path, raw: bytes) -> object: + structured = path.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"} + text = raw.decode("utf-8", errors="strict" if structured else "ignore") + if path.suffix == ".toml": + return tomllib.loads(text) + if path.suffix == ".json": + return json.loads(text) + if path.suffix == ".jsonl": + return [json.loads(line) for line in text.splitlines() if line.strip()] + if path.suffix in {".yaml", ".yml"}: + documents = list(yaml.safe_load_all(text)) + return documents[0] if len(documents) == 1 else documents + return text + + +@dataclass +class UsedSet: + sources: dict[str, set[str]] = field(default_factory=lambda: {label: set() for label in SOURCES}) + instrument: dict[str, dict[str, int]] = field(default_factory=dict) + + def add(self, label: str, names: set[str], size: int, files: int = 1, skipped_values: int = 0) -> None: + self.sources.setdefault(label, set()).update(names) + counts = self.instrument.setdefault(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0}) + counts["files"] += files + counts["bytes"] += size + counts["rows"] += len(names) + counts["skipped_values"] += skipped_values + + def report(self, known_empty: set[str] | None = None) -> dict: + known_empty = known_empty or set() + if known_empty - self.sources.keys(): + raise ValueError("unknown_source_label") + records = {} + for label, names in self.sources.items(): + counts = self.instrument.get(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0}) + if not counts["rows"] and label not in known_empty: + raise ValueError("empty_source:" + label) + records[label] = {**counts, "count": len(names), "known_empty": label in known_empty} + names = set().union(*self.sources.values()) + return {"sources": records, "count": len(names), "sha256": hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest()} + + +def build_used(root: Path, *, stores: Mapping[str, MemoryStore] | None = None, pointer_manifests: Iterable[Path] = ()) -> UsedSet: + used = UsedSet() + + def read(label: str, paths: Iterable[Path], structured: bool = True) -> list[object]: + values = [] + for path in sorted(set(paths)): + if excluded(path): + continue + raw = path.read_bytes() # failure is an instrument failure, never a quiet empty source + value = parse(path, raw) if structured else raw.decode("utf-8", errors="ignore") + counts: dict[str, int] = {} + names = repositories(value, counts) + used.add(label, names, len(raw), skipped_values=counts.get("skipped_values", 0)) + if label == "pairs": + pointer_counts: dict[str, int] = {} + pointers = pointer_repositories(value, pointer_counts) + if pointers or pointer_counts: + used.add("pointers", pointers, len(raw), skipped_values=pointer_counts.get("skipped_values", 0)) + values.append(value) + return values + + pairs = root / "benchmarks/pairs" + read("pairs", (p for p in pairs.rglob("*") if p.is_file() and p.suffix in {".toml", ".json", ".jsonl", ".csv", ".yaml", ".yml"})) + read("pointers", pointer_manifests) + read("populations", (p for p in (root / "benchmarks/independent").glob("population-v[12].toml"))) + read("harvest", (p for directory in pairs.glob("advisory-fixes*") for p in directory.rglob("*.toml"))) + units = set((root / "plane/experiments").glob("*.units.jsonl")) + for path in (root / "plane/experiments").glob("*.y*ml"): + if excluded(path): + continue + raw = path.read_bytes() + manifest = parse(path, raw) + counts = {} + names = repositories(manifest, counts) + used.add("units", names, len(raw), skipped_values=counts.get("skipped_values", 0)) + if isinstance(manifest, dict) and manifest.get("units_file"): + declared = root / Path(manifest["units_file"]).expanduser() + if declared.exists(): # an unregistered plan may not have frozen units yet + units.add(declared) + for label, store in (stores or {}).items(): + if label not in {"local", "s3"}: + raise ValueError("unknown_store_label") + source = "memory_" + label + used.add(source, set(), 0, 0) + # Read actual objects, not a serialized row size masquerading as bytes read. + for key in store._keys("repos/"): + if not key.endswith(".jsonl") or excluded(Path(key)): + continue + result = store._get(key) + if result is None: + raise ValueError("missing_memory_object") + raw = result[0] + rows = parse(Path("rows.jsonl"), raw) + names = set() + counts = {} + for row in rows: + if row.get("kind") == "example" and "repo" in row: + names.update(repositories({"repo": row["repo"]}, counts)) + if row.get("kind") == "experiment" and row.get("units_file"): + registered = root / Path(row["units_file"]).expanduser() + if row.get("units_digest") or registered.exists(): + units.add(registered) + used.add(source, names, len(raw), skipped_values=counts.get("skipped_values", 0)) + read("units", units) + read("measurements", (p for top in ("measurements", "experiments") for p in (root / "benchmarks" / top).rglob("*") if p.is_file())) + read( + "sweep", + ( + p + for top in ("benchmarks", "src", "tests", "plane", ".kiro") + for p in (root / top).rglob("*") + if p.is_file() and not p.is_relative_to(root / "benchmarks/unseen") and not excluded(p) + ), + structured=False, + ) + return used + + +@dataclass(frozen=True) +class Resolution: + full_name: str + names: frozenset[str] + network: str + + +def resolve(candidate: str, api) -> Resolution: + name = normalize(candidate) + if name.count("/") > 1: + return Resolution(name, frozenset({name}), name) + data = api.get_repo(name) + canonical = normalize(data["full_name"]) + names = {name, canonical, *(normalize(n) for n in data.get("former_names", []))} + for key in ("parent", "source"): + if data.get(key): + names.add(normalize(data[key]["full_name"])) + network = normalize((data.get("source") or data.get("parent") or data)["full_name"]) + return Resolution(canonical, frozenset(names), network) + + +def canonicalize_used(used: UsedSet, api) -> None: + """Resolve old used names too: GitHub exposes redirects, not a complete rename history.""" + cache = {} + for names in used.sources.values(): + for name in sorted(names): + if name not in cache: + try: + cache[name] = resolve(name, api) + except RepositoryNotFound: + cache[name] = Resolution(name, frozenset({name}), name) + names.update(cache[name].names) + + +def rejection(candidate: Resolution, used: UsedSet, selected: set[str]) -> str | None: + for label, names in used.sources.items(): + if candidate.names & names: + return label + if candidate.network in selected: + return "fork_network" + selected.add(candidate.network) + return None + + +class RepositoryNotFound(ValueError): + """A historical or synthetic used identity no longer resolves; retain it verbatim.""" + + +class GitHub: + """Coordinator-only GET client. Exceptions never disclose request URLs or tokens.""" + + def __init__(self, token: str): + if not token: + raise ValueError("missing_gh_token") + self.token = token + + def get_repo(self, name: str) -> dict: + from urllib.error import HTTPError + from urllib.request import Request, urlopen + + request = Request( + "https://api.github.com/repos/" + normalize(name), + headers={ + "Authorization": "Bearer " + self.token, + "Accept": "application/vnd.github+json", + }, + ) + try: + with urlopen(request, timeout=30) as response: + return json.load(response) + except HTTPError as exc: + if exc.code == 404: + raise RepositoryNotFound("github_not_found") from None + raise ValueError("github_resolution_failed") from None + except Exception: + raise ValueError("github_resolution_failed") from None + + +def verify_tests(root: Path) -> dict: + """The coordinator runs the guard as a black box; only exit codes and counts escape.""" + import subprocess + import sys + import tempfile + from xml.etree import ElementTree + + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "tests.xml" + done = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + "tests/test_unseen_eligibility.py", + "tests/test_unseen_ledger.py", + "tests/test_independent_population.py", + "--junitxml=" + str(report), + ], + cwd=root, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if done.returncode != 0: + raise ValueError("guard_or_tests_failed") + suites = ElementTree.parse(report).getroot().iter("testsuite") + counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")} + for suite in suites: + for key in counts: + counts[key] += int(suite.attrib[key]) + if not counts["tests"]: + raise ValueError("empty_test_run") + return {"guard": "green", "test_exit_code": done.returncode, "tests": counts} + + +def main(argv: list[str] | None = None) -> int: + import os + + from openultrasast import config + from openultrasast.plane import memory + from openultrasast.plane.reconciler import results_root + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--local-store", default=None) + parser.add_argument("--s3-store", default="s3://") + parser.add_argument("--pointer-manifest", type=Path, action="append", default=[]) + parser.add_argument("--known-empty", choices=(*SOURCES, "memory_local", "memory_s3"), action="append", default=[]) + parser.add_argument("--output", type=Path) + parser.add_argument("--verify-tests", action="store_true") + args = parser.parse_args(argv) + step = "open_stores" + try: + config.load_dotenv(args.root / ".env") + stores = { + "local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True), + "s3": memory.open_store(args.s3_store, read_only=True), + } + step = "build_used" + used = build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest) + step = "report" + used.report(set(args.known_empty)) # prove inputs before API work + step = "canonicalize" + canonicalize_used(used, GitHub(os.environ.get("GH_TOKEN", ""))) + step = "report" + report = used.report(set(args.known_empty)) + if args.verify_tests: + step = "verify_tests" + report.update(verify_tests(args.root)) + step = "report" + text = json.dumps(report, sort_keys=True, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as handle: + handle.write(text) + print(text, end="") + return 0 + except Exception as exc: + # Source paths, store exceptions and JSON parser errors can contain repository names. + report = {"status": "instrument_failure", "step": step, "type": type(exc).__name__} + if str(exc) in {"empty_source:" + label for label in (*SOURCES, "memory_local", "memory_s3")}: + report["source"] = str(exc).partition(":")[2] + print(json.dumps(report)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/unseen/ledger.py b/benchmarks/unseen/ledger.py new file mode 100644 index 0000000..17dcb88 --- /dev/null +++ b/benchmarks/unseen/ledger.py @@ -0,0 +1,88 @@ +"""Append-only slice usage; a hash chain detects rewritten rows, not a rebuilt entire history. + +Keep the published final digest to authenticate a history against truncation or wholesale replacement. +Only opaque decision/experiment labels belong here; repository identities are never accepted. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import re +from datetime import date +from pathlib import Path + +FIELDS = {"slice", "decision", "purpose", "date", "result_digest", "freeze_digest"} +PURPOSES = {"baseline", "exploratory", "informed", "qualifies"} +ZERO = "0" * 64 + + +def digest(row: dict) -> str: + return hashlib.sha256(json.dumps(row, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def validate(row: dict) -> None: + if set(row) != FIELDS or row["purpose"] not in PURPOSES: + raise ValueError("invalid_usage_fields") + if type(row["slice"]) is not int or row["slice"] < 1: + raise ValueError("invalid_slice") + if not isinstance(row["decision"], str) or not re.fullmatch(r"[A-Za-z0-9_-]+", row["decision"]): + raise ValueError("invalid_decision_label") + for key in ("result_digest", "freeze_digest"): + if not isinstance(row[key], str) or not re.fullmatch(r"[0-9a-f]{64}", row[key]): + raise ValueError("invalid_digest") + try: + if date.fromisoformat(row["date"]).isoformat() != row["date"]: + raise ValueError + except (ValueError, TypeError): + raise ValueError("invalid_date") from None + + +def _read(text: str) -> list[dict]: + previous = ZERO + rows = [] + if text and not text.endswith("\n"): + raise ValueError("broken_digest_chain") + for line in text.splitlines(): + try: + row = json.loads(line) + stored = row.pop("digest") + if row["previous"] != previous or digest(row) != stored: + raise ValueError + validate({k: v for k, v in row.items() if k != "previous"}) + except (ValueError, KeyError, TypeError, AttributeError): + raise ValueError("broken_digest_chain") from None + rows.append({**row, "digest": stored}) + previous = stored + return rows + + +def read(path: Path) -> list[dict]: + if not path.exists(): + return [] + with path.open() as handle: + fcntl.flock(handle, fcntl.LOCK_SH) + return _read(handle.read()) + + +def append(path: Path, row: dict) -> str: + validate(row) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + handle.seek(0) + rows = _read(handle.read()) + if any(all(old[key] == row[key] for key in FIELDS) for old in rows): + raise ValueError("duplicate_usage") + if rows and rows[0]["freeze_digest"] != row["freeze_digest"]: + raise ValueError("freeze_digest_changed") + if row["purpose"] == "qualifies" and any( + old["slice"] == row["slice"] and old["decision"] == row["decision"] and old["purpose"] == "informed" for old in rows + ): + raise ValueError("slice_informed_decision") + entry = {**row, "previous": rows[-1]["digest"] if rows else ZERO} + sha = digest(entry) + handle.write(json.dumps({**entry, "digest": sha}, sort_keys=True) + "\n") + handle.flush() + return sha diff --git a/src/openultrasast/plane/memory.py b/src/openultrasast/plane/memory.py index f601024..4e1afd6 100644 --- a/src/openultrasast/plane/memory.py +++ b/src/openultrasast/plane/memory.py @@ -615,9 +615,10 @@ class S3Store(MemoryStore): presigned URLs. An admin configures the bucket once; opening a store verifies it (:meth:`verify_bucket`) and refuses a bucket without versioning, the ``runs/`` expiry rule or S3 Select. There is no local fallback.""" - def __init__(self, client: ObjectClient, bucket: str, prefix: str = "") -> None: + def __init__(self, client: ObjectClient, bucket: str, prefix: str = "", *, read_only: bool = False) -> None: self.client, self.bucket, self.prefix = client, bucket, prefix.strip("/") - if self.describe() not in _VERIFIED: + self.read_only = read_only + if not read_only and self.describe() not in _VERIFIED: self.verify_bucket() _VERIFIED.add(self.describe()) @@ -631,9 +632,11 @@ class S3Store(MemoryStore): return self.client.get(self._k(key)) def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: + self._writable() self.client.put(self._k(key), data, dict(labels or {})) def _delete(self, key: str) -> None: + self._writable() self.client.delete(self._k(key)) def _keys(self, prefix: str) -> list[str]: @@ -647,11 +650,16 @@ class S3Store(MemoryStore): tagged = self.client.tags(self._k(key)).get("kind") return tagged is None or tagged == "*" or kind in tagged.split() + def _writable(self) -> None: + if self.read_only: + raise MemoryStoreError("read-only store") + def verify_bucket(self) -> None: - """Read-only check that the admin's one-time setup is in place: versioning ``Enabled``, an enabled + """Check the admin's one-time setup, writing only the capability probe: versioning ``Enabled``, an enabled lifecycle rule expiring ``/runs/`` and none expiring the whole store, S3 Select answering a probe over JSON Lines, and the probe's object tags readable. Every missing piece is named with its admin action in one :class:`MemoryStoreError`.""" + self._writable() runs = self._k(RUNS_PREFIX) endpoint = '--endpoint-url "$S3_ENDPOINT"' missing: list[str] = [] @@ -739,6 +747,7 @@ class S3Store(MemoryStore): ) from exc def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + self._writable() return self.client.presign("PUT", self._k(key), expires) def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: @@ -915,9 +924,10 @@ def s3_settings(environ: Mapping[str, str] | None = None) -> dict[str, Any]: return settings -def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None) -> MemoryStore: +def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None, *, read_only: bool = False) -> MemoryStore: """The store ``spec`` (else ``OUSAST_MEMORY``, else ``file:///memory``) names. ``s3://`` with no - bucket uses ``S3_BUCKET``.""" + bucket uses ``S3_BUCKET``. ``read_only`` skips the S3 write/Select capability probe and refuses S3 mutations; + it is intended for GET/list inventory readers, which do not need Select. Default opening is unchanged.""" from .reconciler import results_root env = environ if environ is not None else os.environ @@ -932,7 +942,7 @@ def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None if not bucket: raise MemoryStoreError("OUSAST_MEMORY=s3:// names no bucket and S3_BUCKET is unset: set one of them") settings = s3_settings(environ) - return S3Store(S3Client(bucket=bucket, **settings), bucket, parts.path) + return S3Store(S3Client(bucket=bucket, **settings), bucket, parts.path, read_only=read_only) raise MemoryStoreError(f"OUSAST_MEMORY must be file:/// or s3://[/], got {text!r}") diff --git a/tests/test_independent_population.py b/tests/test_independent_population.py index 859ed16..b5b0095 100644 --- a/tests/test_independent_population.py +++ b/tests/test_independent_population.py @@ -8,6 +8,8 @@ anywhere else, which is how an evaluation set quietly becomes a development one. from __future__ import annotations +import hashlib +import json import re import tomllib from pathlib import Path @@ -64,17 +66,131 @@ def test_a_multi_language_repository_is_reserved(path: Path) -> None: assert members and all(by_id[m]["language"] == "php" and "javascript" in by_id[m].get("also", []) for m in members) -def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None: +def _pool_manifests() -> list[Path]: + directory = ROOT / "benchmarks/unseen" + return sorted([*directory.glob("*.toml"), *directory.glob("private/*.toml")]) + + +def _pool_names() -> set[str]: + names = set() + for path in _pool_manifests(): + text = path.read_text().lower() + names.update(m.rstrip("/").removesuffix(".git") for m in re.findall(r"github\.com/([\w.-]+/[\w.-]+)", text)) + return names + + +def _population_names() -> set[str]: reserved = {case["repo"].rstrip("/").lower() for path in POPULATIONS for case in _population(path)["case"]} - names = {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved} - offenders = [] - for top in ("benchmarks", "src", "tests"): + return {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved} + + +def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None: + reservations = [(ROOT / "benchmarks/independent", _population_names()), (ROOT / "benchmarks/unseen", _pool_names())] + offenders = 0 + counts = {} + for top in ("benchmarks", "src", "tests", "plane"): for path in (ROOT / top).rglob("*"): - if not path.is_file() or POPULATION.parent in path.parents or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}: + if not path.is_file() or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}: + continue + # Preserve the population guard's existing scope; plane is newly reserved for pools. + active = [ + (directory, names) + for directory, names in reservations + if directory not in path.parents and names and not (top == "plane" and directory == ROOT / "benchmarks/independent") + ] + if not active: continue try: text = path.read_text(errors="ignore").lower() except OSError: continue - offenders += [f"{path.relative_to(ROOT)}: {name}" for name in names if f"github.com/{name}" in text] - assert not offenders, "a reserved repository is referenced outside the reservation: " + "; ".join(offenders[:10]) + count = sum(1 for _, names in active for name in names if f"github.com/{name}" in text) + offenders += count + counts[top] = counts.get(top, 0) + count + assert not offenders, f"reserved repository referenced outside its reservation: count={offenders}, sources={counts}" + + +def test_pool_and_population_names_are_disjoint() -> None: + count = len(_population_names() & _pool_names()) + assert not count, f"overlapping reservations: count={count}" + + +def manifest_digest(path: Path) -> str: + """Canonical parsed TOML, UTF-8 JSON with sorted keys and compact separators. + + Private entries are covered by their opaque pointers' digests in the public manifest. + The freeze writer must use the same encoding (task 3.2). + """ + raw = json.dumps(tomllib.loads(path.read_text()), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + return hashlib.sha256(raw).hexdigest() + + +def check_pool_digest(path: Path) -> None: + record = path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json") + expected = json.loads(record.read_text())["freeze_digest"] + assert manifest_digest(path) == expected, "frozen pool digest mismatch" + + +def test_frozen_pool_manifest_digest() -> None: + pools = sorted((ROOT / "benchmarks/unseen").glob("pool-p*.toml")) + if not pools: + pytest.skip("no pool frozen") + for path in pools: + check_pool_digest(path) + + +def test_pool_guard_controls(tmp_path, monkeypatch) -> None: + """A synthetic reservation proves green, then red outside its own directory.""" + population = tmp_path / "benchmarks/independent/population-v1.toml" + pool = tmp_path / "benchmarks/unseen/pool-p1.toml" + population.parent.mkdir(parents=True) + pool.parent.mkdir(parents=True) + prefix = "https://" + "github.com/" + population.write_text('[[case]]\nrepo = "' + prefix + 'synthetic/population"\n') + pool.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/pool"\n') + monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path) + monkeypatch.setattr(__import__(__name__), "POPULATIONS", [population]) + test_no_reserved_repository_is_referenced_by_the_development_tree() + leak = tmp_path / "benchmarks/measurements/fixture.txt" + leak.parent.mkdir(parents=True) + leak.write_text(prefix + "synthetic/pool") + with pytest.raises(AssertionError, match="count=1"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + leak.unlink() + plane = tmp_path / "plane/fixture.txt" + plane.parent.mkdir() + plane.write_text(prefix + "synthetic/pool") + with pytest.raises(AssertionError, match="count=1"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + plane.write_text(prefix + "synthetic/population") + test_no_reserved_repository_is_referenced_by_the_development_tree() + plane.unlink() + private = pool.parent / "private/draft-p1.toml" + private.parent.mkdir() + private.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/population"\n') + with pytest.raises(AssertionError, match="count=2"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + + +def test_pool_digest_control(tmp_path) -> None: + pool = tmp_path / "pool-p1.toml" + pool.write_text('status = "frozen"\n[[repository]]\nid = "opaque"\n') + record = tmp_path / "freeze-p1.json" + record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)})) + check_pool_digest(pool) + pool.write_text(pool.read_text() + "slice = 2\n") + with pytest.raises(AssertionError, match="digest"): + check_pool_digest(pool) + + +def test_frozen_pool_discovery_cannot_be_disabled(tmp_path, monkeypatch) -> None: + pool = tmp_path / "benchmarks/unseen/pool-p1.toml" + pool.parent.mkdir(parents=True) + pool.write_text('status = "frozen"\n') + record = pool.with_name("freeze-p1.json") + record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)})) + monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path) + test_frozen_pool_manifest_digest() + pool.write_text('status = "draft"\n') + with pytest.raises(AssertionError, match="digest"): + test_frozen_pool_manifest_digest() diff --git a/tests/test_unseen_eligibility.py b/tests/test_unseen_eligibility.py new file mode 100644 index 0000000..a6fe632 --- /dev/null +++ b/tests/test_unseen_eligibility.py @@ -0,0 +1,293 @@ +"""Offline eligibility fixtures; no real repository identities are needed.""" + +import io +import json +from pathlib import Path + +import pytest + +from openultrasast.plane.memory import FileStore + +with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen")) + import eligibility as e + + +def plant(root, path, text): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + return target + + +def test_sources_and_reserved_input_never_opened(tmp_path, monkeypatch, capsys): + sources = { + "benchmarks/pairs/a/catalog.toml": ("pairs", "pair"), + "benchmarks/pairs/a/recipes.toml": ("pairs", "recipe"), + "benchmarks/pairs/a/dataset.json": ("pairs", "dataset"), + "benchmarks/pairs/advisory-fixes-b/catalog.toml": ("harvest", "harvest"), + "benchmarks/independent/population-v1.toml": ("populations", "first"), + "benchmarks/independent/population-v2.toml": ("populations", "second"), + "plane/experiments/a.units.jsonl": ("units", "unit"), + "benchmarks/measurements/a/record.json": ("measurements", "measurement"), + "benchmarks/experiments/a.json": ("measurements", "experiment"), + "src/example.txt": ("sweep", "sweep"), + } + for path, (_, name) in sources.items(): + text = f'repo = "https://github.com/fixture/{name}"' if path.endswith(".toml") else json.dumps({"repo": f"fixture/{name}"}) + if name == "sweep": + text = "https://github.com/fixture/sweep" + plant(tmp_path, path, text) + pointer = plant(tmp_path, "local/pointers.toml", '[[pair]]\nrepo = "fixture/pointer"\nfix_repo = "fixture/fix"\nvendored = false') + plant(tmp_path, "benchmarks/independent/population-v3.toml", "do not read") + plant(tmp_path, "benchmarks/independent/results-v3.json", "do not read") + original = io.open + + def guarded(file, *args, **kwargs): + if "v3" in str(file): + pytest.fail("reserved input opened") + return original(file, *args, **kwargs) + + monkeypatch.setattr(io, "open", guarded) + store = FileStore(tmp_path / "memory") + plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"id": "1", "kind": "example", "repo": "fixture/memory"}) + "\n") + used = e.build_used(tmp_path, stores={"local": store}, pointer_manifests=[pointer]) + for label, name in sources.values(): + assert f"fixture/{name}" in used.sources[label] + assert {"fixture/pointer", "fixture/fix"} <= used.sources["pointers"] + assert "fixture/memory" in used.sources["memory_local"] + report = used.report() + assert report["sources"]["memory_local"]["bytes"] > 0 + assert all(item["files"] > 0 and item["bytes"] > 0 for item in report["sources"].values()) + assert "fixture/" not in json.dumps(report) + assert capsys.readouterr().out == "" + assert e.normalize("HTTPS://GitHub.com/Fixture/Pair.GIT/") == "fixture/pair" + assert len({e.normalize(s) for s in ["fixture/pair", "Fixture/Pair.git/", "github.com/fixture/pair"]}) == 1 + + +def test_empty_sources_fail_unless_declared(tmp_path): + used = e.build_used(tmp_path) + with pytest.raises(ValueError, match="empty_source"): + used.report() + assert used.report(known_empty=set(used.sources))["count"] == 0 + + +class FakeAPI: + def __init__(self, records): + self.records = records + + def get_repo(self, name): + return self.records[name] + + +def test_resolution_rejects_used_renames_and_forks(): + used = e.UsedSet() + used.add("pairs", {"fixture/used"}, 10) + api = FakeAPI( + { + "fixture/used": {"full_name": "fixture/used"}, + "fixture/old": {"full_name": "fixture/new", "former_names": ["fixture/used"]}, + "fixture/fork": {"full_name": "fixture/fork", "parent": {"full_name": "fixture/used"}, "source": {"full_name": "fixture/used"}}, + "fixture/a": {"full_name": "fixture/a", "source": {"full_name": "fixture/root"}}, + "fixture/b": {"full_name": "fixture/b", "source": {"full_name": "fixture/root"}}, + } + ) + selected = set() + for name in ("used", "old", "fork"): + resolved = e.resolve("fixture/" + name, api) + assert e.rejection(resolved, used, selected) == "pairs" + assert e.rejection(e.resolve("fixture/a", api), used, selected) is None + assert e.rejection(e.resolve("fixture/b", api), used, selected) == "fork_network" + + +def test_registered_units_and_read_only_store(tmp_path): + units = plant(tmp_path, "local/units.jsonl", json.dumps({"repo": "fixture/registered"})) + store = FileStore(tmp_path / "memory") + plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"kind": "experiment", "units_file": str(units)})) + used = e.build_used(tmp_path, stores={"s3": store}) + assert "fixture/registered" in used.sources["units"] + + +def test_cli_injected_store_and_credentials(tmp_path, monkeypatch, capsys): + from openultrasast import config + from openultrasast.plane import memory + + calls = [] + monkeypatch.setattr(config, "load_dotenv", lambda path: calls.append("dotenv")) + monkeypatch.setenv("GH_TOKEN", "test-token") + store = FileStore(tmp_path / "store") + monkeypatch.setattr(memory, "open_store", lambda spec, **kwargs: calls.append(spec) or store) + monkeypatch.setattr(e, "GitHub", lambda token: calls.append(token) or FakeAPI({})) + args = ["--root", str(tmp_path), "--s3-store", "s3://", "--local-store", "file:///unused"] + for label in (*e.SOURCES, "memory_local", "memory_s3"): + args += ["--known-empty", label] + assert e.main(args) == 0 + assert calls == ["dotenv", "file:///unused", "s3://", "test-token"] + output = capsys.readouterr().out + assert json.loads(output)["count"] == 0 + assert "test-token" not in output + + +def test_s3_open_is_read_only(tmp_path, monkeypatch): + from openultrasast.plane import memory + + class Objects: + def keys(self, prefix): + return ["repos/opaque/pin.jsonl"] + + def get(self, key): + return json.dumps({"kind": "example", "repo": "fixture/stored"}).encode(), "1" + + def __getattr__(self, name): + raise AssertionError("unexpected_store_operation") + + monkeypatch.setattr(memory, "S3Client", lambda **kwargs: Objects()) + monkeypatch.setattr(memory, "s3_settings", lambda env: {}) + store = memory.open_store("s3://fixture-bucket", read_only=True) + used = e.build_used(tmp_path, stores={"s3": store}) + assert used.sources["memory_s3"] == {"fixture/stored"} + with pytest.raises(memory.MemoryStoreError, match="read.only"): + store._put("key", b"value") + with pytest.raises(memory.MemoryStoreError, match="read.only"): + store._delete("key") + + +def test_non_repository_provenance_and_missing_used_identity(tmp_path): + plant(tmp_path, "benchmarks/pairs/recipes.toml", 'repo = "provenance-label"') + assert not e.build_used(tmp_path).sources["pairs"] + used = e.UsedSet() + used.add("pairs", {"fixture/old", "fixture/nonexistent"}, 1) + + class API: + def get_repo(self, name): + if name.endswith("nonexistent"): + raise e.RepositoryNotFound + return {"full_name": "fixture/new"} + + e.canonicalize_used(used, API()) + assert "fixture/nonexistent" in used.sources["pairs"] + assert e.rejection(e.resolve("fixture/new", API()), used, set()) == "pairs" + + +def test_binary_measurement_and_local_pointer_discovery(tmp_path): + target = tmp_path / "benchmarks/measurements/a/image.bin" + target.parent.mkdir(parents=True) + target.write_bytes(b"\xff\x00") + plant(tmp_path, "benchmarks/pairs/a/recipes.toml", '[[pair]]\nrepo = "fixture/pointer"\nvendored = false') + used = e.build_used(tmp_path) + assert used.instrument["measurements"]["bytes"] == 2 + assert used.sources["pointers"] == {"fixture/pointer"} + + +def test_coordinator_verification_is_black_box(tmp_path, monkeypatch): + import subprocess + from types import SimpleNamespace + + def run(command, **kwargs): + assert kwargs["stdout"] == kwargs["stderr"] == kwargs["stdin"] == subprocess.DEVNULL + path = Path(next(part.split("=", 1)[1] for part in command if part.startswith("--junitxml="))) + path.write_text('') + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(subprocess, "run", run) + assert e.verify_tests(tmp_path)["tests"]["tests"] == 25 + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: SimpleNamespace(returncode=2)) + with pytest.raises(ValueError, match="guard_or_tests_failed"): + e.verify_tests(tmp_path) + + +def test_registered_unit_group_is_repository(tmp_path): + plant(tmp_path, "plane/experiments/a.units.jsonl", json.dumps({"unit": "opaque", "group": "fixture/unit", "label": 1})) + assert "fixture/unit" in e.build_used(tmp_path).sources["units"] + + +def test_multidocument_measurement_and_missing_registered_units(tmp_path): + plant(tmp_path, "benchmarks/measurements/a/run.yaml", "repo: fixture/first\n---\nrepo: fixture/second\n") + assert e.build_used(tmp_path).sources["measurements"] == {"fixture/first", "fixture/second"} + store = FileStore(tmp_path / "memory") + plant( + tmp_path, + "memory/repos/opaque/pin.jsonl", + json.dumps( + { + "kind": "experiment", + "units_file": "absent.units.jsonl", + "units_digest": "a" * 64, + } + ), + ) + with pytest.raises(FileNotFoundError): + e.build_used(tmp_path, stores={"local": store}) + + +def test_catalog_skips_fixture_and_preserves_host_identity(tmp_path, capsys): + plant( + tmp_path, + "benchmarks/pairs/catalog.toml", + '[[pair]]\norigin = "in-tree-fixture"\nvendored = false\n[[pair]]\nrepo = "https://GitLab.com/Fixture/Private.git"\n', + ) + used = e.build_used(tmp_path) + assert used.sources["pairs"] == {"gitlab.com/fixture/private"} + assert used.instrument["pairs"]["skipped_values"] == 1 + assert used.instrument["pointers"]["skipped_values"] == 1 + assert used.sources["sweep"] == set() + e.canonicalize_used(used, FakeAPI({})) # Non-GitHub identities never reach this API. + assert e.rejection(e.resolve("https://gitlab.com/fixture/private", FakeAPI({})), used, set()) == "pairs" + report = used.report(set(e.SOURCES) - {"pairs"}) + assert report["sources"]["pairs"]["skipped_values"] == 1 + assert "fixture" not in json.dumps(report) + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("https://bitbucket.org/Fixture/Repo.git", "bitbucket.org/fixture/repo"), + ("https://git.example.org/Fixture/Repo", "git.example.org/fixture/repo"), + ("https://github.com/Fixture/Repo/tree/main", "fixture/repo"), + ("Fixture/Repo.git/", "fixture/repo"), + ], +) +def test_repository_hosts(value, expected): + assert e.normalize(value) == expected + assert e.normalize(expected) == expected + + +def test_memory_values_are_skipped_and_counted(tmp_path, capsys): + rows = [ + {"kind": "example", "repo": value} + for value in ["local-slice", "free text https://github.com/fixture/prose", None, "https://bitbucket.org/fixture/repo"] + ] + plant(tmp_path, "memory/repos/opaque/pin.jsonl", "\n".join(json.dumps(row) for row in rows)) + used = e.build_used(tmp_path, stores={"local": FileStore(tmp_path / "memory")}) + assert used.sources["memory_local"] == {"bitbucket.org/fixture/repo"} + assert used.instrument["memory_local"]["skipped_values"] == 3 + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize("step", ["open_stores", "build_used", "report", "canonicalize", "verify_tests"]) +def test_cli_failure_reports_only_step_and_exception_type(tmp_path, monkeypatch, capsys, step): + from openultrasast import config + from openultrasast.plane import memory + + def fail(*args, **kwargs): + raise RuntimeError("sensitive/identity") + + monkeypatch.setattr(config, "load_dotenv", lambda path: None) + monkeypatch.setattr(memory, "open_store", lambda *args, **kwargs: object()) + monkeypatch.setattr(e, "build_used", lambda *args, **kwargs: e.UsedSet()) + monkeypatch.setattr(e.UsedSet, "report", lambda *args, **kwargs: {}) + monkeypatch.setattr(e, "GitHub", lambda token: object()) + monkeypatch.setattr(e, "canonicalize_used", lambda *args: None) + monkeypatch.setattr(e, "verify_tests", lambda *args: {}) + if step == "open_stores": + monkeypatch.setattr(memory, "open_store", fail) + elif step == "report": + monkeypatch.setattr(e.UsedSet, "report", fail) + else: + monkeypatch.setattr(e, "canonicalize_used" if step == "canonicalize" else step, fail) + assert e.main(["--root", str(tmp_path), "--verify-tests"]) == 1 + output = capsys.readouterr() + assert output.err == "" + assert json.loads(output.out) == {"status": "instrument_failure", "step": step, "type": "RuntimeError"} + assert "sensitive" not in output.out diff --git a/tests/test_unseen_ledger.py b/tests/test_unseen_ledger.py new file mode 100644 index 0000000..4895d41 --- /dev/null +++ b/tests/test_unseen_ledger.py @@ -0,0 +1,63 @@ +"""Append-only use and tamper detection, with opaque fixture identifiers.""" + +import json +from pathlib import Path + +import pytest + +with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen")) + import ledger + + +def row(**extra): + return { + "slice": 1, + "decision": "decision-001", + "purpose": "baseline", + "date": "2026-10-04", + "result_digest": "a" * 64, + "freeze_digest": "b" * 64, + **extra, + } + + +def test_append_and_refuse_rewrite(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row()) + original = path.read_bytes() + ledger.append(path, row(purpose="exploratory")) + assert path.read_bytes().startswith(original) + assert len(ledger.read(path)) == 2 + with pytest.raises(ValueError, match="duplicate"): + ledger.append(path, row()) + + +def test_informed_slice_cannot_qualify_same_decision(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row(purpose="informed")) + before = path.read_bytes() + with pytest.raises(ValueError, match="informed"): + ledger.append(path, row(purpose="qualifies")) + assert path.read_bytes() == before + ledger.append(path, row(purpose="qualifies", slice=2)) + ledger.append(path, row(purpose="qualifies", decision="decision-002")) + + +def test_rewritten_row_breaks_digest_chain(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row()) + ledger.append(path, row(purpose="exploratory")) + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["purpose"] = "informed" + path.write_text("".join(json.dumps(r) + "\n" for r in rows)) + with pytest.raises(ValueError, match="chain"): + ledger.read(path) + with pytest.raises(ValueError, match="chain"): + ledger.append(path, row(slice=2)) + + +@pytest.mark.parametrize("extra", [{"decision": "owner/name"}, {"purpose": "other"}, {"freeze_digest": "bad"}, {"slice": 0}]) +def test_invalid_rows_refused(tmp_path, extra): + with pytest.raises(ValueError): + ledger.append(tmp_path / "usage-p1.jsonl", row(**extra))