feat(pair-corpus-honesty): score non-vendored pointer pairs through a local cache

Task 8.3 (Req 10.1-10.5).

- pairs.py: catalog rows with vendored = false carry their harvest recipe
  and no excerpts; evaluate_catalog harvests them into
  OPENULTRASAST_PAIR_CACHE (default ~/.cache/openultrasast/pairs) only
  when `pairs --pointers` or OPENULTRASAST_PAIRS_NETWORK=1 allows the
  network, otherwise records pointer_pair_skipped (a failed fetch is
  pointer_pair_fetch_failed with detail); a warm cache never scores
  without the switch. select_vendored for tests and gates.
- harvest.py: materialize_pointer harvests into the cache without the
  license requirement (nothing is redistributed); redaction applies.
- catalog_gen.py: emits pointer rows with recipe fields; training
  manifests skip them.
- pair_gate and improve/evolve.evaluate_profiles score vendored pairs
  only; cli `pairs --pointers`; .gitignore covers the cache path.
- tests: loader, skip/degradation, cache hit, fetch failure, warm cache
  with network off, profile gate exclusion, harvest and generator paths;
  conftest keeps every test offline whatever the shell exports.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-05 10:45:43 +02:00
co-authored by Claude Fable 5.1
parent 7d238f2400
commit 4102404442
13 changed files with 405 additions and 21 deletions
+4 -1
View File
@@ -156,6 +156,7 @@ def main(argv: list[str] | None = None) -> int:
pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split")
pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)")
pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)")
pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)")
pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
@@ -190,6 +191,7 @@ def main(argv: list[str] | None = None) -> int:
split=args.split,
hunter=args.hunter,
hunter_model=args.hunter_model,
pointers=args.pointers,
)
if args.command == "mcp":
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
@@ -787,6 +789,7 @@ def _pairs(
split: str = "all",
hunter: bool = False,
hunter_model: str | None = None,
pointers: bool = False,
) -> int:
if not catalog.exists() or not catalog.is_file():
raise SystemExit(f"pair catalog is not a file: {catalog}")
@@ -799,7 +802,7 @@ def _pairs(
client = tool_hunter.resolve_hunter_client()
if model and client is not None:
scan = make_hunter_scan(client, model)
result = evaluate_catalog(cases, hunter=scan)
result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None)
if json_out:
print(json.dumps(result_payload(result), indent=2, sort_keys=True))
return 0