mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-18 04:29:27 +00:00
feat(pair-corpus-honesty): score non-vendored pointer pairs through a local cache
Task 8.3 (Req 10.1-10.5). - pairs.py: catalog rows with vendored = false carry their harvest recipe and no excerpts; evaluate_catalog harvests them into OPENULTRASAST_PAIR_CACHE (default ~/.cache/openultrasast/pairs) only when `pairs --pointers` or OPENULTRASAST_PAIRS_NETWORK=1 allows the network, otherwise records pointer_pair_skipped (a failed fetch is pointer_pair_fetch_failed with detail); a warm cache never scores without the switch. select_vendored for tests and gates. - harvest.py: materialize_pointer harvests into the cache without the license requirement (nothing is redistributed); redaction applies. - catalog_gen.py: emits pointer rows with recipe fields; training manifests skip them. - pair_gate and improve/evolve.evaluate_profiles score vendored pairs only; cli `pairs --pointers`; .gitignore covers the cache path. - tests: loader, skip/degradation, cache hit, fetch failure, warm cache with network off, profile gate exclusion, harvest and generator paths; conftest keeps every test offline whatever the shell exports. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
7d238f2400
commit
4102404442
@@ -156,6 +156,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
pairs.add_argument("--split", choices=("all", "train", "holdout"), default="all", help="filter pairs by declared split")
|
||||
pairs.add_argument("--hunter", action="store_true", help="also score the LLM tool hunter on overlay slices (needs a hunter model)")
|
||||
pairs.add_argument("--hunter-model", default=None, help="model id for --hunter (default: [models].hunter from openultrasast.toml)")
|
||||
pairs.add_argument("--pointers", action="store_true", help="allow network for non-vendored pointer pairs this run (nightly; CI never)")
|
||||
pairs.add_argument("--json", action="store_true", help="print the pair scoreboard as JSON")
|
||||
|
||||
subparsers.add_parser("mcp", help="run the narrow MCP server over stdio for OpenCode integration")
|
||||
@@ -190,6 +191,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
split=args.split,
|
||||
hunter=args.hunter,
|
||||
hunter_model=args.hunter_model,
|
||||
pointers=args.pointers,
|
||||
)
|
||||
if args.command == "mcp":
|
||||
from .mcp import serve # lazy: keeps the import cycle (mcp -> cli) one-directional
|
||||
@@ -787,6 +789,7 @@ def _pairs(
|
||||
split: str = "all",
|
||||
hunter: bool = False,
|
||||
hunter_model: str | None = None,
|
||||
pointers: bool = False,
|
||||
) -> int:
|
||||
if not catalog.exists() or not catalog.is_file():
|
||||
raise SystemExit(f"pair catalog is not a file: {catalog}")
|
||||
@@ -799,7 +802,7 @@ def _pairs(
|
||||
client = tool_hunter.resolve_hunter_client()
|
||||
if model and client is not None:
|
||||
scan = make_hunter_scan(client, model)
|
||||
result = evaluate_catalog(cases, hunter=scan)
|
||||
result = evaluate_catalog(cases, hunter=scan, pointers=True if pointers else None)
|
||||
if json_out:
|
||||
print(json.dumps(result_payload(result), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user