feat(pair-corpus-honesty): LLM-generated Real-Vuln repos and unlicensed agent fixes as pointer pairs

Task 8.4 (Req 10.6, 5.1).

- harvest.merge_recipes appends recipes by name and never rewrites
  existing rows.
- vibe-py/build_recipes.py --pointers: the 40 LLM-generated repositories
  without a license become vendored = false rows (80 pairs, agent,
  seeded) appended to recipes.toml; unlicensed human repos stay skipped.
- agent-vfc/build_recipes.py --from ... --pointers [--cache]: unlicensed
  agent-authored fixes become title-tier pointer rows whose function
  label is derived from the cached excerpt (30 added from a fresh
  662-candidate search); TOML booleans rendered correctly.
- catalogs regenerated: vibe-py 115 rows (35 vendored + 80 pointer),
  agent-vfc 59 (29 + 30); no excerpt is vendored for any pointer row.
- `pairs --slice vibe-py --pointers` scores both provenances (human 35,
  agent 80); the default run skips 80 pairs with pointer_pair_skipped;
  suite and gates unchanged. README/docs name the nightly command.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-05 10:45:43 +02:00
co-authored by Claude Fable 5.1
parent 4102404442
commit 3c36f13e03
15 changed files with 7380 additions and 42 deletions
+24 -2
View File
@@ -464,9 +464,10 @@ def test_new_slices_load_offline_and_score_with_profiles() -> None:
assert all(case.license for case in select_vendored(cases)), slice_name # pointer rows may be unlicensed (Req 10)
assert all(case.provenance == expected_provenance for case in select_vendored(cases)), slice_name
assert {case.split for case in cases} <= {"train", "holdout"}
header = cases[0].vuln_file.read_text()[:600]
vendored = select_vendored(cases) # pointer rows have no excerpt in the repository (Req 10)
header = vendored[0].vuln_file.read_text()[:600]
assert "Provenance:" in header and "license:" in header
result = evaluate_catalog(cases[:2])
result = evaluate_catalog(vendored[:2])
assert result.per_slice[slice_name].pairs == 2
assert set(result.per_profile) <= PROVENANCES and expected_provenance in result.per_profile
assert result.per_mechanism and slice_name in result.loss
@@ -963,3 +964,24 @@ def test_warm_cache_never_scores_pointer_pairs_when_network_is_off(tmp_path: Pat
assert result.outcomes == ()
assert {"stage": "pairs", "reason": "pointer_pair_skipped", "pair": "ptr"} in result.degradations
assert evaluate_catalog([case], pointers=True).outcomes[0].pair_correct # warm cache, network on: no fetch, scored
def test_vibe_py_and_agent_vfc_carry_pointer_rows_and_the_default_run_skips_them() -> None:
"""Req 10.6: LLM-generated Real-Vuln repos are agent/seeded pointer rows; unlicensed agent fixes are title pointer rows."""
import io
from contextlib import redirect_stdout
catalog = load_pair_catalog()
vibe = select_slice(catalog, "vibe-py")
pointers = [case for case in vibe if not case.vendored]
assert len(pointers) >= 40 and len({case.repo for case in pointers}) >= 40
assert all(case.provenance == "agent" and case.review_tier == "seeded" and not case.vuln_file.exists() for case in pointers)
assert len(select_vendored(vibe)) == 35 and all(case.provenance == "human" for case in select_vendored(vibe))
agent_pointers = [case for case in select_slice(catalog, "agent-vfc") if not case.vendored]
assert len(agent_pointers) >= 20 and all(case.review_tier == "title" and case.license == "unlicensed" for case in agent_pointers)
buf = io.StringIO()
with redirect_stdout(buf):
assert main(["pairs", "--slice", "vibe-py", "--json"]) == 0
payload = json.loads(buf.getvalue())
skipped = [item for item in payload["degradations"] if item["reason"] == "pointer_pair_skipped"]
assert len(skipped) == len(pointers) and payload["overall"]["pairs"] == 35