mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
feat(pair-corpus-honesty): LLM-generated Real-Vuln repos and unlicensed agent fixes as pointer pairs
Task 8.4 (Req 10.6, 5.1). - harvest.merge_recipes appends recipes by name and never rewrites existing rows. - vibe-py/build_recipes.py --pointers: the 40 LLM-generated repositories without a license become vendored = false rows (80 pairs, agent, seeded) appended to recipes.toml; unlicensed human repos stay skipped. - agent-vfc/build_recipes.py --from ... --pointers [--cache]: unlicensed agent-authored fixes become title-tier pointer rows whose function label is derived from the cached excerpt (30 added from a fresh 662-candidate search); TOML booleans rendered correctly. - catalogs regenerated: vibe-py 115 rows (35 vendored + 80 pointer), agent-vfc 59 (29 + 30); no excerpt is vendored for any pointer row. - `pairs --slice vibe-py --pointers` scores both provenances (human 35, agent 80); the default run skips 80 pairs with pointer_pair_skipped; suite and gates unchanged. README/docs name the nightly command. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
4102404442
commit
3c36f13e03
@@ -464,9 +464,10 @@ def test_new_slices_load_offline_and_score_with_profiles() -> None:
|
||||
assert all(case.license for case in select_vendored(cases)), slice_name # pointer rows may be unlicensed (Req 10)
|
||||
assert all(case.provenance == expected_provenance for case in select_vendored(cases)), slice_name
|
||||
assert {case.split for case in cases} <= {"train", "holdout"}
|
||||
header = cases[0].vuln_file.read_text()[:600]
|
||||
vendored = select_vendored(cases) # pointer rows have no excerpt in the repository (Req 10)
|
||||
header = vendored[0].vuln_file.read_text()[:600]
|
||||
assert "Provenance:" in header and "license:" in header
|
||||
result = evaluate_catalog(cases[:2])
|
||||
result = evaluate_catalog(vendored[:2])
|
||||
assert result.per_slice[slice_name].pairs == 2
|
||||
assert set(result.per_profile) <= PROVENANCES and expected_provenance in result.per_profile
|
||||
assert result.per_mechanism and slice_name in result.loss
|
||||
@@ -963,3 +964,24 @@ def test_warm_cache_never_scores_pointer_pairs_when_network_is_off(tmp_path: Pat
|
||||
assert result.outcomes == ()
|
||||
assert {"stage": "pairs", "reason": "pointer_pair_skipped", "pair": "ptr"} in result.degradations
|
||||
assert evaluate_catalog([case], pointers=True).outcomes[0].pair_correct # warm cache, network on: no fetch, scored
|
||||
|
||||
|
||||
def test_vibe_py_and_agent_vfc_carry_pointer_rows_and_the_default_run_skips_them() -> None:
|
||||
"""Req 10.6: LLM-generated Real-Vuln repos are agent/seeded pointer rows; unlicensed agent fixes are title pointer rows."""
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
catalog = load_pair_catalog()
|
||||
vibe = select_slice(catalog, "vibe-py")
|
||||
pointers = [case for case in vibe if not case.vendored]
|
||||
assert len(pointers) >= 40 and len({case.repo for case in pointers}) >= 40
|
||||
assert all(case.provenance == "agent" and case.review_tier == "seeded" and not case.vuln_file.exists() for case in pointers)
|
||||
assert len(select_vendored(vibe)) == 35 and all(case.provenance == "human" for case in select_vendored(vibe))
|
||||
agent_pointers = [case for case in select_slice(catalog, "agent-vfc") if not case.vendored]
|
||||
assert len(agent_pointers) >= 20 and all(case.review_tier == "title" and case.license == "unlicensed" for case in agent_pointers)
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
assert main(["pairs", "--slice", "vibe-py", "--json"]) == 0
|
||||
payload = json.loads(buf.getvalue())
|
||||
skipped = [item for item in payload["degradations"] if item["reason"] == "pointer_pair_skipped"]
|
||||
assert len(skipped) == len(pointers) and payload["overall"]["pairs"] == 35
|
||||
|
||||
Reference in New Issue
Block a user