mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-20 13:39:27 +00:00
feat(pair-corpus-honesty): LLM-generated Real-Vuln repos and unlicensed agent fixes as pointer pairs
Task 8.4 (Req 10.6, 5.1). - harvest.merge_recipes appends recipes by name and never rewrites existing rows. - vibe-py/build_recipes.py --pointers: the 40 LLM-generated repositories without a license become vendored = false rows (80 pairs, agent, seeded) appended to recipes.toml; unlicensed human repos stay skipped. - agent-vfc/build_recipes.py --from ... --pointers [--cache]: unlicensed agent-authored fixes become title-tier pointer rows whose function label is derived from the cached excerpt (30 added from a fresh 662-candidate search); TOML booleans rendered correctly. - catalogs regenerated: vibe-py 115 rows (35 vendored + 80 pointer), agent-vfc 59 (29 + 30); no excerpt is vendored for any pointer row. - `pairs --slice vibe-py --pointers` scores both provenances (human 35, agent 80); the default run skips 80 pairs with pointer_pair_skipped; suite and gates unchanged. README/docs name the nightly command. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
4102404442
commit
3c36f13e03
@@ -131,11 +131,41 @@ def classify(title: str) -> tuple[str, str, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def build(source: Path, *, limit: int, max_lines: int) -> tuple[list[dict[str, object]], dict[str, int]]:
|
||||
def _pair_tools() -> tuple[object, object]:
|
||||
"""The shared harvest library and catalog generator (pointer recipes need a cached excerpt to derive their function)."""
|
||||
import importlib.util
|
||||
|
||||
modules = []
|
||||
for name, path in (("pair_harvest_lib", ROOT.parent / "harvest.py"), ("catalog_gen_lib", ROOT.parent / "catalog_gen.py")):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
modules.append(module)
|
||||
return modules[0], modules[1]
|
||||
|
||||
|
||||
def build(
|
||||
source: Path, *, limit: int, max_lines: int, pointers: bool = False, cache: Path | None = None, known: set[str] | None = None
|
||||
) -> tuple[list[dict[str, object]], dict[str, int]]:
|
||||
"""``pointers=True`` inverts the license rule: only unlicensed repositories are kept, as ``vendored = false`` rows whose
|
||||
excerpts live in ``cache`` (fetched here to derive the function label; Req 10.6). ``known`` commits/repos are skipped."""
|
||||
recipes: list[dict[str, object]] = []
|
||||
skipped = {"noise_title": 0, "unclassified": 0, "unlicensed": 0, "no_code_file": 0, "too_big": 0, "api": 0, "duplicate_repo": 0}
|
||||
skipped = {
|
||||
"noise_title": 0,
|
||||
"unclassified": 0,
|
||||
"unlicensed": 0,
|
||||
"licensed": 0,
|
||||
"no_code_file": 0,
|
||||
"too_big": 0,
|
||||
"api": 0,
|
||||
"duplicate_repo": 0,
|
||||
"harvest": 0,
|
||||
"no_function": 0,
|
||||
}
|
||||
per_repo: dict[str, int] = {}
|
||||
seen_commits: set[str] = set() # the same fix commit shows up in every fork
|
||||
seen_commits: set[str] = set(known or ()) # the same fix commit shows up in every fork
|
||||
harvest, catalog_gen = _pair_tools() if pointers else (None, None)
|
||||
rows = [json.loads(line) for line in source.read_text().splitlines() if line.strip()]
|
||||
for row in sorted(rows, key=lambda r: (r["repo"], r["date"])):
|
||||
if len(recipes) >= limit:
|
||||
@@ -148,7 +178,7 @@ def build(source: Path, *, limit: int, max_lines: int) -> tuple[list[dict[str, o
|
||||
if classified is None:
|
||||
skipped["unclassified"] += 1
|
||||
continue
|
||||
if per_repo.get(row["repo"], 0) >= 2 or row["sha"] in seen_commits:
|
||||
if per_repo.get(row["repo"], 0) >= (1 if pointers else 2) or row["sha"] in seen_commits or row["repo"] in seen_commits:
|
||||
skipped["duplicate_repo"] += 1
|
||||
continue
|
||||
seen_commits.add(row["sha"])
|
||||
@@ -161,7 +191,10 @@ def build(source: Path, *, limit: int, max_lines: int) -> tuple[list[dict[str, o
|
||||
continue
|
||||
assert isinstance(info, dict) and isinstance(commit, dict)
|
||||
license_id = (info.get("license") or {}).get("spdx_id") or ""
|
||||
if license_id not in ACCEPTED_LICENSES:
|
||||
if pointers and license_id in ACCEPTED_LICENSES:
|
||||
skipped["licensed"] += 1 # licensed fixes are vendored by the default mode, not pointed at
|
||||
continue
|
||||
if not pointers and license_id not in ACCEPTED_LICENSES:
|
||||
skipped["unlicensed"] += 1
|
||||
continue
|
||||
files = [f for f in commit.get("files", []) if _CODE.search(f["filename"]) and not _SKIP.search(f["filename"]) and f.get("patch")]
|
||||
@@ -183,33 +216,47 @@ def build(source: Path, *, limit: int, max_lines: int) -> tuple[list[dict[str, o
|
||||
language = {".py": "python", ".ts": "typescript", ".tsx": "typescript"}.get(suffix, "javascript")
|
||||
base = re.sub(r"[^a-z0-9]+", "-", f"{row['repo'].split('/')[-1]}-{Path(target['filename']).stem}".lower()).strip("-")[:56]
|
||||
name = f"{base}-{row['sha'][:6]}"
|
||||
recipe: dict[str, object] = {
|
||||
"name": name,
|
||||
"host": "github",
|
||||
"repo": row["repo"],
|
||||
"parent": parents[0]["sha"],
|
||||
"commit": commit["sha"],
|
||||
"path": target["filename"],
|
||||
"mode": "hunk",
|
||||
"language": language,
|
||||
"relpath": target["filename"],
|
||||
"license": license_id,
|
||||
"cve": "",
|
||||
"year": int(str(commit["commit"]["author"]["date"])[:4]),
|
||||
"cwe": cwe,
|
||||
"class": cls,
|
||||
"mechanism": mechanism,
|
||||
"provenance": "agent",
|
||||
"origin": "github-agent-trailer",
|
||||
"project": row["repo"].split("/")[-1],
|
||||
"trailer": trailer[:120],
|
||||
"reviewer": "pending",
|
||||
"evidence": title[:200],
|
||||
"commit_url": f"https://github.com/{row['repo']}/commit/{commit['sha']}",
|
||||
}
|
||||
if pointers:
|
||||
assert harvest is not None and catalog_gen is not None and cache is not None
|
||||
recipe["license"] = "unlicensed"
|
||||
recipe["vendored"] = False
|
||||
try:
|
||||
harvest.materialize_pointer(recipe, cache, slice_name="agent-vfc") # type: ignore[attr-defined]
|
||||
except Exception as exc: # noqa: BLE001 - a candidate that cannot be fetched is skipped, not fatal
|
||||
skipped["harvest"] += 1
|
||||
print(f"skip {name}: {str(exc)[:100]}")
|
||||
continue
|
||||
function = catalog_gen.derive_function(cache / "agent-vfc", recipe) # type: ignore[attr-defined]
|
||||
if not function:
|
||||
skipped["no_function"] += 1
|
||||
continue
|
||||
recipe["function"] = function
|
||||
per_repo[row["repo"]] = per_repo.get(row["repo"], 0) + 1
|
||||
recipes.append(
|
||||
{
|
||||
"name": name,
|
||||
"host": "github",
|
||||
"repo": row["repo"],
|
||||
"parent": parents[0]["sha"],
|
||||
"commit": commit["sha"],
|
||||
"path": target["filename"],
|
||||
"mode": "hunk",
|
||||
"language": language,
|
||||
"relpath": target["filename"],
|
||||
"license": license_id,
|
||||
"cve": "",
|
||||
"year": int(str(commit["commit"]["author"]["date"])[:4]),
|
||||
"cwe": cwe,
|
||||
"class": cls,
|
||||
"mechanism": mechanism,
|
||||
"provenance": "agent",
|
||||
"origin": "github-agent-trailer",
|
||||
"project": row["repo"].split("/")[-1],
|
||||
"trailer": trailer[:120],
|
||||
"reviewer": "pending",
|
||||
"evidence": title[:200],
|
||||
"commit_url": f"https://github.com/{row['repo']}/commit/{commit['sha']}",
|
||||
}
|
||||
)
|
||||
recipes.append(recipe)
|
||||
for index, recipe in enumerate(sorted(recipes, key=lambda r: str(r["name"]))):
|
||||
recipe["split"] = "holdout" if index % 2 else "train"
|
||||
return recipes, skipped
|
||||
@@ -224,7 +271,9 @@ def to_toml(recipes: list[dict[str, object]]) -> str:
|
||||
for recipe in sorted(recipes, key=lambda r: str(r["name"])):
|
||||
lines.append("[[recipe]]")
|
||||
for key, value in recipe.items():
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
if isinstance(value, bool):
|
||||
lines.append(f"{key} = {'true' if value else 'false'}")
|
||||
elif isinstance(value, int):
|
||||
lines.append(f"{key} = {value}")
|
||||
else:
|
||||
lines.append(f'{key} = "{str(value).replace(chr(92), chr(92) * 2).replace(chr(34), chr(92) + chr(34))}"')
|
||||
@@ -239,11 +288,29 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.add_argument("--limit", type=int, default=40)
|
||||
parser.add_argument("--max-lines", type=int, default=80)
|
||||
parser.add_argument("--out", type=Path, default=ROOT / "recipes.toml")
|
||||
parser.add_argument(
|
||||
"--pointers", action="store_true", help="unlicensed repos only, as vendored = false rows appended to --out (Req 10.6)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache", type=Path, default=None, help="pointer cache root (default: OPENULTRASAST_PAIR_CACHE or ~/.cache/openultrasast/pairs)"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if args.search:
|
||||
print(f"searched {search(args.search)} candidates -> {args.search}")
|
||||
if args.source:
|
||||
recipes, skipped = build(args.source, limit=args.limit, max_lines=args.max_lines)
|
||||
import os
|
||||
import tomllib
|
||||
|
||||
existing = [dict(r) for r in tomllib.loads(args.out.read_text()).get("recipe", [])] if args.out.is_file() else []
|
||||
if args.pointers:
|
||||
cache = args.cache or Path(os.environ.get("OPENULTRASAST_PAIR_CACHE") or Path.home() / ".cache" / "openultrasast" / "pairs")
|
||||
known = {str(r["commit"]) for r in existing} | {str(r["repo"]) for r in existing}
|
||||
new, skipped = build(args.source, limit=args.limit, max_lines=args.max_lines, pointers=True, cache=cache, known=known)
|
||||
harvest, _ = _pair_tools()
|
||||
recipes, added = harvest.merge_recipes(existing, new) # type: ignore[attr-defined]
|
||||
print(f"appended {len(added)} pointer recipes")
|
||||
else:
|
||||
recipes, skipped = build(args.source, limit=args.limit, max_lines=args.max_lines)
|
||||
args.out.write_text(to_toml(recipes), encoding="utf-8")
|
||||
print(f"wrote {args.out} ({len(recipes)} recipes); skipped {skipped}")
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user