#!/usr/bin/env python3 """Rebuild catalog.toml and training/manifest.jsonl from recipes.toml + excerpts.""" from __future__ import annotations import json import tomllib from pathlib import Path ROOT = Path(__file__).resolve().parent RECIPES = ROOT / "recipes.toml" CATALOG = ROOT / "catalog.toml" TRAINING = ROOT / "training" / "manifest.jsonl" LANG_TO_PAIR = {"c": "c_cpp", "cpp": "c_cpp", "perl": "unknown", "javascript": "javascript", "python": "python", "java": "java"} def excerpt_rel(recipe: dict[str, object]) -> tuple[str, str]: name = str(recipe["name"]) ext = Path(str(recipe.get("path", "a.c"))).suffix or ".c" return f"{name}/vuln{ext}", f"{name}/fixed{ext}" def catalog_text(recipes: list[dict[str, object]]) -> str: lines = [ "# Real-world VFC slice. Reviewed isolated functions.", "# Honesty dashboard — not the local 100% CI gate.", "# Do not add these names to LANGUAGE_MANIFESTS.", "# Regenerated by generate_catalog.py from recipes.toml.", "", ] for recipe in recipes: vuln, fixed = excerpt_rel(recipe) lang = LANG_TO_PAIR.get(str(recipe.get("language", "c")), "c_cpp") lines += [ "[[pair]]", f'name = "{recipe["name"]}"', 'slice = "vfc"', f'language = "{lang}"', f'origin = "{recipe.get("origin", "")}"', f'license = "{recipe.get("license", "")}"', f'repo = "{recipe.get("repo", "")}"', f'cve = "{recipe.get("cve", "")}"', f'cwe = "{recipe.get("cwe", "")}"', f'commit = "{recipe.get("commit", "")}"', f'parent = "{recipe.get("parent", "")}"', f'commit_url = "{recipe.get("commit_url", "")}"', f'vuln = "{vuln}"', f'fixed = "{fixed}"', f'relpath = "{recipe.get("relpath", recipe.get("path", ""))}"', "min_recall = 1.0", 'fix_policy = "silent"', "", "[[pair.expected]]", f'cwe = "{recipe.get("cwe", "")}"', f'class = "{recipe.get("class", "")}"', f'path = "{recipe.get("relpath", recipe.get("path", ""))}"', ] if recipe.get("rule_id"): lines.append(f'rule_id = "{recipe["rule_id"]}"') if recipe.get("sink"): lines.append(f'sink = "{recipe["sink"]}"') if recipe.get("evidence"): evidence = str(recipe["evidence"]).replace('"', '\\"') lines.append(f'evidence = "{evidence}"') lines.append("") return "\n".join(lines) def training_rows(recipes: list[dict[str, object]]) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] for recipe in recipes: vuln, fixed = excerpt_rel(recipe) base = { "pair": recipe["name"], "cve": recipe.get("cve", ""), "year": int(recipe.get("year", 0) or 0), "cwe": recipe.get("cwe", ""), "class": recipe.get("class", ""), "project": recipe.get("project", recipe.get("origin", "")), "repo": recipe.get("repo", ""), "license": recipe.get("license", ""), "function": recipe.get("function", ""), "relpath": recipe.get("relpath", recipe.get("path", "")), "commit_url": recipe.get("commit_url", ""), } rows.append({**base, "id": f"{recipe['name']}:vuln", "label": "vuln", "file": vuln, "commit": recipe.get("parent", "")}) rows.append({**base, "id": f"{recipe['name']}:fixed", "label": "fixed", "file": fixed, "commit": recipe.get("commit", "")}) return rows def main() -> int: recipes = list(tomllib.loads(RECIPES.read_text()).get("recipe", [])) CATALOG.write_text(catalog_text(recipes), encoding="utf-8") TRAINING.parent.mkdir(parents=True, exist_ok=True) with TRAINING.open("w", encoding="utf-8") as handle: for row in training_rows(recipes): handle.write(json.dumps(row, sort_keys=True) + "\n") print(f"wrote {CATALOG} ({len(recipes)} pairs)") print(f"wrote {TRAINING} ({len(recipes) * 2} examples)") return 0 if __name__ == "__main__": raise SystemExit(main())