mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-21 14:09:27 +00:00
feat(pair-corpus-honesty): add vibe-py, vfc-js and agent-vfc slices with mechanism vocabulary
Tasks 1.1, 5.1, 5.2, 5.4 and the data half of 5.3. - mechanisms.toml: closed vocabulary v1 (20 ids) every expected row cites. - vibe-py: 35 Real-Vuln-Benchmark pairs (Apache-2.0), human provenance only because the LLM-generated repositories carry no license (escalated). - vfc-js: 17 Node fix-commit pairs harvested upstream (MIT/Apache-2.0), SecBench.js itself is not vendored (no license); sinks derived on 13/17. - agent-vfc: 29 agent-trailer fixes as catalog-candidates.toml plus REVIEW.md; the loaded catalog stays empty until a human reviews them. - datasets.toml and README document every slice and its review status. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
c3e841003c
commit
8ab4207618
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build vibe-py recipes from Real-Vuln-Benchmark ground truth (maintainer, offline once downloaded).
|
||||
|
||||
Download first (network, maintainer only):
|
||||
|
||||
mkdir -p /tmp/realvuln
|
||||
gh api repos/kolega-ai/Real-Vuln-Benchmark/contents/benchmark-manifest.json --jq .content | base64 -d > /tmp/realvuln/manifest.json
|
||||
for r in $(gh api repos/kolega-ai/Real-Vuln-Benchmark/contents/ground-truth --jq '.[] | select(.type=="dir") | .name'); do
|
||||
gh api "repos/kolega-ai/Real-Vuln-Benchmark/contents/ground-truth/$r/ground-truth.json" --jq .content | base64 -d > "/tmp/realvuln/$r.json"
|
||||
done
|
||||
|
||||
Then:
|
||||
|
||||
python benchmarks/pairs/vibe-py/build_recipes.py --ground-truth /tmp/realvuln --licenses /tmp/realvuln/licenses.json
|
||||
python benchmarks/pairs/harvest.py --slice vibe-py --all --fetch
|
||||
python benchmarks/pairs/catalog_gen.py --slice vibe-py
|
||||
|
||||
Pair shape: the vulnerable function (enclosing the labeled sink line) is the vuln side; a
|
||||
false-positive trap function from the same repository is the fixed twin. Repositories
|
||||
without a stated license are skipped (Req 5.4). Every recipe is a candidate until the
|
||||
catalog row is reviewed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
MECHANISM = {
|
||||
"idor": "identity_from_request_body",
|
||||
"broken_access_control": "missing_auth_guard",
|
||||
"mass_assignment": "identity_from_request_body",
|
||||
"path_traversal": "path_join_user_input",
|
||||
"unrestricted_file_upload": "path_join_user_input",
|
||||
"hardcoded_credentials": "secret_in_client",
|
||||
"sensitive_data_exposure": "secret_in_client",
|
||||
"security_misconfiguration": "permissive_default",
|
||||
"insecure_cookie_flags": "permissive_default",
|
||||
"csrf": "permissive_default",
|
||||
"weak_prng": "config_sink_weak_literal",
|
||||
"weak_crypto": "config_sink_weak_literal",
|
||||
"weak_hashing": "config_sink_weak_literal",
|
||||
"sql_injection": "source_reaches_sink",
|
||||
"nosql_injection": "source_reaches_sink",
|
||||
"command_injection": "source_reaches_sink",
|
||||
"code_injection": "source_reaches_sink",
|
||||
"ssti": "source_reaches_sink",
|
||||
"ssrf": "source_reaches_sink",
|
||||
"open_redirect": "source_reaches_sink",
|
||||
"reflected_xss": "source_reaches_sink",
|
||||
"stored_xss": "source_reaches_sink",
|
||||
"xxe": "source_reaches_sink",
|
||||
"insecure_deserialization": "source_reaches_sink",
|
||||
"sensitive_data_in_logs": "secret_in_client",
|
||||
"business_logic_validation_gap": "validation_strength",
|
||||
"user_enumeration": "validation_strength",
|
||||
"missing_rate_limiting": "permissive_default",
|
||||
"missing_rate_limiting_on_auth": "permissive_default",
|
||||
"denial_of_service": "validation_strength",
|
||||
}
|
||||
# Classes an intra-file engine can talk about; the rest are recorded as candidates but not selected by default.
|
||||
DEFAULT_CLASSES = (
|
||||
"sql_injection",
|
||||
"command_injection",
|
||||
"code_injection",
|
||||
"ssti",
|
||||
"ssrf",
|
||||
"path_traversal",
|
||||
"open_redirect",
|
||||
"insecure_deserialization",
|
||||
"idor",
|
||||
"broken_access_control",
|
||||
"hardcoded_credentials",
|
||||
"security_misconfiguration",
|
||||
"reflected_xss",
|
||||
"xxe",
|
||||
"nosql_injection",
|
||||
)
|
||||
ACCEPTED_LICENSES = {"MIT", "Apache-2.0", "BSD-3-Clause", "BSD-2-Clause", "GPL-3.0", "GPL-2.0", "MPL-2.0", "Unlicense", "ISC"}
|
||||
|
||||
|
||||
def _slug(text: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
||||
|
||||
|
||||
def build(
|
||||
ground_truth: Path, licenses: Path, *, per_repo: int, classes: set[str], allow_unlicensed: bool
|
||||
) -> tuple[list[dict[str, object]], dict[str, int]]:
|
||||
manifest = json.loads((ground_truth / "manifest.json").read_text())
|
||||
license_by_repo = json.loads(licenses.read_text()) if licenses.is_file() else {}
|
||||
recipes: list[dict[str, object]] = []
|
||||
skipped = {"unlicensed": 0, "no_trap": 0, "no_function": 0, "class": 0}
|
||||
for repo_id, meta in sorted(manifest["repos"].items()):
|
||||
gt_path = ground_truth / f"{repo_id}.json"
|
||||
if not gt_path.is_file():
|
||||
continue
|
||||
gt = json.loads(gt_path.read_text())
|
||||
info = license_by_repo.get(repo_id, {})
|
||||
license_id = info.get("license") or ""
|
||||
if license_id not in ACCEPTED_LICENSES and not allow_unlicensed:
|
||||
skipped["unlicensed"] += 1
|
||||
continue
|
||||
full = info.get("full") or "/".join(meta["repo_url"].rstrip("/").split("/")[-2:])
|
||||
commit = meta["commit_sha"]
|
||||
provenance = "agent" if meta["authorship"] == "llm_generated" else "human"
|
||||
findings = gt.get("findings", [])
|
||||
traps = [
|
||||
f
|
||||
for f in findings
|
||||
if not f.get("is_vulnerable") and f.get("location", {}).get("function") and f.get("file", "").endswith(".py")
|
||||
]
|
||||
if not traps:
|
||||
skipped["no_trap"] += 1
|
||||
continue
|
||||
taken = 0
|
||||
seen_functions: set[tuple[str, str]] = set()
|
||||
for finding in findings:
|
||||
if taken >= per_repo:
|
||||
break
|
||||
if not finding.get("is_vulnerable"):
|
||||
continue
|
||||
location = finding.get("location", {})
|
||||
function = location.get("function")
|
||||
file = finding.get("file", "")
|
||||
if not function or not file.endswith(".py"):
|
||||
skipped["no_function"] += 1
|
||||
continue
|
||||
vclass = str(finding.get("vulnerability_class", ""))
|
||||
if vclass not in classes:
|
||||
skipped["class"] += 1
|
||||
continue
|
||||
if (file, function) in seen_functions:
|
||||
continue
|
||||
seen_functions.add((file, function))
|
||||
trap = next((t for t in traps if t["file"] == file), traps[0])
|
||||
name = f"{_slug(repo_id.replace('realvuln-', '').replace('vc-', ''))}-{_slug(Path(file).stem)}-{_slug(function)}"[:60]
|
||||
while any(r["name"] == name for r in recipes):
|
||||
name = name[:57] + "-" + str(sum(1 for r in recipes if str(r["name"]).startswith(name[:57])) + 1)
|
||||
recipes.append(
|
||||
{
|
||||
"name": name,
|
||||
"host": "github",
|
||||
"repo": full,
|
||||
"parent": commit,
|
||||
"commit": commit,
|
||||
"path": file,
|
||||
"fix_path": trap["file"],
|
||||
"mode": "enclosing",
|
||||
"line": int(location["start_line"]),
|
||||
"fix_line": int(trap["location"]["start_line"]),
|
||||
"function": function,
|
||||
"language": "python",
|
||||
"relpath": file,
|
||||
"license": license_id or "unlicensed",
|
||||
"cve": str(finding.get("evidence", {}).get("cve_id") or ""),
|
||||
"year": 2026,
|
||||
"cwe": str(finding.get("primary_cwe", "")),
|
||||
"class": vclass,
|
||||
"mechanism": MECHANISM.get(vclass, "other"),
|
||||
"provenance": provenance,
|
||||
"origin": "real-vuln-benchmark",
|
||||
"project": repo_id,
|
||||
"framework": str(meta.get("framework", "")),
|
||||
"gt_id": str(finding.get("id", "")),
|
||||
"trap_id": str(trap.get("id", "")),
|
||||
"acceptable_cwes": list(finding.get("acceptable_cwes", [])),
|
||||
"evidence": str(finding.get("evidence", {}).get("description", ""))[:240].replace("\n", " "),
|
||||
"commit_url": f"https://github.com/{full}/blob/{commit}/{file}#L{int(location['start_line'])}",
|
||||
}
|
||||
)
|
||||
taken += 1
|
||||
# deterministic holdout: every other recipe by sorted name
|
||||
for index, recipe in enumerate(sorted(recipes, key=lambda r: str(r["name"]))):
|
||||
recipe["split"] = "holdout" if index % 2 else "train"
|
||||
return recipes, skipped
|
||||
|
||||
|
||||
def to_toml(recipes: list[dict[str, object]]) -> str:
|
||||
lines = [
|
||||
"# vibe-py harvest recipes generated from Real-Vuln-Benchmark ground truth by build_recipes.py.",
|
||||
"# Vulnerable function = vuln side; a false-positive trap from the same repo = fixed twin.",
|
||||
"# Review each row before trusting it as a label; regenerate with build_recipes.py.",
|
||||
"",
|
||||
]
|
||||
for recipe in sorted(recipes, key=lambda r: str(r["name"])):
|
||||
lines.append("[[recipe]]")
|
||||
for key, value in recipe.items():
|
||||
if isinstance(value, bool):
|
||||
lines.append(f"{key} = {'true' if value else 'false'}")
|
||||
elif isinstance(value, int):
|
||||
lines.append(f"{key} = {value}")
|
||||
elif isinstance(value, list):
|
||||
inner = ", ".join('"' + str(item).replace('"', '\\"') + '"' for item in value)
|
||||
lines.append(f"{key} = [{inner}]")
|
||||
else:
|
||||
lines.append(f'{key} = "{str(value).replace(chr(92), chr(92) * 2).replace(chr(34), chr(92) + chr(34))}"')
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--ground-truth", type=Path, required=True, help="directory with manifest.json and <repo>.json ground truth")
|
||||
parser.add_argument("--licenses", type=Path, default=None, help="json map repo_id -> {full, license}")
|
||||
parser.add_argument("--per-repo", type=int, default=2)
|
||||
parser.add_argument("--classes", default=",".join(DEFAULT_CLASSES))
|
||||
parser.add_argument("--allow-unlicensed", action="store_true", help="include repositories without a stated license (NOT for vendoring)")
|
||||
parser.add_argument("--out", type=Path, default=ROOT / "recipes.toml")
|
||||
args = parser.parse_args(argv)
|
||||
recipes, skipped = build(
|
||||
args.ground_truth,
|
||||
args.licenses or (args.ground_truth / "licenses.json"),
|
||||
per_repo=args.per_repo,
|
||||
classes=set(args.classes.split(",")),
|
||||
allow_unlicensed=args.allow_unlicensed,
|
||||
)
|
||||
args.out.write_text(to_toml(recipes), encoding="utf-8")
|
||||
print(f"wrote {args.out} ({len(recipes)} recipes); skipped {skipped}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user