Open_Ultra_SAST/benchmarks/pairs/datasets.toml

# External vulnerability-fixing-commit (VFC) datasets. Metadata only — these
# dumps are too large / too noisy to vendor. Use them to grow the pair catalog.
# Label accuracy on auto-mined function datasets is often well below 100%
# (DiverseVul sampled ~60% correct "vulnerable" labels). Prefer manually
# vetted sets (CWE-Bench-Java, Vul4J, SVEN) when evolving the harness.

[[dataset]]
name = "OWASP Benchmark Java"
url = "https://github.com/OWASP-Benchmark/BenchmarkJava"
data = "benchmarks/pairs/sast/owasp-java"
level = "file"
languages = ["Java"]
notes = "Official SAST/DAST/IAST scorecard. 2740 labeled true/false cases, 11 CWEs. Score = TPR - FPR (Youden). Vendored slice under sast/owasp-java."

[[dataset]]
name = "OWASP Benchmark Python"
url = "https://github.com/OWASP-Benchmark/BenchmarkPython"
data = "benchmarks/pairs/sast/owasp-python"
level = "file"
languages = ["Python"]
notes = "Python counterpart of OWASP Benchmark (expectedresults-0.1.csv, Flask testcode/). Same true/false scoring protocol. Vendored slice under sast/owasp-python."

[[dataset]]
name = "Juliet Test Suite C/C++"
url = "https://github.com/arichardson/juliet-test-suite-c"
data = "benchmarks/pairs/sast/juliet-c"
level = "function"
languages = ["C/C++"]
notes = "NIST SARD Juliet 1.3. Each case has bad() and goodG2B/goodB2G. CC0. The arichardson mirror is a memory-safety subset; CWE-78 lives in the full NIST zip / Juliet Java."

[[dataset]]
name = "Juliet Test Suite Java"
url = "https://github.com/UnitTestBot/juliet-java-test-suite"
data = "benchmarks/pairs/sast/juliet-java"
level = "function"
languages = ["Java"]
notes = "NIST Juliet Java 1.3, 112 CWEs including CWE-78 and CWE-89. CC0. Vendored slice extracts bad vs goodG2B sinks."

[[dataset]]
name = "CVEfixes"
url = "https://github.com/secureIT-project/CVEfixes"
data = "https://doi.org/10.5281/zenodo.4476563"
level = "commit/file/function"
languages = ["multi"]
notes = "NVD-linked CVE fix commits. v1.0.8: 11873 CVEs, 12107 commits, 4249 projects. CC BY 4.0 data."

[[dataset]]
name = "MoreFixes"
url = "https://github.com/secureIT-project/MoreFixes"
level = "commit/file/method"
languages = ["C/C++", "Java", "Python", "JavaScript"]
notes = "Largest language-independent CVE-fix set (~26617 CVEs, ~31883 commits). Method-level CSVs also on Kaggle."

[[dataset]]
name = "DiverseVul"
url = "https://github.com/wagner-group/diversevul"
level = "function"
languages = ["C/C++"]
notes = "18945 vuln functions / 150 CWEs. Sampled label accuracy ~60%. Do not treat as ground truth without review."

[[dataset]]
name = "PrimeVul"
url = "https://github.com/DLVulDet/PrimeVul"
level = "function"
languages = ["C/C++"]
notes = "Relabeled DiverseVul/CVEFixes-style pairs with stricter commit-function alignment."

[[dataset]]
name = "MegaVul"
url = "https://github.com/Icyrockton/MegaVul"
level = "function"
languages = ["C/C++"]
notes = "Large C/C++ function pair set (vuln vs after-fix)."

[[dataset]]
name = "CWE-Bench-Java"
url = "https://github.com/iris-sast/cwe-bench-java"
data = "benchmarks/pairs/datasets/cwe-bench-java-project_info.csv"
level = "project/commit"
languages = ["Java"]
notes = "120 manually vetted CVEs (CWE-22/78/79/94) with buggy_commit_id and fix_commit_ids. IRIS paper."

[[dataset]]
name = "Vul4J"
url = "https://github.com/tuhh-softsec/vul4j"
level = "project"
languages = ["Java"]
notes = "Reproducible Java vulns with human patches and PoV tests. CC-BY-4.0 dataset."

[[dataset]]
name = "SVEN"
url = "https://github.com/eth-sri/sven"
data = "https://github.com/eth-sri/sven/tree/master/data_train_val"
level = "function"
languages = ["Python", "C/C++"]
notes = "Manually curated before/after functions (CWE-22/78/79/89/...). Train jsonl is the pair source for several vendored cases."

[[dataset]]
name = "vfc_datasets"
url = "https://github.com/UzL-ITS/vfc_datasets"
level = "toolkit"
languages = ["multi"]
notes = "Unified loader over 22 commit-level and 8 function-level VFC datasets (Devign, BigVul, CrossVul, PatchDB, CleanVul, SecVulEval, ...)."

[[dataset]]
name = "GitHub Advisory Database"
url = "https://github.com/github/advisory-database"
level = "advisory"
languages = ["multi"]
notes = "GHSA JSON with affected packages; pair with OSV/NVD for fix commits. Not code itself."

[[dataset]]
name = "OSV"
url = "https://github.com/google/osv.dev"
level = "advisory"
languages = ["multi"]
notes = "Cross-ecosystem advisories with introduced/fixed commit ranges when present."

[[dataset]]
name = "security-patches-dataset"
url = "https://github.com/tqrg/security-patches-dataset"
level = "commit"
languages = ["multi"]
notes = "Ground-truth security patches mined from NVD, CVE Details, OSV."

[[dataset]]
name = "OpenSSL"
url = "https://github.com/openssl/openssl"
level = "commit/function"
languages = ["C"]
notes = "Public git. Seed: CVE-2014-0160 tls1_process_heartbeat. Do not vendor the whole tree; harvest isolated functions."

[[dataset]]
name = "Firefox mozilla-central"
url = "https://hg.mozilla.org/mozilla-central"
data = "https://github.com/mozilla-firefox/firefox"
level = "commit/function"
languages = ["C/C++", "JavaScript"]
notes = "Canonical VCS is hg.mozilla.org; git mirrors are gecko-dev (archived) and mozilla-firefox/firefox. Filter CVEfixes/MoreFixes for gecko-dev. Do not clone mozilla-central. FixFox is embargoed until 2030. Seed: CVE-2020-15667 mar_insert_item."

[[dataset]]
name = "Chromium"
url = "https://github.com/chromium/chromium"
level = "commit/function"
languages = ["C/C++", "JavaScript"]
notes = "Public git mirror of chromium/src. Seed: CVE-2019-5786 FileReaderLoader::ArrayBufferResult. Do not vendor the whole tree."

[[dataset]]
name = "curl"
url = "https://github.com/curl/curl"
level = "commit/function"
languages = ["C"]
notes = "Public git. Seed: CVE-2023-38545 do_SOCKS5. Isolated functions only; do not vendor the tree."

# --- pair-corpus-honesty: web, agent-authored, and LLM-generated corpora ---

[[dataset]]
name = "Real-Vuln-Benchmark"
url = "https://github.com/kolega-ai/Real-Vuln-Benchmark"
data = "benchmarks/pairs/vibe-py"
level = "repo/function/line"
languages = ["Python"]
notes = "Apache-2.0 (v2). 66 pinned Python web repos: 26 human-authored, 40 LLM-generated (Claude, GPT, Kimi) seeded with reviewed vulns. 1896 vulnerable findings + 280 false-positive traps with file, line range, function, primary/acceptable CWEs, severity, authorship. Educational/CTF style, not production. Source of the `vibe-py` slice; traps are the fixed-side twins."

[[dataset]]
name = "SecBench.js"
url = "https://github.com/cristianstaicu/SecBench.js"
level = "package/sink-line"
languages = ["JavaScript"]
notes = "600 real npm vulnerabilities (prototype pollution, ReDoS, command injection, path traversal, code injection) with Jest exploit oracles, fixCommit, fixedVersion, and sinkLocation. No license file: never vendor its test files; harvest function pairs from the upstream package repositories named in fixCommit (`vfc-js` slice) and record each package's own license."

[[dataset]]
name = "AIDev"
url = "https://huggingface.co/datasets/hao-li/AIDev"
level = "pull request"
languages = ["multi"]
notes = "932k agent-authored PRs (Claude Code, Codex, Copilot, Cursor, Devin, Jules) with patches and language; CC-BY-4.0 index, source repos keep their licenses. No security labels. Candidate source for `agent-vfc` together with GitHub commit search on agent co-author trailers."

[[dataset]]
name = "BaxBench"
url = "https://github.com/logic-star-ai/baxbench"
data = "https://huggingface.co/datasets/LogicStar/BaxBench"
level = "scenario/exploit"
languages = ["Python", "JavaScript", "TypeScript", "Go", "Rust", "PHP"]
notes = "MIT. 28 backend scenarios x 14 frameworks (Express, Flask, FastAPI, Django, Nest, ...) with functional tests and executable exploits per CWE. Generated code is not shipped; generate with any model, the exploit harness labels each sample. Deferred slice (`baxgen`) until deep mode can run its exploits."

[[dataset]]
name = "SecurityEval"
url = "https://github.com/s2e-lab/SecurityEval"
level = "prompt/sample"
languages = ["Python"]
notes = "121 prompts over 69 CWEs with Copilot and InCoder outputs. Vibe-textbook only; not web recall."

[[dataset]]
name = "CodeSecEval"
url = "https://arxiv.org/abs/2407.02395"
level = "sample"
languages = ["Python"]
notes = "180 samples, 44 vulnerability types, insecure and secure executable code with tests. Pointer only."

[[dataset]]
name = "CWEval"
url = "https://github.com/Co1lin/CWEval"
level = "task"
languages = ["C", "Python", "JavaScript"]
notes = "Functionality plus security evaluation of LLM-generated code on the same tasks. Pointer only."

[[dataset]]
name = "CyberSecEval insecure code"
url = "https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks"
level = "test case"
languages = ["C", "C++", "C#", "Java", "JavaScript", "PHP", "Python", "Rust"]
notes = "MIT. Labels are DEFINED by its Insecure Code Detector regex/semgrep/weggli rules. Circular for a rule-proposing tool: coverage pointer only, never recall ground truth."