mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-19 04:59:28 +00:00
feat(pair-corpus-honesty): shared function naming, harvest library, catalog generator
Tasks 3.1, 3.2 (harvest library) and the Scorer-owned naming helper. - semantic/functions.py: one source of function names for scorer, overlay and catalog generator (declarator wins over parser name; JS/TS bare-param arrows and undeclared methods are <anon>; C++ Class::method resolves to its unqualified tail; Perl sub; statements never name). - benchmarks/pairs/harvest.py: shared library with name / line_range / hunk / enclosing modes, comment-blind masking (triple-quote aware), upstream_start headers, code-safe quoted-only redaction. - benchmarks/pairs/catalog_gen.py: derive function labels and sinks from the declarator at the mapped sink location, CWE-family sink tokens, --prune with refusal when nothing derived, fails loud without the package; vfc/harvest.py and vfc/generate_catalog.py become shims. - tests/test_pair_harvest.py replaces test_vfc_harvest.py. Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
0116f9bea6
commit
071dea4712
@@ -1,104 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild catalog.toml and training/manifest.jsonl from recipes.toml + excerpts."""
|
||||
"""Thin shim: ``python benchmarks/pairs/catalog_gen.py --slice vfc``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tomllib
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
RECIPES = ROOT / "recipes.toml"
|
||||
CATALOG = ROOT / "catalog.toml"
|
||||
TRAINING = ROOT / "training" / "manifest.jsonl"
|
||||
|
||||
LANG_TO_PAIR = {"c": "c_cpp", "cpp": "c_cpp", "perl": "unknown", "javascript": "javascript", "python": "python", "java": "java"}
|
||||
|
||||
|
||||
def excerpt_rel(recipe: dict[str, object]) -> tuple[str, str]:
|
||||
name = str(recipe["name"])
|
||||
ext = Path(str(recipe.get("path", "a.c"))).suffix or ".c"
|
||||
return f"{name}/vuln{ext}", f"{name}/fixed{ext}"
|
||||
|
||||
|
||||
def catalog_text(recipes: list[dict[str, object]]) -> str:
|
||||
lines = [
|
||||
"# Real-world VFC slice. Reviewed isolated functions.",
|
||||
"# Honesty dashboard — not the local 100% CI gate.",
|
||||
"# Do not add these names to LANGUAGE_MANIFESTS.",
|
||||
"# Regenerated by generate_catalog.py from recipes.toml.",
|
||||
"",
|
||||
]
|
||||
for recipe in recipes:
|
||||
vuln, fixed = excerpt_rel(recipe)
|
||||
lang = LANG_TO_PAIR.get(str(recipe.get("language", "c")), "c_cpp")
|
||||
lines += [
|
||||
"[[pair]]",
|
||||
f'name = "{recipe["name"]}"',
|
||||
'slice = "vfc"',
|
||||
f'language = "{lang}"',
|
||||
f'origin = "{recipe.get("origin", "")}"',
|
||||
f'license = "{recipe.get("license", "")}"',
|
||||
f'repo = "{recipe.get("repo", "")}"',
|
||||
f'cve = "{recipe.get("cve", "")}"',
|
||||
f'cwe = "{recipe.get("cwe", "")}"',
|
||||
f'commit = "{recipe.get("commit", "")}"',
|
||||
f'parent = "{recipe.get("parent", "")}"',
|
||||
f'commit_url = "{recipe.get("commit_url", "")}"',
|
||||
f'vuln = "{vuln}"',
|
||||
f'fixed = "{fixed}"',
|
||||
f'relpath = "{recipe.get("relpath", recipe.get("path", ""))}"',
|
||||
"min_recall = 1.0",
|
||||
'fix_policy = "silent"',
|
||||
"",
|
||||
"[[pair.expected]]",
|
||||
f'cwe = "{recipe.get("cwe", "")}"',
|
||||
f'class = "{recipe.get("class", "")}"',
|
||||
f'path = "{recipe.get("relpath", recipe.get("path", ""))}"',
|
||||
]
|
||||
if recipe.get("rule_id"):
|
||||
lines.append(f'rule_id = "{recipe["rule_id"]}"')
|
||||
if recipe.get("sink"):
|
||||
lines.append(f'sink = "{recipe["sink"]}"')
|
||||
if recipe.get("evidence"):
|
||||
evidence = str(recipe["evidence"]).replace('"', '\\"')
|
||||
lines.append(f'evidence = "{evidence}"')
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def training_rows(recipes: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
rows: list[dict[str, object]] = []
|
||||
for recipe in recipes:
|
||||
vuln, fixed = excerpt_rel(recipe)
|
||||
base = {
|
||||
"pair": recipe["name"],
|
||||
"cve": recipe.get("cve", ""),
|
||||
"year": int(recipe.get("year", 0) or 0),
|
||||
"cwe": recipe.get("cwe", ""),
|
||||
"class": recipe.get("class", ""),
|
||||
"project": recipe.get("project", recipe.get("origin", "")),
|
||||
"repo": recipe.get("repo", ""),
|
||||
"license": recipe.get("license", ""),
|
||||
"function": recipe.get("function", ""),
|
||||
"relpath": recipe.get("relpath", recipe.get("path", "")),
|
||||
"commit_url": recipe.get("commit_url", ""),
|
||||
}
|
||||
rows.append({**base, "id": f"{recipe['name']}:vuln", "label": "vuln", "file": vuln, "commit": recipe.get("parent", "")})
|
||||
rows.append({**base, "id": f"{recipe['name']}:fixed", "label": "fixed", "file": fixed, "commit": recipe.get("commit", "")})
|
||||
return rows
|
||||
_LIB = Path(__file__).resolve().parents[1] / "catalog_gen.py"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
recipes = list(tomllib.loads(RECIPES.read_text()).get("recipe", []))
|
||||
CATALOG.write_text(catalog_text(recipes), encoding="utf-8")
|
||||
TRAINING.parent.mkdir(parents=True, exist_ok=True)
|
||||
with TRAINING.open("w", encoding="utf-8") as handle:
|
||||
for row in training_rows(recipes):
|
||||
handle.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
print(f"wrote {CATALOG} ({len(recipes)} pairs)")
|
||||
print(f"wrote {TRAINING} ({len(recipes) * 2} examples)")
|
||||
return 0
|
||||
return int(runpy.run_path(str(_LIB), run_name="catalog_gen_lib")["main"](["--slice", "vfc"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+11
-188
@@ -1,203 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract one function from public parent/fix blobs. Maintainer-only; CI stays offline.
|
||||
"""Thin shim: the shared harvest library lives in ``benchmarks/pairs/harvest.py``.
|
||||
|
||||
Does not clone mozilla-central, gecko-dev, or Chromium. Advisory-only recipes
|
||||
(no parent and commit) are rejected. FixFox is not a fetch source.
|
||||
Kept so existing maintainer commands and tests keep working:
|
||||
``python benchmarks/pairs/vfc/harvest.py --name openssl-cve-2014-0160``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import runpy
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ALLOWED_HOSTS = {
|
||||
"raw.githubusercontent.com",
|
||||
"github.com",
|
||||
"phabricator.services.mozilla.com",
|
||||
"hg.mozilla.org",
|
||||
"hg-edge.mozilla.org",
|
||||
}
|
||||
BLOCKED_TOKENS = ("fixfox", "zenodo.org/record")
|
||||
|
||||
|
||||
class RecipeError(ValueError):
|
||||
"""Invalid harvest recipe."""
|
||||
|
||||
|
||||
def extract_function(source: str, function_name: str) -> str:
|
||||
"""Copy the named function through balanced braces."""
|
||||
needle = function_name
|
||||
idx = 0
|
||||
while True:
|
||||
found = source.find(needle, idx)
|
||||
if found < 0:
|
||||
raise RecipeError(f"function not found: {function_name}")
|
||||
before = source[found - 1] if found else " "
|
||||
after_name = source[found + len(needle) : found + len(needle) + 1]
|
||||
if before.isalnum() or before == "_":
|
||||
idx = found + len(needle)
|
||||
continue
|
||||
rest = source[found + len(needle) :].lstrip()
|
||||
if rest.startswith("(") or rest.startswith("<") or rest.startswith("{") or after_name in "(<{":
|
||||
brace = source.find("{", found)
|
||||
semi = source.find(";", found)
|
||||
if brace >= 0 and (semi < 0 or brace < semi):
|
||||
break
|
||||
idx = found + len(needle)
|
||||
line_start = source.rfind("\n", 0, found) + 1
|
||||
prev_nl = source.rfind("\n", 0, line_start - 1)
|
||||
prev_line = source[prev_nl + 1 : line_start]
|
||||
start = line_start
|
||||
stripped_prev = prev_line.strip()
|
||||
if (
|
||||
stripped_prev
|
||||
and not stripped_prev.startswith(("/", "*", "#", "//"))
|
||||
and "{" not in prev_line
|
||||
and ";" not in prev_line
|
||||
and not stripped_prev.startswith(("sub ", "def ", "class "))
|
||||
):
|
||||
start = prev_nl + 1
|
||||
brace = source.find("{", found)
|
||||
if brace < 0:
|
||||
raise RecipeError(f"no body for function: {function_name}")
|
||||
depth = 0
|
||||
for pos, char in enumerate(source[brace:], brace):
|
||||
if char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return source[start : pos + 1].strip() + "\n"
|
||||
raise RecipeError(f"unbalanced braces for function: {function_name}")
|
||||
|
||||
|
||||
def provenance_header(recipe: dict[str, Any], *, side: str, sha: str) -> str:
|
||||
lines = [
|
||||
f"Provenance: {recipe.get('repo', '')} {recipe.get('function', '')} ({side}).",
|
||||
f"repo: {recipe.get('repo', '')}",
|
||||
f"commit: {sha}",
|
||||
f"parent: {recipe.get('parent', '')}",
|
||||
f"commit_url: {recipe.get('commit_url', '')}",
|
||||
f"cve: {recipe.get('cve', '')}",
|
||||
f"license: {recipe.get('license', '')}",
|
||||
f"function: {recipe.get('function', '')}",
|
||||
f"relpath: {recipe.get('relpath', recipe.get('path', ''))}",
|
||||
]
|
||||
prefix = "//" if str(recipe.get("language", "c")).startswith("c") and str(recipe.get("path", "")).endswith((".cc", ".cpp", ".java", ".js")) else "/*"
|
||||
if prefix == "//":
|
||||
return "\n".join(f"// {line}" for line in lines) + "\n"
|
||||
return "/* " + "\n * ".join(lines) + "\n */\n"
|
||||
|
||||
|
||||
def validate_recipe(recipe: dict[str, Any]) -> None:
|
||||
name = str(recipe.get("name", ""))
|
||||
if any(token in str(recipe).lower() for token in BLOCKED_TOKENS):
|
||||
raise RecipeError(f"{name}: embargoed or blocked dataset")
|
||||
if not recipe.get("parent") or not recipe.get("commit"):
|
||||
raise RecipeError(f"{name}: advisory-only recipe rejected; parent and commit required")
|
||||
if not recipe.get("path") or not recipe.get("function"):
|
||||
raise RecipeError(f"{name}: path and function required")
|
||||
if not recipe.get("license"):
|
||||
raise RecipeError(f"{name}: license required")
|
||||
|
||||
|
||||
def github_raw_url(repo: str, sha: str, path: str) -> str:
|
||||
return f"https://raw.githubusercontent.com/{repo}/{sha}/{path}"
|
||||
|
||||
|
||||
def excerpt_paths(recipe: dict[str, Any], root: Path) -> tuple[Path, Path]:
|
||||
name = str(recipe["name"])
|
||||
ext = Path(str(recipe.get("path", "a.c"))).suffix or ".c"
|
||||
folder = root / name
|
||||
return folder / f"vuln{ext}", folder / f"fixed{ext}"
|
||||
|
||||
|
||||
def write_excerpt(path: Path, recipe: dict[str, Any], *, side: str, sha: str, body: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
header = provenance_header(recipe, side=side, sha=sha)
|
||||
path.write_text(header + "\n" + body, encoding="utf-8")
|
||||
|
||||
|
||||
def harvest_recipe(recipe: dict[str, Any], root: Path) -> tuple[Path, Path]:
|
||||
validate_recipe(recipe)
|
||||
if str(recipe.get("host", "github")) != "github":
|
||||
raise RecipeError(f"{recipe.get('name')}: live fetch currently implemented for host=github only")
|
||||
parent_src = fetch_url(github_raw_url(str(recipe["repo"]), str(recipe["parent"]), str(recipe["path"])))
|
||||
fix_src = fetch_url(github_raw_url(str(recipe["repo"]), str(recipe["commit"]), str(recipe["path"])))
|
||||
function = str(recipe["function"])
|
||||
parent_fn = extract_function(parent_src, function)
|
||||
fix_fn = extract_function(fix_src, function)
|
||||
vuln_path, fixed_path = excerpt_paths(recipe, root)
|
||||
write_excerpt(vuln_path, recipe, side="vuln", sha=str(recipe["parent"]), body=parent_fn)
|
||||
write_excerpt(fixed_path, recipe, side="fixed", sha=str(recipe["commit"]), body=fix_fn)
|
||||
return vuln_path, fixed_path
|
||||
|
||||
|
||||
def fetch_url(url: str, timeout: int = 60) -> str:
|
||||
host = urlparse(url).hostname or ""
|
||||
if host not in ALLOWED_HOSTS:
|
||||
raise RecipeError(f"host not allowed: {host}")
|
||||
request = Request(url, headers={"User-Agent": "OpenUltraSAST-vfc-harvest"})
|
||||
with urlopen(request, timeout=timeout) as response: # noqa: S310 — host allowlisted
|
||||
return response.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def load_recipes(path: Path) -> tuple[dict[str, Any], ...]:
|
||||
payload = tomllib.loads(path.read_text())
|
||||
return tuple(dict(item) for item in payload.get("recipe", []))
|
||||
_LIB = Path(__file__).resolve().parents[1] / "harvest.py"
|
||||
_NS = runpy.run_path(str(_LIB), run_name="pair_harvest_lib")
|
||||
globals().update({key: value for key, value in _NS.items() if not key.startswith("__") and key != "main"})
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--recipes", type=Path, default=Path(__file__).with_name("recipes.toml"))
|
||||
parser.add_argument("--name", help="recipe name to harvest (omit with --all)")
|
||||
parser.add_argument("--all", action="store_true", help="process every recipe")
|
||||
parser.add_argument("--fetch", action="store_true", help="HTTP-fetch and write excerpts; default is dry-run validate")
|
||||
parser.add_argument("--force", action="store_true", help="overwrite existing excerpts")
|
||||
parser.add_argument("--out", type=Path, default=Path(__file__).resolve().parent, help="excerpt root")
|
||||
args = parser.parse_args(argv)
|
||||
loaded = load_recipes(args.recipes)
|
||||
recipes = {str(item["name"]): item for item in loaded}
|
||||
selected: list[dict[str, Any]]
|
||||
if args.all:
|
||||
selected = list(loaded)
|
||||
elif args.name:
|
||||
if args.name not in recipes:
|
||||
print(f"unknown recipe: {args.name}", file=sys.stderr)
|
||||
return 2
|
||||
selected = [recipes[args.name]]
|
||||
else:
|
||||
print("pass --name RECIPE or --all", file=sys.stderr)
|
||||
return 2
|
||||
failures = 0
|
||||
for recipe in selected:
|
||||
name = str(recipe["name"])
|
||||
try:
|
||||
validate_recipe(recipe)
|
||||
except RecipeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
failures += 1
|
||||
continue
|
||||
if not args.fetch:
|
||||
print(f"ok {name} parent={recipe['parent']} commit={recipe['commit']}")
|
||||
continue
|
||||
vuln_path, fixed_path = excerpt_paths(recipe, args.out)
|
||||
if vuln_path.is_file() and fixed_path.is_file() and not args.force:
|
||||
print(f"skip {name} (exists)")
|
||||
continue
|
||||
try:
|
||||
harvest_recipe(recipe, args.out)
|
||||
except RecipeError as exc:
|
||||
print(f"{name}: {exc}", file=sys.stderr)
|
||||
failures += 1
|
||||
continue
|
||||
print(f"wrote {name} -> {vuln_path.parent}")
|
||||
return 1 if failures else 0
|
||||
args = list(sys.argv[1:] if argv is None else argv)
|
||||
if "--slice" not in args:
|
||||
args = ["--slice", "vfc", *args]
|
||||
return int(_NS["main"](args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user