mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
Marius CiepluchandClaude Opus 5
e219dd553d
Task 4.1. A new excerpt mode keeps the labeled handler with its decorators and every statement outside it that registers the handler by name, so guards living in decorators, routers and middleware travel with the pair. Comment and string mentions never count. The closed mechanism vocabulary gains the three obligation ids, catalog rows accept an additive obligation field validated against the closed operation kinds, and the pair scorer counts a finding tagged with that obligation inside the labeled function as a detection. Rows without the label run the same code as before. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
649 lines
28 KiB
Python
649 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Shared pair harvest library: extract one function from public parent/fix blobs.
|
|
|
|
Maintainer-only; CI stays offline (tests run the extractors on local strings).
|
|
Does not clone mozilla-central, gecko-dev, or Chromium. Advisory-only recipes
|
|
(no parent and commit) are rejected. FixFox is not a fetch source.
|
|
|
|
Modes (recipe ``mode``, default ``name``):
|
|
- ``name`` anchor on the declarator of ``function`` (comment and string mentions ignored)
|
|
- ``line_range`` copy ``line_start..line_end`` from the parent blob and
|
|
``fix_line_start..fix_line_end`` (default: same) from the fix blob
|
|
- ``hunk`` take the function enclosing the first changed hunk on each side
|
|
- ``enclosing`` take the function enclosing ``line`` on the parent blob and
|
|
``fix_line`` (default: same) on the fix blob; ``fix_path`` may differ
|
|
|
|
Languages: c, cpp, java, javascript, typescript (brace-balanced), python (indentation), perl (``sub``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import difflib
|
|
import re
|
|
import sys
|
|
import tomllib
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
ALLOWED_HOSTS = {
|
|
"raw.githubusercontent.com",
|
|
"github.com",
|
|
"phabricator.services.mozilla.com",
|
|
"hg.mozilla.org",
|
|
"hg-edge.mozilla.org",
|
|
}
|
|
BLOCKED_TOKENS = ("fixfox", "zenodo.org/record")
|
|
MODES = ("name", "line_range", "hunk", "enclosing", "handler_context")
|
|
_SUFFIX_LANGUAGE = {
|
|
".c": "c",
|
|
".h": "c",
|
|
".cc": "cpp",
|
|
".cpp": "cpp",
|
|
".hpp": "cpp",
|
|
".java": "java",
|
|
".js": "javascript",
|
|
".mjs": "javascript",
|
|
".cjs": "javascript",
|
|
".ts": "typescript",
|
|
".tsx": "typescript",
|
|
".py": "python",
|
|
".pl": "perl",
|
|
".pm": "perl",
|
|
}
|
|
_HASH_COMMENT = {"python", "perl"}
|
|
_DEF_RE = re.compile(r"^(\s*)(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(")
|
|
|
|
|
|
class RecipeError(ValueError):
|
|
"""Invalid harvest recipe or extraction failure."""
|
|
|
|
|
|
# --- masking -----------------------------------------------------------------
|
|
|
|
|
|
def mask_comments_and_strings(source: str, language: str = "c") -> str:
|
|
"""Return ``source`` with comment and string-literal bodies replaced by spaces.
|
|
|
|
Same length, newlines preserved, so indexes and line numbers line up with the
|
|
original. Braces and identifiers inside comments or strings therefore never
|
|
anchor an extraction.
|
|
"""
|
|
out = list(source)
|
|
n = len(source)
|
|
i = 0
|
|
hash_comments = language in _HASH_COMMENT
|
|
slash_comments = not hash_comments
|
|
while i < n:
|
|
ch = source[i]
|
|
nxt = source[i + 1] if i + 1 < n else ""
|
|
if hash_comments and ch == "#":
|
|
while i < n and source[i] != "\n":
|
|
out[i] = " "
|
|
i += 1
|
|
continue
|
|
if slash_comments and ch == "/" and nxt == "/":
|
|
while i < n and source[i] != "\n":
|
|
out[i] = " "
|
|
i += 1
|
|
continue
|
|
if slash_comments and ch == "/" and nxt == "*":
|
|
out[i] = " "
|
|
out[i + 1] = " "
|
|
i += 2
|
|
while i < n and not (source[i] == "*" and i + 1 < n and source[i + 1] == "/"):
|
|
if source[i] != "\n":
|
|
out[i] = " "
|
|
i += 1
|
|
if i < n:
|
|
out[i] = " "
|
|
if i + 1 < n:
|
|
out[i + 1] = " "
|
|
i += 2
|
|
continue
|
|
if language == "python" and ch in "\"'" and source.startswith(ch * 3, i):
|
|
quote = ch * 3
|
|
for k in range(i, min(i + 3, n)):
|
|
out[k] = " " # blank the delimiters too: a closing triple quote at column 0 is not a dedent
|
|
i += 3
|
|
while i < n and not source.startswith(quote, i):
|
|
if source[i] != "\n":
|
|
out[i] = " "
|
|
i += 1
|
|
for k in range(i, min(i + 3, n)):
|
|
out[k] = " "
|
|
i += 3
|
|
continue
|
|
if ch in "\"'`":
|
|
quote = ch
|
|
i += 1
|
|
while i < n and source[i] != quote:
|
|
if source[i] == "\\" and i + 1 < n:
|
|
out[i] = " "
|
|
if source[i + 1] != "\n":
|
|
out[i + 1] = " "
|
|
i += 2
|
|
continue
|
|
if source[i] == "\n" and quote != "`":
|
|
break # unterminated single-line literal; stop masking at EOL
|
|
if source[i] != "\n":
|
|
out[i] = " "
|
|
i += 1
|
|
i += 1
|
|
continue
|
|
i += 1
|
|
return "".join(out)
|
|
|
|
|
|
# --- extraction --------------------------------------------------------------
|
|
|
|
|
|
def language_for(recipe: dict[str, Any]) -> str:
|
|
language = str(recipe.get("language") or "").lower()
|
|
if language:
|
|
return {"c++": "cpp", "js": "javascript", "ts": "typescript"}.get(language, language)
|
|
return _SUFFIX_LANGUAGE.get(Path(str(recipe.get("path", ""))).suffix.lower(), "c")
|
|
|
|
|
|
def extract_function(source: str, function_name: str, language: str = "c") -> str:
|
|
"""Copy the named function from its declarator through its balanced body."""
|
|
if language == "python":
|
|
return extract_python_block(source, function_name)
|
|
masked = mask_comments_and_strings(source, language)
|
|
needle = function_name
|
|
idx = 0
|
|
while True:
|
|
found = masked.find(needle, idx)
|
|
if found < 0:
|
|
raise RecipeError(f"function not found: {function_name}")
|
|
idx = found + len(needle)
|
|
before = masked[found - 1] if found else " "
|
|
if before.isalnum() or before == "_" or before == ".":
|
|
continue
|
|
rest = masked[idx:].lstrip()
|
|
if not (rest.startswith("(") or rest.startswith("<") or rest.startswith("{")):
|
|
continue
|
|
brace = masked.find("{", found)
|
|
semi = masked.find(";", found)
|
|
if brace < 0 or (0 <= semi < brace):
|
|
continue # prototype / declaration, keep searching
|
|
break
|
|
start = _declarator_start(source, masked, found)
|
|
end = _balanced_end(masked, brace)
|
|
return source[start : end + 1].strip() + "\n"
|
|
|
|
|
|
def extract_python_block(source: str, function_name: str) -> str:
|
|
lines = source.splitlines(keepends=True)
|
|
masked = mask_comments_and_strings(source, "python").splitlines(keepends=True)
|
|
for index, line in enumerate(lines):
|
|
match = _DEF_RE.match(line)
|
|
if match is None or match.group(2) != function_name:
|
|
continue
|
|
indent = len(match.group(1))
|
|
start = index
|
|
while (
|
|
start > 0 and lines[start - 1].lstrip().startswith("@") and (len(lines[start - 1]) - len(lines[start - 1].lstrip())) == indent
|
|
):
|
|
start -= 1 # decorators belong to the function
|
|
end = _python_block_end(masked, index, indent)
|
|
return "".join(lines[start:end]).rstrip("\n") + "\n"
|
|
raise RecipeError(f"function not found: {function_name}")
|
|
|
|
|
|
def extract_handler_context(source: str, function_name: str, language: str = "c") -> str:
|
|
"""The named function (with its decorators) followed by every statement outside it that names the function as an
|
|
identifier: `app.add_url_rule(..., view_func=fn)`, `router.get('/x', fn)`, `path('x', fn)`, `app.use(fn)`. Comment and
|
|
string mentions never count (the masked text is searched). One excerpt per side, so guards living in decorators,
|
|
routers and middleware are visible to the teacher and the scorer (authorization-obligations, Req 7.1)."""
|
|
body = extract_function(source, function_name, language)
|
|
lines = source.splitlines()
|
|
start = _start_line_of(source, body) - 1
|
|
end = start + len(body.splitlines())
|
|
masked_lines = mask_comments_and_strings(source, language).splitlines()
|
|
mention = re.compile(rf"(?<![\w.]){re.escape(function_name)}(?!\w)")
|
|
declaration = re.compile(rf"^\s*(?:async\s+)?(?:def|function|sub)\s+{re.escape(function_name)}\b")
|
|
registrations = [
|
|
lines[index]
|
|
for index, masked in enumerate(masked_lines)
|
|
if not (start <= index < end) and index < len(lines) and mention.search(masked) and not declaration.match(masked)
|
|
]
|
|
if not registrations:
|
|
return body
|
|
return body.rstrip("\n") + "\n\n" + "\n".join(line.strip() for line in registrations) + "\n"
|
|
|
|
|
|
def extract_line_range(source: str, start: int, end: int) -> str:
|
|
"""Inclusive 1-based line range."""
|
|
lines = source.splitlines(keepends=True)
|
|
if start < 1 or end < start or end > len(lines):
|
|
raise RecipeError(f"line range {start}..{end} outside 1..{len(lines)}")
|
|
return "".join(lines[start - 1 : end]).rstrip("\n") + "\n"
|
|
|
|
|
|
def function_bounds_at(source: str, line: int, language: str = "c") -> tuple[int, int] | None:
|
|
"""1-based inclusive (start, end) of the function enclosing ``line``; None when not inside one."""
|
|
lines = source.splitlines(keepends=True)
|
|
if line < 1 or line > len(lines):
|
|
return None
|
|
if language == "python":
|
|
return _python_bounds_at(lines, line)
|
|
masked = mask_comments_and_strings(source, language)
|
|
offsets = _line_offsets(source)
|
|
target = offsets[line - 1]
|
|
depth = 0
|
|
open_at: int | None = None
|
|
for pos, ch in enumerate(masked):
|
|
if ch == "{":
|
|
if depth == 0:
|
|
open_at = pos
|
|
depth += 1
|
|
elif ch == "}":
|
|
depth -= 1
|
|
if depth == 0 and open_at is not None:
|
|
if open_at <= target <= pos or _line_of(offsets, open_at) <= line <= _line_of(offsets, pos):
|
|
start = _declarator_start(source, masked, open_at)
|
|
return _line_of(offsets, start), _line_of(offsets, pos)
|
|
open_at = None
|
|
return None
|
|
|
|
|
|
WHOLE_FILE_MAX_LINES = 150
|
|
|
|
|
|
def extract_hunk(parent_source: str, fix_source: str, language: str = "c") -> tuple[str, str]:
|
|
"""Functions enclosing the first changed hunk that sits inside a function on both sides.
|
|
|
|
Fallback for small modules (npm one-file packages, route files) whose change is at
|
|
module level: when both blobs are at most ``WHOLE_FILE_MAX_LINES`` lines, the whole
|
|
file is the excerpt. Larger module-level changes are a RecipeError.
|
|
"""
|
|
parent_lines = parent_source.splitlines()
|
|
fix_lines = fix_source.splitlines()
|
|
matcher = difflib.SequenceMatcher(a=parent_lines, b=fix_lines, autojunk=False)
|
|
changed = [(tag, i1, i2, j1, j2) for tag, i1, i2, j1, j2 in matcher.get_opcodes() if tag != "equal"]
|
|
if not changed:
|
|
raise RecipeError("parent and fix blobs are identical")
|
|
for _tag, i1, i2, j1, j2 in changed:
|
|
parent_line = i1 + 1 if i2 > i1 else max(i1, 1)
|
|
fix_line = j1 + 1 if j2 > j1 else max(j1, 1)
|
|
parent_bounds = function_bounds_at(parent_source, parent_line, language)
|
|
fix_bounds = function_bounds_at(fix_source, fix_line, language)
|
|
if parent_bounds is not None and fix_bounds is not None:
|
|
return (
|
|
extract_line_range(parent_source, *parent_bounds),
|
|
extract_line_range(fix_source, *fix_bounds),
|
|
)
|
|
if len(parent_lines) <= WHOLE_FILE_MAX_LINES and len(fix_lines) <= WHOLE_FILE_MAX_LINES:
|
|
return parent_source.rstrip("\n") + "\n", fix_source.rstrip("\n") + "\n"
|
|
raise RecipeError("no changed hunk is inside a function on both sides and the file is too large to vendor whole")
|
|
|
|
|
|
def _declarator_start(source: str, masked: str, anchor: int) -> int:
|
|
"""Start offset of the declarator statement that owns ``anchor`` (walk back to the previous ';', '}' or preprocessor line)."""
|
|
pos = anchor - 1
|
|
while pos >= 0:
|
|
ch = masked[pos]
|
|
if ch in ";}":
|
|
break
|
|
if ch == "\n":
|
|
line_start = masked.rfind("\n", 0, pos) + 1
|
|
if masked[line_start:pos].lstrip().startswith("#"):
|
|
break
|
|
pos -= 1
|
|
start = pos + 1
|
|
# Skip blank and comment-only lines between the boundary and the declarator.
|
|
while True:
|
|
line_end = source.find("\n", start)
|
|
if line_end < 0:
|
|
break
|
|
if masked[start:line_end].strip():
|
|
break
|
|
start = line_end + 1
|
|
return start
|
|
|
|
|
|
def _balanced_end(masked: str, brace: int) -> int:
|
|
depth = 0
|
|
for pos in range(brace, len(masked)):
|
|
ch = masked[pos]
|
|
if ch == "{":
|
|
depth += 1
|
|
elif ch == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
return pos
|
|
raise RecipeError("unbalanced braces")
|
|
|
|
|
|
def _line_offsets(source: str) -> list[int]:
|
|
offsets = [0]
|
|
for index, ch in enumerate(source):
|
|
if ch == "\n":
|
|
offsets.append(index + 1)
|
|
return offsets
|
|
|
|
|
|
def _line_of(offsets: list[int], pos: int) -> int:
|
|
low, high = 0, len(offsets) - 1
|
|
while low < high:
|
|
mid = (low + high + 1) // 2
|
|
if offsets[mid] <= pos:
|
|
low = mid
|
|
else:
|
|
high = mid - 1
|
|
return low + 1
|
|
|
|
|
|
def _python_block_end(lines: list[str], def_index: int, indent: int) -> int:
|
|
"""End index of the block starting at ``def_index``. Pass masked lines so the bodies of
|
|
triple-quoted strings (blank after masking) never terminate the block."""
|
|
end = def_index + 1
|
|
last_code = def_index + 1
|
|
while end < len(lines):
|
|
line = lines[end]
|
|
stripped = line.strip()
|
|
if stripped and not stripped.startswith("#"):
|
|
if len(line) - len(line.lstrip()) <= indent:
|
|
break
|
|
last_code = end + 1
|
|
end += 1
|
|
return last_code
|
|
|
|
|
|
def _python_bounds_at(lines: list[str], line: int) -> tuple[int, int] | None:
|
|
index = line - 1
|
|
masked = mask_comments_and_strings("".join(lines), "python").splitlines(keepends=True)
|
|
for def_index in range(index, -1, -1):
|
|
match = _DEF_RE.match(lines[def_index])
|
|
if match is None:
|
|
continue
|
|
indent = len(match.group(1))
|
|
end = _python_block_end(masked, def_index, indent)
|
|
if def_index <= index < end:
|
|
return def_index + 1, end
|
|
return None
|
|
|
|
|
|
# --- recipes and IO ----------------------------------------------------------
|
|
|
|
|
|
def provenance_header(recipe: dict[str, Any], *, side: str, sha: str, upstream_start: int | None = None) -> str:
|
|
lines = [
|
|
f"Provenance: {recipe.get('repo', '')} {recipe.get('function', '')} ({side}).",
|
|
f"repo: {recipe.get('repo', '')}",
|
|
f"commit: {sha}",
|
|
f"parent: {recipe.get('parent', '')}",
|
|
f"commit_url: {recipe.get('commit_url', '')}",
|
|
f"cve: {recipe.get('cve', '')}",
|
|
f"license: {recipe.get('license', '')}",
|
|
f"function: {recipe.get('function', '')}",
|
|
f"relpath: {recipe.get('relpath', recipe.get('path', ''))}",
|
|
f"provenance: {recipe.get('provenance', 'human')}",
|
|
f"mechanism: {recipe.get('mechanism', 'other')}",
|
|
]
|
|
if upstream_start is not None:
|
|
lines.append(f"upstream_start: {upstream_start}") # 1-based line of the excerpt body in the upstream file
|
|
language = language_for(recipe)
|
|
if language in _HASH_COMMENT:
|
|
return "\n".join(f"# {line}" for line in lines) + "\n"
|
|
if language in {"cpp", "java", "javascript", "typescript"}:
|
|
return "\n".join(f"// {line}" for line in lines) + "\n"
|
|
return "/* " + "\n * ".join(lines) + "\n */\n"
|
|
|
|
|
|
def validate_recipe(recipe: dict[str, Any], *, require_license: bool = True) -> None:
|
|
name = str(recipe.get("name", ""))
|
|
if any(token in str(recipe).lower() for token in BLOCKED_TOKENS):
|
|
raise RecipeError(f"{name}: embargoed or blocked dataset")
|
|
if not recipe.get("parent") or not recipe.get("commit"):
|
|
raise RecipeError(f"{name}: advisory-only recipe rejected; parent and commit required")
|
|
mode = str(recipe.get("mode", "name"))
|
|
if mode not in MODES:
|
|
raise RecipeError(f"{name}: unknown mode {mode!r}; expected one of {MODES}")
|
|
if not recipe.get("path"):
|
|
raise RecipeError(f"{name}: path required")
|
|
if mode in {"name", "handler_context"} and not recipe.get("function"):
|
|
raise RecipeError(f"{name}: path and function required")
|
|
if mode == "line_range" and not (isinstance(recipe.get("line_start"), int) and isinstance(recipe.get("line_end"), int)):
|
|
raise RecipeError(f"{name}: line_range mode needs integer line_start and line_end")
|
|
if mode == "enclosing" and not isinstance(recipe.get("line"), int):
|
|
raise RecipeError(f"{name}: enclosing mode needs an integer line")
|
|
if require_license and not recipe.get("license"):
|
|
raise RecipeError(f"{name}: license required")
|
|
|
|
|
|
def github_raw_url(repo: str, sha: str, path: str) -> str:
|
|
return f"https://raw.githubusercontent.com/{repo}/{sha}/{path}"
|
|
|
|
|
|
def excerpt_paths(recipe: dict[str, Any], root: Path) -> tuple[Path, Path]:
|
|
name = str(recipe["name"])
|
|
ext = Path(str(recipe.get("path", "a.c"))).suffix or ".c"
|
|
folder = root / name
|
|
return folder / f"vuln{ext}", folder / f"fixed{ext}"
|
|
|
|
|
|
_TOKEN_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
|
|
(re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.DOTALL), "***REDACTED***"),
|
|
(re.compile(r"(Bearer\s+)[A-Za-z0-9._\-]{12,}"), r"\1***REDACTED***"),
|
|
(re.compile(r"\bsk-[A-Za-z0-9._\-]{16,}"), "***REDACTED***"),
|
|
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "***REDACTED***"),
|
|
(re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), "***REDACTED***"),
|
|
(re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}"), "***REDACTED***"),
|
|
(re.compile(r"\bAIza[A-Za-z0-9_\-]{20,}"), "***REDACTED***"),
|
|
(re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s:/@]+:)[^\s:/@]{3,}(@)"), r"\1***REDACTED***\2"),
|
|
# Generic sensitive assignment: ONLY quoted literal values, so redaction never breaks code
|
|
# (`password = request.form[...]` and `token = getToken()` are code, not secrets).
|
|
(
|
|
re.compile(r"(?i)\b(api[_-]?key|secret|token|password|passwd|pwd|access[_-]?key)\b(\s*[=:]\s*)(['\"])[A-Za-z0-9/+_\-]{8,}\3"),
|
|
r"\1\2\3***REDACTED***\3",
|
|
),
|
|
)
|
|
|
|
|
|
def redact(body: str) -> str:
|
|
"""Mask credential-shaped literals in an excerpt without changing code shape (mirrors openultrasast.redaction, quoted-only generic rule)."""
|
|
for pattern, replacement in _TOKEN_PATTERNS:
|
|
body = pattern.sub(replacement, body)
|
|
return body
|
|
|
|
|
|
def write_excerpt(path: Path, recipe: dict[str, Any], *, side: str, sha: str, body: str, upstream_start: int | None = None) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
header = provenance_header(recipe, side=side, sha=sha, upstream_start=upstream_start)
|
|
path.write_text(header + "\n" + redact(body), encoding="utf-8")
|
|
|
|
|
|
def extract_pair_with_lines(recipe: dict[str, Any], parent_src: str, fix_src: str) -> tuple[tuple[str, int], tuple[str, int]]:
|
|
"""Like ``extract_pair`` but also returns the 1-based upstream start line of each excerpt body."""
|
|
language = language_for(recipe)
|
|
mode = str(recipe.get("mode", "name"))
|
|
if mode == "line_range":
|
|
start, end = int(recipe["line_start"]), int(recipe["line_end"])
|
|
fix_start = int(recipe.get("fix_line_start", start))
|
|
fix_end = int(recipe.get("fix_line_end", end))
|
|
return (extract_line_range(parent_src, start, end), start), (extract_line_range(fix_src, fix_start, fix_end), fix_start)
|
|
if mode == "hunk":
|
|
return _hunk_with_lines(parent_src, fix_src, language)
|
|
if mode == "enclosing":
|
|
line = int(recipe["line"])
|
|
fix_line = int(recipe.get("fix_line", line))
|
|
parent_bounds = function_bounds_at(parent_src, line, language)
|
|
fix_bounds = function_bounds_at(fix_src, fix_line, language)
|
|
if parent_bounds is None or fix_bounds is None:
|
|
raise RecipeError(f"{recipe.get('name')}: line {line}/{fix_line} is not inside a function")
|
|
return (extract_line_range(parent_src, *parent_bounds), parent_bounds[0]), (extract_line_range(fix_src, *fix_bounds), fix_bounds[0])
|
|
function = str(recipe["function"])
|
|
extract = extract_handler_context if mode == "handler_context" else extract_function
|
|
parent_text = extract(parent_src, function, language)
|
|
fix_text = extract(fix_src, function, language)
|
|
return (parent_text, _start_line_of(parent_src, parent_text)), (fix_text, _start_line_of(fix_src, fix_text))
|
|
|
|
|
|
def _start_line_of(source: str, excerpt: str) -> int:
|
|
first = excerpt.splitlines()[0] if excerpt.strip() else ""
|
|
for index, line in enumerate(source.splitlines(), start=1):
|
|
if line.rstrip() == first.rstrip():
|
|
return index
|
|
return 1
|
|
|
|
|
|
def _hunk_with_lines(parent_source: str, fix_source: str, language: str) -> tuple[tuple[str, int], tuple[str, int]]:
|
|
parent_lines = parent_source.splitlines()
|
|
fix_lines = fix_source.splitlines()
|
|
matcher = difflib.SequenceMatcher(a=parent_lines, b=fix_lines, autojunk=False)
|
|
changed = [(tag, i1, i2, j1, j2) for tag, i1, i2, j1, j2 in matcher.get_opcodes() if tag != "equal"]
|
|
if not changed:
|
|
raise RecipeError("parent and fix blobs are identical")
|
|
for _tag, i1, i2, j1, j2 in changed:
|
|
parent_line = i1 + 1 if i2 > i1 else max(i1, 1)
|
|
fix_line = j1 + 1 if j2 > j1 else max(j1, 1)
|
|
parent_bounds = function_bounds_at(parent_source, parent_line, language)
|
|
fix_bounds = function_bounds_at(fix_source, fix_line, language)
|
|
if parent_bounds is not None and fix_bounds is not None:
|
|
return (extract_line_range(parent_source, *parent_bounds), parent_bounds[0]), (
|
|
extract_line_range(fix_source, *fix_bounds),
|
|
fix_bounds[0],
|
|
)
|
|
if len(parent_lines) <= WHOLE_FILE_MAX_LINES and len(fix_lines) <= WHOLE_FILE_MAX_LINES:
|
|
return (parent_source.rstrip("\n") + "\n", 1), (fix_source.rstrip("\n") + "\n", 1)
|
|
raise RecipeError("no changed hunk is inside a function on both sides and the file is too large to vendor whole")
|
|
|
|
|
|
def extract_pair(recipe: dict[str, Any], parent_src: str, fix_src: str) -> tuple[str, str]:
|
|
"""Apply the recipe's mode to two blobs. Pure; unit-tested offline."""
|
|
language = language_for(recipe)
|
|
mode = str(recipe.get("mode", "name"))
|
|
if mode == "line_range":
|
|
start, end = int(recipe["line_start"]), int(recipe["line_end"])
|
|
fix_start = int(recipe.get("fix_line_start", start))
|
|
fix_end = int(recipe.get("fix_line_end", end))
|
|
return extract_line_range(parent_src, start, end), extract_line_range(fix_src, fix_start, fix_end)
|
|
if mode == "hunk":
|
|
return extract_hunk(parent_src, fix_src, language)
|
|
if mode == "enclosing":
|
|
line = int(recipe["line"])
|
|
fix_line = int(recipe.get("fix_line", line))
|
|
parent_bounds = function_bounds_at(parent_src, line, language)
|
|
fix_bounds = function_bounds_at(fix_src, fix_line, language)
|
|
if parent_bounds is None or fix_bounds is None:
|
|
raise RecipeError(f"{recipe.get('name')}: line {line}/{fix_line} is not inside a function")
|
|
return extract_line_range(parent_src, *parent_bounds), extract_line_range(fix_src, *fix_bounds)
|
|
function = str(recipe["function"])
|
|
extract = extract_handler_context if mode == "handler_context" else extract_function
|
|
return extract(parent_src, function, language), extract(fix_src, function, language)
|
|
|
|
|
|
def harvest_recipe(recipe: dict[str, Any], root: Path, *, require_license: bool = True) -> tuple[Path, Path]:
|
|
validate_recipe(recipe, require_license=require_license)
|
|
if str(recipe.get("host", "github")) != "github":
|
|
raise RecipeError(f"{recipe.get('name')}: live fetch currently implemented for host=github only")
|
|
parent_src = fetch_url(github_raw_url(str(recipe["repo"]), str(recipe["parent"]), str(recipe["path"])))
|
|
# fix_path lets a pair take its fixed side from another file of the same snapshot
|
|
# (Real-Vuln-Benchmark false-positive traps are the fixed twins of vulnerable findings).
|
|
fix_src = fetch_url(
|
|
github_raw_url(str(recipe.get("fix_repo", recipe["repo"])), str(recipe["commit"]), str(recipe.get("fix_path", recipe["path"])))
|
|
)
|
|
(parent_fn, parent_start), (fix_fn, fix_start) = extract_pair_with_lines(recipe, parent_src, fix_src)
|
|
vuln_path, fixed_path = excerpt_paths(recipe, root)
|
|
write_excerpt(vuln_path, recipe, side="vuln", sha=str(recipe["parent"]), body=parent_fn, upstream_start=parent_start)
|
|
write_excerpt(fixed_path, recipe, side="fixed", sha=str(recipe["commit"]), body=fix_fn, upstream_start=fix_start)
|
|
return vuln_path, fixed_path
|
|
|
|
|
|
def materialize_pointer(recipe: dict[str, Any], cache_root: Path, *, slice_name: str) -> tuple[Path, Path]:
|
|
"""Harvest a non-vendored pair into ``cache_root/<slice>/<name>/`` (Req 10.2).
|
|
|
|
The cache lives outside the repository; a missing upstream license is allowed here because nothing is redistributed,
|
|
the excerpts stay on the operator's machine. Redaction applies as for vendored excerpts.
|
|
"""
|
|
root = Path(cache_root) / slice_name
|
|
vuln_path, fixed_path = excerpt_paths(recipe, root)
|
|
if vuln_path.is_file() and fixed_path.is_file():
|
|
return vuln_path, fixed_path
|
|
return harvest_recipe(recipe, root, require_license=False)
|
|
|
|
|
|
def fetch_url(url: str, timeout: int = 60) -> str:
|
|
host = urlparse(url).hostname or ""
|
|
if host not in ALLOWED_HOSTS:
|
|
raise RecipeError(f"host not allowed: {host}")
|
|
request = Request(url, headers={"User-Agent": "OpenUltraSAST-pair-harvest"})
|
|
with urlopen(request, timeout=timeout) as response: # noqa: S310 — host allowlisted
|
|
return response.read().decode("utf-8", "replace")
|
|
|
|
|
|
def merge_recipes(existing: list[dict[str, Any]], new: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[str]]:
|
|
"""Append ``new`` recipes whose name is not already present; existing rows are never rewritten (vendored labels stay put)."""
|
|
names = {str(r["name"]) for r in existing}
|
|
added = [r for r in new if str(r["name"]) not in names]
|
|
return [*existing, *added], [str(r["name"]) for r in added]
|
|
|
|
|
|
def load_recipes(path: Path) -> tuple[dict[str, Any], ...]:
|
|
payload = tomllib.loads(path.read_text())
|
|
return tuple(dict(item) for item in payload.get("recipe", []))
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--slice", default="vfc", help="slice directory under benchmarks/pairs (default: vfc)")
|
|
parser.add_argument("--recipes", type=Path, default=None, help="recipes file (default: <slice>/recipes.toml)")
|
|
parser.add_argument("--name", help="recipe name to harvest (omit with --all)")
|
|
parser.add_argument("--all", action="store_true", help="process every recipe")
|
|
parser.add_argument("--prefix", help="process every recipe whose name starts with this prefix")
|
|
parser.add_argument("--fetch", action="store_true", help="HTTP-fetch and write excerpts; default is dry-run validate")
|
|
parser.add_argument("--force", action="store_true", help="overwrite existing excerpts")
|
|
parser.add_argument("--out", type=Path, default=None, help="excerpt root (default: <slice>/)")
|
|
args = parser.parse_args(argv)
|
|
slice_root = ROOT / args.slice
|
|
recipes_path = args.recipes or slice_root / "recipes.toml"
|
|
out = args.out or slice_root
|
|
loaded = load_recipes(recipes_path)
|
|
recipes = {str(item["name"]): item for item in loaded}
|
|
selected: list[dict[str, Any]]
|
|
if args.all:
|
|
selected = list(loaded)
|
|
elif args.prefix:
|
|
selected = [item for item in loaded if str(item["name"]).startswith(args.prefix)]
|
|
elif args.name:
|
|
if args.name not in recipes:
|
|
print(f"unknown recipe: {args.name}", file=sys.stderr)
|
|
return 2
|
|
selected = [recipes[args.name]]
|
|
else:
|
|
print("pass --name RECIPE, --prefix PREFIX, or --all", file=sys.stderr)
|
|
return 2
|
|
failures = 0
|
|
for recipe in selected:
|
|
name = str(recipe["name"])
|
|
try:
|
|
validate_recipe(recipe)
|
|
except RecipeError as exc:
|
|
print(str(exc), file=sys.stderr)
|
|
failures += 1
|
|
continue
|
|
if not args.fetch:
|
|
print(f"ok {name} parent={recipe['parent']} commit={recipe['commit']} mode={recipe.get('mode', 'name')}")
|
|
continue
|
|
vuln_path, fixed_path = excerpt_paths(recipe, out)
|
|
if vuln_path.is_file() and fixed_path.is_file() and not args.force:
|
|
print(f"skip {name} (exists)")
|
|
continue
|
|
try:
|
|
harvest_recipe(recipe, out)
|
|
except (RecipeError, OSError) as exc:
|
|
print(f"{name}: {exc}", file=sys.stderr)
|
|
failures += 1
|
|
continue
|
|
print(f"wrote {name} -> {vuln_path.parent}")
|
|
return 1 if failures else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|