diff --git a/benchmarks/measurements/2026-10-01-decision-engine-labels/labels.json b/benchmarks/measurements/2026-10-01-decision-engine-labels/labels.json new file mode 100644 index 0000000..5da0edb --- /dev/null +++ b/benchmarks/measurements/2026-10-01-decision-engine-labels/labels.json @@ -0,0 +1,477 @@ +{ + "assumed_benign_history": { + "accepted": 969, + "commits_scanned": 36000, + "rejected": { + "bulk change": 278, + "dependency": 2206, + "english": 4251, + "later fix touches its files": 3168, + "near_fix": 4911, + "no changed function": 50, + "no product source": 2368, + "other_language": 2, + "protocol pin": 55, + "reference": 77 + }, + "repositories": 28 + }, + "conditional_negatives": { + "assumed_benign": { + "by_language": { + "go": 18, + "javascript": 2136, + "php": 832, + "python": 1049, + "typescript": 636 + }, + "groups": 27, + "pushes": 969, + "rows": 4671 + }, + "benign_control": { + "by_language": { + "javascript": 7, + "php": 27, + "python": 23, + "typescript": 15 + }, + "groups": 24, + "pushes": 24, + "rows": 72 + } + }, + "created": "2026-10-01", + "excluded_unread": [ + "candidates-v3-php.json", + "freeze-v3-php.json", + "population-v3-php.toml", + "protocol-v3.md", + "research-2026-09-30-php.md" + ], + "floors": { + "per_family": 20, + "pooled": 10, + "unit": "repository groups with positive and with negative pin labels" + }, + "group_merges": { + "advisory": 0, + "alias": 0, + "commit": 0 + }, + "groups": 105, + "labels_sha256": "e71815fffefcc59d6323979eb09abbbcb5b01f9e8570fe3a32a825977a8d2ba4", + "note": "First label snapshot of the learned decision engine (task 4), COUNTS ONLY: rows and repository groups per source, family, language, framework, split and unit. No candidate, path, function, commit or repository name. Built by `ousast learn labels` from src/openultrasast/learn/sources.toml (fail-closed; population v3 excluded unread). verified_pin_labels exclude conditional rows (benign_control, assumed_benign), which need a candidate signal at the tip and none at the base. `model` applies the design's floors to repository groups with positive and with negative pin labels. The full rows (labels_sha256) stay local: they name functions. Rebuilt 2026-10-01 after the fixed-side fix: a pair gives a fixed-side negative only where the fixed side's source declares the labelled function (or a deterministic rename at a distinct fix commit, provenance fixed_side_moved); otherwise it is counted under pair_fixed_side.absent_fixed_side. 108 pair labels lost their negative: 80 seeded pointer pairs whose fixed side is a guard helper of the same commit, and 28 Real-Vuln-Benchmark traps whose fixed side is a sibling handler of the same snapshot (fix_function; 4 of those handlers are labelled positives themselves). No renamed equivalent exists (fixed_side_moved = 0). Supersedes 2026-09-30 for the per-family floors.", + "pair_fixed_side": { + "absent_fixed_side": { + "by_family": { + "access_control": 13, + "config_secrets": 4, + "deserialization": 20, + "injection": 18, + "output_encoding": 16, + "path": 11, + "untrusted_destination": 26 + }, + "labels": 108 + }, + "fixed_side_moved": 0 + }, + "rows": 6201, + "schema": 1, + "skipped": { + "pairs: label names no function": 44, + "pairs: no fixed-side negative (fixed side is another function of the same snapshot)": 108, + "pairs: title tier (sensitivity arm)": 58, + "pairs: unscorable pair": 7, + "population: positive the fix did not change (no negative)": 22 + }, + "sources": [ + "adjudications", + "assumed_benign", + "dev-php", + "pairs", + "population-v1", + "population-v2" + ], + "verified_delta_labels": { + "access_control:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 12, + "positive_rows": 13 + }, + "config_secrets:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 5, + "positive_rows": 18 + }, + "config_secrets:repair": { + "negative_groups": 2, + "negative_rows": 12, + "positive_groups": 0, + "positive_rows": 0 + }, + "deserialization:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 20, + "positive_rows": 20 + }, + "injection:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 42, + "positive_rows": 77 + }, + "injection:repair": { + "negative_groups": 33, + "negative_rows": 48, + "positive_groups": 0, + "positive_rows": 0 + }, + "memory:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 5, + "positive_rows": 173 + }, + "memory:repair": { + "negative_groups": 5, + "negative_rows": 173, + "positive_groups": 0, + "positive_rows": 0 + }, + "output_encoding:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 15, + "positive_rows": 56 + }, + "output_encoding:repair": { + "negative_groups": 1, + "negative_rows": 40, + "positive_groups": 0, + "positive_rows": 0 + }, + "path:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 14, + "positive_rows": 14 + }, + "path:repair": { + "negative_groups": 3, + "negative_rows": 3, + "positive_groups": 0, + "positive_rows": 0 + }, + "unknown:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 2, + "positive_rows": 2 + }, + "unknown:repair": { + "negative_groups": 2, + "negative_rows": 2, + "positive_groups": 0, + "positive_rows": 0 + }, + "untrusted_destination:introduce": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 35, + "positive_rows": 45 + }, + "untrusted_destination:repair": { + "negative_groups": 7, + "negative_rows": 10, + "positive_groups": 0, + "positive_rows": 0 + } + }, + "verified_pin_labels": { + "by_family": { + "access_control": { + "model": "insufficient", + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 12, + "positive_rows": 13 + }, + "config_secrets": { + "model": "insufficient", + "negative_groups": 2, + "negative_rows": 13, + "positive_groups": 5, + "positive_rows": 18 + }, + "deserialization": { + "model": "insufficient", + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 20, + "positive_rows": 20 + }, + "injection": { + "model": "per_family", + "negative_groups": 33, + "negative_rows": 71, + "positive_groups": 44, + "positive_rows": 81 + }, + "memory": { + "model": "insufficient", + "negative_groups": 5, + "negative_rows": 173, + "positive_groups": 5, + "positive_rows": 173 + }, + "output_encoding": { + "model": "insufficient", + "negative_groups": 1, + "negative_rows": 40, + "positive_groups": 15, + "positive_rows": 56 + }, + "path": { + "model": "insufficient", + "negative_groups": 3, + "negative_rows": 3, + "positive_groups": 15, + "positive_rows": 15 + }, + "unknown": { + "model": "insufficient", + "negative_groups": 2, + "negative_rows": 2, + "positive_groups": 2, + "positive_rows": 2 + }, + "untrusted_destination": { + "model": "insufficient", + "negative_groups": 8, + "negative_rows": 26, + "positive_groups": 35, + "positive_rows": 46 + } + }, + "by_framework": { + "django": { + "negative_groups": 1, + "negative_rows": 2, + "positive_groups": 1, + "positive_rows": 2 + }, + "express": { + "negative_groups": 3, + "negative_rows": 6, + "positive_groups": 4, + "positive_rows": 12 + }, + "flask": { + "negative_groups": 2, + "negative_rows": 3, + "positive_groups": 2, + "positive_rows": 3 + }, + "httpx": { + "negative_groups": 3, + "negative_rows": 4, + "positive_groups": 3, + "positive_rows": 6 + }, + "lodash": { + "negative_groups": 5, + "negative_rows": 9, + "positive_groups": 6, + "positive_rows": 15 + }, + "node-http-clients": { + "negative_groups": 4, + "negative_rows": 8, + "positive_groups": 5, + "positive_rows": 9 + }, + "none": { + "negative_groups": 8, + "negative_rows": 25, + "positive_groups": 9, + "positive_rows": 36 + }, + "pyyaml": { + "negative_groups": 3, + "negative_rows": 5, + "positive_groups": 3, + "positive_rows": 6 + }, + "react": { + "negative_groups": 4, + "negative_rows": 4, + "positive_groups": 4, + "positive_rows": 5 + }, + "requests": { + "negative_groups": 5, + "negative_rows": 8, + "positive_groups": 5, + "positive_rows": 10 + }, + "sanitize-html": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 1, + "positive_rows": 1 + }, + "sqlalchemy": { + "negative_groups": 2, + "negative_rows": 2, + "positive_groups": 2, + "positive_rows": 3 + }, + "unknown": { + "negative_groups": 40, + "negative_rows": 275, + "positive_groups": 77, + "positive_rows": 346 + }, + "vue": { + "negative_groups": 3, + "negative_rows": 5, + "positive_groups": 4, + "positive_rows": 6 + }, + "wordpress": { + "negative_groups": 5, + "negative_rows": 8, + "positive_groups": 9, + "positive_rows": 13 + } + }, + "by_language": { + "c_cpp": { + "negative_groups": 5, + "negative_rows": 173, + "positive_groups": 5, + "positive_rows": 173 + }, + "java": { + "negative_groups": 1, + "negative_rows": 40, + "positive_groups": 1, + "positive_rows": 40 + }, + "javascript": { + "negative_groups": 25, + "negative_rows": 30, + "positive_groups": 22, + "positive_rows": 31 + }, + "php": { + "negative_groups": 9, + "negative_rows": 26, + "positive_groups": 13, + "positive_rows": 22 + }, + "python": { + "negative_groups": 10, + "negative_rows": 23, + "positive_groups": 59, + "positive_rows": 132 + }, + "typescript": { + "negative_groups": 7, + "negative_rows": 33, + "positive_groups": 7, + "positive_rows": 23 + }, + "unknown": { + "negative_groups": 1, + "negative_rows": 3, + "positive_groups": 1, + "positive_rows": 3 + } + }, + "by_source": { + "adjudications": { + "negative_groups": 15, + "negative_rows": 40, + "positive_groups": 3, + "positive_rows": 3 + }, + "dev-php": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 3, + "positive_rows": 3 + }, + "pairs": { + "negative_groups": 25, + "negative_rows": 235, + "positive_groups": 74, + "positive_rows": 343 + }, + "population-v1": { + "negative_groups": 11, + "negative_rows": 35, + "positive_groups": 11, + "positive_rows": 50 + }, + "population-v2": { + "negative_groups": 14, + "negative_rows": 18, + "positive_groups": 17, + "positive_rows": 25 + } + }, + "by_split": { + "dev-php": { + "negative_groups": 0, + "negative_rows": 0, + "positive_groups": 3, + "positive_rows": 3 + }, + "holdout": { + "negative_groups": 12, + "negative_rows": 135, + "positive_groups": 63, + "positive_rows": 189 + }, + "population-v1": { + "negative_groups": 11, + "negative_rows": 46, + "positive_groups": 11, + "positive_rows": 51 + }, + "population-v2": { + "negative_groups": 15, + "negative_rows": 47, + "positive_groups": 17, + "positive_rows": 27 + }, + "train": { + "negative_groups": 16, + "negative_rows": 100, + "positive_groups": 65, + "positive_rows": 154 + } + }, + "design_informed": { + "adjudications": { + "negative_groups": 13, + "negative_rows": 29, + "positive_groups": 2, + "positive_rows": 2 + }, + "population-v2": { + "negative_groups": 14, + "negative_rows": 18, + "positive_groups": 17, + "positive_rows": 25 + } + }, + "privileged_positives": 2 + } +} diff --git a/src/openultrasast/learn/cli.py b/src/openultrasast/learn/cli.py index f388f2b..c05a5a9 100644 --- a/src/openultrasast/learn/cli.py +++ b/src/openultrasast/learn/cli.py @@ -211,7 +211,6 @@ def _memory_build(args: argparse.Namespace) -> int: try: build = build_examples( labels, records_by_key(rows), reader, sources=sources, store=store, profile=args.profile, license_of=reader.license, - function_of=reader.function, ) # fmt: skip except ExampleBuildError as exc: print(f"learn memory build: {exc}", file=sys.stderr) diff --git a/src/openultrasast/learn/examples.py b/src/openultrasast/learn/examples.py index f3294c6..c439419 100644 --- a/src/openultrasast/learn/examples.py +++ b/src/openultrasast/learn/examples.py @@ -15,8 +15,11 @@ Memory examples carry **no rationale** (code, signals, roles, family and label o - conditional rows (``benign_control``, ``assumed_benign``) are skipped until their condition is evaluated; - a candidate whose excerpt is empty or whose function is not found is never stored; when more than 10% of a source's recorded candidates have no excerpt the build fails that source (exit 2 in the CLI): an unread clone is - not a small memory. A fixed side whose file was read but no longer declares the labelled function (a pointer - pair whose fix lives in another function) is a label defect, counted apart as ``absent_fixed_side``. + not a small memory. A fixed side whose file was read but no longer declares the labelled function is a label + defect, counted apart as ``absent_fixed_side`` (the label builder emits a fixed-side negative only where the fixed + side declares its candidate function, so this count should stay 0). A pair label names the function its side + declares -- the labelled one, or the renamed one of a ``fixed_side_moved`` negative -- never the catalog's + ``fix_function`` (a different handler of the same snapshot). Pair labels have no commit pin (the corpus holds excerpts); their envelope pin is :func:`label_pin`, a stable 40-hex digest of the pair and its side, so the vulnerable and the fixed side stay distinct rows. @@ -255,13 +258,6 @@ class SideReader: self._pairs = {case.name: case for case in load_pair_catalog(self.root / source.files[0])} return self._pairs - def function(self, label: Mapping[str, Any], pin_role: str) -> str | None: - """The function a pair's fixed side holds when it is not the labelled one (``fix_function``).""" - if label.get("source") != "pairs" or pin_role != "fixed": - return None - case = self._pair_cases().get(str(label.get("source_ref"))) - return str(getattr(case, "fix_function", "") or "") or None - def license(self, label: Mapping[str, Any]) -> str: if label.get("source") != "pairs": return "" diff --git a/src/openultrasast/learn/labels.py b/src/openultrasast/learn/labels.py index 5005641..142f700 100644 --- a/src/openultrasast/learn/labels.py +++ b/src/openultrasast/learn/labels.py @@ -13,7 +13,11 @@ Rules per source (design section 3): protocol assumption, **conditional** on a candidate there having a signal at the tip and none at the base (``benign_control``). A vulnerable-pin candidate matching no site is unlabelled, never negative. - *Pairs*: the labelled function on the vulnerable side (positive) and on the fixed side (negative); unscorable - pairs and the ``title`` tier are excluded (the tier is a sensitivity arm, ``--include-title``). + pairs and the ``title`` tier are excluded (the tier is a sensitivity arm, ``--include-title``). The negative is + emitted only where the fixed side's source declares the function (the excerpt builder's matcher) or a + deterministic rename at a distinct fix commit replaces it (``provenance = fixed_side_moved``); a fixed side that + holds another function of the same snapshot (a guard helper, a sibling handler) gives no negative and is counted + as ``absent_fixed_side``. - *Recipes* (dev-php): the reviewed ``[[known]]`` function at the recipe commit (positive). - *Adjudications*: recorded ``True``/``TP`` verdicts positive (``privileged`` kept), ``False``/``FP`` negative, with the recorded reason as evidence. @@ -41,7 +45,7 @@ from pathlib import Path from typing import Any from ..model.taxonomy import load_families -from ..plane.tasks.repo_facts import _EXCLUDED_DIRS, _NOT_PRODUCT, DECLARATION, GLOBAL, enclosing +from ..plane.tasks.repo_facts import _EXCLUDED_DIRS, _NOT_PRODUCT, DECLARATION, GLOBAL, _declared_name, declared_functions, enclosing from ..preprocess import detect_language from ..ruleset.frameworks import load_frameworks @@ -134,6 +138,10 @@ class SourceGuard: self.files.add(resolved) if source.kind == "pairs": self.trees.add(resolved.parent) + if any(source.kind == "pairs" for source in sources.sources): + from ..pairs import pair_cache_dir + + self.trees.add(pair_cache_dir().resolve()) # pointer pairs: the catalog's sides, materialised outside the tree self.opened: list[Path] = [] def check(self, path: Path) -> Path: @@ -317,6 +325,7 @@ class Label: conditional: str | None = None design_informed: bool = False privileged: bool = False + provenance: str = "" # how a negative was derived when it is not the labelled function itself (``fixed_side_moved``) def _family_of(cwe: str | None, family: str | None) -> str: @@ -432,6 +441,50 @@ def changed_functions(clone: Clone, pin: str, ranges: Mapping[str, Sequence[tupl return out +# --- the fixed side of a pair ---------------------------------------------------------------------------------------- + +FIXED_DECLARED = "declared" +FIXED_MOVED = "fixed_side_moved" +ABSENT_UNREAD = "fixed side unread" +ABSENT_UNMATCHED = "fixed side does not declare the function" +ABSENT_SAME_SNAPSHOT = "fixed side is another function of the same snapshot" +ABSENT_NO_EQUIVALENT = "fixed side does not declare the function, no renamed equivalent" +_HEADER = re.compile(r"^\s*(?:#|//|/\*|\*)") # comment lines: the excerpt header names the labelled function +_PIN_LINE = re.compile(r"^\W*(commit|parent):\s*([0-9a-f]{7,40})\b", re.IGNORECASE) + + +@dataclass(frozen=True) +class FixedSide: + verdict: str + function: str = "" + + +def _fix_commits(case: Any, fixed_lines: Sequence[str]) -> tuple[str, str] | None: + """``(vulnerable, fix)`` commits of a pair: the recipe's ``parent``/``commit`` (pointer pairs) or the excerpt + header's; ``None`` when either is unrecorded (a fix that cannot be told apart from its snapshot).""" + recorded = dict(getattr(case, "recipe", ()) or ()) + header = {m.group(1).lower(): m.group(2) for line in fixed_lines[:20] if (m := _PIN_LINE.match(line))} + parent = str(recorded.get("parent") or header.get("parent") or "") + commit = str(getattr(case, "commit", "") or recorded.get("commit") or header.get("commit") or "") + return (parent.lower(), commit.lower()) if parent and commit else None + + +def _signature(lines: Sequence[str], start: int, function: str) -> str | None: + """The parameter list after ``function`` on its declaration (up to 5 lines), whitespace-normalised.""" + text = "\n".join(lines[start : start + 5]) + at = re.search(rf"(? pair labels with no fixed-side negative def skip(self, reason: str, n: int = 1) -> None: self.skipped[reason] = self.skipped.get(reason, 0) + n @@ -584,8 +638,71 @@ class LabelBuilder: } # fmt: skip self._row(label=1, unit="pin", **common) self._row(label=1, unit="delta", direction="introduce", **common) - self._row(label=0, unit="pin", pin_role="fixed", **common) - self._row(label=0, unit="delta", direction="repair", pin_role="fixed", **common) + fixed = self._fixed_side(case, expected.function) + if fixed.verdict == FIXED_DECLARED: + self._row(label=0, unit="pin", pin_role="fixed", **common) + self._row(label=0, unit="delta", direction="repair", pin_role="fixed", **common) + elif fixed.verdict == FIXED_MOVED: + rename = f"{FIXED_MOVED}: {expected.function} -> {fixed.function} (same file, same signature, body changed)" + moved = { + **common, "candidate": f"{path}::{fixed.function}", "provenance": FIXED_MOVED, + "evidence": f"{common['evidence']}; {rename}", + } # fmt: skip + self._row(label=0, unit="pin", pin_role="fixed", **moved) + self._row(label=0, unit="delta", direction="repair", pin_role="fixed", **moved) + else: + family = str(common["family"]) + self.build.absent_fixed_side[family] = self.build.absent_fixed_side.get(family, 0) + 1 + self.build.skip(f"pairs: no fixed-side negative ({fixed.verdict})") + + def _pair_lines(self, file: Path) -> list[str] | None: + """A pair side's lines through the guard (catalog excerpts, or pointer files in the pair cache); ``None`` unread.""" + try: + return self.guard.text(file).splitlines() if Path(file).is_file() else None + except (OSError, UnicodeDecodeError): + return None + + def _fixed_side(self, case: Any, function: str) -> FixedSide: + """Whether the fixed side gives a defensible negative for ``function`` (Req 2.1: the same function at the fix). + + Verified from the fixed side's source with the excerpt builder's own matcher: the fixed side must declare the + labelled function. Where it does not, the only other negative accepted is a deterministic rename at a real + fix (a fix commit distinct from the vulnerable one): exactly one function the fixed side declares that the + vulnerable side does not, with the labelled function's parameter list and a changed body (``fixed_side_moved``). + A different function of the same snapshot (a guard helper, a sibling handler) is never a negative.""" + from .excerpt import LANGUAGE_ALIASES, function_bounds + + fixed_lines, vuln_lines = self._pair_lines(case.fixed_file), self._pair_lines(case.vuln_file) + if not fixed_lines: + return FixedSide(ABSENT_UNREAD) + language = LANGUAGE_ALIASES.get(case.language, case.language) + if function_bounds(fixed_lines, language, function): + return FixedSide(FIXED_DECLARED) + vuln_span = function_bounds(vuln_lines or (), language, function) + if vuln_span is None: + # the matcher resolves the function on neither side: its limit, not evidence of absence. The fixed side + # still declares it when its name is on a code line (the excerpt header names it in a comment). + named = re.compile(rf"(? dict[str, Any]: for source in sorted({r.source for r in conditional}) }, "assumed_benign_history": build.history, + "pair_fixed_side": { + "absent_fixed_side": { + "labels": sum(build.absent_fixed_side.values()), + "by_family": dict(sorted(build.absent_fixed_side.items())), + }, + "fixed_side_moved": sum(1 for r in pins if r.provenance == FIXED_MOVED), + }, "skipped": dict(sorted(build.skipped.items())), "floors": { "per_family": PER_FAMILY_FLOOR, diff --git a/tests/test_learn_labels.py b/tests/test_learn_labels.py index c133405..d506463 100644 --- a/tests/test_learn_labels.py +++ b/tests/test_learn_labels.py @@ -247,11 +247,34 @@ def test_repository_groups_merge_urls_forks_and_advisories() -> None: assert L.advisories("fix for cve-2024-0001 and GHSA-mh7h-gpmx-fggj") == {"CVE-2024-0001", "GHSA-MH7H-GPMX-FGGJ"} -def _pair(name: str, *, tier: str = "advisory", unscorable: str | None = None, function: str | None = "f", repo: str = "") -> PairCase: +VULN_PY = "def f(q):\n return db.execute('SELECT ' + q)\n" +FIXED_PY = "def f(q):\n return db.execute('SELECT ?', (q,))\n" + + +def _pair( + name: str, + *, + tier: str = "advisory", + unscorable: str | None = None, + function: str | None = "f", + repo: str = "", + root: Path | None = None, + vuln: str = VULN_PY, + fixed: str = FIXED_PY, + parent: str = "a" * 40, + commit: str = "b" * 40, +) -> PairCase: expected = (ExpectedFinding(cwe="CWE-89", vulnerability_class="sqli", path="a.py", evidence="e", function=function),) + folder = (root or Path("/nonexistent")) / "pairs" / name + if root is not None: + folder.mkdir(parents=True, exist_ok=True) + header = f"# Provenance: fixture\n# commit: {commit}\n# parent: {parent}\n# function: {function}\n" + (folder / "vuln.py").write_text(header + vuln) + (folder / "fixed.py").write_text(header + fixed) return PairCase( - name=name, slice="vfc", language="python", origin="o", vuln_file=Path("v"), fixed_file=Path("f"), relpath="a.py", - expected=expected, min_recall=1.0, fix_policy="p", repo=repo, cve="CVE-2024-0001", review_tier=tier, unscorable=unscorable, + name=name, slice="vfc", language="python", origin="o", vuln_file=folder / "vuln.py", fixed_file=folder / "fixed.py", + relpath="a.py", expected=expected, min_recall=1.0, fix_policy="p", repo=repo, commit=commit, cve="CVE-2024-0001", + review_tier=tier, unscorable=unscorable, ) # fmt: skip @@ -262,8 +285,9 @@ def test_pairs_exclude_unscorable_and_title_and_group_by_repository(tmp_path: Pa '[[source]]\nid = "pairs"\nkind = "pairs"\nfile = "pairs/catalog.toml"\nrecorded = "x"\nstatus = "x"\nexclude_tiers = ["title"]\n' ) cases = ( - _pair("ok", repo="https://github.com/o/r"), _pair("twin", unscorable="identical_twin"), _pair("titled", tier="title"), - _pair("nofn", function=None), _pair("fork", repo="https://github.com/x/r-fork"), + _pair("ok", repo="https://github.com/o/r", root=tmp_path), _pair("twin", unscorable="identical_twin"), + _pair("titled", tier="title", root=tmp_path), _pair("nofn", function=None), + _pair("fork", repo="https://github.com/x/r-fork", root=tmp_path), ) # fmt: skip monkeypatch.setattr("openultrasast.pairs.load_pair_catalog", lambda path: cases) build, record = L.build_labels(tmp_path, sources_path=tmp_path / "s.toml", cache=tmp_path, assumed_benign=False) @@ -283,6 +307,78 @@ def test_pairs_exclude_unscorable_and_title_and_group_by_repository(tmp_path: Pa assert "titled" in {r.source_ref for r in titled.labels} +def _pair_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *cases: PairCase) -> tuple[L.Build, dict[str, Any]]: + (tmp_path / "pairs").mkdir(exist_ok=True) + (tmp_path / "pairs" / "catalog.toml").write_text("") + source = '[[source]]\nid = "pairs"\nkind = "pairs"\nfile = "pairs/catalog.toml"\nrecorded = "x"\nstatus = "x"\n' + (tmp_path / "s.toml").write_text(source) + monkeypatch.setattr("openultrasast.pairs.load_pair_catalog", lambda path: cases) + return L.build_labels(tmp_path, sources_path=tmp_path / "s.toml", cache=tmp_path, assumed_benign=False) + + +def _fixed_rows(build: L.Build, name: str) -> list[L.Label]: + return [r for r in build.labels if r.source_ref == name and r.pin_role == "fixed"] + + +GATE = "def _gate(q):\n return bool(q)\n" # a guard helper of the same snapshot, same parameter list as the labelled f + + +def test_a_fixed_side_without_the_function_gives_no_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + snapshot = "c" * 40 + cases = ( + _pair("gate", root=tmp_path, fixed=GATE, parent=snapshot, commit=snapshot), # the kolega-ai-dev pointer shape + _pair("other", root=tmp_path, fixed="def g(a, b):\n return a\n"), # a real fix, but no renamed equivalent + _pair("unread", root=None), + ) + build, record = _pair_build(tmp_path, monkeypatch, *cases) + for name in ("gate", "other", "unread"): + assert _fixed_rows(build, name) == [], name + assert {(r.unit, r.label) for r in build.labels if r.source_ref == name} == {("pin", 1), ("delta", 1)}, "positives stay" + assert record["pair_fixed_side"] == {"absent_fixed_side": {"labels": 3, "by_family": {"injection": 3}}, "fixed_side_moved": 0} + assert record["skipped"] == { + f"pairs: no fixed-side negative ({L.ABSENT_SAME_SNAPSHOT})": 1, + f"pairs: no fixed-side negative ({L.ABSENT_NO_EQUIVALENT})": 1, + f"pairs: no fixed-side negative ({L.ABSENT_UNREAD})": 1, + } + assert record["verified_pin_labels"]["by_family"]["injection"]["negative_groups"] == 0 + + +def test_a_renamed_function_at_a_real_fix_is_a_moved_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + moved = "def f_safe(q):\n return db.execute('SELECT ?', (q,))\n" + build, record = _pair_build(tmp_path, monkeypatch, _pair("moved", root=tmp_path, fixed=moved, repo="https://github.com/o/r")) + rows = _fixed_rows(build, "moved") + assert {(r.unit, r.label, r.candidate, r.provenance) for r in rows} == { + ("pin", 0, "a.py::f_safe", L.FIXED_MOVED), + ("delta", 0, "a.py::f_safe", L.FIXED_MOVED), + } + assert all("f -> f_safe" in r.evidence for r in rows) + assert record["pair_fixed_side"]["fixed_side_moved"] == 1 and record["pair_fixed_side"]["absent_fixed_side"]["labels"] == 0 + # the same rename in one snapshot (no fix commit) is a sibling function, never a negative + snapshot = "d" * 40 + same, _ = _pair_build(tmp_path, monkeypatch, _pair("sibling", root=tmp_path, fixed=moved, parent=snapshot, commit=snapshot)) + assert _fixed_rows(same, "sibling") == [] + # two candidate renames are ambiguous: no negative + two = moved + "\n\ndef f_other(q):\n return q\n" + ambiguous, _ = _pair_build(tmp_path, monkeypatch, _pair("two", root=tmp_path, fixed=two)) + assert _fixed_rows(ambiguous, "two") == [] + + +def test_ordinary_pairs_and_matcher_limits_keep_their_negative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + js_vuln, js_fixed = ( + "module.exports = {\n ps: function(pid) {\n exec('ps ' + pid)\n }\n}\n", + ("module.exports = {\n ps: function(pid) {\n execFile('ps', [pid])\n }\n}\n"), + ) + cache = tmp_path / "pointer-cache" + monkeypatch.setenv("OPENULTRASAST_PAIR_CACHE", str(cache)) + js = _pair("js", root=cache, function="ps", vuln=js_vuln, fixed=js_fixed) + js = PairCase(**{**js.__dict__, "language": "javascript"}) + build, record = _pair_build(tmp_path, monkeypatch, _pair("plain", root=tmp_path), js) + assert {(r.unit, r.candidate, r.provenance) for r in _fixed_rows(build, "plain")} == {("pin", "a.py::f", ""), ("delta", "a.py::f", "")} + assert {r.candidate for r in _fixed_rows(build, "js")} == {"a.py::ps"}, "the pointer file in the pair cache was read" + assert record["pair_fixed_side"]["absent_fixed_side"]["labels"] == 0 + assert record["verified_pin_labels"]["by_family"]["injection"]["negative_rows"] == 2 + + def test_the_snapshot_carries_counts_only(tmp_path: Path) -> None: _, record, _, pins = _build(tmp_path) text = json.dumps(record)