From 8213d3baad7b885db1fa7120b759ac927488809f Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:17:35 +0200 Subject: [PATCH 1/3] unseen-eval group 1: used set (1,029 repositories, every source read), extended guard, usage ledger; record 01 Implemented with Codex; task 1.5 run by the coordinator against the real sources (S3 memory read-only). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + .kiro/specs/unseen-repo-evaluation/tasks.md | 27 +- .../2026-10-04-unseen-01-used-set/record.json | 86 ++++ benchmarks/unseen/README.md | 62 +++ benchmarks/unseen/eligibility.py | 379 ++++++++++++++++++ benchmarks/unseen/ledger.py | 88 ++++ src/openultrasast/plane/memory.py | 22 +- tests/test_independent_population.py | 130 +++++- tests/test_unseen_eligibility.py | 293 ++++++++++++++ tests/test_unseen_ledger.py | 63 +++ 10 files changed, 1134 insertions(+), 19 deletions(-) create mode 100644 benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json create mode 100644 benchmarks/unseen/README.md create mode 100644 benchmarks/unseen/eligibility.py create mode 100644 benchmarks/unseen/ledger.py create mode 100644 tests/test_unseen_eligibility.py create mode 100644 tests/test_unseen_ledger.py diff --git a/.gitignore b/.gitignore index 9dd8031..82df916 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ workspace/ # MkDocs build output (mkdocs build) site/ + +# Unseen evaluation pointers for non-permissive repositories. +benchmarks/unseen/private/ diff --git a/.kiro/specs/unseen-repo-evaluation/tasks.md b/.kiro/specs/unseen-repo-evaluation/tasks.md index 124060e..0ad2cb4 100644 --- a/.kiro/specs/unseen-repo-evaluation/tasks.md +++ b/.kiro/specs/unseen-repo-evaluation/tasks.md @@ -29,8 +29,8 @@ Execution rules: - **Instrument first** (AGENTS.md). A zero, an empty draft or a fast run is treated as an instrument failure until bytes read and wall time prove otherwise. -- [ ] 1. Used set and guard (design 1.2, 6; increment 1) -- [ ] 1.1 Used-set builder +- [x] 1. Used set and guard (design 1.2, 6; increment 1) +- [x] 1.1 Used-set builder - `benchmarks/unseen/eligibility.py`: normalises names (lower case, no `.git`, no trailing slash) and builds the used set from every source of design table 1.2: pair catalogs, recipes and datasets under `benchmarks/pairs/` plus the local pointer-pair manifests as `benchmarks/pairs/catalog_gen.py` reads them; `population-v1.toml` @@ -43,14 +43,14 @@ Execution rules: and trailing-slash variants collapse; stdout and the output JSON contain no name; a v3-shaped file planted in the synthetic tree is not opened (open is patched to fail on it). - _Requirements: 1.1_ -- [ ] 1.2 Canonical resolution, renames and forks +- [x] 1.2 Canonical resolution, renames and forks - `eligibility.py`: `resolve(candidate, api)` with an injected GitHub client returns the canonical `full_name`, former names (redirects) and the fork network (`parent`, `source`). A candidate is ineligible when any of them is in the used set; one candidate per fork network is kept. - Tests with a fake client: a planted used repository, a renamed one and a fork of a used one are each rejected; two forks of one network keep one; the rejection reason is a source label, not a name. - _Requirements: 1.1_ -- [ ] 1.3 Guard extended to pools +- [x] 1.3 Guard extended to pools - `tests/test_independent_population.py`: the sweep becomes a list of (reservation directory, names): `benchmarks/independent/` for populations, `benchmarks/unseen/` for pools (public manifests, plus `benchmarks/unseen/private/*.toml` when present). The sweep also covers `plane/`. New tests: pool and @@ -62,13 +62,13 @@ Execution rules: - Evidence: the guard is green with an empty `benchmarks/unseen/`, and red when a test fixture names a pool repository in `benchmarks/measurements/` (shown once in the record). - _Requirements: 1.2, 1.3, 1.4_ -- [ ] 1.4 Usage ledger +- [x] 1.4 Usage ledger - `benchmarks/unseen/ledger.py`: append-only `usage-pN.jsonl`, one row per use (slice, decision or experiment id, purpose `baseline | exploratory | informed | qualifies`, date, result digest, freeze digest). It refuses a `qualifies` row for a change whose slice already has an `informed` row for it, and refuses to rewrite rows. - `tests/test_unseen_ledger.py`: append, the refusal, a rewritten row detected by a digest chain. - _Requirements: 5.2_ -- [ ] 1.5 Used-set measurement and record 01 **(coordinator, network)** +- [x] 1.5 Used-set measurement and record 01 **(coordinator, network)** - Run `eligibility.py` against the real sources, the S3 memory store included (read only). - Instrument: each source reports files read and bytes; a source with zero rows fails the run unless it is known empty. @@ -319,3 +319,18 @@ Execution rules: false-alarm rates, coverage per state, spend; `exploratory` rows in `usage-p1.jsonl`. - Observable: one committed table of arms on slice 1; G1's table exists. - _Requirements: 3.4, 5.2, 5.3_ + +## Implementation Notes + +- 2026-10-04: Tasks 1.1–1.4 implemented offline and independently reviewed. Task 1.5 remains for the + coordinator; no networked measurement or commit was performed. Command and API contracts: + `benchmarks/unseen/README.md`. Offline evidence: + `benchmarks/measurements/2026-10-04-unseen-01-offline/record.json`. +- Population guard scope is preserved under the maintainer's hard rule: benchmarks/src/tests for + populations, plus plane for pools. Each directory exemption is reservation-specific, including private + pool manifests. Extending population scope to plane exposed 30 existing references; no historical + records were changed or exempted inside the existing population scope. +- S3 inventory opens through `open_store(read_only=True)` because ordinary store construction writes a + Select capability probe. Default store behavior is unchanged; inventory uses LIST/GET only. +- The freeze writer must match the canonical TOML digest encoding documented in the pool README and + exercised by `test_independent_population.py`; final pool discovery does not trust mutable status. diff --git a/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json b/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json new file mode 100644 index 0000000..2554207 --- /dev/null +++ b/benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json @@ -0,0 +1,86 @@ +{ + "count": 1029, + "guard": "green", + "sha256": "5a644a1e93c68a0a35c794e653286d0532b7eb8d757123ccce5f08b11bf9a4a2", + "sources": { + "harvest": { + "bytes": 501031, + "count": 467, + "files": 3, + "known_empty": false, + "rows": 446, + "skipped_values": 446 + }, + "measurements": { + "bytes": 21169903, + "count": 278, + "files": 221, + "known_empty": false, + "rows": 376, + "skipped_values": 681 + }, + "memory_local": { + "bytes": 8084523, + "count": 192, + "files": 776, + "known_empty": false, + "rows": 759, + "skipped_values": 2 + }, + "memory_s3": { + "bytes": 15928676, + "count": 572, + "files": 1499, + "known_empty": false, + "rows": 1490, + "skipped_values": 2 + }, + "pairs": { + "bytes": 1680480, + "count": 728, + "files": 28, + "known_empty": false, + "rows": 900, + "skipped_values": 1260 + }, + "pointers": { + "bytes": 892448, + "count": 537, + "files": 7, + "known_empty": false, + "rows": 586, + "skipped_values": 666 + }, + "populations": { + "bytes": 32431, + "count": 28, + "files": 2, + "known_empty": false, + "rows": 28, + "skipped_values": 0 + }, + "sweep": { + "bytes": 34006439, + "count": 1029, + "files": 1619, + "known_empty": false, + "rows": 2155, + "skipped_values": 0 + }, + "units": { + "bytes": 319641, + "count": 229, + "files": 9, + "known_empty": false, + "rows": 642, + "skipped_values": 0 + } + }, + "test_exit_code": 0, + "tests": { + "errors": 0, + "failures": 0, + "skipped": 1, + "tests": 41 + } +} diff --git a/benchmarks/unseen/README.md b/benchmarks/unseen/README.md new file mode 100644 index 0000000..fdc1ff8 --- /dev/null +++ b/benchmarks/unseen/README.md @@ -0,0 +1,62 @@ +# Unseen evaluation, group 1 + +The coordinator runs task 1.5 from the repository root: + +```sh +export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH +export PYTHONPATH=$PWD/src +python -m benchmarks.unseen.eligibility --s3-store s3:// --verify-tests --output benchmarks/measurements/2026-10-04-unseen-01-used-set/record.json +``` + +This is the networked command; implementation tests use fakes. It loads `.env` with +`openultrasast.config.load_dotenv` (exported values win), opens the local FileStore and +S3 MemoryStore through `openultrasast.plane.memory.open_store(..., read_only=True)`, +and uses `GH_TOKEN` for GitHub GET resolution. `s3://` selects `S3_BUCKET`; pass +`--s3-store s3://BUCKET/PREFIX` for a prefixed store and `--local-store file:///PATH` +for a different local store. Read-only S3 opening skips the normal write/Select probe; +collection uses LIST/GET only. Existing normal store opening is unchanged. + +Pair catalogs/recipes include local pointer rows (`vendored = false`, including +`fix_repo`). Additional external manifests can be supplied with repeatable +`--pointer-manifest PATH`. Registered units in memory and experiment YAML are followed; +missing units with a registered digest fail. Plans without frozen units are not used data. +Experiment units identify repositories in `group`. Binary measurement artifacts are +read and counted; structured JSON, TOML and multi-document YAML are parsed strictly. + +Each source records files/objects read, bytes read, repository-bearing rows (distinct +names per input), and distinct repository counts. The digest is SHA-256 of sorted, +normalized names joined with newlines. Source counts overlap. An empty source fails; +only after establishing it is empty may the coordinator pass `--known-empty LABEL`. +The output records that declaration. Missing/unreadable inputs still fail. + +GitHub redirects add the requested and canonical names, plus parent/source fork +identities. Used names are also resolved so an old used name excludes a renamed +candidate. GitHub cannot enumerate all historical names: injected clients may supply +`former_names`; actual historical aliases come from used references and redirects. +A 404 for a historical/synthetic used identity retains its literal identity; other +API failures stop collection. Candidate resolution always fails closed on a 404. +`resolve(candidate, api)` and `rejection(resolution, used, selected_networks)` are +available to the later sourcing task. Names stay in memory; output contains only +counts, labels and SHA-256 digests. Output files are created exclusively, never overwritten. + +`--verify-tests` runs the three group-1 test modules with both output streams discarded, +including the existing population guard. Only aggregate test counts and the exit code +enter the record; a failed test prevents the record. V3 paths are excluded before +eligibility reads; only the existing guard handles v3 inside its black-box test process. + +Population reservations retain their existing sweep over `benchmarks/`, `src/`, and +`tests/`. Pool reservations also cover `plane/`. Each reservation exempts only its own +directory; population checks include all of `benchmarks/unseen/private/`. This preserves +the existing population behavior while adding pools (the hard rule for this increment). + +All `pool-pN.toml` files require a matching `freeze-pN.json`; drafts use `draft-pN.toml`. +The freeze digest is SHA-256 of parsed public TOML encoded as UTF-8 JSON with sorted +keys, compact separators and `ensure_ascii=False`. Private entries must be represented +by opaque digests in that public manifest, as task 3.2 requires. + +The ledger exposes `append(path, row)` and `read(path)`. Rows contain `slice`, `decision` +(an opaque decision or experiment label), `purpose`, ISO `date`, `result_digest`, and +`freeze_digest`. Appends lock and validate the existing chain, reject duplicates or a +changed freeze, and reject qualification of an informed slice for the same decision. +Keep the published final digest to authenticate history: the chain detects row edits, +but cannot independently prove that an entire history was not replaced or truncated. diff --git a/benchmarks/unseen/eligibility.py b/benchmarks/unseen/eligibility.py new file mode 100644 index 0000000..ec1186f --- /dev/null +++ b/benchmarks/unseen/eligibility.py @@ -0,0 +1,379 @@ +"""Read-only used-set collection. Only counts, source labels and digests leave this module's CLI. + +Run from the repository root: python -m benchmarks.unseen.eligibility --help. +Population v3 is exclusively the existing guard's responsibility. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tomllib +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import urlsplit + +import yaml + +from openultrasast.plane.memory import MemoryStore + +URL = re.compile(r"github\.com/([\w.-]+/[\w.-]+)", re.I) +NAME = re.compile(r"[\w.-]+/[\w.-]+") +SOURCES = ("pairs", "pointers", "populations", "harvest", "units", "measurements", "sweep") + + +def normalize(value: str) -> str: + text = value.strip().lower().rstrip("/") + if NAME.fullmatch(text): + name = text.removesuffix(".git") + else: + # Structured fields must be identities, not prose containing a URL. + if any(char.isspace() for char in text): + raise ValueError("invalid_repository") + url = urlsplit(text if "://" in text else "https://" + text) + if url.scheme not in {"http", "https", "ssh", "git"} or not url.hostname or "." not in url.hostname: + raise ValueError("invalid_repository") + parts = url.path.strip("/").split("/") + if len(parts) < 2: + raise ValueError("invalid_repository") + name = "/".join(parts[:2]).removesuffix(".git") + if not NAME.fullmatch(name): + raise ValueError("invalid_repository") + if url.hostname != "github.com": + return url.hostname + "/" + name + if not NAME.fullmatch(name) or any(part in {".", ".."} for part in name.split("/")): + raise ValueError("invalid_repository") + return name + + +def excluded(path: Path) -> bool: + return any("v3" in part.lower() or part in {".git", "__pycache__", ".venv"} for part in path.parts) + + +def repositories(value: object, counts: dict[str, int] | None = None) -> set[str]: + """Structured repo fields include bare owner/name pointers; URLs also occur in prose.""" + found = set() + if isinstance(value, dict): + for key, item in value.items(): + if key in {"repo", "fix_repo", "repository", "full_name", "origin"} or (key == "group" and "unit" in value): + try: + if not isinstance(item, str): + raise ValueError("invalid_repository") + found.add(normalize(item)) + except ValueError: + if counts is not None: + counts["skipped_values"] = counts.get("skipped_values", 0) + 1 + else: + found.update(repositories(item, counts)) + elif isinstance(value, list): + for item in value: + found.update(repositories(item, counts)) + elif isinstance(value, str): + found.update(normalize(m[1]) for m in URL.finditer(value)) + return found + + +def pointer_repositories(value: object, counts: dict[str, int] | None = None) -> set[str]: + if isinstance(value, dict): + if value.get("vendored") is False: + return repositories(value, counts) + return set().union(*(pointer_repositories(v, counts) for v in value.values())) + if isinstance(value, list): + return set().union(*(pointer_repositories(v, counts) for v in value)) + return set() + + +def parse(path: Path, raw: bytes) -> object: + structured = path.suffix in {".toml", ".json", ".jsonl", ".yaml", ".yml"} + text = raw.decode("utf-8", errors="strict" if structured else "ignore") + if path.suffix == ".toml": + return tomllib.loads(text) + if path.suffix == ".json": + return json.loads(text) + if path.suffix == ".jsonl": + return [json.loads(line) for line in text.splitlines() if line.strip()] + if path.suffix in {".yaml", ".yml"}: + documents = list(yaml.safe_load_all(text)) + return documents[0] if len(documents) == 1 else documents + return text + + +@dataclass +class UsedSet: + sources: dict[str, set[str]] = field(default_factory=lambda: {label: set() for label in SOURCES}) + instrument: dict[str, dict[str, int]] = field(default_factory=dict) + + def add(self, label: str, names: set[str], size: int, files: int = 1, skipped_values: int = 0) -> None: + self.sources.setdefault(label, set()).update(names) + counts = self.instrument.setdefault(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0}) + counts["files"] += files + counts["bytes"] += size + counts["rows"] += len(names) + counts["skipped_values"] += skipped_values + + def report(self, known_empty: set[str] | None = None) -> dict: + known_empty = known_empty or set() + if known_empty - self.sources.keys(): + raise ValueError("unknown_source_label") + records = {} + for label, names in self.sources.items(): + counts = self.instrument.get(label, {"files": 0, "bytes": 0, "rows": 0, "skipped_values": 0}) + if not counts["rows"] and label not in known_empty: + raise ValueError("empty_source:" + label) + records[label] = {**counts, "count": len(names), "known_empty": label in known_empty} + names = set().union(*self.sources.values()) + return {"sources": records, "count": len(names), "sha256": hashlib.sha256("\n".join(sorted(names)).encode()).hexdigest()} + + +def build_used(root: Path, *, stores: Mapping[str, MemoryStore] | None = None, pointer_manifests: Iterable[Path] = ()) -> UsedSet: + used = UsedSet() + + def read(label: str, paths: Iterable[Path], structured: bool = True) -> list[object]: + values = [] + for path in sorted(set(paths)): + if excluded(path): + continue + raw = path.read_bytes() # failure is an instrument failure, never a quiet empty source + value = parse(path, raw) if structured else raw.decode("utf-8", errors="ignore") + counts: dict[str, int] = {} + names = repositories(value, counts) + used.add(label, names, len(raw), skipped_values=counts.get("skipped_values", 0)) + if label == "pairs": + pointer_counts: dict[str, int] = {} + pointers = pointer_repositories(value, pointer_counts) + if pointers or pointer_counts: + used.add("pointers", pointers, len(raw), skipped_values=pointer_counts.get("skipped_values", 0)) + values.append(value) + return values + + pairs = root / "benchmarks/pairs" + read("pairs", (p for p in pairs.rglob("*") if p.is_file() and p.suffix in {".toml", ".json", ".jsonl", ".csv", ".yaml", ".yml"})) + read("pointers", pointer_manifests) + read("populations", (p for p in (root / "benchmarks/independent").glob("population-v[12].toml"))) + read("harvest", (p for directory in pairs.glob("advisory-fixes*") for p in directory.rglob("*.toml"))) + units = set((root / "plane/experiments").glob("*.units.jsonl")) + for path in (root / "plane/experiments").glob("*.y*ml"): + if excluded(path): + continue + raw = path.read_bytes() + manifest = parse(path, raw) + counts = {} + names = repositories(manifest, counts) + used.add("units", names, len(raw), skipped_values=counts.get("skipped_values", 0)) + if isinstance(manifest, dict) and manifest.get("units_file"): + declared = root / Path(manifest["units_file"]).expanduser() + if declared.exists(): # an unregistered plan may not have frozen units yet + units.add(declared) + for label, store in (stores or {}).items(): + if label not in {"local", "s3"}: + raise ValueError("unknown_store_label") + source = "memory_" + label + used.add(source, set(), 0, 0) + # Read actual objects, not a serialized row size masquerading as bytes read. + for key in store._keys("repos/"): + if not key.endswith(".jsonl") or excluded(Path(key)): + continue + result = store._get(key) + if result is None: + raise ValueError("missing_memory_object") + raw = result[0] + rows = parse(Path("rows.jsonl"), raw) + names = set() + counts = {} + for row in rows: + if row.get("kind") == "example" and "repo" in row: + names.update(repositories({"repo": row["repo"]}, counts)) + if row.get("kind") == "experiment" and row.get("units_file"): + registered = root / Path(row["units_file"]).expanduser() + if row.get("units_digest") or registered.exists(): + units.add(registered) + used.add(source, names, len(raw), skipped_values=counts.get("skipped_values", 0)) + read("units", units) + read("measurements", (p for top in ("measurements", "experiments") for p in (root / "benchmarks" / top).rglob("*") if p.is_file())) + read( + "sweep", + ( + p + for top in ("benchmarks", "src", "tests", "plane", ".kiro") + for p in (root / top).rglob("*") + if p.is_file() and not p.is_relative_to(root / "benchmarks/unseen") and not excluded(p) + ), + structured=False, + ) + return used + + +@dataclass(frozen=True) +class Resolution: + full_name: str + names: frozenset[str] + network: str + + +def resolve(candidate: str, api) -> Resolution: + name = normalize(candidate) + if name.count("/") > 1: + return Resolution(name, frozenset({name}), name) + data = api.get_repo(name) + canonical = normalize(data["full_name"]) + names = {name, canonical, *(normalize(n) for n in data.get("former_names", []))} + for key in ("parent", "source"): + if data.get(key): + names.add(normalize(data[key]["full_name"])) + network = normalize((data.get("source") or data.get("parent") or data)["full_name"]) + return Resolution(canonical, frozenset(names), network) + + +def canonicalize_used(used: UsedSet, api) -> None: + """Resolve old used names too: GitHub exposes redirects, not a complete rename history.""" + cache = {} + for names in used.sources.values(): + for name in sorted(names): + if name not in cache: + try: + cache[name] = resolve(name, api) + except RepositoryNotFound: + cache[name] = Resolution(name, frozenset({name}), name) + names.update(cache[name].names) + + +def rejection(candidate: Resolution, used: UsedSet, selected: set[str]) -> str | None: + for label, names in used.sources.items(): + if candidate.names & names: + return label + if candidate.network in selected: + return "fork_network" + selected.add(candidate.network) + return None + + +class RepositoryNotFound(ValueError): + """A historical or synthetic used identity no longer resolves; retain it verbatim.""" + + +class GitHub: + """Coordinator-only GET client. Exceptions never disclose request URLs or tokens.""" + + def __init__(self, token: str): + if not token: + raise ValueError("missing_gh_token") + self.token = token + + def get_repo(self, name: str) -> dict: + from urllib.error import HTTPError + from urllib.request import Request, urlopen + + request = Request( + "https://api.github.com/repos/" + normalize(name), + headers={ + "Authorization": "Bearer " + self.token, + "Accept": "application/vnd.github+json", + }, + ) + try: + with urlopen(request, timeout=30) as response: + return json.load(response) + except HTTPError as exc: + if exc.code == 404: + raise RepositoryNotFound("github_not_found") from None + raise ValueError("github_resolution_failed") from None + except Exception: + raise ValueError("github_resolution_failed") from None + + +def verify_tests(root: Path) -> dict: + """The coordinator runs the guard as a black box; only exit codes and counts escape.""" + import subprocess + import sys + import tempfile + from xml.etree import ElementTree + + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "tests.xml" + done = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + "tests/test_unseen_eligibility.py", + "tests/test_unseen_ledger.py", + "tests/test_independent_population.py", + "--junitxml=" + str(report), + ], + cwd=root, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if done.returncode != 0: + raise ValueError("guard_or_tests_failed") + suites = ElementTree.parse(report).getroot().iter("testsuite") + counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")} + for suite in suites: + for key in counts: + counts[key] += int(suite.attrib[key]) + if not counts["tests"]: + raise ValueError("empty_test_run") + return {"guard": "green", "test_exit_code": done.returncode, "tests": counts} + + +def main(argv: list[str] | None = None) -> int: + import os + + from openultrasast import config + from openultrasast.plane import memory + from openultrasast.plane.reconciler import results_root + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--local-store", default=None) + parser.add_argument("--s3-store", default="s3://") + parser.add_argument("--pointer-manifest", type=Path, action="append", default=[]) + parser.add_argument("--known-empty", choices=(*SOURCES, "memory_local", "memory_s3"), action="append", default=[]) + parser.add_argument("--output", type=Path) + parser.add_argument("--verify-tests", action="store_true") + args = parser.parse_args(argv) + step = "open_stores" + try: + config.load_dotenv(args.root / ".env") + stores = { + "local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True), + "s3": memory.open_store(args.s3_store, read_only=True), + } + step = "build_used" + used = build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest) + step = "report" + used.report(set(args.known_empty)) # prove inputs before API work + step = "canonicalize" + canonicalize_used(used, GitHub(os.environ.get("GH_TOKEN", ""))) + step = "report" + report = used.report(set(args.known_empty)) + if args.verify_tests: + step = "verify_tests" + report.update(verify_tests(args.root)) + step = "report" + text = json.dumps(report, sort_keys=True, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as handle: + handle.write(text) + print(text, end="") + return 0 + except Exception as exc: + # Source paths, store exceptions and JSON parser errors can contain repository names. + report = {"status": "instrument_failure", "step": step, "type": type(exc).__name__} + if str(exc) in {"empty_source:" + label for label in (*SOURCES, "memory_local", "memory_s3")}: + report["source"] = str(exc).partition(":")[2] + print(json.dumps(report)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/unseen/ledger.py b/benchmarks/unseen/ledger.py new file mode 100644 index 0000000..17dcb88 --- /dev/null +++ b/benchmarks/unseen/ledger.py @@ -0,0 +1,88 @@ +"""Append-only slice usage; a hash chain detects rewritten rows, not a rebuilt entire history. + +Keep the published final digest to authenticate a history against truncation or wholesale replacement. +Only opaque decision/experiment labels belong here; repository identities are never accepted. +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import re +from datetime import date +from pathlib import Path + +FIELDS = {"slice", "decision", "purpose", "date", "result_digest", "freeze_digest"} +PURPOSES = {"baseline", "exploratory", "informed", "qualifies"} +ZERO = "0" * 64 + + +def digest(row: dict) -> str: + return hashlib.sha256(json.dumps(row, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def validate(row: dict) -> None: + if set(row) != FIELDS or row["purpose"] not in PURPOSES: + raise ValueError("invalid_usage_fields") + if type(row["slice"]) is not int or row["slice"] < 1: + raise ValueError("invalid_slice") + if not isinstance(row["decision"], str) or not re.fullmatch(r"[A-Za-z0-9_-]+", row["decision"]): + raise ValueError("invalid_decision_label") + for key in ("result_digest", "freeze_digest"): + if not isinstance(row[key], str) or not re.fullmatch(r"[0-9a-f]{64}", row[key]): + raise ValueError("invalid_digest") + try: + if date.fromisoformat(row["date"]).isoformat() != row["date"]: + raise ValueError + except (ValueError, TypeError): + raise ValueError("invalid_date") from None + + +def _read(text: str) -> list[dict]: + previous = ZERO + rows = [] + if text and not text.endswith("\n"): + raise ValueError("broken_digest_chain") + for line in text.splitlines(): + try: + row = json.loads(line) + stored = row.pop("digest") + if row["previous"] != previous or digest(row) != stored: + raise ValueError + validate({k: v for k, v in row.items() if k != "previous"}) + except (ValueError, KeyError, TypeError, AttributeError): + raise ValueError("broken_digest_chain") from None + rows.append({**row, "digest": stored}) + previous = stored + return rows + + +def read(path: Path) -> list[dict]: + if not path.exists(): + return [] + with path.open() as handle: + fcntl.flock(handle, fcntl.LOCK_SH) + return _read(handle.read()) + + +def append(path: Path, row: dict) -> str: + validate(row) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + handle.seek(0) + rows = _read(handle.read()) + if any(all(old[key] == row[key] for key in FIELDS) for old in rows): + raise ValueError("duplicate_usage") + if rows and rows[0]["freeze_digest"] != row["freeze_digest"]: + raise ValueError("freeze_digest_changed") + if row["purpose"] == "qualifies" and any( + old["slice"] == row["slice"] and old["decision"] == row["decision"] and old["purpose"] == "informed" for old in rows + ): + raise ValueError("slice_informed_decision") + entry = {**row, "previous": rows[-1]["digest"] if rows else ZERO} + sha = digest(entry) + handle.write(json.dumps({**entry, "digest": sha}, sort_keys=True) + "\n") + handle.flush() + return sha diff --git a/src/openultrasast/plane/memory.py b/src/openultrasast/plane/memory.py index f601024..4e1afd6 100644 --- a/src/openultrasast/plane/memory.py +++ b/src/openultrasast/plane/memory.py @@ -615,9 +615,10 @@ class S3Store(MemoryStore): presigned URLs. An admin configures the bucket once; opening a store verifies it (:meth:`verify_bucket`) and refuses a bucket without versioning, the ``runs/`` expiry rule or S3 Select. There is no local fallback.""" - def __init__(self, client: ObjectClient, bucket: str, prefix: str = "") -> None: + def __init__(self, client: ObjectClient, bucket: str, prefix: str = "", *, read_only: bool = False) -> None: self.client, self.bucket, self.prefix = client, bucket, prefix.strip("/") - if self.describe() not in _VERIFIED: + self.read_only = read_only + if not read_only and self.describe() not in _VERIFIED: self.verify_bucket() _VERIFIED.add(self.describe()) @@ -631,9 +632,11 @@ class S3Store(MemoryStore): return self.client.get(self._k(key)) def _put(self, key: str, data: bytes, labels: Mapping[str, str] | None = None) -> None: + self._writable() self.client.put(self._k(key), data, dict(labels or {})) def _delete(self, key: str) -> None: + self._writable() self.client.delete(self._k(key)) def _keys(self, prefix: str) -> list[str]: @@ -647,11 +650,16 @@ class S3Store(MemoryStore): tagged = self.client.tags(self._k(key)).get("kind") return tagged is None or tagged == "*" or kind in tagged.split() + def _writable(self) -> None: + if self.read_only: + raise MemoryStoreError("read-only store") + def verify_bucket(self) -> None: - """Read-only check that the admin's one-time setup is in place: versioning ``Enabled``, an enabled + """Check the admin's one-time setup, writing only the capability probe: versioning ``Enabled``, an enabled lifecycle rule expiring ``/runs/`` and none expiring the whole store, S3 Select answering a probe over JSON Lines, and the probe's object tags readable. Every missing piece is named with its admin action in one :class:`MemoryStoreError`.""" + self._writable() runs = self._k(RUNS_PREFIX) endpoint = '--endpoint-url "$S3_ENDPOINT"' missing: list[str] = [] @@ -739,6 +747,7 @@ class S3Store(MemoryStore): ) from exc def presign_put(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: + self._writable() return self.client.presign("PUT", self._k(key), expires) def presign_get(self, key: str, expires: timedelta = timedelta(hours=1)) -> str: @@ -915,9 +924,10 @@ def s3_settings(environ: Mapping[str, str] | None = None) -> dict[str, Any]: return settings -def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None) -> MemoryStore: +def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None, *, read_only: bool = False) -> MemoryStore: """The store ``spec`` (else ``OUSAST_MEMORY``, else ``file:///memory``) names. ``s3://`` with no - bucket uses ``S3_BUCKET``.""" + bucket uses ``S3_BUCKET``. ``read_only`` skips the S3 write/Select capability probe and refuses S3 mutations; + it is intended for GET/list inventory readers, which do not need Select. Default opening is unchanged.""" from .reconciler import results_root env = environ if environ is not None else os.environ @@ -932,7 +942,7 @@ def open_store(spec: str | None = None, environ: Mapping[str, str] | None = None if not bucket: raise MemoryStoreError("OUSAST_MEMORY=s3:// names no bucket and S3_BUCKET is unset: set one of them") settings = s3_settings(environ) - return S3Store(S3Client(bucket=bucket, **settings), bucket, parts.path) + return S3Store(S3Client(bucket=bucket, **settings), bucket, parts.path, read_only=read_only) raise MemoryStoreError(f"OUSAST_MEMORY must be file:/// or s3://[/], got {text!r}") diff --git a/tests/test_independent_population.py b/tests/test_independent_population.py index 859ed16..b5b0095 100644 --- a/tests/test_independent_population.py +++ b/tests/test_independent_population.py @@ -8,6 +8,8 @@ anywhere else, which is how an evaluation set quietly becomes a development one. from __future__ import annotations +import hashlib +import json import re import tomllib from pathlib import Path @@ -64,17 +66,131 @@ def test_a_multi_language_repository_is_reserved(path: Path) -> None: assert members and all(by_id[m]["language"] == "php" and "javascript" in by_id[m].get("also", []) for m in members) -def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None: +def _pool_manifests() -> list[Path]: + directory = ROOT / "benchmarks/unseen" + return sorted([*directory.glob("*.toml"), *directory.glob("private/*.toml")]) + + +def _pool_names() -> set[str]: + names = set() + for path in _pool_manifests(): + text = path.read_text().lower() + names.update(m.rstrip("/").removesuffix(".git") for m in re.findall(r"github\.com/([\w.-]+/[\w.-]+)", text)) + return names + + +def _population_names() -> set[str]: reserved = {case["repo"].rstrip("/").lower() for path in POPULATIONS for case in _population(path)["case"]} - names = {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved} - offenders = [] - for top in ("benchmarks", "src", "tests"): + return {url.rsplit("/", 2)[-2] + "/" + url.rsplit("/", 1)[-1] for url in reserved} + + +def test_no_reserved_repository_is_referenced_by_the_development_tree() -> None: + reservations = [(ROOT / "benchmarks/independent", _population_names()), (ROOT / "benchmarks/unseen", _pool_names())] + offenders = 0 + counts = {} + for top in ("benchmarks", "src", "tests", "plane"): for path in (ROOT / top).rglob("*"): - if not path.is_file() or POPULATION.parent in path.parents or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}: + if not path.is_file() or path == Path(__file__) or path.suffix in {".pyc", ".bin", ".gz"}: + continue + # Preserve the population guard's existing scope; plane is newly reserved for pools. + active = [ + (directory, names) + for directory, names in reservations + if directory not in path.parents and names and not (top == "plane" and directory == ROOT / "benchmarks/independent") + ] + if not active: continue try: text = path.read_text(errors="ignore").lower() except OSError: continue - offenders += [f"{path.relative_to(ROOT)}: {name}" for name in names if f"github.com/{name}" in text] - assert not offenders, "a reserved repository is referenced outside the reservation: " + "; ".join(offenders[:10]) + count = sum(1 for _, names in active for name in names if f"github.com/{name}" in text) + offenders += count + counts[top] = counts.get(top, 0) + count + assert not offenders, f"reserved repository referenced outside its reservation: count={offenders}, sources={counts}" + + +def test_pool_and_population_names_are_disjoint() -> None: + count = len(_population_names() & _pool_names()) + assert not count, f"overlapping reservations: count={count}" + + +def manifest_digest(path: Path) -> str: + """Canonical parsed TOML, UTF-8 JSON with sorted keys and compact separators. + + Private entries are covered by their opaque pointers' digests in the public manifest. + The freeze writer must use the same encoding (task 3.2). + """ + raw = json.dumps(tomllib.loads(path.read_text()), sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + return hashlib.sha256(raw).hexdigest() + + +def check_pool_digest(path: Path) -> None: + record = path.with_name("freeze-" + path.stem.removeprefix("pool-") + ".json") + expected = json.loads(record.read_text())["freeze_digest"] + assert manifest_digest(path) == expected, "frozen pool digest mismatch" + + +def test_frozen_pool_manifest_digest() -> None: + pools = sorted((ROOT / "benchmarks/unseen").glob("pool-p*.toml")) + if not pools: + pytest.skip("no pool frozen") + for path in pools: + check_pool_digest(path) + + +def test_pool_guard_controls(tmp_path, monkeypatch) -> None: + """A synthetic reservation proves green, then red outside its own directory.""" + population = tmp_path / "benchmarks/independent/population-v1.toml" + pool = tmp_path / "benchmarks/unseen/pool-p1.toml" + population.parent.mkdir(parents=True) + pool.parent.mkdir(parents=True) + prefix = "https://" + "github.com/" + population.write_text('[[case]]\nrepo = "' + prefix + 'synthetic/population"\n') + pool.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/pool"\n') + monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path) + monkeypatch.setattr(__import__(__name__), "POPULATIONS", [population]) + test_no_reserved_repository_is_referenced_by_the_development_tree() + leak = tmp_path / "benchmarks/measurements/fixture.txt" + leak.parent.mkdir(parents=True) + leak.write_text(prefix + "synthetic/pool") + with pytest.raises(AssertionError, match="count=1"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + leak.unlink() + plane = tmp_path / "plane/fixture.txt" + plane.parent.mkdir() + plane.write_text(prefix + "synthetic/pool") + with pytest.raises(AssertionError, match="count=1"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + plane.write_text(prefix + "synthetic/population") + test_no_reserved_repository_is_referenced_by_the_development_tree() + plane.unlink() + private = pool.parent / "private/draft-p1.toml" + private.parent.mkdir() + private.write_text('[[repository]]\nrepo = "' + prefix + 'synthetic/population"\n') + with pytest.raises(AssertionError, match="count=2"): + test_no_reserved_repository_is_referenced_by_the_development_tree() + + +def test_pool_digest_control(tmp_path) -> None: + pool = tmp_path / "pool-p1.toml" + pool.write_text('status = "frozen"\n[[repository]]\nid = "opaque"\n') + record = tmp_path / "freeze-p1.json" + record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)})) + check_pool_digest(pool) + pool.write_text(pool.read_text() + "slice = 2\n") + with pytest.raises(AssertionError, match="digest"): + check_pool_digest(pool) + + +def test_frozen_pool_discovery_cannot_be_disabled(tmp_path, monkeypatch) -> None: + pool = tmp_path / "benchmarks/unseen/pool-p1.toml" + pool.parent.mkdir(parents=True) + pool.write_text('status = "frozen"\n') + record = pool.with_name("freeze-p1.json") + record.write_text(json.dumps({"freeze_digest": manifest_digest(pool)})) + monkeypatch.setattr(__import__(__name__), "ROOT", tmp_path) + test_frozen_pool_manifest_digest() + pool.write_text('status = "draft"\n') + with pytest.raises(AssertionError, match="digest"): + test_frozen_pool_manifest_digest() diff --git a/tests/test_unseen_eligibility.py b/tests/test_unseen_eligibility.py new file mode 100644 index 0000000..a6fe632 --- /dev/null +++ b/tests/test_unseen_eligibility.py @@ -0,0 +1,293 @@ +"""Offline eligibility fixtures; no real repository identities are needed.""" + +import io +import json +from pathlib import Path + +import pytest + +from openultrasast.plane.memory import FileStore + +with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen")) + import eligibility as e + + +def plant(root, path, text): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + return target + + +def test_sources_and_reserved_input_never_opened(tmp_path, monkeypatch, capsys): + sources = { + "benchmarks/pairs/a/catalog.toml": ("pairs", "pair"), + "benchmarks/pairs/a/recipes.toml": ("pairs", "recipe"), + "benchmarks/pairs/a/dataset.json": ("pairs", "dataset"), + "benchmarks/pairs/advisory-fixes-b/catalog.toml": ("harvest", "harvest"), + "benchmarks/independent/population-v1.toml": ("populations", "first"), + "benchmarks/independent/population-v2.toml": ("populations", "second"), + "plane/experiments/a.units.jsonl": ("units", "unit"), + "benchmarks/measurements/a/record.json": ("measurements", "measurement"), + "benchmarks/experiments/a.json": ("measurements", "experiment"), + "src/example.txt": ("sweep", "sweep"), + } + for path, (_, name) in sources.items(): + text = f'repo = "https://github.com/fixture/{name}"' if path.endswith(".toml") else json.dumps({"repo": f"fixture/{name}"}) + if name == "sweep": + text = "https://github.com/fixture/sweep" + plant(tmp_path, path, text) + pointer = plant(tmp_path, "local/pointers.toml", '[[pair]]\nrepo = "fixture/pointer"\nfix_repo = "fixture/fix"\nvendored = false') + plant(tmp_path, "benchmarks/independent/population-v3.toml", "do not read") + plant(tmp_path, "benchmarks/independent/results-v3.json", "do not read") + original = io.open + + def guarded(file, *args, **kwargs): + if "v3" in str(file): + pytest.fail("reserved input opened") + return original(file, *args, **kwargs) + + monkeypatch.setattr(io, "open", guarded) + store = FileStore(tmp_path / "memory") + plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"id": "1", "kind": "example", "repo": "fixture/memory"}) + "\n") + used = e.build_used(tmp_path, stores={"local": store}, pointer_manifests=[pointer]) + for label, name in sources.values(): + assert f"fixture/{name}" in used.sources[label] + assert {"fixture/pointer", "fixture/fix"} <= used.sources["pointers"] + assert "fixture/memory" in used.sources["memory_local"] + report = used.report() + assert report["sources"]["memory_local"]["bytes"] > 0 + assert all(item["files"] > 0 and item["bytes"] > 0 for item in report["sources"].values()) + assert "fixture/" not in json.dumps(report) + assert capsys.readouterr().out == "" + assert e.normalize("HTTPS://GitHub.com/Fixture/Pair.GIT/") == "fixture/pair" + assert len({e.normalize(s) for s in ["fixture/pair", "Fixture/Pair.git/", "github.com/fixture/pair"]}) == 1 + + +def test_empty_sources_fail_unless_declared(tmp_path): + used = e.build_used(tmp_path) + with pytest.raises(ValueError, match="empty_source"): + used.report() + assert used.report(known_empty=set(used.sources))["count"] == 0 + + +class FakeAPI: + def __init__(self, records): + self.records = records + + def get_repo(self, name): + return self.records[name] + + +def test_resolution_rejects_used_renames_and_forks(): + used = e.UsedSet() + used.add("pairs", {"fixture/used"}, 10) + api = FakeAPI( + { + "fixture/used": {"full_name": "fixture/used"}, + "fixture/old": {"full_name": "fixture/new", "former_names": ["fixture/used"]}, + "fixture/fork": {"full_name": "fixture/fork", "parent": {"full_name": "fixture/used"}, "source": {"full_name": "fixture/used"}}, + "fixture/a": {"full_name": "fixture/a", "source": {"full_name": "fixture/root"}}, + "fixture/b": {"full_name": "fixture/b", "source": {"full_name": "fixture/root"}}, + } + ) + selected = set() + for name in ("used", "old", "fork"): + resolved = e.resolve("fixture/" + name, api) + assert e.rejection(resolved, used, selected) == "pairs" + assert e.rejection(e.resolve("fixture/a", api), used, selected) is None + assert e.rejection(e.resolve("fixture/b", api), used, selected) == "fork_network" + + +def test_registered_units_and_read_only_store(tmp_path): + units = plant(tmp_path, "local/units.jsonl", json.dumps({"repo": "fixture/registered"})) + store = FileStore(tmp_path / "memory") + plant(tmp_path, "memory/repos/opaque/pin.jsonl", json.dumps({"kind": "experiment", "units_file": str(units)})) + used = e.build_used(tmp_path, stores={"s3": store}) + assert "fixture/registered" in used.sources["units"] + + +def test_cli_injected_store_and_credentials(tmp_path, monkeypatch, capsys): + from openultrasast import config + from openultrasast.plane import memory + + calls = [] + monkeypatch.setattr(config, "load_dotenv", lambda path: calls.append("dotenv")) + monkeypatch.setenv("GH_TOKEN", "test-token") + store = FileStore(tmp_path / "store") + monkeypatch.setattr(memory, "open_store", lambda spec, **kwargs: calls.append(spec) or store) + monkeypatch.setattr(e, "GitHub", lambda token: calls.append(token) or FakeAPI({})) + args = ["--root", str(tmp_path), "--s3-store", "s3://", "--local-store", "file:///unused"] + for label in (*e.SOURCES, "memory_local", "memory_s3"): + args += ["--known-empty", label] + assert e.main(args) == 0 + assert calls == ["dotenv", "file:///unused", "s3://", "test-token"] + output = capsys.readouterr().out + assert json.loads(output)["count"] == 0 + assert "test-token" not in output + + +def test_s3_open_is_read_only(tmp_path, monkeypatch): + from openultrasast.plane import memory + + class Objects: + def keys(self, prefix): + return ["repos/opaque/pin.jsonl"] + + def get(self, key): + return json.dumps({"kind": "example", "repo": "fixture/stored"}).encode(), "1" + + def __getattr__(self, name): + raise AssertionError("unexpected_store_operation") + + monkeypatch.setattr(memory, "S3Client", lambda **kwargs: Objects()) + monkeypatch.setattr(memory, "s3_settings", lambda env: {}) + store = memory.open_store("s3://fixture-bucket", read_only=True) + used = e.build_used(tmp_path, stores={"s3": store}) + assert used.sources["memory_s3"] == {"fixture/stored"} + with pytest.raises(memory.MemoryStoreError, match="read.only"): + store._put("key", b"value") + with pytest.raises(memory.MemoryStoreError, match="read.only"): + store._delete("key") + + +def test_non_repository_provenance_and_missing_used_identity(tmp_path): + plant(tmp_path, "benchmarks/pairs/recipes.toml", 'repo = "provenance-label"') + assert not e.build_used(tmp_path).sources["pairs"] + used = e.UsedSet() + used.add("pairs", {"fixture/old", "fixture/nonexistent"}, 1) + + class API: + def get_repo(self, name): + if name.endswith("nonexistent"): + raise e.RepositoryNotFound + return {"full_name": "fixture/new"} + + e.canonicalize_used(used, API()) + assert "fixture/nonexistent" in used.sources["pairs"] + assert e.rejection(e.resolve("fixture/new", API()), used, set()) == "pairs" + + +def test_binary_measurement_and_local_pointer_discovery(tmp_path): + target = tmp_path / "benchmarks/measurements/a/image.bin" + target.parent.mkdir(parents=True) + target.write_bytes(b"\xff\x00") + plant(tmp_path, "benchmarks/pairs/a/recipes.toml", '[[pair]]\nrepo = "fixture/pointer"\nvendored = false') + used = e.build_used(tmp_path) + assert used.instrument["measurements"]["bytes"] == 2 + assert used.sources["pointers"] == {"fixture/pointer"} + + +def test_coordinator_verification_is_black_box(tmp_path, monkeypatch): + import subprocess + from types import SimpleNamespace + + def run(command, **kwargs): + assert kwargs["stdout"] == kwargs["stderr"] == kwargs["stdin"] == subprocess.DEVNULL + path = Path(next(part.split("=", 1)[1] for part in command if part.startswith("--junitxml="))) + path.write_text('') + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(subprocess, "run", run) + assert e.verify_tests(tmp_path)["tests"]["tests"] == 25 + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: SimpleNamespace(returncode=2)) + with pytest.raises(ValueError, match="guard_or_tests_failed"): + e.verify_tests(tmp_path) + + +def test_registered_unit_group_is_repository(tmp_path): + plant(tmp_path, "plane/experiments/a.units.jsonl", json.dumps({"unit": "opaque", "group": "fixture/unit", "label": 1})) + assert "fixture/unit" in e.build_used(tmp_path).sources["units"] + + +def test_multidocument_measurement_and_missing_registered_units(tmp_path): + plant(tmp_path, "benchmarks/measurements/a/run.yaml", "repo: fixture/first\n---\nrepo: fixture/second\n") + assert e.build_used(tmp_path).sources["measurements"] == {"fixture/first", "fixture/second"} + store = FileStore(tmp_path / "memory") + plant( + tmp_path, + "memory/repos/opaque/pin.jsonl", + json.dumps( + { + "kind": "experiment", + "units_file": "absent.units.jsonl", + "units_digest": "a" * 64, + } + ), + ) + with pytest.raises(FileNotFoundError): + e.build_used(tmp_path, stores={"local": store}) + + +def test_catalog_skips_fixture_and_preserves_host_identity(tmp_path, capsys): + plant( + tmp_path, + "benchmarks/pairs/catalog.toml", + '[[pair]]\norigin = "in-tree-fixture"\nvendored = false\n[[pair]]\nrepo = "https://GitLab.com/Fixture/Private.git"\n', + ) + used = e.build_used(tmp_path) + assert used.sources["pairs"] == {"gitlab.com/fixture/private"} + assert used.instrument["pairs"]["skipped_values"] == 1 + assert used.instrument["pointers"]["skipped_values"] == 1 + assert used.sources["sweep"] == set() + e.canonicalize_used(used, FakeAPI({})) # Non-GitHub identities never reach this API. + assert e.rejection(e.resolve("https://gitlab.com/fixture/private", FakeAPI({})), used, set()) == "pairs" + report = used.report(set(e.SOURCES) - {"pairs"}) + assert report["sources"]["pairs"]["skipped_values"] == 1 + assert "fixture" not in json.dumps(report) + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("https://bitbucket.org/Fixture/Repo.git", "bitbucket.org/fixture/repo"), + ("https://git.example.org/Fixture/Repo", "git.example.org/fixture/repo"), + ("https://github.com/Fixture/Repo/tree/main", "fixture/repo"), + ("Fixture/Repo.git/", "fixture/repo"), + ], +) +def test_repository_hosts(value, expected): + assert e.normalize(value) == expected + assert e.normalize(expected) == expected + + +def test_memory_values_are_skipped_and_counted(tmp_path, capsys): + rows = [ + {"kind": "example", "repo": value} + for value in ["local-slice", "free text https://github.com/fixture/prose", None, "https://bitbucket.org/fixture/repo"] + ] + plant(tmp_path, "memory/repos/opaque/pin.jsonl", "\n".join(json.dumps(row) for row in rows)) + used = e.build_used(tmp_path, stores={"local": FileStore(tmp_path / "memory")}) + assert used.sources["memory_local"] == {"bitbucket.org/fixture/repo"} + assert used.instrument["memory_local"]["skipped_values"] == 3 + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize("step", ["open_stores", "build_used", "report", "canonicalize", "verify_tests"]) +def test_cli_failure_reports_only_step_and_exception_type(tmp_path, monkeypatch, capsys, step): + from openultrasast import config + from openultrasast.plane import memory + + def fail(*args, **kwargs): + raise RuntimeError("sensitive/identity") + + monkeypatch.setattr(config, "load_dotenv", lambda path: None) + monkeypatch.setattr(memory, "open_store", lambda *args, **kwargs: object()) + monkeypatch.setattr(e, "build_used", lambda *args, **kwargs: e.UsedSet()) + monkeypatch.setattr(e.UsedSet, "report", lambda *args, **kwargs: {}) + monkeypatch.setattr(e, "GitHub", lambda token: object()) + monkeypatch.setattr(e, "canonicalize_used", lambda *args: None) + monkeypatch.setattr(e, "verify_tests", lambda *args: {}) + if step == "open_stores": + monkeypatch.setattr(memory, "open_store", fail) + elif step == "report": + monkeypatch.setattr(e.UsedSet, "report", fail) + else: + monkeypatch.setattr(e, "canonicalize_used" if step == "canonicalize" else step, fail) + assert e.main(["--root", str(tmp_path), "--verify-tests"]) == 1 + output = capsys.readouterr() + assert output.err == "" + assert json.loads(output.out) == {"status": "instrument_failure", "step": step, "type": "RuntimeError"} + assert "sensitive" not in output.out diff --git a/tests/test_unseen_ledger.py b/tests/test_unseen_ledger.py new file mode 100644 index 0000000..4895d41 --- /dev/null +++ b/tests/test_unseen_ledger.py @@ -0,0 +1,63 @@ +"""Append-only use and tamper detection, with opaque fixture identifiers.""" + +import json +from pathlib import Path + +import pytest + +with pytest.MonkeyPatch.context() as patch: + patch.syspath_prepend(str(Path(__file__).resolve().parents[1] / "benchmarks/unseen")) + import ledger + + +def row(**extra): + return { + "slice": 1, + "decision": "decision-001", + "purpose": "baseline", + "date": "2026-10-04", + "result_digest": "a" * 64, + "freeze_digest": "b" * 64, + **extra, + } + + +def test_append_and_refuse_rewrite(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row()) + original = path.read_bytes() + ledger.append(path, row(purpose="exploratory")) + assert path.read_bytes().startswith(original) + assert len(ledger.read(path)) == 2 + with pytest.raises(ValueError, match="duplicate"): + ledger.append(path, row()) + + +def test_informed_slice_cannot_qualify_same_decision(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row(purpose="informed")) + before = path.read_bytes() + with pytest.raises(ValueError, match="informed"): + ledger.append(path, row(purpose="qualifies")) + assert path.read_bytes() == before + ledger.append(path, row(purpose="qualifies", slice=2)) + ledger.append(path, row(purpose="qualifies", decision="decision-002")) + + +def test_rewritten_row_breaks_digest_chain(tmp_path): + path = tmp_path / "usage-p1.jsonl" + ledger.append(path, row()) + ledger.append(path, row(purpose="exploratory")) + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["purpose"] = "informed" + path.write_text("".join(json.dumps(r) + "\n" for r in rows)) + with pytest.raises(ValueError, match="chain"): + ledger.read(path) + with pytest.raises(ValueError, match="chain"): + ledger.append(path, row(slice=2)) + + +@pytest.mark.parametrize("extra", [{"decision": "owner/name"}, {"purpose": "other"}, {"freeze_digest": "bad"}, {"slice": 0}]) +def test_invalid_rows_refused(tmp_path, extra): + with pytest.raises(ValueError): + ledger.append(tmp_path / "usage-p1.jsonl", row(**extra)) From 48ae918dec25119142ae781607ba99306ae5ec17 Mon Sep 17 00:00:00 2001 From: Marius Ciepluch <11855163+norandom@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:51:30 +0200 Subject: [PATCH 2/3] unseen-eval group 2 (2.1-2.4): advisory sourcing, vulnerability-introducing ranges, ordinary-commit draw, draft manifest and slices; journal with resume Implemented with Codex. Sourcing run (2.5) next, by the coordinator. Co-Authored-By: Claude Opus 5.5 --- .kiro/specs/unseen-repo-evaluation/tasks.md | 28 +- benchmarks/unseen/README.md | 137 +++++ benchmarks/unseen/draft.py | 472 ++++++++++++++++++ benchmarks/unseen/extract.py | 474 ++++++++++++++++++ benchmarks/unseen/fixtures/advisory.json | 9 + benchmarks/unseen/journal.py | 184 +++++++ benchmarks/unseen/source.py | 280 +++++++++++ tests/test_unseen_draft.py | 527 ++++++++++++++++++++ tests/test_unseen_extract.py | 314 ++++++++++++ tests/test_unseen_source.py | 182 +++++++ 10 files changed, 2603 insertions(+), 4 deletions(-) create mode 100644 benchmarks/unseen/draft.py create mode 100644 benchmarks/unseen/extract.py create mode 100644 benchmarks/unseen/fixtures/advisory.json create mode 100644 benchmarks/unseen/journal.py create mode 100644 benchmarks/unseen/source.py create mode 100644 tests/test_unseen_draft.py create mode 100644 tests/test_unseen_extract.py create mode 100644 tests/test_unseen_source.py diff --git a/.kiro/specs/unseen-repo-evaluation/tasks.md b/.kiro/specs/unseen-repo-evaluation/tasks.md index 0ad2cb4..9195df9 100644 --- a/.kiro/specs/unseen-repo-evaluation/tasks.md +++ b/.kiro/specs/unseen-repo-evaluation/tasks.md @@ -77,7 +77,7 @@ Execution rules: - _Requirements: 1.1, 1.3_ - [ ] 2. Sourcing and extraction (design 1.1, 1.3, 1.4, 2, 3; increment 2) -- [ ] 2.1 Advisory candidates and repository filters +- [x] 2.1 Advisory candidates and repository filters - `benchmarks/unseen/source.py`: from GHSA advisory JSON (pip, npm, maven, composer; published 2019 or later) keep those whose CWE maps to a family through `src/openultrasast/model/taxonomy.py` and that link one fix commit with one parent in one repository; attach the OSV first affected version; flag `post_cutoff` when @@ -87,7 +87,7 @@ Execution rules: multi-repository fixes dropped, cutoff flag, licence classing (GPL, AGPL, LGPL, none and source-available go private). - _Requirements: 1.4, 2.1_ -- [ ] 2.2 Vulnerability-introducing range +- [x] 2.2 Vulnerability-introducing range - `benchmarks/unseen/extract.py`: the `introducing` method of design 2.1 (blame of the sink line, or the declaration for an absence bug; absence at V's first parent; no merge or root; at most 50 files and 2,000 lines; OSV version bracket when present; walk-back of at most five steps) and the `last_touch` fallback. Each @@ -96,7 +96,7 @@ Execution rules: - `tests/test_unseen_extract.py` on a synthetic repository built in a temp dir: accepted V; a reformat walked back; a merge rejected; size cap; bracket rejection; fallback chosen when no V passes. - _Requirements: 2.1_ -- [ ] 2.3 Ordinary-commit draw +- [x] 2.3 Ordinary-commit draw - `extract.py`: first-parent, non-merge commits touching an analysed language; the exclusions of design 2.2 (security words of the narrow pattern, reverts, bot and bump commits, manifest- or lock-only, generated or vendored, advisory fix commits, commits touching a known vulnerable function, more than 1,000 source lines, @@ -105,7 +105,7 @@ Execution rules: - Tests: each exclusion; the narrow pattern keeps "query", "token", "session"; allocation and the seed reproduce the same draw; the 40-commit floor. - _Requirements: 2.2, 2.3_ -- [ ] 2.4 Draft manifest and slices +- [x] 2.4 Draft manifest and slices - `benchmarks/unseen/draft.py`: writes `benchmarks/unseen/draft-p1.toml` (permissive entries by name) and `benchmarks/unseen/private/draft-p1.toml` (the rest); the public file holds an opaque id and sha256(URL, commits) per private entry. Seeded shuffle into three slices of 100 repositories, stratified by @@ -334,3 +334,23 @@ Execution rules: Select capability probe. Default store behavior is unchanged; inventory uses LIST/GET only. - The freeze writer must match the canonical TOML digest encoding documented in the pool README and exercised by `test_independent_population.py`; final pool discovery does not trust mutable status. + +### Group 2 implementation evidence (2026-10-04) + +Tasks 2.1–2.4 are implemented offline and independently reviewed. Task 2.5 is +`python -m benchmarks.unseen.draft --root "$PWD" --s3-store s3:// --seed 20260601` +with the documented PATH/PYTHONPATH and GH_TOKEN environment. No networked draw, +label check, group-2 measurement record, guard bisection, or commit was performed. +The group header remains pending its coordinator exits 2.5/2.6. + +The command rebuilds the group-1 used set, resolves renamed/fork repositories, +uses disposable external blobless clones, re-reads licences at extracted heads, +and emits counts/digests only. The README documents injected clients, method bias, +the hard 300-repository floor, and an explicitly unmeasured resource estimate. +Independent review caught and fixed ambiguous parent-function absence and stale +candidate-404 handling; regression tests cover both. + +Validation: requested pytest selection 103 passed, 1 skipped, exit 0 (population-test +output discarded); `ruff check .` exit 0; `ruff format --check .` exit 0 (411 files); +`mypy src/openultrasast` exit 0 (160 source files). The CLI `--help` smoke exited 0. +Source-read instrumentation printed 20,101 bytes for the extraction implementation. diff --git a/benchmarks/unseen/README.md b/benchmarks/unseen/README.md index fdc1ff8..9274f76 100644 --- a/benchmarks/unseen/README.md +++ b/benchmarks/unseen/README.md @@ -60,3 +60,140 @@ The ledger exposes `append(path, row)` and `read(path)`. Rows contain `slice`, ` changed freeze, and reject qualification of an informed slice for the same decision. Keep the published final digest to authenticate history: the chain detects row edits, but cannot independently prove that an entire history was not replaced or truncated. + +## Sourcing and extraction (tasks 2.1–2.4) + +The coordinator's networked task **2.5** is now: + +```sh +export PATH=/home/mc/Source/OpenUltraSAST/.venv/bin:$PATH +export PYTHONPATH=$PWD/src +python -m benchmarks.unseen.draft --root "$PWD" --s3-store s3:// --seed 20260601 \ + --cache /home/mc/ousast-results/plane/unseen-draft --max-hours 8 +``` + +Export `GH_TOKEN` first, or supply it through the root `.env` (exported values win). +The same read-only local/S3 stores, `--pointer-manifest`, and explicit `--known-empty` +options as task 1.5 apply. Missing or unreadable sources fail. The used set is rebuilt +and canonicalized before candidate selection, including redirects and fork networks. +No population v3 file is opened and no guard/bisection is run by this command; that is +reserved for group 3. An existing draft is never overwritten. + +The command writes `--cache/journal/draft-p1.toml` and +`--cache/journal/private-draft-p1.toml` only after all sourcing stages complete. +Both stay outside the repository, with mode 0600. The public-format file contains permissive entries +and, for every private entry, only an opaque ID, slice, private marker, and SHA-256 +of canonical JSON containing its URL and ordered `[base, head]` pairs. Private entries +carry the complete records, including licences, in the private-format file. +Licensing is re-read through GitHub's licence endpoint **at the actual extracted +vulnerability replay head**; the candidate's earlier fix-parent classification is +provisional. An explicit SPDX declaration in the pinned file takes precedence over +the API's classification. Missing/unknown/compound licences remain private. + +Three seeded slices each contain 100 repositories, one vulnerability range and twelve +ordinary changes per repository. Each ecosystem/post-cutoff stratum differs from its +proportional per-slice target by less than one repository. Post-cutoff candidates are +attempted first within each ecosystem. Below 300 eligible repositories the command +exits 1 with counts and does not write a partial draft or relax any selection bar. +The group-2 measured exit and label confirmation remain coordinator tasks 2.5/2.6; +passing implementation tests does not assert that 300 eligible repositories exist. + +Clones and checkpoints default to `results_root()/unseen-draft` (normally +`~/ousast-results/plane/unseen-draft`), overridable by `--cache /outside/path`. +The cache cannot lie inside `--root`. One opaque, disposable clone exists at a time; +it is removed on success, rejection, or failure. `--filter=blob:none --no-checkout` +keeps the commit graph needed for temporal thirds, blame, and release-tag ancestry, +and fetches source blobs on demand. This is a **partial clone**, not a shallow history. +The 500 MiB disk cap is checked after Git commands, including lazy blob fetches; +one in-flight command can transiently exceed it before rejection and cleanup. +Git output is captured, never forwarded. Each repository must read a nonempty source +blob before extraction. Summary instrumentation includes source read operations and +actual bytes, clone count, cumulative/peak disk bytes, wall time, and logical API calls +and HTTP attempts. Zero source bytes or a failed Git read is an instrument failure, +not an eligibility drop. Stdout is a JSON summary of counts/digests only; no names or +URLs enter stdout or a measurement record. Raw per-change data stays in the external journal and manifests. + +Extraction uses Git and syntax boundaries, with no scanner, engine, or model. Python +uses stdlib AST; Java/PHP/JS/TS use conservative named-function brace matching. Ambiguous +or unparseable functions can reject a candidate. The first removed code line per +fix-touched function is the SZZ sink proxy; pure-addition fixes use its declaration. +Normalized text absence, non-root/non-merge ancestry, file/line limits, and the OSV +release bracket are checked before accepting `introducing`. Matching tags are `VERSION` +or `vVERSION`; an absent/ambiguous tag or missing prior release cannot certify the +bracket. After at most five walk-backs, `last_touch` uses `git log -L` with the resolved +numeric function extent (also works where Git lacks a language-specific function driver). +Method, bias, fallback reasons, head function/range, files/hunks, family, advisory, and +cutoff flag are recorded. Task 2.6 must review every third `introducing` label. + +All reviewed advisories are enumerated to exclude **all** known repository fixes from +ordinary changes, including old advisories and advisories outside the four candidate +ecosystems. Fix URLs are canonicalized before indexing so renames cannot hide them. +The ordinary filter uses only the design's narrowed security pattern. It counts +rejections (including oversized commits and repositories failing the 40-commit floor), +checks both sides for touches to known vulnerable functions, and inspects generated +headers and inherited `.gitattributes`. Sampling allocates twelve draws proportionally +across three history-time thirds and four source-size buckets, with one per third. + +For offline tests, inject GitHub/OSV clients, the clone context, and `UsedSet` into +`draft.run`; HTTP transports and Git subprocess runners also accept fakes. Tests use +sanitized REST-shaped data and a temporary synthetic local Git history. No implementation +test contacts GitHub, OSV, a model, or a cluster. HTTP retries are bounded to five attempts; +rate-limit reset/Retry-After waits and transient 5xx backoff are tested with fake clocks +and sleepers. A stale candidate 404 is counted; other service failures fail the run. + +### Checkpoint and resume + +The command above uses journal directory +`/home/mc/ousast-results/plane/unseen-draft/journal`. Run the same command to resume. +`--max-hours` is a per-invocation budget; changing it does not invalidate checkpoints. +Create `journal/STOP` to stop between units, then remove it before resuming. A page, +repository resolution, candidate decision, or repository extraction already running +finishes first, so the time budget is not a hard timeout on a large repository. +Clean stops exit 0 with `status: stopped`; failures and shortfalls exit 1. Completion +exits 0 with `status: draft`. No partial manifest is published. + +`inputs.json` binds the seed, source-labelled used-set digest, known-empty declarations, +clone cap, and design fingerprints (sourcing/extraction/eligibility/publication code and +family taxonomy). A mismatch refuses resume and prints only the changed parameter +labels. Use a new external cache for a deliberately changed design or population. +The used set is rebuilt each invocation; repository resolutions, including 404s, are +reused from the journal. Live GitHub metadata for unfinished units can still change; +completed pages and decisions are the durable sourcing snapshot. + +Each completed unit is appended to its stage JSONL and fsynced before work advances: +`advisories.jsonl` (pages), `resolution.jsonl` (alias/fork metadata), +`eligibility.jsonl` (candidate decisions), `extraction.jsonl` (repository/advisory +results, vulnerable range, ordinary draws, pinned licence, method and counts), and +`completion.jsonl` (final manifest digest). A lock permits only one writer. +`progress.json` is atomically rewritten after every unit with stage counts, last unit, +last observed GitHub rate-limit headers, and cumulative active elapsed seconds. +These files may contain identities and must stay outside version control. Stdout and +repository records contain only counts, digests and fixed labels. + +Replaying completed units makes no GitHub/OSV or clone calls. Final manifests are +reconstructed deterministically from journal results; an interrupted publication can +be retried, but a different existing manifest is refused. Completed JSONL rows are +never rewritten. Malformed or torn records fail closed instead of being interpreted +as completed work. Tests interrupt each stage with fakes, then compare resumed and +uninterrupted manifests byte for byte. + +### Planning estimate for task 2.5 (not measured) + +Let A be all reviewed advisories, R distinct fix-linked repository identities, U used +identities not already cached, C candidate advisories passing early filters, and K +repositories surviving extraction. GitHub requests are approximately +`ceil(A/100) + R + U + 2*C + K` (enumeration, canonical repositories, fix metadata, +provisional licence, final licence), plus retries. OSV requests are at most C. Git +fetches and lazy blob transfers are additional network operations, not GitHub API calls. + +For a planning scenario of A=40,000, R=15,000, U≈1,000, C=600–1,500, K=300: +about **18,000–20,000 GitHub calls** and **600–1,500 OSV calls**. The all-advisory index +makes this substantially more than 300 repository lookups. At an assumed 5,000-request +hourly allowance, allow several rate-limit windows. Actual token limits can differ. +For 300–1,000 attempted clones averaging 20–100 MB after source hydration, plan roughly +**6–100 GB transferred**, with one clone occupying about 500 MiB at most between commands +plus the retained journal and manifests after completion. Transfer volume is an estimate; disk bytes +are only a proxy. A working planning range is **8–48 hours**, potentially longer on +large histories: every first-parent commit is inspected and lazy blobs can dominate. +These are assumptions, not evidence from a network run. The command reports actual +counts and wall time; task 2.5 establishes the measured estimate. diff --git a/benchmarks/unseen/draft.py b/benchmarks/unseen/draft.py new file mode 100644 index 0000000..6597699 --- /dev/null +++ b/benchmarks/unseen/draft.py @@ -0,0 +1,472 @@ +"""Coordinator-only sourcing command: python -m benchmarks.unseen.draft --help. + +All network and git operations are injected at the run() boundary. Only counts, +digests and fixed failure codes go to stdout; identity-bearing output stays in the +private journal outside the repository. No population guard is run here. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import random +import subprocess +import tempfile +import time +from collections import Counter, defaultdict +from contextlib import contextmanager +from dataclasses import asdict +from pathlib import Path + +from . import eligibility, extract, source +from .extract import Git, InstrumentFailure +from .journal import Journal, ResolvedGitHub, ResumeMismatch, Stopped, atomic, digest, external + + +def canonical(value) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def private_digest(row: dict) -> str: + return hashlib.sha256(canonical({"url": row["url"], "commits": [[c["base"], c["head"]] for c in row["changes"]]})).hexdigest() + + +def assign_slices(rows: list[dict], *, seed: int) -> list[dict]: + if len(rows) != 300 or len({r["repository"] for r in rows}) != 300: + raise ValueError("pool_size") + for row in rows: + kinds = Counter(c["kind"] for c in row["changes"]) + if kinds != {"vulnerability": 1, "ordinary": 12}: + raise ValueError("change_count") + if row["license_class"] not in {"permissive", "private"}: + raise ValueError("license_class") + if row["license_class"] == "permissive" and row["license_spdx"] not in source.PERMISSIVE: + raise ValueError("license_class") + rng = random.Random(seed) + groups = defaultdict(list) + for row in sorted(rows, key=lambda r: r["repository"]): + groups[row["ecosystem"], row["post_cutoff"]].append(row) + sizes = Counter({1: 0, 2: 0, 3: 0}) + assigned = [] + for key in sorted(groups): + group = groups[key] + rng.shuffle(group) + quotient, remainder = divmod(len(group), 3) + # Give each stratum its floor on all slices, then fill globally least-full slices. + quotas = {i: quotient for i in sizes} + order = list(sizes) + rng.shuffle(order) + order.sort(key=lambda i: sizes[i]) + for i in order[:remainder]: + quotas[i] += 1 + offset = 0 + for slice_id in sorted(quotas): + for row in group[offset : offset + quotas[slice_id]]: + digest = private_digest(row) + assigned.append({**row, "id": "r-" + digest[:24], "slice": slice_id}) + sizes[slice_id] += quotas[slice_id] + offset += quotas[slice_id] + if set(sizes.values()) != {100}: + raise ValueError("slice_size") + return sorted(assigned, key=lambda r: (r["slice"], r["id"])) + + +def toml(value) -> str: + """Inline TOML values keep nested change records structured without a new dependency.""" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, int): + return str(value) + if isinstance(value, list): + return "[" + ", ".join(toml(v) for v in value) + "]" + if isinstance(value, dict): + return "{" + ", ".join(json.dumps(k) + " = " + toml(v) for k, v in sorted(value.items()) if v is not None) + "}" + raise ValueError("unsupported_manifest_type") + + +def manifest(rows: list[dict], seed: int) -> str: + lines = ['status = "draft"', 'pool = "p1"', f"seed = {seed}"] + for row in rows: + lines.append("\n[[repository]]") + lines.extend(f"{key} = {toml(value)}" for key, value in sorted(row.items()) if value is not None) + return "\n".join(lines) + "\n" + + +def write_draft(root: Path, assigned: list[dict], *, seed: int) -> tuple[Path, Path]: + public_path = root / "benchmarks/unseen/draft-p1.toml" + private_path = root / "benchmarks/unseen/private/draft-p1.toml" + if public_path.exists() or private_path.exists(): + raise FileExistsError("draft_exists") + public, private = [], [] + for row in assigned: + if row["license_class"] == "permissive": + public.append(row) + else: + private.append(row) + public.append({"id": row["id"], "sha256": private_digest(row), "slice": row["slice"], "private": True}) + private_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + # Exclusive creates protect drafts awaiting human label review. Private first: never + # publish a pointer whose private manifest failed to write. + with private_path.open("x") as handle: + os.chmod(private_path, 0o600) + handle.write(manifest(private, seed)) + with public_path.open("x") as handle: + handle.write(manifest(public, seed)) + return public_path, private_path + + +class Clones: + """One disposable blobless clone at a time; no name in cache paths or subprocess output.""" + + def __init__(self, root: Path, cache: Path, *, runner=None, limit_bytes=500 * 1024 * 1024): + self.cache = external(root, cache) + self.runner = runner or subprocess.run + self.limit_bytes = limit_bytes + self.counts = Counter() + + @contextmanager + def open(self, url: str): + self.cache.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="clone-", dir=self.cache) as directory: + path = Path(directory) + git = Git(path, runner=self.runner, limit_bytes=self.limit_bytes) + try: + # Full commit graph is needed for temporal thirds and OSV brackets; blobs + # are fetched lazily. This is a partial clone, not a truncated history. + done = self.runner( + ["git", "-c", "credential.helper=", "clone", "--quiet", "--filter=blob:none", "--no-checkout", url, str(path)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=180, + env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, + ) + if done.returncode: + raise InstrumentFailure("clone_failed") + self.counts["clones"] += 1 + if git.disk_bytes() > self.limit_bytes: + raise InstrumentFailure("clone_size_cap") + yield git + finally: + size = git.disk_bytes() + self.counts["clone_bytes"] += size + self.counts["peak_clone_bytes"] = max(self.counts["peak_clone_bytes"], size) + self.counts["source_bytes_read"] += git.bytes_read + self.counts["source_files_read"] += git.files_read + self.counts["clone_wall_seconds"] += round(time.monotonic() - started, 3) + + +def prove_source(git, revision: str) -> dict: + files = git.run("ls-tree", "-r", "--name-only", revision).splitlines() + for path in files: + if Path(path).suffix in extract.LANGUAGES and not extract.generated_path(path): + text = git.blob(revision, path) + if text: + return {"files": 1, "bytes": getattr(git, "last_blob_bytes", len(text.encode()))} + raise InstrumentFailure("zero_source_bytes") + + +def advisory_index(rows: list[dict], github) -> dict[str, set[str]]: + """Include ALL enumerated fixes, even old, unmapped, multi-fix and other ecosystems. + + Canonicalize link names as well, so an old advisory URL survives repository renames. + A 404 is not treated as proof that an advisory is unrelated to a selected repository. + """ + index = defaultdict(set) + for row in rows: + for name, sha in source.fix_links(row): + try: + resolved = eligibility.resolve(name, github) + except eligibility.RepositoryNotFound: + index[name].add(sha) + continue + index[resolved.full_name].add(sha) + return dict(index) + + +def repository_changes(git, candidate, fixes: set[str], *, seed: int): + proof = prove_source(git, candidate.parent) + change = extract.introducing(git, candidate) + known = set() + expanded = set() + for fix in sorted(fixes | {candidate.fix}): + # Explicitly fetch detached advisory commits unavailable on default-branch history. + # All git including this possible network call still goes through the injected client. + git.run("fetch", "--quiet", "--filter=blob:none", "origin", fix) + full = git.run("rev-parse", fix).strip() + expanded.add(full) + parents = git.parents(full) + for parent in parents: + sites = extract.fix_sites(git, full, parent, global_ok=True) + known.update((site["path"], site["function"]) for site in sites) + rows = extract.ordinary_history(git, "HEAD") + ordinary, exclusions = extract.draw_ordinary(rows, expanded, known, seed=seed) + return [change, *ordinary], exclusions, proof + + +def inputs(used, seed, known_empty, clones): + from openultrasast.model.taxonomy import DEFAULT_FAMILIES_PATH + + parameters = { + "seed": seed, + "used_set_digest": digest({label: sorted(names) for label, names in used.sources.items()}), + "known_empty": sorted(known_empty), + "clone_limit_bytes": getattr(clones, "limit_bytes", 500 * 1024 * 1024), + "journal_version": 1, + } + # Hardcoded design thresholds and algorithms are covered as well as taxonomy data. + for name in ("draft", "source", "extract", "eligibility", "journal"): + parameters["design_" + name] = hashlib.sha256(Path(__file__).with_name(name + ".py").read_bytes()).hexdigest() + parameters["design_taxonomy"] = hashlib.sha256(DEFAULT_FAMILIES_PATH.read_bytes()).hexdigest() + return parameters + + +def enumerate_advisories(github, journal): + page, records = 1, {} + while True: + key = str(page) + if key not in journal.rows["advisories"]: + journal.boundary() + result = github.advisory_page(page) + journal.put("advisories", key, result) + result = journal.rows["advisories"][key] + records.update((row["ghsa_id"], row) for row in result["rows"]) + if result["done"]: + break + page += 1 + if not records: + raise InstrumentFailure("empty_advisories") + return list(records.values()) + + +def decide(row, github, osv, used, selected_networks): + try: + links = source.fix_links(row) + if len(links) != 1: + raise source.Rejected("fix_links") + name, _ = next(iter(links)) + resolved = eligibility.resolve(name, github) + rejected_by = eligibility.rejection(resolved, used, set(selected_networks)) + if rejected_by: + raise source.Rejected("used_" + rejected_by) + candidate = source.candidate(row, github, osv) + return {"candidate": asdict(candidate), "network": resolved.network} + except source.Rejected as exc: + return {"rejected": str(exc)} + except eligibility.RepositoryNotFound: + return {"rejected": "repository_or_commit_not_found"} + + +def extract_repository(candidate, github, clones, fixes, seed): + before = dict(clones.counts) + try: + with clones.open(candidate.url) as git: + git.run("fetch", "--quiet", "--filter=blob:none", "origin", candidate.fix) + changes, exclusions, proof = repository_changes(git, candidate, fixes, seed=seed) + pinned = changes[0]["head"] + license_record = github.get_license(candidate.repository, pinned) + spdx, license_class = source.license_info(license_record) + entry = { + **asdict(candidate), + "license_spdx": spdx, + "license_class": license_class, + "license_ref": pinned, + "license_path": license_record.get("path", ""), + "changes": changes, + "instrument": proof, + } + result = {"entry": entry, "ordinary_exclusions": exclusions} + except source.Rejected as exc: + result = {"rejected": str(exc), "ordinary_exclusions": getattr(exc, "counts", {})} + except eligibility.RepositoryNotFound: + result = {"rejected": "repository_or_commit_not_found", "ordinary_exclusions": {}} + result["counts"] = {key: value - before.get(key, 0) for key, value in clones.counts.items()} + return result + + +def publish(journal, assigned, seed): + """Idempotent publication from completed journal data, never inside the checkout.""" + public, private = [], [] + for row in assigned: + if row["license_class"] == "permissive": + public.append(row) + else: + private.append(row) + public.append({"id": row["id"], "sha256": private_digest(row), "slice": row["slice"], "private": True}) + for name, rows in (("private-draft-p1.toml", private), ("draft-p1.toml", public)): + path = journal.path / name + data = manifest(rows, seed).encode() + if path.exists(): + if path.read_bytes() != data: + raise ValueError("draft_exists") + else: + atomic(path, data) + + +def run( + root: Path, + *, + github, + osv, + clones, + used: eligibility.UsedSet, + seed: int, + known_empty=None, + journal_path=None, + max_hours=None, + clock=None, +) -> dict: + started = time.monotonic() + report = {"status": "sourcing", "seed": seed, "candidates": {}, "rejections": {}, "ordinary_exclusions": {}} + counts, rejected, ordinary_counts = Counter(), Counter(), Counter() + checkpoint = None + try: + known_empty = set(known_empty or ()) + used.report(known_empty) + cache = getattr(clones, "cache", root.parent / (root.name + "-cache")) + path = external(root, journal_path or cache / "journal") + checkpoint = Journal( + path, max_hours=max_hours, clock=clock, rate_limit=lambda: getattr(getattr(github, "transport", None), "rate_limit", {}) + ) + with checkpoint.open(inputs(used, seed, known_empty, clones)): + checkpoint.boundary() + api = ResolvedGitHub(github, checkpoint) + used = copy.deepcopy(used) # canonicalization must not mutate the resume input + eligibility.canonicalize_used(used, api) + report["used_set"] = used.report(known_empty) + rows = enumerate_advisories(api, checkpoint) + report["advisories"] = len(rows) + fixes_by_repo = advisory_index(rows, api) + selected_networks, entries = set(), [] + for row in source.ordered_advisories(rows, seed): + checkpoint.boundary() + key = row["ghsa_id"] + counts[source.ecosystem(row)] += 1 + if key not in checkpoint.rows["eligibility"]: + decision = decide(row, api, osv, used, selected_networks) + checkpoint.put("eligibility", key, decision) + decision = checkpoint.rows["eligibility"][key] + if "rejected" in decision: + rejected[decision["rejected"]] += 1 + continue + candidate = source.Candidate(**decision["candidate"]) + # Multiple advisories for one repository can have different fix/range inputs. + extraction_key = digest({"repository": candidate.repository, "advisory": key, "fix": candidate.fix}) + if extraction_key not in checkpoint.rows["extraction"]: + checkpoint.boundary() + result = extract_repository(candidate, api, clones, fixes_by_repo.get(candidate.repository, set()), seed) + checkpoint.put("extraction", extraction_key, result) + result = checkpoint.rows["extraction"][extraction_key] + ordinary_counts.update(result["ordinary_exclusions"]) + if "rejected" in result: + rejected[result["rejected"]] += 1 + continue + selected_networks.add(decision["network"]) + entries.append(result["entry"]) + if len(entries) == 300: + break + report.update( + { + "repositories": len(entries), + "candidates": dict(counts), + "rejections": dict(rejected), + "ordinary_exclusions": dict(ordinary_counts), + } + ) + checkpoint.boundary() + if len(entries) != 300: + report["status"] = "insufficient_repositories" + checkpoint.progress(report["status"]) + return report + assigned = assign_slices(entries, seed=seed) + completion = {"manifest_sha256": digest(assigned), "repositories": len(entries)} + if "all" not in checkpoint.rows["completion"]: + checkpoint.put("completion", "all", completion) + elif checkpoint.rows["completion"]["all"] != completion: + raise ValueError("journal_completion_mismatch") + checkpoint.boundary() + publish(checkpoint, assigned, seed) + report.update( + { + "status": "draft", + "changes": 3900, + "slices": 3, + "license_split": dict(Counter(e["license_class"] for e in entries)), + "methods": dict(Counter(e["changes"][0]["method"] for e in entries)), + "post_cutoff": sum(e["post_cutoff"] for e in entries), + **completion, + } + ) + checkpoint.progress("draft") + return report + except ResumeMismatch as exc: + report.update({"status": "resume_refused", "changed_parameters": exc.parameters}) + return report + except Stopped as exc: + report.update({"status": "stopped", "reason": str(exc)}) + return report + except Exception as exc: + report.update({"status": "instrument_failure", "error_type": type(exc).__name__}) + return report + finally: + report.update({"candidates": dict(counts), "rejections": dict(rejected), "ordinary_exclusions": dict(ordinary_counts)}) + report["instrument"] = dict(clones.counts) + report["wall_seconds"] = round(time.monotonic() - started, 3) + report["github_calls"] = github.calls + report["osv_calls"] = osv.calls + report["github_http_attempts"] = getattr(getattr(github, "transport", None), "attempts", github.calls) + report["osv_http_attempts"] = getattr(getattr(osv, "transport", None), "attempts", osv.calls) + + +def main(argv: list[str] | None = None) -> int: + from openultrasast import config + from openultrasast.plane import memory + from openultrasast.plane.reconciler import results_root + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--cache", type=Path) + parser.add_argument("--max-hours", type=float, help="Stop between units after this many hours per invocation") + parser.add_argument("--seed", type=int, default=20260601) + parser.add_argument("--local-store") + parser.add_argument("--s3-store", default="s3://") + parser.add_argument("--pointer-manifest", type=Path, action="append", default=[]) + parser.add_argument("--known-empty", choices=(*eligibility.SOURCES, "memory_local", "memory_s3"), action="append", default=[]) + args = parser.parse_args(argv) + try: + config.load_dotenv(args.root / ".env") + # Refuse an existing draft before doing any network work. + if any((args.root / p).exists() for p in ("benchmarks/unseen/draft-p1.toml", "benchmarks/unseen/private/draft-p1.toml")): + raise FileExistsError("draft_exists") + clones = Clones(args.root, args.cache or results_root() / "unseen-draft") + stores = { + "local": memory.open_store(args.local_store or "file://" + str(results_root() / "memory"), read_only=True), + "s3": memory.open_store(args.s3_store, read_only=True), + } + used = eligibility.build_used(args.root, stores=stores, pointer_manifests=args.pointer_manifest) + report = run( + args.root, + github=source.GitHub(os.environ.get("GH_TOKEN", "")), + osv=source.OSV(), + clones=clones, + used=used, + seed=args.seed, + known_empty=set(args.known_empty), + journal_path=clones.cache / "journal", + max_hours=args.max_hours, + ) + except Exception as exc: + report = {"status": "instrument_failure", "error_type": type(exc).__name__} + print(json.dumps(report, sort_keys=True)) + return 0 if report["status"] in {"draft", "stopped"} else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/unseen/extract.py b/benchmarks/unseen/extract.py new file mode 100644 index 0000000..0e7a5a5 --- /dev/null +++ b/benchmarks/unseen/extract.py @@ -0,0 +1,474 @@ +"""Git-only change extraction; clients and subprocess execution are injectable. + +No scanner, engine or model runs here. Function boundaries are syntax, not a security +judgement: the first removed code line of a fix is the SZZ sink proxy. Ambiguous sites +are dropped, and the recorded SZZ bias remains subject to task 2.6's label check. +""" + +from __future__ import annotations + +import ast +import json +import re +import subprocess +from pathlib import Path + +from .source import Candidate, Rejected + +LANGUAGES = {".py", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".php", ".java"} +HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.M) +SHA = re.compile(r"[a-f0-9]{40}") +BIAS = { + "introducing": "SZZ last-writer proxy: refactors, moves and reformats can misidentify the introduction in either direction.", + "last_touch": "Pessimistic lower bound: the flaw may already exist in base, so a delta hook can correctly stay quiet.", +} + + +class InstrumentFailure(ValueError): + """A failed read or command must never masquerade as an empty history.""" + + +class Git: + def __init__(self, path: Path, *, runner=None, limit_bytes: int | None = None): + self.path = path + self.runner = runner or subprocess.run + self.bytes_read = 0 + self.files_read = 0 + self.last_blob_bytes = 0 + self.limit_bytes = limit_bytes + + def run(self, *args: str, allowed=(0,)) -> str: + try: + done = self.runner( + ["git", "-c", "core.quotePath=false", "-C", str(self.path), *args], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=180, + check=False, + ) + except Exception: + raise InstrumentFailure("git_launch_failed") from None + if done.returncode not in allowed: + raise InstrumentFailure("git_exit_" + str(done.returncode)) + if self.limit_bytes is not None and self.disk_bytes() > self.limit_bytes: + raise InstrumentFailure("clone_size_cap") + self.last_blob_bytes = len(done.stdout) + return done.stdout.decode("utf-8", errors="replace") + + def disk_bytes(self) -> int: + return sum(p.stat().st_size for p in self.path.rglob("*") if p.is_file() and not p.is_symlink()) + + def parents(self, sha: str) -> list[str]: + return self.run("rev-list", "--parents", "-n", "1", sha).split()[1:] + + def blob(self, sha: str, path: str) -> str | None: + # Missing is established by a successful tree read, never by swallowing a show error. + if not self.run("ls-tree", sha, "--", path).strip(): + return None + text = self.run("show", f"{sha}:{path}") + if Path(path).suffix in LANGUAGES: + self.files_read += 1 + self.bytes_read += self.last_blob_bytes + return text + + def stats(self, base: str, head: str) -> list[tuple[str, int, int]]: + rows = [] + for line in self.run("diff", "--no-renames", "--numstat", base, head, "--").splitlines(): + added, removed, path = line.split("\t", 2) + rows.append((path, int(added) if added.isdigit() else 0, int(removed) if removed.isdigit() else 0)) + return rows + + def hunks(self, base: str, head: str, path: str) -> list[tuple[int, int, int, int]]: + diff = self.run("diff", "--no-ext-diff", "--no-renames", "-U0", base, head, "--", path) + return [(int(m[1]), int(m[2] or 1), int(m[3]), int(m[4] or 1)) for m in HUNK.finditer(diff)] + + def ancestor(self, older: str, newer: str) -> bool: + # rev-list avoids interpreting a command failure as a negative ancestry result. + return not self.run("rev-list", "--max-count=1", older, "--not", newer).strip() + + def blame(self, revision: str, path: str, line: int) -> tuple[str, str, int]: + text = self.run("blame", "-w", "-M", "-C", "--porcelain", f"-L{line},{line}", revision, "--", path) + header = text.splitlines()[0].split() + if len(header) < 3 or not SHA.fullmatch(header[0]): + raise InstrumentFailure("invalid_blame") + filename = next((row[9:] for row in text.splitlines() if row.startswith("filename ")), path) + if filename.startswith('"'): + filename = json.loads(filename) + return header[0], filename, int(header[1]) + + +def functions(path: str, text: str) -> list[tuple[str, int, int]]: + """Conservative syntax boundaries. No scanner import; unresolved sites are rejected.""" + if Path(path).suffix == ".py": + try: + tree = ast.parse(text) + except SyntaxError: + raise Rejected("function_parse") from None + return [ + (n.name, n.lineno, n.end_lineno or n.lineno) for n in ast.walk(tree) if isinstance(n, ast.FunctionDef | ast.AsyncFunctionDef) + ] + # Blank comments and quoted strings before brace matching. Keep newlines/offsets. + masked = re.sub( + r'/\*.*?\*/|//[^\n]*|\#[^\n]*|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|`(?:\\.|[^`\\])*`', + lambda m: "".join("\n" if c == "\n" else " " for c in m[0]), + text, + flags=re.S, + ) + declaration = re.compile( + r"\bfunction\s*&?\s*(\w+)\s*\([^)]*\)\s*(?::[^\n{]+)?\s*\{|" + r"\b(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|\w+)\s*=>\s*\{|" + r"\b([\w$]+)\s*\([^;{}]*\)\s*(?:throws\s+[\w., ]+)?\s*\{" + ) + found = [] + for match in declaration.finditer(masked): + name = next(g for g in match.groups() if g) + if name in {"if", "while", "for", "switch", "catch", "with", "synchronized"}: + continue + depth = 1 + end = match.end() + while end < len(masked) and depth: + depth += (masked[end] == "{") - (masked[end] == "}") + end += 1 + if depth == 0: + found.append((name, masked.count("\n", 0, match.start()) + 1, masked.count("\n", 0, end) + 1)) + return found + + +def enclosing(path: str, text: str, line: int, *, global_ok=False) -> tuple[str, int, int] | None: + spans = [s for s in functions(path, text) if s[1] <= line <= s[2]] + if spans: + return min(spans, key=lambda s: s[2] - s[1]) + return ("", 1, max(1, len(text.splitlines()))) if global_ok else None + + +def normalized(text: str) -> str: + return re.sub(r"\s+", "", text) + + +def site_at(git, revision: str, path: str, name: str): + text = git.blob(revision, path) + if text is None: + return None + spans = [("", 1, max(1, len(text.splitlines())))] if name == "" else [s for s in functions(path, text) if s[0] == name] + if len(spans) != 1: + return None + return text, spans[0] + + +def fix_sites(git, fix: str, parent: str, *, global_ok=False) -> list[dict]: + sites = [] + for path, _, _ in git.stats(parent, fix): + if Path(path).suffix not in LANGUAGES: + continue + text = git.blob(parent, path) + if text is None: + continue + lines = text.splitlines() + for start, count, _, _ in git.hunks(parent, fix, path): + for line in range(max(1, start), max(1, start) + max(1, count)): + if line > len(lines): + continue + span = enclosing(path, text, line, global_ok=global_ok) + if span is None: + continue + sink_line = line if count else span[1] + sink = lines[sink_line - 1] + if not sink.strip() or sink.lstrip().startswith(("#", "//", "/*", "*")): + continue + key = (path, span[0]) + if any((s["path"], s["function"]) == key for s in sites): + continue + sites.append({"path": path, "function": span[0], "line": sink_line, "text": sink, "absence": count == 0, "span": span}) + return sites + + +def version_bracket(git, candidate: Candidate, revision: str) -> bool: + if not candidate.first_affected: + return True + tags = git.run("tag", "--sort=version:refname").splitlines() + matching = [t for t in tags if t.removeprefix("v") == candidate.first_affected] + if len(matching) != 1: + return False + affected = matching[0] + earlier = [t for t in tags[: tags.index(affected)] if git.ancestor(t, affected)] + return bool(earlier) and git.ancestor(revision, affected) and not git.ancestor(revision, earlier[-1]) + + +def change_files(git, base: str, head: str) -> list[dict]: + return [ + {"path": path, "head_lines": [[max(1, new), max(1, new + max(count, 1) - 1)] for _, _, new, count in git.hunks(base, head, path)]} + for path, _, _ in git.stats(base, head) + ] + + +def introducing(git, candidate: Candidate) -> dict: + from collections import Counter + + if git.parents(candidate.fix) != [candidate.parent]: + raise Rejected("fix_parents") + sites = fix_sites(git, candidate.fix, candidate.parent, global_ok=candidate.family == "config_secrets") + reasons = Counter() + + def record(method, base, head, path, name): + resolved = site_at(git, head, path, name) + if resolved is None: + return None + return { + "kind": "vulnerability", + "base": base, + "head": head, + "method": method, + "family": candidate.family, + "function": name, + "path": path, + "function_lines": list(resolved[1][1:]), + "files": change_files(git, base, head), + "post_cutoff": candidate.post_cutoff, + "advisory": candidate.advisory, + "bias": BIAS[method], + "fallback_reasons": dict(reasons), + } + + for site in sites: + revision, path, line = candidate.parent, site["path"], site["line"] + for _ in range(6): # initial blame plus at most five walk-back steps + blamed, blamed_path, blamed_line = git.blame(revision, path, line) + parents = git.parents(blamed) + if len(parents) != 1: + reasons["merge_or_root"] += 1 + break + base = parents[0] + head_site = site_at(git, blamed, blamed_path, site["function"]) + if head_site is None: + reasons["unresolved_function"] += 1 + break + parent_text = git.blob(base, blamed_path) + if parent_text is not None and site["function"] != "": + matching = [span for span in functions(blamed_path, parent_text) if span[0] == site["function"]] + if len(matching) > 1: + reasons["ambiguous_parent"] += 1 + break + old_site = site_at(git, base, blamed_path, site["function"]) + occurrences = [] + if old_site: + text, (_, start, end) = old_site + occurrences = [ + i for i, value in enumerate(text.splitlines(), 1) if start <= i <= end and normalized(value) == normalized(site["text"]) + ] + if occurrences: + reasons["text_present"] += 1 + revision, path, line = base, blamed_path, occurrences[0] + continue + stats = git.stats(base, blamed) + if len(stats) > 50 or sum(a + d for _, a, d in stats) > 2000: + reasons["size_cap"] += 1 + break + if not version_bracket(git, candidate, blamed): + reasons["version_bracket"] += 1 + break + result = record("introducing", base, blamed, blamed_path, site["function"]) + if result: + return result + else: + reasons["walkback_limit"] += 1 + for site in sites: + # Numeric function extent implements log -L for all five languages, including + # languages for which Git has no built-in :function: driver. + _, start, end = site["span"] + history = git.run("log", "--format=%H", "--no-patch", f"-L{start},{end}:{site['path']}", candidate.parent).splitlines() + if not history: + continue + latest = history[0] + parents = git.parents(latest) + if len(parents) != 1: + continue + result = record("last_touch", parents[0], candidate.parent, site["path"], site["function"]) + if result: + return result + raise Rejected("no_range") + + +SECURITY = re.compile(r"secur|cve|ghsa|vuln|inject|xss|csrf|ssrf|exploit|sanitiz|escap", re.I) +DEPENDENCY = re.compile(r"\bbump\b|\bupgrade .* from\b", re.I) +MANIFESTS = { + "package.json", + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", + "composer.json", + "composer.lock", + "pom.xml", + "build.gradle", + "build.gradle.kts", + "gradle.lockfile", + "pyproject.toml", + "setup.py", + "setup.cfg", + "Pipfile", + "Pipfile.lock", + "poetry.lock", + "uv.lock", +} + + +def generated_path(path: str) -> bool: + return bool({"vendor", "node_modules", "dist"} & set(Path(path).parts)) or path.endswith(".min.js") + + +def manifest(path: str) -> bool: + name = Path(path).name + return name in MANIFESTS or name.endswith(".lock") or bool(re.fullmatch(r"requirements[^/]*\.txt", name)) + + +def ordinary_rejection(row: dict, fixes: set[str], known_functions: set[tuple[str, str]]) -> str | None: + if len(row["parents"]) != 1: + return "merge_or_root" + if row["head"] in fixes: + return "advisory_fix" + if SECURITY.search(row["message"]): + return "security" + if re.search(r"^Revert|This reverts commit", row["message"], re.I | re.M): + return "revert" + if re.search(r"dependabot|renovate|\[bot\]", row["author"], re.I) or DEPENDENCY.search(row["message"]): + return "dependency" + files = row["files"] + if files and all(manifest(f["path"]) for f in files): + return "manifest_only" + source = [f for f in files if Path(f["path"]).suffix in LANGUAGES and not manifest(f["path"])] + if not source: + return "no_source" + if all(f["generated"] or generated_path(f["path"]) for f in source): + return "generated" + if any((f["path"], name) in known_functions for f in files for name in f["functions"]): + return "vulnerable_function" + if any((f["path"], "") in known_functions for f in files): + return "vulnerable_function" + if sum(f["added"] + f["removed"] for f in source) > 1000: + return "size_cap" + return None + + +def generated(git, revision: str, path: str, text: str) -> bool: + import fnmatch + + if generated_path(path): + return True + header = "\n".join(text.splitlines()[:20]) + if re.search(r"(?im)^\s*(?:#|//|/\*|\*|