diff --git a/src/openultrasast/preprocess.py b/src/openultrasast/preprocess.py index fe0ca1e..8e7cb39 100644 --- a/src/openultrasast/preprocess.py +++ b/src/openultrasast/preprocess.py @@ -1,7 +1,7 @@ from __future__ import annotations -import fnmatch import json +import re import subprocess from dataclasses import asdict, dataclass, field from pathlib import Path @@ -95,13 +95,14 @@ def write_preprocess_artifact(snapshot: RepoSnapshot, targets: list[FileTarget], def enumerate_source_files(root: Path) -> list[Path]: patterns = _load_ignore_patterns(root) + tracked = _tracked_files(root) if patterns else None paths: list[Path] = [] for path in root.rglob("*"): if ( path.is_symlink() or not path.is_file() or _has_ignored_dir(root, path) - or _is_ignored(root, path, patterns) + or _is_ignored(root, path, patterns, tracked) or _is_vendored(root, path) ): continue @@ -166,16 +167,80 @@ def detect_tags(relative_path: str, language: str, text: str) -> list[str]: return sorted(tags) -def _load_ignore_patterns(root: Path) -> list[str]: +# One `.gitignore` rule: whether it re-includes (`!`), whether it names directories only (trailing `/`), and +# the path it matches as a regular expression over the repository-relative POSIX path. +_IgnoreRule = tuple[bool, bool, "re.Pattern[str]"] + + +def _glob_regex(body: str) -> str: + """Git's wildmatch for one pattern body: `*` and `?` stay inside a path component, `**` crosses them.""" + out, i = "", 0 + while i < len(body): + if body.startswith("**/", i): + out, i = out + "(?:.*/)?", i + 3 + elif body.startswith("/**", i) and i + 3 == len(body): + out, i = out + "/.*", i + 3 + elif body.startswith("**", i): + out, i = out + ".*", i + 2 + elif body[i] == "*": + out, i = out + "[^/]*", i + 1 + elif body[i] == "?": + out, i = out + "[^/]", i + 1 + elif body[i] == "[" and "]" in body[i + 1 :]: + end = body.index("]", i + 1) + members = body[i + 1 : end] + members = "^" + members[1:] if members.startswith("!") else members + out, i = out + "[" + members.replace("\\", "\\\\") + "]", end + 1 + elif body[i] == "\\" and i + 1 < len(body): + out, i = out + re.escape(body[i + 1]), i + 2 + else: + out, i = out + re.escape(body[i]), i + 1 + return out + + +def _load_ignore_patterns(root: Path) -> list[_IgnoreRule]: + """The root `.gitignore`, with git's semantics. + + The first version dropped every `!` line and matched with `fnmatch`, whose `*` crosses `/`. YesWiki's + `tools/*` followed by `!tools/bazar` therefore removed all 583 of its tool PHP files -- the vulnerable one + among them -- and the scan read 243 files of 803, completed every question it had, and reported the + answer as clean. + """ ignore_file = root / ".gitignore" if not ignore_file.exists(): return [] - patterns: list[str] = [] + rules: list[_IgnoreRule] = [] for line in ignore_file.read_text(errors="ignore").splitlines(): - stripped = line.strip() - if stripped and not stripped.startswith("#") and not stripped.startswith("!"): - patterns.append(stripped.rstrip("/")) - return patterns + text = line.rstrip() + if not text or text.startswith("#"): + continue + negate = text.startswith("!") + text = text[1:] if negate else text.removeprefix("\\") + directory_only = text.endswith("/") + text = text.rstrip("/") + if not text: + continue + anchored = "/" in text + body = _glob_regex(text.lstrip("/")) + rules.append((negate, directory_only, re.compile("^" + ("" if anchored else "(?:.*/)?") + body + "$"))) + return rules + + +def _ignored_by(rules: list[_IgnoreRule], relative: str, is_dir: bool) -> bool: + verdict = False + for negate, directory_only, pattern in rules: + if (is_dir or not directory_only) and pattern.match(relative): + verdict = not negate + return verdict + + +def _tracked_files(root: Path) -> set[str] | None: + """What git tracks under `root`, or None where `root` is not a work tree (a `git archive` export).""" + try: + done = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], capture_output=True, check=True, timeout=60) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return None + return {name for name in done.stdout.decode(errors="replace").split("\0") if name} def _has_ignored_dir(root: Path, path: Path) -> bool: @@ -192,9 +257,17 @@ def _is_vendored(root: Path, path: Path) -> bool: return is_vendored(path.relative_to(root).as_posix()) -def _is_ignored(root: Path, path: Path, patterns: list[str]) -> bool: +def _is_ignored(root: Path, path: Path, rules: list[_IgnoreRule], tracked: set[str] | None = None) -> bool: relative = path.relative_to(root).as_posix() - return any(fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(path.name, pattern) for pattern in patterns) + # Git never ignores a tracked file; `.gitignore` is about what is NOT in the repository. + if not rules or (tracked is not None and relative in tracked): + return False + parts = relative.split("/") + # Nothing under an ignored directory can be re-included, which is git's rule too. + for depth in range(1, len(parts)): + if _ignored_by(rules, "/".join(parts[:depth]), is_dir=True): + return True + return _ignored_by(rules, relative, is_dir=False) def _read_text(path: Path) -> str: diff --git a/tests/test_gitignore_semantics.py b/tests/test_gitignore_semantics.py new file mode 100644 index 0000000..caeb07d --- /dev/null +++ b/tests/test_gitignore_semantics.py @@ -0,0 +1,94 @@ +"""`.gitignore` is read with git's own semantics, checked against git itself. + +The first reader dropped every `!` line and matched with `fnmatch`, whose `*` crosses `/`. YesWiki's +`tools/*` + `!tools/bazar` removed all 583 of its tool PHP files, the vulnerable one among them, and the scan +completed every question it had over the 243 files left -- a clean-looking zero from an unread tree. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +from openultrasast.preprocess import _is_ignored, _load_ignore_patterns, enumerate_source_files + +GITIGNORE = """\ +# YesWiki's shape +tools/* +!tools/README.md +!tools/bazar +!tools/bazar/* +# anchoring, depth, directory-only, double star +/build +cache/ +*.min.js +docs/**/generated.py +!keep.min.js +secret?.php +logs/ +!logs/keep.php +""" + +FILES = [ + "tools/bazar/services/FormManager.php", + "tools/bazar/FormController.php", + "tools/attach/attach.php", + "tools/README.md", + "build/out.py", + "src/build/inner.py", + "cache/x.py", + "src/cache/y.py", + "src/app.min.js", + "keep.min.js", + "docs/a/b/generated.py", + "docs/generated.py", + "secret1.php", + "secret12.php", + "logs/keep.php", + "app.py", +] + + +def _git_ignored(root: Path, relative: str) -> bool: + return subprocess.run(["git", "-C", str(root), "check-ignore", "-q", relative], check=False).returncode == 0 + + +@pytest.mark.skipif(shutil.which("git") is None, reason="git is the oracle") +def test_ignore_rules_agree_with_git(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + (tmp_path / ".gitignore").write_text(GITIGNORE) + for name in FILES: + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text("x\n") + rules = _load_ignore_patterns(tmp_path) + disagreements = { + name: (_git_ignored(tmp_path, name), _is_ignored(tmp_path, tmp_path / name, rules)) + for name in FILES + if _git_ignored(tmp_path, name) != _is_ignored(tmp_path, tmp_path / name, rules) + } + assert not disagreements, f"(git, ours) per file: {disagreements}" + assert not _is_ignored(tmp_path, tmp_path / "tools/bazar/services/FormManager.php", rules) + + +def test_an_export_keeps_what_the_ignore_file_re_includes(tmp_path: Path) -> None: + """A `git archive` export has no `.git`: the re-included tree must still be read.""" + (tmp_path / ".gitignore").write_text("tools/*\n!tools/bazar\n!tools/bazar/*\n") + for name in ("tools/bazar/services/FormManager.php", "tools/attach/attach.php", "includes/App.php"): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text(" None: + """Git ignores only what it does not track; a committed file under an ignored pattern is source.""" + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + (tmp_path / "gen").mkdir() + (tmp_path / "gen" / "committed.py").write_text("x = 1\n") + subprocess.run(["git", "-C", str(tmp_path), "add", "gen/committed.py"], check=True) + (tmp_path / ".gitignore").write_text("gen/\n") + (tmp_path / "gen" / "untracked.py").write_text("y = 2\n") + read = {p.relative_to(tmp_path).as_posix() for p in enumerate_source_files(tmp_path)} + assert read == {"gen/committed.py"}, read