fix(preprocess): read .gitignore with git's semantics; never ignore a tracked file

The enumeration dropped every `!` line and matched with fnmatch, whose `*` crosses
`/`. YesWiki's `tools/*` followed by `!tools/bazar` therefore removed all 583 PHP
files under tools/, the vulnerable FormManager.php among them. The scan read 243 of
803 files, completed 1128 of 1128 questions and looked clean: a zero from an unread
tree.

Rules now follow git:
- the last matching rule wins, and `!` re-includes;
- `*` and `?` stay inside one path component, and `**` crosses components;
- a rule with an inner `/` is anchored to the root;
- a trailing `/` matches directories only;
- nothing under an ignored directory can be re-included.
In a work tree, a tracked file is never ignored, because `.gitignore` describes what
the repository does not hold.

The test uses `git check-ignore` as the oracle on 16 paths covering negation,
anchoring, depth, directory-only and `**`. It also covers a git-archive export with
no .git, and a tracked file under an ignored directory. All three tests fail on the
old reader.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo

src/openultrasast/preprocess.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-import fnmatch
 import json
+import re
 import subprocess
 from dataclasses import asdict, dataclass, field
 from pathlib import Path
@@ -95,13 +95,14 @@ def write_preprocess_artifact(snapshot: RepoSnapshot, targets: list[FileTarget],
 
 def enumerate_source_files(root: Path) -> list[Path]:
     patterns = _load_ignore_patterns(root)
+    tracked = _tracked_files(root) if patterns else None
     paths: list[Path] = []
     for path in root.rglob("*"):
         if (
             path.is_symlink()
             or not path.is_file()
             or _has_ignored_dir(root, path)
-            or _is_ignored(root, path, patterns)
+            or _is_ignored(root, path, patterns, tracked)
             or _is_vendored(root, path)
         ):
             continue
@@ -166,16 +167,80 @@ def detect_tags(relative_path: str, language: str, text: str) -> list[str]:
     return sorted(tags)
 
 
-def _load_ignore_patterns(root: Path) -> list[str]:
+# One `.gitignore` rule: whether it re-includes (`!`), whether it names directories only (trailing `/`), and
+# the path it matches as a regular expression over the repository-relative POSIX path.
+_IgnoreRule = tuple[bool, bool, "re.Pattern[str]"]
+
+
+def _glob_regex(body: str) -> str:
+    """Git's wildmatch for one pattern body: `*` and `?` stay inside a path component, `**` crosses them."""
+    out, i = "", 0
+    while i < len(body):
+        if body.startswith("**/", i):
+            out, i = out + "(?:.*/)?", i + 3
+        elif body.startswith("/**", i) and i + 3 == len(body):
+            out, i = out + "/.*", i + 3
+        elif body.startswith("**", i):
+            out, i = out + ".*", i + 2
+        elif body[i] == "*":
+            out, i = out + "[^/]*", i + 1
+        elif body[i] == "?":
+            out, i = out + "[^/]", i + 1
+        elif body[i] == "[" and "]" in body[i + 1 :]:
+            end = body.index("]", i + 1)
+            members = body[i + 1 : end]
+            members = "^" + members[1:] if members.startswith("!") else members
+            out, i = out + "[" + members.replace("\\", "\\\\") + "]", end + 1
+        elif body[i] == "\\" and i + 1 < len(body):
+            out, i = out + re.escape(body[i + 1]), i + 2
+        else:
+            out, i = out + re.escape(body[i]), i + 1
+    return out
+
+
+def _load_ignore_patterns(root: Path) -> list[_IgnoreRule]:
+    """The root `.gitignore`, with git's semantics.
+
+    The first version dropped every `!` line and matched with `fnmatch`, whose `*` crosses `/`. YesWiki's
+    `tools/*` followed by `!tools/bazar` therefore removed all 583 of its tool PHP files -- the vulnerable one
+    among them -- and the scan read 243 files of 803, completed every question it had, and reported the
+    answer as clean.
+    """
     ignore_file = root / ".gitignore"
     if not ignore_file.exists():
         return []
-    patterns: list[str] = []
+    rules: list[_IgnoreRule] = []
     for line in ignore_file.read_text(errors="ignore").splitlines():
-        stripped = line.strip()
-        if stripped and not stripped.startswith("#") and not stripped.startswith("!"):
-            patterns.append(stripped.rstrip("/"))
-    return patterns
+        text = line.rstrip()
+        if not text or text.startswith("#"):
+            continue
+        negate = text.startswith("!")
+        text = text[1:] if negate else text.removeprefix("\\")
+        directory_only = text.endswith("/")
+        text = text.rstrip("/")
+        if not text:
+            continue
+        anchored = "/" in text
+        body = _glob_regex(text.lstrip("/"))
+        rules.append((negate, directory_only, re.compile("^" + ("" if anchored else "(?:.*/)?") + body + "$")))
+    return rules
+
+
+def _ignored_by(rules: list[_IgnoreRule], relative: str, is_dir: bool) -> bool:
+    verdict = False
+    for negate, directory_only, pattern in rules:
+        if (is_dir or not directory_only) and pattern.match(relative):
+            verdict = not negate
+    return verdict
+
+
+def _tracked_files(root: Path) -> set[str] | None:
+    """What git tracks under `root`, or None where `root` is not a work tree (a `git archive` export)."""
+    try:
+        done = subprocess.run(["git", "-C", str(root), "ls-files", "-z"], capture_output=True, check=True, timeout=60)
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        return None
+    return {name for name in done.stdout.decode(errors="replace").split("\0") if name}
 
 
 def _has_ignored_dir(root: Path, path: Path) -> bool:
@@ -192,9 +257,17 @@ def _is_vendored(root: Path, path: Path) -> bool:
     return is_vendored(path.relative_to(root).as_posix())
 
 
-def _is_ignored(root: Path, path: Path, patterns: list[str]) -> bool:
+def _is_ignored(root: Path, path: Path, rules: list[_IgnoreRule], tracked: set[str] | None = None) -> bool:
     relative = path.relative_to(root).as_posix()
-    return any(fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(path.name, pattern) for pattern in patterns)
+    # Git never ignores a tracked file; `.gitignore` is about what is NOT in the repository.
+    if not rules or (tracked is not None and relative in tracked):
+        return False
+    parts = relative.split("/")
+    # Nothing under an ignored directory can be re-included, which is git's rule too.
+    for depth in range(1, len(parts)):
+        if _ignored_by(rules, "/".join(parts[:depth]), is_dir=True):
+            return True
+    return _ignored_by(rules, relative, is_dir=False)
 
 
 def _read_text(path: Path) -> str:

tests/test_gitignore_semantics.py

@@ -0,0 +1,94 @@
+"""`.gitignore` is read with git's own semantics, checked against git itself.
+
+The first reader dropped every `!` line and matched with `fnmatch`, whose `*` crosses `/`. YesWiki's
+`tools/*` + `!tools/bazar` removed all 583 of its tool PHP files, the vulnerable one among them, and the scan
+completed every question it had over the 243 files left -- a clean-looking zero from an unread tree.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from openultrasast.preprocess import _is_ignored, _load_ignore_patterns, enumerate_source_files
+
+GITIGNORE = """\
+# YesWiki's shape
+tools/*
+!tools/README.md
+!tools/bazar
+!tools/bazar/*
+# anchoring, depth, directory-only, double star
+/build
+cache/
+*.min.js
+docs/**/generated.py
+!keep.min.js
+secret?.php
+logs/
+!logs/keep.php
+"""
+
+FILES = [
+    "tools/bazar/services/FormManager.php",
+    "tools/bazar/FormController.php",
+    "tools/attach/attach.php",
+    "tools/README.md",
+    "build/out.py",
+    "src/build/inner.py",
+    "cache/x.py",
+    "src/cache/y.py",
+    "src/app.min.js",
+    "keep.min.js",
+    "docs/a/b/generated.py",
+    "docs/generated.py",
+    "secret1.php",
+    "secret12.php",
+    "logs/keep.php",
+    "app.py",
+]
+
+
+def _git_ignored(root: Path, relative: str) -> bool:
+    return subprocess.run(["git", "-C", str(root), "check-ignore", "-q", relative], check=False).returncode == 0
+
+
+@pytest.mark.skipif(shutil.which("git") is None, reason="git is the oracle")
+def test_ignore_rules_agree_with_git(tmp_path: Path) -> None:
+    subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
+    (tmp_path / ".gitignore").write_text(GITIGNORE)
+    for name in FILES:
+        (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)
+        (tmp_path / name).write_text("x\n")
+    rules = _load_ignore_patterns(tmp_path)
+    disagreements = {
+        name: (_git_ignored(tmp_path, name), _is_ignored(tmp_path, tmp_path / name, rules))
+        for name in FILES
+        if _git_ignored(tmp_path, name) != _is_ignored(tmp_path, tmp_path / name, rules)
+    }
+    assert not disagreements, f"(git, ours) per file: {disagreements}"
+    assert not _is_ignored(tmp_path, tmp_path / "tools/bazar/services/FormManager.php", rules)
+
+
+def test_an_export_keeps_what_the_ignore_file_re_includes(tmp_path: Path) -> None:
+    """A `git archive` export has no `.git`: the re-included tree must still be read."""
+    (tmp_path / ".gitignore").write_text("tools/*\n!tools/bazar\n!tools/bazar/*\n")
+    for name in ("tools/bazar/services/FormManager.php", "tools/attach/attach.php", "includes/App.php"):
+        (tmp_path / name).parent.mkdir(parents=True, exist_ok=True)
+        (tmp_path / name).write_text("<?php\n")
+    read = {p.relative_to(tmp_path).as_posix() for p in enumerate_source_files(tmp_path)}
+    assert read == {"tools/bazar/services/FormManager.php", "includes/App.php"}, read
+
+
+@pytest.mark.skipif(shutil.which("git") is None, reason="needs a work tree")
+def test_a_tracked_file_is_never_ignored(tmp_path: Path) -> None:
+    """Git ignores only what it does not track; a committed file under an ignored pattern is source."""
+    subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
+    (tmp_path / "gen").mkdir()
+    (tmp_path / "gen" / "committed.py").write_text("x = 1\n")
+    subprocess.run(["git", "-C", str(tmp_path), "add", "gen/committed.py"], check=True)
+    (tmp_path / ".gitignore").write_text("gen/\n")
+    (tmp_path / "gen" / "untracked.py").write_text("y = 2\n")
+    read = {p.relative_to(tmp_path).as_posix() for p in enumerate_source_files(tmp_path)}
+    assert read == {"gen/committed.py"}, read