Fix gitignore pattern handling

Author: blhsing

pythongit/cli.py

@@ -1506,12 +1506,8 @@ def cmd_clean(argv: list[str]) -> int:
     if not (args.force or args.dry_run):
         _err("fatal: clean.requireForce; use -f or -n")
         return 1
-    from . import ignore as _ig
-    ig = None if args.x else _ig.load(repo.path)
-    s = workdir.status(repo)
+    s = workdir.status(repo, include_ignored=args.x)
     for u in s["untracked"]:
-        if ig and ig.is_ignored(u):
-            continue
         p = repo.path / u
         if args.dry_run:
             _print(f"Would remove {u}")
@@ -1884,7 +1880,9 @@ def cmd_check_ignore(argv: list[str]) -> int:
     ig = _ig.load(repo.path)
     rc = 1
     for p in args.paths:
-        if ig.is_ignored(p):
+        match_path = p.replace(os.sep, "/")
+        is_dir = match_path.endswith("/") or (repo.path / p).is_dir()
+        if ig.is_ignored(match_path, is_dir=is_dir):
             _print(p)
             rc = 0
     return rc

pythongit/ignore.py

@@ -1,107 +1,96 @@
-"""gitignore parsing.
-
-Each directory may have its own .gitignore. Patterns:
-  - lines starting with # are comments
-  - leading ! negates
-  - trailing / matches directories only
-  - leading / anchors to the .gitignore's directory
-  - ** matches any number of path components
-"""
+"""gitignore parsing."""
 from __future__ import annotations
 
-import fnmatch
 import os
 import re
 from pathlib import Path
-from typing import Optional
 
 
 class IgnoreRule:
-    __slots__ = ("pattern", "negate", "dir_only", "anchored", "base", "regex")
+    __slots__ = ("pattern", "negate", "dir_only", "pathname", "base", "regex")
 
     def __init__(self, raw: str, base: str):
         self.negate = False
         self.dir_only = False
-        self.anchored = False
-        self.base = base
+        self.pathname = False
+        self.base = base.strip("/")
         s = raw
         if s.startswith("!"):
             self.negate = True
             s = s[1:]
+        elif s.startswith("\\!") or s.startswith("\\#"):
+            s = s[1:]
         if s.endswith("/"):
             self.dir_only = True
             s = s[:-1]
         if s.startswith("/"):
-            self.anchored = True
+            self.pathname = True
             s = s[1:]
+        if "/" in s:
+            self.pathname = True
         self.pattern = s
-        # Translate glob to regex anchored to base
-        regex = ""
-        i = 0
-        while i < len(s):
-            c = s[i]
-            if c == "*" and i + 1 < len(s) and s[i + 1] == "*":
-                # **
-                regex += ".*"
-                i += 2
-                if i < len(s) and s[i] == "/":
-                    i += 1
-            elif c == "*":
-                regex += "[^/]*"
-                i += 1
-            elif c == "?":
-                regex += "[^/]"
-                i += 1
-            elif c in ".(){}+|^$\\":
-                regex += "\\" + c
-                i += 1
-            elif c == "[":
-                end = s.find("]", i)
-                if end == -1:
-                    regex += "\\["
-                    i += 1
-                else:
-                    regex += s[i : end + 1]
-                    i = end + 1
-            else:
-                regex += c
-                i += 1
-        self.regex = re.compile("^" + regex + "$")
+        self.regex = re.compile("^" + _wildmatch_to_regex(s) + "$")
 
     def match(self, rel: str, is_dir: bool) -> bool:
-        if self.dir_only and not is_dir:
+        sub = self._relative_path(rel)
+        if not sub:
             return False
-        target = rel
-        if self.anchored:
-            if self.base and not target.startswith(self.base + "/") and target != self.base:
-                return False
-            sub = target[len(self.base) + 1 :] if self.base else target
-            return self.regex.match(sub) is not None
-        # match against any suffix
-        if self.regex.match(target):
+        if self._match_path(sub, is_dir):
             return True
-        for i, ch in enumerate(target):
-            if ch == "/" and self.regex.match(target[i + 1 :]):
+
+        # A pattern that matches a directory also applies to everything below
+        # that directory.  This is what makes "build/" ignore "build/out.o".
+        parts = sub.split("/")
+        for i in range(1, len(parts)):
+            parent = "/".join(parts[:i])
+            if self._match_path(parent, True):
                 return True
-        # also match against final component
-        last = target.rsplit("/", 1)[-1]
-        return self.regex.match(last) is not None
+        return False
+
+    def _relative_path(self, rel: str) -> str | None:
+        if self.base:
+            if rel == self.base:
+                return None
+            prefix = self.base + "/"
+            if not rel.startswith(prefix):
+                return None
+            return rel[len(prefix) :]
+        return rel
+
+    def _match_path(self, sub: str, is_dir: bool) -> bool:
+        if self.dir_only and not is_dir:
+            return False
+        if self.pathname:
+            return self.regex.match(sub) is not None
+        if "/" in sub:
+            sub = sub.rsplit("/", 1)[-1]
+        return self.regex.match(sub) is not None
 
 
 class IgnoreSet:
     def __init__(self) -> None:
         self.rules: list[IgnoreRule] = []
 
     def add_file(self, gitignore_path: Path, base_rel: str) -> None:
-        if not gitignore_path.exists():
+        if not gitignore_path.exists() or gitignore_path.is_symlink():
             return
         for line in gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines():
-            line = line.rstrip()
+            line = _trim_trailing_spaces(line)
             if not line or line.startswith("#"):
                 continue
             self.rules.append(IgnoreRule(line, base_rel))
 
     def is_ignored(self, rel_path: str, is_dir: bool = False) -> bool:
+        rel_path, is_dir = _normalize_path(rel_path, is_dir)
+        if not rel_path:
+            return False
+        parts = rel_path.split("/")
+        for i in range(1, len(parts)):
+            if self._last_match("/".join(parts[:i]), True):
+                return True
+        return self._last_match(rel_path, is_dir)
+
+    def _last_match(self, rel_path: str, is_dir: bool) -> bool:
         ignored = False
         for r in self.rules:
             if r.match(rel_path, is_dir):
@@ -111,10 +100,9 @@ def is_ignored(self, rel_path: str, is_dir: bool = False) -> bool:
 
 def load(repo_path: Path) -> IgnoreSet:
     s = IgnoreSet()
-    # Root .gitignore + .git/info/exclude
-    s.add_file(repo_path / ".gitignore", "")
+    # Lower-precedence excludes are loaded first; later matches override them.
     s.add_file(repo_path / ".git" / "info" / "exclude", "")
-    # Walk for nested .gitignore files
+    s.add_file(repo_path / ".gitignore", "")
     for root, dirs, files in os.walk(repo_path):
         dirs[:] = [d for d in dirs if d != ".git"]
         if ".gitignore" in files:
@@ -123,3 +111,103 @@ def load(repo_path: Path) -> IgnoreSet:
                 continue
             s.add_file(Path(root) / ".gitignore", rel_root)
     return s
+
+
+def _normalize_path(path: str, is_dir: bool) -> tuple[str, bool]:
+    path = path.replace(os.sep, "/")
+    if path.endswith("/"):
+        is_dir = True
+        path = path.rstrip("/")
+    parts = [p for p in path.split("/") if p and p != "."]
+    return "/".join(parts), is_dir
+
+
+def _trim_trailing_spaces(line: str) -> str:
+    last_space: int | None = None
+    i = 0
+    while i < len(line):
+        c = line[i]
+        if c == " ":
+            if last_space is None:
+                last_space = i
+        elif c == "\\":
+            i += 1
+            if i >= len(line):
+                return line
+            last_space = None
+        else:
+            last_space = None
+        i += 1
+    if last_space is not None:
+        return line[:last_space]
+    return line
+
+
+def _wildmatch_to_regex(pattern: str) -> str:
+    out: list[str] = []
+    i = 0
+    while i < len(pattern):
+        c = pattern[i]
+        if c == "*":
+            start = i
+            while i < len(pattern) and pattern[i] == "*":
+                i += 1
+            count = i - start
+            prev_is_sep = start == 0 or pattern[start - 1] == "/"
+            next_is_sep = i == len(pattern) or pattern[i] == "/"
+            if count >= 2 and prev_is_sep and next_is_sep:
+                if i < len(pattern) and pattern[i] == "/":
+                    out.append("(?:.*/)?")
+                    i += 1
+                else:
+                    out.append(".*")
+            else:
+                out.append("[^/]*")
+            continue
+        if c == "?":
+            out.append("[^/]")
+        elif c == "[":
+            token, end = _translate_char_class(pattern, i)
+            out.append(token)
+            i = end
+            continue
+        elif c == "\\":
+            i += 1
+            if i >= len(pattern):
+                return r"(?!)"
+            out.append(re.escape(pattern[i]))
+        else:
+            out.append(re.escape(c))
+        i += 1
+    return "".join(out)
+
+
+def _translate_char_class(pattern: str, start: int) -> tuple[str, int]:
+    i = start + 1
+    if i >= len(pattern):
+        return r"\[", start + 1
+
+    negate = False
+    if pattern[i] in ("!", "^"):
+        negate = True
+        i += 1
+    if i < len(pattern) and pattern[i] == "]":
+        i += 1
+
+    body = ""
+    while i < len(pattern) and pattern[i] != "]":
+        if pattern[i] == "\\" and i + 1 < len(pattern):
+            i += 1
+            body += re.escape(pattern[i])
+        else:
+            body += pattern[i]
+        i += 1
+    if i >= len(pattern):
+        return r"\[", start + 1
+
+    body = body.replace("\\", r"\\")
+    if negate:
+        token = f"(?:(?!/)[^{body}])"
+    else:
+        token = f"(?:(?!/)[{body}])"
+    return token, i + 1

pythongit/workdir.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from typing import Iterable, Iterator, Optional
 
+from . import ignore as ignore_mod
 from . import objects as objs
 from . import refs as refs_mod
 from .index import (
@@ -33,6 +34,13 @@ def _ignored(rel: str) -> bool:
     return ".git" in parts
 
 
+def _is_dir_no_follow(path: Path) -> bool:
+    try:
+        return st_mod.S_ISDIR(path.lstat().st_mode)
+    except FileNotFoundError:
+        return False
+
+
 def iter_worktree(repo: Repository) -> list[str]:
     out: list[str] = []
     base = repo.path
@@ -116,6 +124,8 @@ def tree_path_entry(repo: Repository, tree_sha: str, path: str) -> Optional[objs
 
 def add_paths(repo: Repository, paths: Iterable[str]) -> None:
     idx = read_index(repo)
+    tracked = set(idx.by_path())
+    ignores = ignore_mod.load(repo.path)
     to_add: list[str] = []
     for p in paths:
         ap = (repo.path / p).resolve()
@@ -124,10 +134,12 @@ def add_paths(repo: Repository, paths: Iterable[str]) -> None:
                 dirs[:] = [d for d in dirs if d != ".git"]
                 for f in files:
                     rel = _norm(os.path.relpath(os.path.join(root, f), repo.path))
-                    if not _ignored(rel):
+                    if not _ignored(rel) and (rel in tracked or not ignores.is_ignored(rel)):
                         to_add.append(rel)
         else:
-            to_add.append(_norm(os.path.relpath(ap, repo.path)))
+            rel = _norm(os.path.relpath(ap, repo.path))
+            if rel in tracked or not ignores.is_ignored(rel, is_dir=_is_dir_no_follow(ap)):
+                to_add.append(rel)
     for rel in sorted(set(to_add)):
         full = repo.path / rel
         if not full.exists() and not full.is_symlink():
@@ -161,10 +173,11 @@ def rm_paths(repo: Repository, paths: Iterable[str], *, cached: bool = False) ->
 # status / diff target lists
 
 
-def status(repo: Repository) -> dict[str, list[str]]:
+def status(repo: Repository, *, include_ignored: bool = False) -> dict[str, list[str]]:
     """Return groups: staged_new, staged_mod, staged_del, mod, untracked."""
     idx = read_index(repo)
     by_path = idx.by_path()
+    ignores = None if include_ignored else ignore_mod.load(repo.path)
 
     head_tree = _head_tree_map(repo)
 
@@ -187,6 +200,8 @@ def status(repo: Repository) -> dict[str, list[str]]:
             sha, _ = objs.hash_bytes("blob", data, repo)
             if sha != by_path[rel].sha:
                 modified.append(rel)
+        elif ignores is not None and ignores.is_ignored(rel, is_dir=_is_dir_no_follow(full)):
+            pass
         else:
             untracked.append(rel)
         seen.discard(rel)