Dramatically speed up git ignored file checking using pathspec directly and make getting files mentions roughly ~100 faster

Author: dwash96

cecli/coders/base_coder.py

@@ -2809,73 +2809,57 @@ def add_assistant_reply_to_cur_messages(self):
             )
 
     def get_file_mentions(self, content, ignore_current=False):
-        # Get file-like words from content (contiguous strings containing slashes or periods)
+        # 1. Extract words once: O(N)
         words = set()
         for word in content.split():
-            # Strip quotes and punctuation
             word = word.strip("\"'`*_,.!;:?")
             if re.search(r"[\\\/._-]", word):
                 words.add(word)
 
-        # Also check basenames of file-like words
-        basename_words = set()
-        for word in words:
-            basename = os.path.basename(word)
-            if basename and basename != word:  # Only add if basename is different
-                basename_words.add(basename)
-
-        # Combine all words to check
+        basename_words = {os.path.basename(w) for w in words if os.path.basename(w) != w}
         all_words = words | basename_words
 
-        if ignore_current:
-            files_to_check = self.get_all_relative_files()
-            existing_basenames = set()
-        else:
-            files_to_check = self.get_addable_relative_files()
-            # Get basenames of files already in chat or read-only
+        # Pre-normalize for O(1) lookups: O(W)
+        normalized_words = {w.replace("\\", "/") for w in all_words}
+
+        # 2. Get files and filter ignored once: O(F)
+        raw_files = (
+            self.get_all_relative_files() if ignore_current else self.get_addable_relative_files()
+        )
+
+        # Filter ignored files once to avoid repeated expensive calls
+        files_to_check = [f for f in raw_files if not (self.repo and self.repo.git_ignored_file(f))]
+
+        # 3. Existing basenames setup
+        existing_basenames = set()
+
+        if not ignore_current:
             existing_basenames = {os.path.basename(f) for f in self.get_inchat_relative_files()} | {
                 os.path.basename(self.get_rel_fname(f))
                 for f in self.abs_read_only_fnames | self.abs_read_only_stubs_fnames
             }
 
-        # Build map of basenames to files for uniqueness check
-        # Only consider basenames that look like filenames (contain /, \, ., _, or -)
-        # to avoid false matches on common words like "run" or "make"
+        # 4. Build map: O(F)
         basename_to_files = {}
         for rel_fname in files_to_check:
-            # Skip git-ignored files
-            if self.repo and self.repo.git_ignored_file(rel_fname):
-                continue
-
-            basename = os.path.basename(rel_fname)
-            # Only include basenames that look like filenames
-            if re.search(r"[\\\/._-]", basename):
-                if basename not in basename_to_files:
-                    basename_to_files[basename] = []
-                basename_to_files[basename].append(rel_fname)
+            bn = os.path.basename(rel_fname)
+            if re.search(r"[\\\/._-]", bn):
+                basename_to_files.setdefault(bn, []).append(rel_fname)
 
+        # 5. Final selection: O(F)
         mentioned_rel_fnames = set()
-
         for rel_fname in files_to_check:
-            # Skip git-ignored files
-            if self.repo and self.repo.git_ignored_file(rel_fname):
-                continue
-
-            # Check if full path matches
-            normalized_fname = rel_fname.replace("\\", "/")
-            normalized_words = {w.replace("\\", "/") for w in all_words}
-
-            if normalized_fname in normalized_words:
+            # Full path match
+            if rel_fname.replace("\\", "/") in normalized_words:
                 mentioned_rel_fnames.add(rel_fname)
                 continue
 
-            # Check basename - only add if unique among addable files and not already in chat
-            basename = os.path.basename(rel_fname)
+            # Basename match logic
+            bn = os.path.basename(rel_fname)
             if (
-                basename in all_words
-                and basename not in existing_basenames
-                and len(basename_to_files.get(basename, [])) == 1
-                and basename_to_files[basename][0] == rel_fname
+                bn in all_words
+                and bn not in existing_basenames
+                and len(basename_to_files.get(bn, [])) == 1
             ):
                 mentioned_rel_fnames.add(rel_fname)
 

cecli/repo.py

@@ -58,6 +58,9 @@ class GitRepo:
     subtree_only = False
     ignore_file_cache = {}
     git_repo_error = None
+    gitignore_spec_cache = {}
+    gitignore_file_cache = {}
+    gitignore_last_check = 0
 
     def __init__(
         self,
@@ -525,16 +528,81 @@ def refresh_cecli_ignore(self):
                 lines,
             )
 
+    def _get_gitignore_spec(self, dir_path):
+        """Get or create a GitIgnoreSpec for a directory, caching for performance."""
+        dir_path = Path(dir_path).resolve()
+
+        # Check cache first
+        if dir_path in self.gitignore_spec_cache:
+            return self.gitignore_spec_cache[dir_path]
+
+        # Read .gitignore from this directory
+        patterns = []
+        gitignore_path = dir_path / ".gitignore"
+        if gitignore_path.is_file():
+            try:
+                with open(gitignore_path, "r") as f:
+                    patterns = [
+                        line.rstrip("\n") for line in f if line.strip() and not line.startswith("#")
+                    ]
+            except (OSError, IOError):
+                pass
+
+        # Create spec for this directory
+        if patterns:
+            spec = pathspec.GitIgnoreSpec.from_lines(patterns)
+        else:
+            spec = pathspec.GitIgnoreSpec.from_lines([])
+
+        self.gitignore_spec_cache[dir_path] = spec
+        return spec
+
+    def _is_gitignored_by_pathspec(self, path):
+        """Check if a file is ignored by any .gitignore file using pathspec."""
+        if not self.repo:
+            return False
+
+        try:
+            file_path = Path(path).resolve()
+            if not file_path.is_relative_to(self.root):
+                return False
+
+            # Walk up from file's directory to root
+            current_dir = file_path.parent
+            relative_path = file_path.relative_to(self.root)
+
+            # Check each directory level
+            while current_dir.is_relative_to(self.root):
+                spec = self._get_gitignore_spec(current_dir)
+
+                # Get path relative to the directory containing the .gitignore
+                if current_dir == Path(self.root).resolve():
+                    path_to_check = str(relative_path)
+                else:
+                    path_to_check = str(
+                        relative_path.relative_to(current_dir.relative_to(self.root))
+                    )
+
+                if spec.match_file(path_to_check):
+                    return True
+
+                # Move up one directory
+                if current_dir == Path(self.root).resolve():
+                    break
+                current_dir = current_dir.parent
+
+            return False
+        except (ValueError, OSError):
+            return False
+
     def git_ignored_file(self, path):
         if not self.repo:
             return
         try:
-            repo_ignored = self.repo.ignored(path)
-
             if not self.cecli_ignore_file or not self.cecli_ignore_file.is_file():
-                return repo_ignored
-
-            return self.ignored_file(path)
+                return self._is_gitignored_by_pathspec(path)
+            else:
+                return self.ignored_file(path)
         except ANY_GIT_ERROR:
             return False