fix: address PR review -- 4 verified findings

1. Add -> int return annotation to index_repository_with_progress
2. Skip incremental indexing when include_paths is set (incremental_index_repository
   uses git diff which doesn't understand subset boundaries)
3. Move blocking rglob() out of async handler into _scan_directories() sync helper,
   called via asyncio.to_thread() to avoid blocking the event loop on large repos
4. Replace full-repo rglob + startswith filter in _discover_code_files with targeted
   subtree walking (only iterate include_paths dirs) -- eliminates prefix collision
   risk (packages/effectx no longer matches packages/effect)

Author: DevanshuNEU

backend/routes/repos.py

@@ -178,28 +178,17 @@ async def delete_repository(
         raise HTTPException(status_code=500, detail="Failed to delete repository")
 
 
-@router.get("/{repo_id}/directories")
-async def get_repo_directories(
-    repo_id: str,
-    auth: AuthContext = Depends(require_auth),
-):
-    """Return the top-level directory tree of a cloned repo.
+def _scan_directories(local_path: Path) -> list:
+    """Scan top-level directories and count code files in each.
 
-    Used for monorepo subset selection -- lets the user pick which
-    directories to index instead of the entire repo.
+    Runs synchronously -- call via asyncio.to_thread() from async handlers
+    to avoid blocking the event loop on large repos.
     """
-    repo = get_repo_or_404(repo_id, auth.user_id)
-    local_path = Path(repo["local_path"])
-
-    if not local_path.exists():
-        raise HTTPException(status_code=404, detail="Repo not cloned yet")
-
     skip = {"node_modules", ".git", "__pycache__", "venv", ".next", "dist", "build"}
+    extensions = {".py", ".js", ".jsx", ".ts", ".tsx"}
     dirs = []
     for item in sorted(local_path.iterdir()):
         if item.is_dir() and item.name not in skip and not item.name.startswith("."):
-            # count code files in this directory
-            extensions = {".py", ".js", ".jsx", ".ts", ".tsx"}
             file_count = sum(
                 1 for f in item.rglob("*")
                 if f.is_file() and f.suffix in extensions
@@ -210,6 +199,26 @@ async def get_repo_directories(
                 "path": str(item.relative_to(local_path)),
                 "file_count": file_count,
             })
+    return dirs
+
+
+@router.get("/{repo_id}/directories")
+async def get_repo_directories(
+    repo_id: str,
+    auth: AuthContext = Depends(require_auth),
+):
+    """Return the top-level directory tree of a cloned repo.
+
+    Used for monorepo subset selection -- lets the user pick which
+    directories to index instead of the entire repo.
+    """
+    repo = get_repo_or_404(repo_id, auth.user_id)
+    local_path = Path(repo["local_path"])
+
+    if not local_path.exists():
+        raise HTTPException(status_code=404, detail="Repo not cloned yet")
+
+    dirs = await asyncio.to_thread(_scan_directories, local_path)
 
     return {
         "repo_id": repo_id,
@@ -341,9 +350,12 @@ async def _run_async_indexing(
             publisher.publish_progress(repo_id, 0, 1, 0, "Starting...")
 
         # Check for incremental
+        # Skip incremental when include_paths is set -- incremental_index_repository
+        # uses git diff which doesn't understand subset boundaries
         last_commit = repo_manager.get_last_indexed_commit(repo_id)
+        can_incremental = incremental and last_commit and not include_paths
 
-        if incremental and last_commit:
+        if can_incremental:
             logger.info("Async INCREMENTAL indexing", repo_id=repo_id, last_commit=last_commit[:8])
             total_functions = await indexer.incremental_index_repository(
                 repo_id,

backend/services/indexer_optimized.py

@@ -125,26 +125,36 @@ def _discover_code_files(
         Args:
             include_paths: If set, only include files under these relative
                 directories (e.g. ['packages/effect', 'packages/schema']).
+                Uses path-component-aware matching and only walks the
+                specified subtrees instead of the entire repo.
         """
         repo_path = Path(repo_path)
         code_files = []
-        
+
         extensions = {'.py', '.js', '.jsx', '.ts', '.tsx'}
         skip_dirs = {'node_modules', '.git', '__pycache__', 'venv', 'env', 'dist', 'build', '.next', '.vscode'}
-        
-        for file_path in repo_path.rglob('*'):
-            if file_path.is_dir():
-                continue
-            if any(skip in file_path.parts for skip in skip_dirs):
-                continue
-            if file_path.suffix in extensions:
-                # Subset filter: only include files under specified paths
-                if include_paths:
-                    rel = str(file_path.relative_to(repo_path))
-                    if not any(rel.startswith(p) for p in include_paths):
-                        continue
-                code_files.append(file_path)
-        
+
+        # When include_paths is set, only walk those subtrees
+        if include_paths:
+            roots = []
+            for p in include_paths:
+                subtree = repo_path / p
+                if subtree.is_dir():
+                    roots.append(subtree)
+                else:
+                    logger.warning("include_path not found, skipping: %s", p)
+        else:
+            roots = [repo_path]
+
+        for root in roots:
+            for file_path in root.rglob('*'):
+                if file_path.is_dir():
+                    continue
+                if any(skip in file_path.parts for skip in skip_dirs):
+                    continue
+                if file_path.suffix in extensions:
+                    code_files.append(file_path)
+
         return code_files
 
     async def _create_embeddings_batch(self, texts: List[str]) -> List[List[float]]:
@@ -704,7 +714,7 @@ async def index_repository_with_progress(
         progress_callback,
         max_files: int = None,
         include_paths: Optional[List[str]] = None,
-    ):
+    ) -> int:
         """Index repository with real-time progress updates
 
         Args: