fix: lazy re-clone working tree on redeploy via ensure_clone chokepoint (#311)

Railway containers have ephemeral disk; a redeploy wipes ./repos while Pinecone and Supabase survive, so every file-reading op 404'd on existing repos. Reframe local_path as a cache hint (git remote is source of truth) and funnel all ~10 consumers through one async ensure_clone chokepoint that lazily re-clones on a cache miss.

Chose lazy on-demand re-clone over eager re-clone-all at startup: eager risks the 300s Railway healthcheck timeout on a cold container (the surface behind silent prod outage F-011). Tradeoff: first access per repo post-redeploy pays the full clone cost; every access after is a sub-ms stat. Correctness over a one-time latency hit vs a permanent 404.

Concurrency: in-process per-repo asyncio.Lock + clone to temp dir then atomic os.rename, so simultaneous/crashed clones never leave a half-populated dir. Clone runs via asyncio.to_thread to keep the event loop free. Failure surfaces as an actionable 503 (RepoCloneError), not an opaque 500. In analyze_impact, input validation runs BEFORE re-clone so a path-traversal attempt is rejected without triggering external work. Warm path unchanged.

Author: DevanshuNEU

backend/routes/analysis.py

@@ -4,7 +4,7 @@
 
 from dependencies import (
     dependency_analyzer, style_analyzer, dna_extractor,
-    get_repo_or_404
+    get_repo_or_404, repo_manager
 )
 from services.input_validator import InputValidator
 from middleware.auth import require_auth, AuthContext
@@ -35,6 +35,7 @@ async def get_dependency_graph(
                 return {**cached_graph, "cached": True}
 
         logger.info("Building fresh dependency graph", repo_id=repo_id, include_paths=repo.get("include_paths"))
+        await repo_manager.ensure_clone(repo)
         graph_data = dependency_analyzer.build_dependency_graph(repo["local_path"], include_paths=repo.get("include_paths"))
         dependency_analyzer.save_to_cache(repo_id, graph_data)
 
@@ -57,12 +58,15 @@ async def analyze_impact(
     try:
         repo = get_repo_or_404(repo_id, auth.user_id)
 
+        # Validate user input BEFORE any expensive/external work (re-clone). Reject traversal first.
         valid_path, path_error = InputValidator.validate_file_path(
             request.file_path, repo["local_path"]
         )
         if not valid_path:
             raise HTTPException(status_code=400, detail=f"Invalid file path: {path_error}")
 
+        await repo_manager.ensure_clone(repo)
+
         graph_data = dependency_analyzer.load_from_cache(repo_id)
         if not graph_data:
             logger.info("Building dependency graph for impact analysis", repo_id=repo_id)
@@ -96,6 +100,7 @@ async def get_repository_insights(
         graph_data = dependency_analyzer.load_from_cache(repo_id)
         if not graph_data:
             logger.info("Building dependency graph for insights", repo_id=repo_id)
+            await repo_manager.ensure_clone(repo)
             graph_data = dependency_analyzer.build_dependency_graph(repo["local_path"], include_paths=repo.get("include_paths"))
             dependency_analyzer.save_to_cache(repo_id, graph_data)
 
@@ -134,6 +139,7 @@ async def get_style_analysis(
             return {**cached_style, "cached": True}
 
         logger.info("Analyzing code style", repo_id=repo_id)
+        await repo_manager.ensure_clone(repo)
         style_data = style_analyzer.analyze_repository_style(repo["local_path"], include_paths=repo.get("include_paths"))
         style_analyzer.save_to_cache(repo_id, style_data)
 
@@ -178,6 +184,7 @@ async def get_codebase_dna(
         logger.info("Extracting codebase DNA", repo_id=repo_id)
         metrics.increment("dna_extractions")
 
+        await repo_manager.ensure_clone(repo)
         dna = dna_extractor.extract_dna(repo["local_path"], repo_id, include_paths=repo.get("include_paths"))
         dna_extractor.save_to_cache(repo_id, dna)
 

backend/routes/repos.py

@@ -470,11 +470,10 @@ async def get_repo_directories(
     directories to index instead of the entire repo.
     """
     repo = get_repo_or_404(repo_id, auth.user_id)
+    # Re-clone if the container was redeployed and wiped the working tree (#311).
+    await repo_manager.ensure_clone(repo)
     local_path = Path(repo["local_path"])
 
-    if not local_path.exists():
-        raise HTTPException(status_code=404, detail="Repo not cloned yet")
-
     dirs = await asyncio.to_thread(_scan_directories, local_path)
 
     return {
@@ -501,7 +500,9 @@ async def index_repository(
 
     try:
         repo = get_repo_or_404(repo_id, user_id)
-        
+        # Re-clone if the container was redeployed and wiped the working tree (#311).
+        await repo_manager.ensure_clone(repo)
+
         # Re-check size limits before indexing (in case tier changed or repo updated)
         analysis = repo_validator.analyze_repo(repo["local_path"])
 
@@ -765,10 +766,13 @@ async def index_repository_async(
 
     try:
         repo = get_repo_or_404(repo_id, user_id)
-        
+        # Re-clone before the size-check (which reads the working tree) so a redeployed
+        # container rehydrates here, before the 202, rather than failing the gate (#311).
+        await repo_manager.ensure_clone(repo)
+
         # Re-check size limits
         analysis = repo_validator.analyze_repo(repo["local_path"])
-        
+
         if not analysis.success:
             raise HTTPException(
                 status_code=500,
@@ -867,7 +871,15 @@ async def websocket_index(websocket: WebSocket, repo_id: str):
     if not repo:
         await websocket.close(code=4004, reason="Repository not found")
         return
-    
+
+    # Re-clone if the container was redeployed and wiped the working tree (#311).
+    try:
+        await repo_manager.ensure_clone(repo)
+    except Exception as e:
+        logger.error("Re-clone failed for websocket indexing", repo_id=repo_id, error=str(e))
+        await websocket.close(code=4005, reason="Repository clone unavailable")
+        return
+
     # Check size limits before WebSocket indexing
     analysis = repo_validator.analyze_repo(repo["local_path"])
 

backend/services/repo_manager.py

@@ -2,22 +2,54 @@
 Repository Manager (Supabase Edition)
 Handles repository CRUD operations with PostgreSQL via Supabase
 """
+import asyncio
+import os
+import shutil
 import uuid
-from typing import List, Optional
+from typing import Dict, List, Optional
 import git
 from pathlib import Path
+from fastapi import HTTPException
 from services.supabase_service import get_supabase_service
 from services.observability import logger, metrics
 
 
+class RepoCloneError(HTTPException):
+    """Repo working tree is missing and could not be restored from its git remote.
+
+    Subclasses HTTPException so it surfaces as an actionable 503 (handlers already re-raise
+    HTTPException) instead of an opaque 500. UX matters here: a redeploy is invisible to the
+    user, so the message has to tell them what to actually do. (#311)
+    """
+
+    def __init__(self, repo_id: str, reason: str = ""):
+        super().__init__(
+            status_code=503,
+            detail={
+                "error": "REPO_UNAVAILABLE",
+                "repo_id": repo_id,
+                "message": (
+                    "Repository source files are temporarily unavailable and could not be "
+                    "restored from the git remote. Private repositories are not yet supported "
+                    "for re-sync; for public repos, please retry shortly."
+                ),
+            },
+        )
+        self.reason = reason
+
+
 class RepositoryManager:
     """Manage repositories with Supabase persistence"""
 
     def __init__(self):
         self.repos_dir = Path("./repos")
         self.repos_dir.mkdir(exist_ok=True)
         self.db = get_supabase_service()
-        
+
+        # Per-repo locks so two concurrent ops on the same missing clone don't both clone.
+        # Single uvicorn worker means an in-process lock is sufficient here.
+        self._clone_locks: Dict[str, asyncio.Lock] = {}
+
         # Discover and sync existing repositories on startup
         self._sync_existing_repos()
 
@@ -126,10 +158,66 @@ def add_repo(self, name: str, git_url: str, branch: str = "main", user_id: Optio
         except Exception as e:
             # Cleanup on failure
             if local_path.exists():
-                import shutil
                 shutil.rmtree(local_path)
             raise Exception(f"Failed to clone repository: {str(e)}")
-    
+
+    async def ensure_clone(self, repo: dict) -> str:
+        """Guarantee the working tree exists on disk, lazily re-cloning from git_url if needed.
+
+        Railway redeploys wipe ./repos (ephemeral disk) but Pinecone/Supabase survive, so
+        local_path is a cache hint, not source of truth -- the git remote is. On a warm hit
+        this is a sub-millisecond stat with no behavior change; on a miss it re-clones.
+        Returns the canonical local path and refreshes repo['local_path'] in place.
+        """
+        repo_id = repo["id"]
+        canonical = self.repos_dir / repo_id
+
+        # Warm path: clone present. No re-clone, no event-loop work.
+        if (canonical / ".git").exists():
+            repo["local_path"] = str(canonical)
+            return str(canonical)
+
+        git_url = repo.get("git_url")
+        if not git_url or git_url == "unknown":
+            raise RepoCloneError(repo_id, "no git_url on record")
+        branch = repo.get("branch") or "main"
+
+        lock = self._clone_locks.setdefault(repo_id, asyncio.Lock())
+        async with lock:
+            # Another coroutine may have cloned while we waited for the lock.
+            if not (canonical / ".git").exists():
+                try:
+                    await asyncio.to_thread(self._clone_into_place, repo_id, git_url, branch, canonical)
+                except Exception as e:
+                    # Private repo (no creds on a fresh container), network failure, deleted
+                    # remote: surface as an actionable 503, not an opaque 500.
+                    logger.error("Re-clone failed", repo_id=repo_id, git_url=git_url, error=str(e))
+                    raise RepoCloneError(repo_id, str(e)) from e
+                logger.info("Re-cloned repo on demand (cache miss)", repo_id=repo_id, git_url=git_url)
+                metrics.increment("repos_recloned")
+
+        repo["local_path"] = str(canonical)
+        return str(canonical)
+
+    def _clone_into_place(self, repo_id: str, git_url: str, branch: str, canonical: Path) -> None:
+        """Clone into a temp dir then atomically rename into the canonical path.
+
+        The rename is the correctness guarantee: a crashed or concurrent clone never leaves a
+        half-populated canonical dir for a reader to trip over. Runs in a worker thread (git is
+        blocking I/O); never call directly on the event loop.
+        """
+        tmp = self.repos_dir / f".{repo_id}.tmp.{uuid.uuid4().hex}"
+        try:
+            git.Repo.clone_from(git_url, tmp, branch=branch, depth=1)
+            # Clear any leftover partial dir before the atomic swap.
+            if canonical.exists():
+                shutil.rmtree(canonical, ignore_errors=True)
+            os.rename(tmp, canonical)  # atomic on the same filesystem
+        except Exception:
+            if tmp.exists():
+                shutil.rmtree(tmp, ignore_errors=True)
+            raise
+
     def update_status(self, repo_id: str, status: str):
         """Update repository status"""
         self.db.update_repository_status(repo_id, status)
@@ -158,8 +246,6 @@ def update_last_commit(self, repo_id: str, commit_sha: str, function_count: int
 
     def delete_repo(self, repo_id: str) -> bool:
         """Delete repository and clean up local files"""
-        import shutil
-        
         repo = self.get_repo(repo_id)
         if not repo:
             return False