fix(playground): add partial indexing support and fix bugs (#125)

- Add partial=true parameter to index first 200 files of large repos
- Fix create_session() call - generate token before creating session
- Fix JSON serialization in validation error handler
- Add missing capture_http_exception function to sentry module
- Add comprehensive tests for anonymous indexing (30 tests)

All 169 tests passing.

Author: DevanshuNEU

backend/main.py

@@ -98,11 +98,22 @@ async def dispatch(self, request: Request, call_next):
 @app.exception_handler(RequestValidationError)
 async def validation_exception_handler(request: Request, exc: RequestValidationError):
     """Handle validation errors with clear messages."""
+    # Convert errors to JSON-serializable format
+    errors = []
+    for err in exc.errors():
+        error_dict = {
+            "type": err.get("type"),
+            "loc": err.get("loc"),
+            "msg": err.get("msg"),
+            "input": str(err.get("input")) if err.get("input") is not None else None,
+        }
+        errors.append(error_dict)
+
     return JSONResponse(
         status_code=422,
         content={
             "detail": "Validation error",
-            "errors": exc.errors()
+            "errors": errors
         }
     )
 

backend/routes/playground.py

@@ -76,6 +76,7 @@ class IndexRepoRequest(BaseModel):
     """
     github_url: str
     branch: Optional[str] = None  # None = use repo's default branch
+    partial: bool = False  # If True, index first 200 files of large repos
 
     @field_validator("github_url")
     @classmethod
@@ -653,8 +654,9 @@ async def start_anonymous_indexing(
     client_ip = _get_client_ip(req)
 
     if not session_token:
-        # Create new session
-        session_token = limiter.create_session()
+        # Create new session - generate token first, then create session
+        session_token = limiter._generate_session_token()
+        limiter.create_session(session_token)
         _set_session_cookie(response, session_token)
         logger.info("Created new session for indexing",
                     session_token=session_token[:8],
@@ -767,18 +769,32 @@ async def start_anonymous_indexing(
         file_count = max(repo_size_kb // 3, 1)
 
     # Check file limit
+    is_partial = False
+    files_to_index = file_count
+
     if file_count > ANONYMOUS_FILE_LIMIT:
-        raise HTTPException(
-            status_code=400,
-            detail={
-                "error": "validation_failed",
-                "reason": "too_large",
-                "message": f"Repository has {file_count:,} code files. "
-                           f"Anonymous limit is {ANONYMOUS_FILE_LIMIT}.",
-                "file_count": file_count,
-                "limit": ANONYMOUS_FILE_LIMIT
-            }
-        )
+        if request.partial:
+            # Partial indexing - cap at limit
+            is_partial = True
+            files_to_index = ANONYMOUS_FILE_LIMIT
+            logger.info("Partial indexing enabled",
+                        total_files=file_count,
+                        indexing=files_to_index)
+        else:
+            # Reject large repos without partial flag
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": "validation_failed",
+                    "reason": "too_large",
+                    "message": f"Repository has {file_count:,} code files. "
+                               f"Anonymous limit is {ANONYMOUS_FILE_LIMIT}. "
+                               f"Use partial=true to index first {ANONYMOUS_FILE_LIMIT} files.",
+                    "file_count": file_count,
+                    "limit": ANONYMOUS_FILE_LIMIT,
+                    "hint": "Set partial=true to index a subset of files"
+                }
+            )
 
     # --- Validation passed! Create job and start background indexing ---
 
@@ -796,7 +812,9 @@ async def start_anonymous_indexing(
         owner=owner,
         repo_name=repo_name,
         branch=branch,
-        file_count=file_count
+        file_count=file_count,
+        is_partial=is_partial,
+        max_files=files_to_index
     )
 
     # Queue background task
@@ -811,25 +829,38 @@ async def start_anonymous_indexing(
         owner=owner,
         repo_name=repo_name,
         branch=branch,
-        file_count=file_count
+        file_count=files_to_index,  # Actual files to index (may be capped)
+        max_files=files_to_index if is_partial else None  # Limit for partial indexing
     )
 
     logger.info("Indexing job queued",
                 job_id=job_id,
                 owner=owner,
                 repo=repo_name,
                 branch=branch,
-                file_count=file_count,
+                file_count=files_to_index,
+                is_partial=is_partial,
                 session_token=session_token[:8],
                 response_time_ms=response_time_ms)
 
     # Estimate time based on file count (~0.3s per file)
-    estimated_seconds = max(10, int(file_count * 0.3))
+    estimated_seconds = max(10, int(files_to_index * 0.3))
 
-    return {
+    response_data = {
         "job_id": job_id,
         "status": "queued",
         "estimated_time_seconds": estimated_seconds,
-        "file_count": file_count,
+        "file_count": files_to_index,
         "message": f"Indexing started. Poll /playground/index/{job_id} for status."
     }
+
+    # Add partial info if applicable
+    if is_partial:
+        response_data["partial"] = True
+        response_data["total_files"] = file_count
+        response_data["message"] = (
+            f"Partial indexing started ({files_to_index} of {file_count} files). "
+            f"Poll /playground/index/{job_id} for status."
+        )
+
+    return response_data

backend/services/anonymous_indexer.py

@@ -93,7 +93,9 @@ def create_job(
         owner: str,
         repo_name: str,
         branch: str,
-        file_count: int
+        file_count: int,
+        is_partial: bool = False,
+        max_files: Optional[int] = None
     ) -> dict:
         """
         Create a new indexing job in Redis.
@@ -111,6 +113,8 @@ def create_job(
             "repo_name": repo_name,
             "branch": branch,
             "file_count": file_count,
+            "is_partial": is_partial,
+            "max_files": max_files,
             "status": JobStatus.QUEUED.value,
             "progress": None,
             "stats": None,
@@ -225,13 +229,17 @@ async def run_indexing_job(
     owner: str,
     repo_name: str,
     branch: str,
-    file_count: int
+    file_count: int,
+    max_files: Optional[int] = None
 ) -> None:
     """
     Background task to clone and index a repository.
 
     This runs asynchronously after the endpoint returns.
     Updates Redis with progress and final status.
+
+    Args:
+        max_files: If set, limit indexing to first N files (for partial indexing)
     """
     import time
     start_time = time.time()
@@ -286,7 +294,8 @@ async def progress_callback(files_processed: int, functions_found: int, total: i
                 indexer.index_repository_with_progress(
                     repo_id,
                     str(temp_path),
-                    progress_callback
+                    progress_callback,
+                    max_files=max_files
                 ),
                 timeout=job_manager.INDEX_TIMEOUT_SECONDS
             )

backend/services/indexer_optimized.py

@@ -464,17 +464,30 @@ async def explain_code(
             return f"Error: {str(e)}"
 
     async def index_repository_with_progress(
-        self, 
-        repo_id: str, 
+        self,
+        repo_id: str,
         repo_path: str,
-        progress_callback
+        progress_callback,
+        max_files: int = None
     ):
-        """Index repository with real-time progress updates"""
+        """Index repository with real-time progress updates
+
+        Args:
+            max_files: If set, limit indexing to first N files (for partial indexing)
+        """
         start_time = time.time()
         logger.info("Starting optimized indexing with progress", repo_id=repo_id)
-        
+
         # Discover code files
         code_files = self._discover_code_files(repo_path)
+
+        # Apply file limit if specified (partial indexing)
+        if max_files and len(code_files) > max_files:
+            logger.info("Limiting files for partial indexing",
+                        total_discovered=len(code_files),
+                        max_files=max_files)
+            code_files = code_files[:max_files]
+
         total_files = len(code_files)
         logger.info("Found code files", repo_id=repo_id, total_files=total_files)
 

backend/services/sentry.py

@@ -12,58 +12,58 @@
 def init_sentry() -> bool:
     """
     Initialize Sentry SDK if SENTRY_DSN is configured.
-    
+
     Returns:
         bool: True if Sentry was initialized, False otherwise
     """
     sentry_dsn = os.getenv("SENTRY_DSN")
-    
+
     if not sentry_dsn:
         print("ℹ️  Sentry DSN not configured - error tracking disabled")
         return False
-    
+
     try:
         import sentry_sdk
         from sentry_sdk.integrations.fastapi import FastApiIntegration
         from sentry_sdk.integrations.starlette import StarletteIntegration
-        
+
         environment = os.getenv("ENVIRONMENT", "development")
-        
+
         sentry_sdk.init(
             dsn=sentry_dsn,
             environment=environment,
-            
+
             # Performance monitoring - sample rate based on environment
             traces_sample_rate=0.1 if environment == "production" else 1.0,
-            
+
             # Profile sampled transactions
             profiles_sample_rate=0.1 if environment == "production" else 1.0,
-            
+
             # Send PII for debugging (user IDs, emails)
             send_default_pii=True,
-            
+
             # Integrations
             integrations=[
                 FastApiIntegration(transaction_style="endpoint"),
                 StarletteIntegration(transaction_style="endpoint"),
             ],
-            
+
             # Filter noisy events
             before_send=_filter_events,
-            
+
             # Debug mode for development
             debug=environment == "development",
-            
+
             # Attach stack traces to messages
             attach_stacktrace=True,
-            
+
             # Include local variables in stack traces
             include_local_variables=True,
         )
-        
+
         print(f"✅ Sentry initialized (environment: {environment})")
         return True
-        
+
     except ImportError:
         print("⚠️  sentry-sdk not installed - error tracking disabled")
         return False
@@ -74,26 +74,26 @@ def init_sentry() -> bool:
 
 def _filter_events(event, hint):
     """Filter out noisy events before sending to Sentry."""
-    
+
     # Don't send health check errors
     request_url = event.get("request", {}).get("url", "")
     if "/health" in request_url:
         return None
-    
+
     # Don't send 404s for common bot paths
     exception_values = event.get("exception", {}).get("values", [])
     if exception_values:
         exception_value = str(exception_values[0].get("value", ""))
         bot_paths = ["/wp-admin", "/wp-login", "/.env", "/config", "/admin", "/phpmyadmin", "/.git"]
         if any(path in exception_value for path in bot_paths):
             return None
-    
+
     # Don't send validation errors (they're expected)
     if exception_values:
         exception_type = exception_values[0].get("type", "")
         if exception_type in ("RequestValidationError", "ValidationError"):
             return None
-    
+
     return event
 
 
@@ -104,7 +104,7 @@ def _filter_events(event, hint):
 def set_user_context(user_id: Optional[str] = None, email: Optional[str] = None):
     """
     Set user context for error tracking.
-    
+
     DEPRECATED: Use from services.observability import set_user_context
     """
     try:
@@ -117,7 +117,7 @@ def set_user_context(user_id: Optional[str] = None, email: Optional[str] = None)
 def capture_exception(error: Exception, **extra_context):
     """
     Manually capture an exception with additional context.
-    
+
     DEPRECATED: Use from services.observability import capture_exception
     """
     try:
@@ -133,7 +133,7 @@ def capture_exception(error: Exception, **extra_context):
 def capture_message(message: str, level: str = "info", **extra_context):
     """
     Capture a message (not an exception) for tracking.
-    
+
     DEPRECATED: Use from services.observability import get_logger
     """
     try:
@@ -149,7 +149,7 @@ def capture_message(message: str, level: str = "info", **extra_context):
 def set_operation_context(operation: str, **tags):
     """
     Set operation context for the current scope.
-    
+
     DEPRECATED: Use from services.observability import trace_operation
     """
     try:
@@ -159,3 +159,19 @@ def set_operation_context(operation: str, **tags):
             sentry_sdk.set_tag(key, str(value))
     except ImportError:
         pass
+
+
+def capture_http_exception(request, exc: Exception, status_code: int):
+    """
+    Capture HTTP exception with request context for error tracking.
+    """
+    try:
+        import sentry_sdk
+        with sentry_sdk.push_scope() as scope:
+            scope.set_extra("status_code", status_code)
+            scope.set_extra("path", str(request.url.path))
+            scope.set_extra("method", request.method)
+            sentry_sdk.capture_exception(exc)
+    except ImportError:
+        pass
+        pass