feat(search-v2): implement function-level chunking with tree-sitter

Issue: #68

This PR implements the foundation for Search V2 by adding function-level
code extraction using tree-sitter AST parsing.

Changes:
- Add TreeSitterExtractor for accurate Python/JS/TS function extraction
- Add FunctionFilter to remove test/junk functions
- Add ExtractedFunction and SearchResult data models
- Add index_repository_v2() method to indexer
- Add comprehensive test suite (15 tests passing)

Key features:
- Qualified names (Class.method) for disambiguation
- Docstring extraction for better search context
- Async function detection
- Decorator extraction
- Quality filtering (removes test_*, mock_*, etc.)

This is PR 1 of 4 in the Search V2 epic.
Next: PR 2 will add AI-generated summaries (#69)

Author: DevanshuNEU

backend/services/indexer_optimized.py

@@ -31,6 +31,9 @@
 # Search enhancement
 from services.search_enhancer import SearchEnhancer
 
+# Search V2 - Function-level extraction (Issue #68)
+from services.search_v2 import TreeSitterExtractor, FunctionFilter, ExtractedFunction
+
 # Observability
 from services.observability import logger, trace_operation, track_time, capture_exception, add_breadcrumb, metrics
 
@@ -89,6 +92,14 @@ def __init__(self):
             'typescript': self._create_parser(Language(tsjavascript.language())),
         }
 
+        # Search V2: Initialize advanced tree-sitter extractor and filter (Issue #68)
+        self.tree_sitter_extractor = TreeSitterExtractor()
+        self.function_filter = FunctionFilter(
+            include_private=False,
+            include_dunders=True,
+            max_name_length=50
+        )
+        
         logger.info("OptimizedCodeIndexer initialized", model=EMBEDDING_MODEL)
 
     def _create_parser(self, language) -> Parser:
@@ -340,6 +351,203 @@ async def _extract_functions_from_file(
             logger.error("Error processing file", file_path=file_path, error=str(e))
             return []
 
+    def extract_functions_v2(
+        self,
+        repo_path: str,
+        max_functions: int = 5000
+    ) -> List[ExtractedFunction]:
+        """
+        Extract functions using Search V2 tree-sitter extractor (Issue #68).
+        
+        This is the improved extraction that:
+        - Uses proper AST parsing for accuracy
+        - Extracts qualified names (Class.method)
+        - Filters out test/junk functions
+        - Captures docstrings and decorators
+        
+        Args:
+            repo_path: Path to repository root
+            max_functions: Maximum functions to extract
+            
+        Returns:
+            List of ExtractedFunction objects
+        """
+        from pathlib import Path
+        
+        # Extract all functions
+        all_functions = self.tree_sitter_extractor.extract_from_repo(
+            Path(repo_path),
+            max_functions=max_functions
+        )
+        
+        # Filter to keep only quality functions
+        quality_functions = self.function_filter.filter_functions(all_functions)
+        
+        logger.info(
+            "V2 extraction complete",
+            total_extracted=len(all_functions),
+            after_filter=len(quality_functions),
+            filtered_out=len(all_functions) - len(quality_functions)
+        )
+        
+        return quality_functions
+    
+    def _function_to_embedding_text(self, func: ExtractedFunction) -> str:
+        """
+        Create rich embedding text from ExtractedFunction (Issue #68).
+        
+        Combines signature, docstring, and code for better semantic matching.
+        """
+        parts = []
+        
+        # Add qualified name for disambiguation
+        parts.append(f"Function: {func.qualified_name}")
+        
+        # Add signature
+        parts.append(f"Signature: {func.signature}")
+        
+        # Add docstring if present
+        if func.docstring:
+            parts.append(f"Description: {func.docstring[:500]}")
+        
+        # Add language context
+        parts.append(f"Language: {func.language}")
+        
+        # Add code (primary content)
+        parts.append(f"Code:\n{func.code[:2000]}")
+        
+        return "\n".join(parts)
+    
+    def _function_to_pinecone_metadata(
+        self, 
+        func: ExtractedFunction, 
+        repo_id: str
+    ) -> Dict:
+        """
+        Convert ExtractedFunction to Pinecone metadata (Issue #68).
+        
+        Updated schema with qualified names and additional fields.
+        """
+        return {
+            "repo_id": repo_id,
+            "file_path": func.file_path,
+            "name": func.name,
+            "qualified_name": func.qualified_name,
+            "type": "method" if func.is_method else "function",
+            "code": func.code[:1000],  # Truncate for metadata limits
+            "signature": func.signature,
+            "start_line": func.start_line,
+            "end_line": func.end_line,
+            "language": func.language,
+            "class_name": func.class_name or "",
+            "docstring": (func.docstring or "")[:500],
+            "is_async": func.is_async,
+        }
+    
+    async def index_repository_v2(
+        self,
+        repo_id: str,
+        repo_path: str,
+        progress_callback=None
+    ) -> int:
+        """
+        Index repository using Search V2 extraction (Issue #68).
+        
+        This is the improved indexing that uses:
+        - Function-level chunking with qualified names
+        - Quality filtering to remove junk
+        - Rich embedding text with docstrings
+        
+        Args:
+            repo_id: Unique repository identifier
+            repo_path: Path to repository root
+            progress_callback: Optional async callback(files, functions, total)
+            
+        Returns:
+            Number of functions indexed
+        """
+        from services.observability import set_operation_context
+        
+        set_operation_context("indexing_v2", repo_id=repo_id)
+        add_breadcrumb("Starting V2 repository indexing", category="indexing", repo_id=repo_id)
+        
+        start_time = time.time()
+        logger.info("Starting V2 indexing", repo_id=repo_id, path=repo_path)
+        
+        # Step 1: Extract functions using V2 extractor
+        functions = self.extract_functions_v2(repo_path)
+        
+        if not functions:
+            logger.warning("No functions extracted", repo_id=repo_id)
+            if progress_callback:
+                await progress_callback(0, 0, 0)
+            return 0
+        
+        logger.info("Functions extracted", repo_id=repo_id, count=len(functions))
+        
+        # Step 2: Generate embeddings in batches
+        embedding_texts = [self._function_to_embedding_text(f) for f in functions]
+        
+        all_embeddings = []
+        with track_time("embedding_generation_v2", repo_id=repo_id, total=len(embedding_texts)):
+            for i in range(0, len(embedding_texts), self.EMBEDDING_BATCH_SIZE):
+                batch_texts = embedding_texts[i:i + self.EMBEDDING_BATCH_SIZE]
+                batch_embeddings = await self._create_embeddings_batch(batch_texts)
+                all_embeddings.extend(batch_embeddings)
+                
+                if progress_callback:
+                    await progress_callback(
+                        len(all_embeddings),
+                        len(functions),
+                        len(functions)
+                    )
+                
+                logger.debug(
+                    "Embeddings generated",
+                    progress=len(all_embeddings),
+                    total=len(embedding_texts)
+                )
+        
+        # Step 3: Prepare vectors for Pinecone
+        vectors_to_upsert = []
+        
+        for func, embedding in zip(functions, all_embeddings):
+            func_id = hashlib.md5(func.id_string.encode()).hexdigest()
+            
+            vectors_to_upsert.append({
+                "id": func_id,
+                "values": embedding,
+                "metadata": self._function_to_pinecone_metadata(func, repo_id)
+            })
+        
+        # Step 4: Upsert to Pinecone in batches
+        add_breadcrumb("Uploading to Pinecone", category="indexing", vector_count=len(vectors_to_upsert))
+        
+        with track_time("pinecone_upload_v2", repo_id=repo_id, vectors=len(vectors_to_upsert)):
+            for i in range(0, len(vectors_to_upsert), self.PINECONE_UPSERT_BATCH):
+                batch = vectors_to_upsert[i:i + self.PINECONE_UPSERT_BATCH]
+                self.index.upsert(vectors=batch)
+                logger.debug(
+                    "Vectors uploaded",
+                    progress=min(i + self.PINECONE_UPSERT_BATCH, len(vectors_to_upsert)),
+                    total=len(vectors_to_upsert)
+                )
+        
+        elapsed = time.time() - start_time
+        speed = len(functions) / elapsed if elapsed > 0 else 0
+        
+        logger.info(
+            "V2 indexing complete",
+            repo_id=repo_id,
+            functions=len(functions),
+            duration_s=round(elapsed, 2),
+            speed=round(speed, 1)
+        )
+        metrics.increment("indexing_v2_completed")
+        metrics.timing("indexing_v2_duration_s", elapsed)
+        
+        return len(functions)
+    
     async def semantic_search(
         self,
         query: str,

backend/services/search_v2/__init__.py

@@ -0,0 +1,38 @@
+"""
+Search V2 Module
+Function-level semantic search with Triple Fusion
+
+This module implements the improved search system from the 
+codeintel-research experiments, achieving 85%+ accuracy.
+
+Key components:
+- TreeSitterExtractor: AST-based function extraction
+- FunctionFilter: Quality filtering to remove junk
+- Types: Data models for functions and search results
+
+Issue: #67 - Semantic Search V2 Epic
+"""
+from services.search_v2.types import (
+    ExtractedFunction,
+    SearchResult,
+    Language,
+)
+from services.search_v2.tree_sitter_extractor import TreeSitterExtractor
+from services.search_v2.function_filter import (
+    FunctionFilter,
+    filter_functions,
+)
+
+__all__ = [
+    # Types
+    "ExtractedFunction",
+    "SearchResult", 
+    "Language",
+    
+    # Extractors
+    "TreeSitterExtractor",
+    
+    # Filters
+    "FunctionFilter",
+    "filter_functions",
+]