Add operation-based indexing pipeline

rust/rubydex-mcp/src/server.rs

@@ -54,7 +54,11 @@ impl RubydexServer {
                 }
 
                 let mut graph = Graph::new();
-                let errors = rubydex::indexing::index_files(&mut graph, file_paths);
+                let errors = rubydex::indexing::index_files(
+                    &mut graph,
+                    file_paths,
+                    rubydex::indexing::IndexerBackend::RubyIndexer,
+                );
                 for error in &errors {
                     eprintln!("Indexing error: {error}");
                 }

rust/rubydex-sys/src/graph_api.rs

@@ -233,7 +233,7 @@ pub unsafe extern "C" fn rdx_index_all(
 
     with_mut_graph(pointer, |graph| {
         let (file_paths, listing_errors) = listing::collect_file_paths(file_paths, graph.excluded_paths());
-        let indexing_errors = indexing::index_files(graph, file_paths);
+        let indexing_errors = indexing::index_files(graph, file_paths, indexing::IndexerBackend::RubyIndexer);
 
         let all_errors: Vec<String> = listing_errors
             .into_iter()

rust/rubydex/src/indexing.rs

@@ -3,6 +3,7 @@ use crate::{
     indexing::{local_graph::LocalGraph, rbs_indexer::RBSIndexer, ruby_indexer::RubyIndexer},
     job_queue::{Job, JobQueue},
     model::graph::Graph,
+    operation::ruby_builder::RubyOperationBuilder,
 };
 use crossbeam_channel::{Sender, unbounded};
 use std::{ffi::OsStr, fs, path::PathBuf, sync::Arc};
@@ -12,6 +13,15 @@ pub mod local_graph;
 pub mod rbs_indexer;
 pub mod ruby_indexer;
 
+/// Which backend to use for indexing Ruby files.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum IndexerBackend {
+    /// The original tree-walking indexer.
+    RubyIndexer,
+    /// The two-phase operation builder + applier pipeline.
+    OperationBuilder,
+}
+
 /// The language of a source document, used to dispatch to the appropriate indexer
 pub enum LanguageId {
     Ruby,
@@ -42,15 +52,22 @@ impl LanguageId {
 /// Job that indexes a single file
 pub struct IndexingJob {
     path: PathBuf,
+    backend: IndexerBackend,
     local_graph_tx: Sender<LocalGraph>,
     errors_tx: Sender<Errors>,
 }
 
 impl IndexingJob {
     #[must_use]
-    pub fn new(path: PathBuf, local_graph_tx: Sender<LocalGraph>, errors_tx: Sender<Errors>) -> Self {
+    pub fn new(
+        path: PathBuf,
+        backend: IndexerBackend,
+        local_graph_tx: Sender<LocalGraph>,
+        errors_tx: Sender<Errors>,
+    ) -> Self {
         Self {
             path,
+            backend,
             local_graph_tx,
             errors_tx,
         }
@@ -84,7 +101,7 @@ impl Job for IndexingJob {
         };
 
         let language = self.path.extension().map_or(LanguageId::Ruby, LanguageId::from);
-        let local_graph = build_local_graph(url.to_string(), &source, &language);
+        let local_graph = build_local_graph(url.to_string(), &source, &language, self.backend);
 
         self.local_graph_tx
             .send(local_graph)
@@ -94,7 +111,7 @@ impl Job for IndexingJob {
 
 /// Indexes a single source string in memory, dispatching to the appropriate indexer based on `language_id`.
 pub fn index_source(graph: &mut Graph, uri: &str, source: &str, language_id: &LanguageId) {
-    let local_graph = build_local_graph(uri.to_string(), source, language_id);
+    let local_graph = build_local_graph(uri.to_string(), source, language_id, IndexerBackend::RubyIndexer);
     graph.consume_document_changes(local_graph);
 }
 
@@ -103,14 +120,15 @@ pub fn index_source(graph: &mut Graph, uri: &str, source: &str, language_id: &La
 /// # Panics
 ///
 /// Will panic if the graph cannot be wrapped in an Arc<Mutex<>>
-pub fn index_files(graph: &mut Graph, paths: Vec<PathBuf>) -> Vec<Errors> {
+pub fn index_files(graph: &mut Graph, paths: Vec<PathBuf>, backend: IndexerBackend) -> Vec<Errors> {
     let queue = Arc::new(JobQueue::new());
     let (local_graphs_tx, local_graphs_rx) = unbounded();
     let (errors_tx, errors_rx) = unbounded();
 
     for path in paths {
         queue.push(Box::new(IndexingJob::new(
             path,
+            backend,
             local_graphs_tx.clone(),
             errors_tx.clone(),
         )));
@@ -134,13 +152,21 @@ pub fn index_files(graph: &mut Graph, paths: Vec<PathBuf>) -> Vec<Errors> {
 }
 
 /// Indexes a source string using the appropriate indexer for the given language.
-fn build_local_graph(uri: String, source: &str, language: &LanguageId) -> LocalGraph {
+#[must_use]
+pub fn build_local_graph(uri: String, source: &str, language: &LanguageId, backend: IndexerBackend) -> LocalGraph {
     match language {
-        LanguageId::Ruby => {
-            let mut indexer = RubyIndexer::new(uri, source);
-            indexer.index();
-            indexer.local_graph()
-        }
+        LanguageId::Ruby => match backend {
+            IndexerBackend::RubyIndexer => {
+                let mut indexer = RubyIndexer::new(uri, source);
+                indexer.index();
+                indexer.local_graph()
+            }
+            IndexerBackend::OperationBuilder => {
+                let builder = RubyOperationBuilder::new(uri, source);
+                let result = builder.build();
+                crate::operation::applier::apply_operations(result)
+            }
+        },
         LanguageId::Rbs => {
             let mut indexer = RBSIndexer::new(uri, source);
             indexer.index();
@@ -175,7 +201,7 @@ mod tests {
         let relative_to_pwd = &dots.join(absolute_path);
 
         let mut graph = Graph::new();
-        let errors = index_files(&mut graph, vec![relative_to_pwd.clone()]);
+        let errors = index_files(&mut graph, vec![relative_to_pwd.clone()], IndexerBackend::RubyIndexer);
 
         assert!(errors.is_empty());
         assert_eq!(graph.documents().len(), 2);
@@ -196,7 +222,7 @@ mod tests {
         let uri = Url::from_file_path(&path).unwrap().to_string();
 
         let mut graph = Graph::new();
-        let errors = index_files(&mut graph, vec![path]);
+        let errors = index_files(&mut graph, vec![path], IndexerBackend::RubyIndexer);
 
         assert!(errors.is_empty(), "Expected no errors, got: {errors:#?}");
         assert_eq!(6, graph.definitions().len());

rust/rubydex/src/indexing/local_graph.rs

@@ -206,6 +206,44 @@ impl LocalGraph {
         &self.name_dependents
     }
 
+    /// Creates a `LocalGraph` from pre-built parts (used by the operation applier pipeline).
+    #[must_use]
+    pub fn from_parts(
+        uri_id: UriId,
+        document: Document,
+        strings: IdentityHashMap<StringId, StringRef>,
+        names: IdentityHashMap<NameId, NameRef>,
+    ) -> Self {
+        let mut name_dependents: IdentityHashMap<NameId, Vec<NameDependent>> = IdentityHashMap::default();
+        for (name_id, name_ref) in &names {
+            if let NameRef::Unresolved(name) = name_ref {
+                if let Some(&parent_scope) = name.parent_scope().as_ref() {
+                    name_dependents
+                        .entry(parent_scope)
+                        .or_default()
+                        .push(NameDependent::ChildName(*name_id));
+                }
+                if let Some(&nesting_id) = name.nesting().as_ref() {
+                    name_dependents
+                        .entry(nesting_id)
+                        .or_default()
+                        .push(NameDependent::NestedName(*name_id));
+                }
+            }
+        }
+
+        Self {
+            uri_id,
+            document,
+            definitions: IdentityHashMap::default(),
+            strings,
+            names,
+            constant_references: IdentityHashMap::default(),
+            method_references: IdentityHashMap::default(),
+            name_dependents,
+        }
+    }
+
     // Into parts
 
     #[must_use]

rust/rubydex/src/indexing/ruby_indexer.rs

@@ -2481,5 +2481,11 @@ impl Visit<'_> for RubyIndexer<'_> {
 }
 
 #[cfg(test)]
+fn backend() -> super::IndexerBackend {
+    super::IndexerBackend::RubyIndexer
+}
+
+#[cfg(test)]
+#[allow(clippy::duplicate_mod)]
 #[path = "ruby_indexer_tests.rs"]
 mod tests;

rust/rubydex/src/indexing/ruby_indexer_tests.rs

@@ -1,3 +1,7 @@
+// This file is included via #[path] by both ruby_indexer.rs and operation/applier.rs
+// to run the same tests against both indexing backends. Each parent module provides
+// a `backend()` function that `index_source` calls via `super::backend()`.
+
 use crate::{
     assert_def_comments_eq, assert_def_mixins_eq, assert_def_name_eq, assert_def_name_offset_eq, assert_def_str_eq,
     assert_def_superclass_ref_eq, assert_definition_at, assert_dependents, assert_local_diagnostics_eq,
@@ -86,7 +90,7 @@ macro_rules! assert_method_references_eq {
 }
 
 fn index_source(source: &str) -> LocalGraphTest {
-    LocalGraphTest::new("file:///foo.rb", source)
+    LocalGraphTest::new_with_backend("file:///foo.rb", source, super::backend())
 }
 
 mod constant_tests {

rust/rubydex/src/lib.rs

@@ -7,6 +7,7 @@ pub mod job_queue;
 pub mod listing;
 pub mod model;
 pub mod offset;
+pub mod operation;
 pub mod position;
 pub mod query;
 pub mod resolution;

rust/rubydex/src/main.rs

@@ -2,7 +2,8 @@ use clap::{Parser, ValueEnum};
 use std::{collections::HashSet, mem};
 
 use rubydex::{
-    indexing, integrity, listing,
+    indexing::{self, IndexerBackend},
+    integrity, listing,
     model::graph::Graph,
     resolution::Resolver,
     stats::{
@@ -31,6 +32,14 @@ struct Args {
     #[arg(long = "check-integrity", help = "Check the integrity of the graph after resolution")]
     check_integrity: bool,
 
+    #[arg(
+        long = "indexer",
+        value_enum,
+        default_value = "ruby-indexer",
+        help = "Which indexer backend to use for Ruby files"
+    )]
+    indexer: Indexer,
+
     #[arg(
         long = "report-orphans",
         value_name = "PATH",
@@ -49,6 +58,21 @@ enum StopAfter {
     Resolution,
 }
 
+#[derive(Debug, Clone, ValueEnum)]
+enum Indexer {
+    RubyIndexer,
+    OperationBuilder,
+}
+
+impl From<&Indexer> for IndexerBackend {
+    fn from(indexer: &Indexer) -> Self {
+        match indexer {
+            Indexer::RubyIndexer => IndexerBackend::RubyIndexer,
+            Indexer::OperationBuilder => IndexerBackend::OperationBuilder,
+        }
+    }
+}
+
 fn exit(print_stats: bool) {
     if print_stats {
         Timer::print_breakdown();
@@ -80,7 +104,8 @@ fn main() {
     // Indexing
 
     let mut graph = Graph::new();
-    let errors = time_it!(indexing, { indexing::index_files(&mut graph, file_paths) });
+    let backend = IndexerBackend::from(&args.indexer);
+    let errors = time_it!(indexing, { indexing::index_files(&mut graph, file_paths, backend) });
 
     for error in errors {
         eprintln!("{error}");

rust/rubydex/src/model/definitions.rs

@@ -29,7 +29,7 @@ use crate::{
     assert_mem_size,
     model::{
         comment::Comment,
-        ids::{ConstantReferenceId, DefinitionId, NameId, StringId, UriId},
+        ids::{self, ConstantReferenceId, DefinitionId, NameId, StringId, UriId},
         visibility::Visibility,
     },
     offset::Offset,
@@ -289,7 +289,7 @@ impl ClassDefinition {
 
     #[must_use]
     pub fn id(&self) -> DefinitionId {
-        DefinitionId::from(&format!("{}{}{}", *self.uri_id, self.offset.start(), *self.name_id))
+        ids::namespace_definition_id(self.uri_id, &self.offset, self.name_id)
     }
 
     #[must_use]
@@ -411,7 +411,7 @@ impl SingletonClassDefinition {
 
     #[must_use]
     pub fn id(&self) -> DefinitionId {
-        DefinitionId::from(&format!("{}{}{}", *self.uri_id, self.offset.start(), *self.name_id))
+        ids::namespace_definition_id(self.uri_id, &self.offset, self.name_id)
     }
 
     #[must_use]
@@ -515,7 +515,7 @@ impl ModuleDefinition {
 
     #[must_use]
     pub fn id(&self) -> DefinitionId {
-        DefinitionId::from(&format!("{}{}{}", *self.uri_id, self.offset.start(), *self.name_id))
+        ids::namespace_definition_id(self.uri_id, &self.offset, self.name_id)
     }
 
     #[must_use]
@@ -611,7 +611,7 @@ impl ConstantDefinition {
 
     #[must_use]
     pub fn id(&self) -> DefinitionId {
-        DefinitionId::from(&format!("{}{}{}", *self.uri_id, self.offset.start(), *self.name_id))
+        ids::namespace_definition_id(self.uri_id, &self.offset, self.name_id)
     }
 
     #[must_use]
@@ -926,7 +926,7 @@ pub struct MethodDefinition {
 assert_mem_size!(MethodDefinition, 96);
 
 /// The receiver of a singleton method definition.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum Receiver {
     /// `def self.foo` - receiver is the enclosing definition (class, module, singleton class or DSL)
     SelfReceiver(DefinitionId),
@@ -965,18 +965,7 @@ impl MethodDefinition {
 
     #[must_use]
     pub fn id(&self) -> DefinitionId {
-        let mut formatted_id = format!("{}{}{}", *self.uri_id, self.offset.start(), *self.str_id);
-
-        if let Some(receiver) = &self.receiver {
-            match receiver {
-                Receiver::SelfReceiver(def_id) => formatted_id.push_str(&def_id.to_string()),
-                Receiver::ConstantReceiver(name_id) => {
-                    formatted_id.push_str(&name_id.to_string());
-                }
-            }
-        }
-
-        DefinitionId::from(&formatted_id)
+        ids::method_definition_id(self.uri_id, &self.offset, self.str_id, self.receiver.as_ref())
     }
 
     #[must_use]