(EAI-1043) (#742)

* add test case for non versioned content

* change vector search filter syntax, update dataset filter to match

Author: yakubova92

packages/datasets/src/bin/huggingFace/uploadCodeExampleDatasetToHuggingFace.ts

@@ -127,9 +127,6 @@ function makeLoadPagesFilter(
     sourceType: { $in: publicDatasetSourceTypes },
     url: { $nin: forbiddenUrls },
     action: { $ne: "deleted" },
-    $or: [
-      { "metadata.version.isCurrent": { $exists: false } },
-      { "metadata.version.isCurrent": true },
-    ],
+    "metadata.version.isCurrent": { $ne: false },
   };
 }

packages/datasets/src/pageDataset/loadPageDataset.ts

@@ -28,10 +28,7 @@ export async function loadPagesDataset({
         sourceType: { $in: dataSourceTypes },
         url: { $nin: forbiddenUrls },
         action: { $ne: "deleted" },
-        $or: [
-          { "metadata.version.isCurrent": { $exists: false } },
-          { "metadata.version.isCurrent": true },
-        ],
+        "metadata.version.isCurrent": { $ne: false },
       },
     },
     {

packages/mongodb-rag-core/src/contentStore/MongoDbEmbeddedContentStore.test.ts

@@ -335,6 +335,26 @@ describe("nearest neighbor search", () => {
     ).toHaveLength(5);
   });
 
+  it("Should be able to handle content that is not versioned", async () => {
+    assert(store);
+    const query = "tell me about MongoDB";
+    const filter = {
+      sourceName: "blog", // blog is not versioned
+    };
+    const { embedding } = await embedder.embed({
+      text: query,
+    });
+
+    const matches = await store.findNearestNeighbors(embedding, {
+      ...findNearestNeighborOptions,
+      filter,
+    });
+
+    expect(
+      matches.filter((match) => match.metadata?.version === undefined)
+    ).toHaveLength(5);
+  });
+
   it("Should filter content to sourceType requested", async () => {
     assert(store);
     const query = "db.collection.insertOne()";

packages/mongodb-rag-core/src/contentStore/MongoDbEmbeddedContentStore.ts

@@ -263,10 +263,7 @@ export function makeMongoDbEmbeddedContentStore({
 type MongoDbAtlasVectorSearchFilter = {
   sourceName?: string;
   "metadata.version.label"?: string;
-  "metadata.version.isCurrent"?: boolean;
-  $or?: {
-    "metadata.version.isCurrent": boolean | null;
-  }[];
+  "metadata.version.isCurrent"?: boolean | { $ne: boolean };
   sourceType?: string;
 };
 
@@ -289,10 +286,7 @@ const handleFilters = (
   // 1. current=true was explicitly requested, or
   // 2. [Default] no version filters were specified (current and label are undefined)
   else if (current === true || current === undefined) {
-    vectorSearchFilter["$or"] = [
-      { "metadata.version.isCurrent": true },
-      { "metadata.version.isCurrent": null },
-    ];
+    vectorSearchFilter["metadata.version.isCurrent"] = { $ne: false }; // Include unversioned embeddings
   }
   // Only find embeddings that are explicitly marked as non-current (isCurrent: false)
   else if (current === false) {