(EAI-428) feature versioned docs (#739)

* (EAI-968) ingest multiple versions (#683)

* remove snooty prefix

* ingesting pages for all branches on each data source

* do not ingest (and delete if already exists) pages on inactive branches

* handle current version override

* cleanup unused code from previous version override implementation, tests

* update SnootyDataSource tests

* remove override for docs current version

* (EAI-969) query multiple versions (#693)

* nearest neighbor search accepts filters, defaults to current version

* parse filters to mdb query

* (EAI-1003) get available versions for data source (#696)

* get versions of a data source

* get versions for multiple data sources

* (EAI-922) ensure only current version on hugging face dataset (#698)

* exclude old versions from dataset

* add test case

* fix other tests

* fix return type of getDataSourceVersions - return object, not array

* move QueryFilters type def to embedded content store

* fix type

* (EAI-1001) enable search by type (#703)

* add sourceType to pages and embedded_content and ability to filter by it

* test case

* change sourceRegex filter to sourceType filter

* lint

Author: yakubova92

packages/datasets/src/bin/huggingFace/uploadCodeExampleDatasetToHuggingFace.ts

@@ -10,7 +10,7 @@ import {
 } from "mongodb-rag-core";
 import {
   forbiddenUrls,
-  publicDatasetSourceName,
+  publicDatasetSourceTypes,
 } from "../../mongoDbDatasetConstants";
 import { uploadDatasetToHuggingFace } from "../../uploadDatasetToHuggingFace";
 import { HUGGINGFACE, HUGGINGFACE_DOCS_CODE_EXAMPLES } from "../../EnvVars";
@@ -21,7 +21,7 @@ import { CodeExampleDatasetEntry } from "../../codeExampleDataset/createCodeExam
 import { Filter } from "mongodb-rag-core/mongodb";
 
 async function uploadCodeExampleDatasetToHuggingFace() {
-  logger.info("Staring upload code example dataset to Hugging Face script");
+  logger.info("Starting upload code example dataset to Hugging Face script");
 
   const {
     HUGGINGFACE_ACCESS_TOKEN,
@@ -56,7 +56,7 @@ async function uploadCodeExampleDatasetToHuggingFace() {
   try {
     const pages = await pageStore.loadPages({
       query: makeLoadPagesFilter(
-        publicDatasetSourceName,
+        publicDatasetSourceTypes,
         Array.from(forbiddenUrls)
       ),
     });
@@ -120,12 +120,16 @@ async function uploadCodeExampleDatasetToHuggingFace() {
 uploadCodeExampleDatasetToHuggingFace();
 
 function makeLoadPagesFilter(
-  publicDatasetSourceName: RegExp,
+  publicDatasetSourceTypes: string[],
   forbiddenUrls: string[]
 ): Filter<PersistedPage> {
   return {
-    sourceName: publicDatasetSourceName,
+    sourceType: { $in: publicDatasetSourceTypes },
     url: { $nin: forbiddenUrls },
     action: { $ne: "deleted" },
+    $or: [
+      { "metadata.version.isCurrent": { $exists: false } },
+      { "metadata.version.isCurrent": true },
+    ],
   };
 }

packages/datasets/src/bin/huggingFace/uploadContentDatasetToHuggingFace.ts

@@ -6,7 +6,7 @@ import {
 } from "mongodb-rag-core";
 import {
   forbiddenUrls,
-  publicDatasetSourceName,
+  publicDatasetSourceTypes,
 } from "../../mongoDbDatasetConstants";
 import { uploadDatasetToHuggingFace } from "../../uploadDatasetToHuggingFace";
 import { HUGGINGFACE, HUGGINGFACE_DOCS_CONTENT } from "../../EnvVars";
@@ -36,7 +36,7 @@ async function uploadContentDatasetToHuggingFace() {
     logger.info("Loading pages dataset from MongoDB");
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceRegex: publicDatasetSourceName,
+      dataSourceTypes: publicDatasetSourceTypes,
       forbiddenUrls: Array.from(forbiddenUrls),
     });
     logger.info(

packages/datasets/src/mongoDbDatasetConstants.ts

@@ -17,6 +17,6 @@ export const forbiddenUrls = new Set([
 ]);
 
 /**
-  The {@link PersistedPage.sourceName} for public datasets should contain `snooty` (Docs) or `devcenter` (Developer Center).
+  The {@link PersistedPage.sourceType} for public datasets should be either `tech-docs` (Docs) or `devcenter` (Developer Center).
  */
-export const publicDatasetSourceName = /snooty|devcenter/;
+export const publicDatasetSourceTypes = ["tech-docs", "devcenter"];

packages/datasets/src/pageDataset/loadPageDataset.test.ts

@@ -20,6 +20,7 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "updated",
+      sourceType: "tech-docs",
     },
     // This page should be filtered out by forbidden url
     {
@@ -31,6 +32,7 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "updated",
+      sourceType: "tech-docs",
     },
     // This page has action deleted, so should not be returned
     {
@@ -42,8 +44,9 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "deleted",
+      sourceType: "tech-docs",
     },
-    // This page does not match regex (even though it is active)
+    // This page does not match sourceType (even though it is active)
     {
       url: "https://example.com/page4",
       body: "Page 4 body",
@@ -53,6 +56,36 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "created",
+      sourceType: "blog",
+    },
+    {
+      url: "https://example.com/page5",
+      body: "Page 5 body",
+      metadata: {
+        extra: "info1",
+        version: { isCurrent: true, label: "current" },
+      },
+      title: "Page 5",
+      sourceName: "SourceE",
+      updated: new Date(),
+      format: "html",
+      action: "updated",
+      sourceType: "tech-docs",
+    },
+    // This page represents an older version and should not be returned
+    {
+      url: "https://example.com/old-version/page5",
+      body: "Page 5 body",
+      metadata: {
+        extra: "info5",
+        version: { isCurrent: false, label: "old-version" },
+      },
+      title: "Page 5",
+      sourceName: "SourceE",
+      updated: new Date(),
+      format: "html",
+      action: "updated",
+      sourceType: "tech-docs",
     },
   ];
 
@@ -72,21 +105,14 @@ describe("loadPagesDataset", () => {
     await pageStore.close();
   });
 
-  it("should only return pages matching a regex", async () => {
+  it("should only return pages matching a sourceType", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceRegex: /SourceA/,
       forbiddenUrls: [],
+      dataSourceTypes: ["tech-docs"],
     });
-    // page1 should be returned, page2 as well if not forbidden & not deleted, page3 is deleted.
-    // page4 is filtered out because dataSource doesn't match.
+    // page1 should be returned as it matches the sourceType, page 4 does not match
     expect(dataset.map((p) => p.url)).toContain("https://example.com/page1");
-    expect(dataset.map((p) => p.url)).not.toContain(
-      "https://example.com/page2"
-    );
-    expect(dataset.map((p) => p.url)).not.toContain(
-      "https://example.com/page3"
-    );
     expect(dataset.map((p) => p.url)).not.toContain(
       "https://example.com/page4"
     );
@@ -95,8 +121,8 @@ describe("loadPagesDataset", () => {
   it("should exclude forbidden urls", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceRegex: /SourceA|SourceB/,
       forbiddenUrls: [samplePages[1].url],
+      dataSourceTypes: ["devcenter", "tech-docs"],
     });
     expect(dataset.map((p) => p.url)).toContain("https://example.com/page1");
     expect(dataset.map((p) => p.url)).not.toContain(
@@ -107,19 +133,32 @@ describe("loadPagesDataset", () => {
   it("should not include pages with action 'deleted'", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceRegex: /foo/,
       forbiddenUrls: [],
+      dataSourceTypes: ["devcenter", "tech-docs"],
     });
     const urls = dataset.map((p) => p.url);
+    expect(urls.length).toBeGreaterThan(0);
     expect(urls).not.toContain("https://example.com/page3");
   });
 
+  it("should not include pages representing a non current version", async () => {
+    const dataset = await loadPagesDataset({
+      pageStore,
+      forbiddenUrls: [],
+      dataSourceTypes: ["tech-docs"],
+    });
+    const urls = dataset.map((p) => p.url);
+    expect(urls).toContain("https://example.com/page5");
+    expect(urls).not.toContain("https://example.com/old-version/page5");
+  });
+
   it("should only return the projected fields", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceRegex: /foo/,
       forbiddenUrls: [],
+      dataSourceTypes: ["devcenter", "tech-docs"],
     });
+    expect(dataset.length).toBeGreaterThan(0);
     for (const page of dataset) {
       const pageKeys = Object.keys(page);
 

packages/datasets/src/pageDataset/loadPageDataset.ts

@@ -10,7 +10,7 @@ export interface LoadPagesDatasetParams {
   /**
     Regular expression to filter pages by `Page.dataSource`
    */
-  dataSourceRegex: RegExp;
+  dataSourceTypes: string[];
   /**
     Set of urls to exclude from the dataset
    */
@@ -19,15 +19,19 @@ export interface LoadPagesDatasetParams {
 
 export async function loadPagesDataset({
   pageStore,
-  dataSourceRegex,
+  dataSourceTypes,
   forbiddenUrls,
 }: LoadPagesDatasetParams): Promise<PageDatasetEntry[]> {
   return pageStore.aggregatePages<PageDatasetEntry>([
     {
       $match: {
-        sourceName: dataSourceRegex,
+        sourceType: { $in: dataSourceTypes },
         url: { $nin: forbiddenUrls },
         action: { $ne: "deleted" },
+        $or: [
+          { "metadata.version.isCurrent": { $exists: false } },
+          { "metadata.version.isCurrent": true },
+        ],
       },
     },
     {

packages/ingest-mongodb-public/src/meta.config.ts

@@ -71,12 +71,10 @@ export const standardConfig = {
   }),
   dataSources: async () => {
     const source = makeSnootyDataSource({
-      name: `snooty-${metaSnootyProject.name}`,
+      name: metaSnootyProject.name,
       project: {
         ...metaSnootyProject,
-        currentBranch: metaSnootyProject.currentBranch,
         type: "snooty",
-        baseUrl: metaSnootyProject.baseUrl?.replace(/\/?$/, "/"),
       },
       snootyDataApiBaseUrl,
     });

packages/ingest-mongodb-public/src/sources/DevCenterDataSource.test.ts

@@ -57,6 +57,7 @@ describe("makeDevCenterPage()", () => {
       }),
       format: "md",
       sourceName: "devcenter",
+      sourceType: "devcenter",
       metadata: {
         tags: ["Realm", "GitHub Actions", "JavaScript"],
         pageDescription: devCenterDoc.description,

packages/ingest-mongodb-public/src/sources/DevCenterDataSource.ts

@@ -79,6 +79,7 @@ export function makeDevCenterPage(
     }),
     format: "md",
     sourceName: name,
+    sourceType: "devcenter",
     metadata: {
       tags: extractTags(document.tags),
       pageDescription: document.description,

packages/ingest-mongodb-public/src/sources/mongoose.ts

@@ -26,7 +26,7 @@ export const mongooseSourceConstructor = async () => {
     metadata: {
       productName: "Mongoose ODM",
       tags: ["node.js", "community library", "mongoose", "odm"],
-      version: "v7.x (current)",
+      versionLabel: "v7.x (current)",
     },
   });
 };