Revert "(EAI-428) versioned docs" (#737)

Revert "(EAI-428) versioned docs (#699)"

This reverts commit 7566854.

Author: yakubova92

packages/datasets/src/bin/huggingFace/uploadCodeExampleDatasetToHuggingFace.ts

@@ -10,7 +10,7 @@ import {
 } from "mongodb-rag-core";
 import {
   forbiddenUrls,
-  publicDatasetSourceTypes,
+  publicDatasetSourceName,
 } from "../../mongoDbDatasetConstants";
 import { uploadDatasetToHuggingFace } from "../../uploadDatasetToHuggingFace";
 import { HUGGINGFACE, HUGGINGFACE_DOCS_CODE_EXAMPLES } from "../../EnvVars";
@@ -21,7 +21,7 @@ import { CodeExampleDatasetEntry } from "../../codeExampleDataset/createCodeExam
 import { Filter } from "mongodb-rag-core/mongodb";
 
 async function uploadCodeExampleDatasetToHuggingFace() {
-  logger.info("Starting upload code example dataset to Hugging Face script");
+  logger.info("Staring upload code example dataset to Hugging Face script");
 
   const {
     HUGGINGFACE_ACCESS_TOKEN,
@@ -56,7 +56,7 @@ async function uploadCodeExampleDatasetToHuggingFace() {
   try {
     const pages = await pageStore.loadPages({
       query: makeLoadPagesFilter(
-        publicDatasetSourceTypes,
+        publicDatasetSourceName,
         Array.from(forbiddenUrls)
       ),
     });
@@ -120,16 +120,12 @@ async function uploadCodeExampleDatasetToHuggingFace() {
 uploadCodeExampleDatasetToHuggingFace();
 
 function makeLoadPagesFilter(
-  publicDatasetSourceTypes: string[],
+  publicDatasetSourceName: RegExp,
   forbiddenUrls: string[]
 ): Filter<PersistedPage> {
   return {
-    sourceType: { $in: publicDatasetSourceTypes },
+    sourceName: publicDatasetSourceName,
     url: { $nin: forbiddenUrls },
     action: { $ne: "deleted" },
-    $or: [
-      { "metadata.version.isCurrent": { $exists: false } },
-      { "metadata.version.isCurrent": true },
-    ],
   };
 }

packages/datasets/src/bin/huggingFace/uploadContentDatasetToHuggingFace.ts

@@ -6,7 +6,7 @@ import {
 } from "mongodb-rag-core";
 import {
   forbiddenUrls,
-  publicDatasetSourceTypes,
+  publicDatasetSourceName,
 } from "../../mongoDbDatasetConstants";
 import { uploadDatasetToHuggingFace } from "../../uploadDatasetToHuggingFace";
 import { HUGGINGFACE, HUGGINGFACE_DOCS_CONTENT } from "../../EnvVars";
@@ -36,7 +36,7 @@ async function uploadContentDatasetToHuggingFace() {
     logger.info("Loading pages dataset from MongoDB");
     const dataset = await loadPagesDataset({
       pageStore,
-      dataSourceTypes: publicDatasetSourceTypes,
+      dataSourceRegex: publicDatasetSourceName,
       forbiddenUrls: Array.from(forbiddenUrls),
     });
     logger.info(

packages/datasets/src/mongoDbDatasetConstants.ts

@@ -17,6 +17,6 @@ export const forbiddenUrls = new Set([
 ]);
 
 /**
-  The {@link PersistedPage.sourceType} for public datasets should be either `tech-docs` (Docs) or `devcenter` (Developer Center).
+  The {@link PersistedPage.sourceName} for public datasets should contain `snooty` (Docs) or `devcenter` (Developer Center).
  */
-export const publicDatasetSourceTypes = ["tech-docs", "devcenter"];
+export const publicDatasetSourceName = /snooty|devcenter/;

packages/datasets/src/pageDataset/loadPageDataset.test.ts

@@ -20,7 +20,6 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "updated",
-      sourceType: "tech-docs",
     },
     // This page should be filtered out by forbidden url
     {
@@ -32,7 +31,6 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "updated",
-      sourceType: "tech-docs",
     },
     // This page has action deleted, so should not be returned
     {
@@ -44,9 +42,8 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "deleted",
-      sourceType: "tech-docs",
     },
-    // This page does not match sourceType (even though it is active)
+    // This page does not match regex (even though it is active)
     {
       url: "https://example.com/page4",
       body: "Page 4 body",
@@ -56,36 +53,6 @@ describe("loadPagesDataset", () => {
       updated: new Date(),
       format: "html",
       action: "created",
-      sourceType: "blog",
-    },
-    {
-      url: "https://example.com/page5",
-      body: "Page 5 body",
-      metadata: {
-        extra: "info1",
-        version: { isCurrent: true, label: "current" },
-      },
-      title: "Page 5",
-      sourceName: "SourceE",
-      updated: new Date(),
-      format: "html",
-      action: "updated",
-      sourceType: "tech-docs",
-    },
-    // This page represents an older version and should not be returned
-    {
-      url: "https://example.com/old-version/page5",
-      body: "Page 5 body",
-      metadata: {
-        extra: "info5",
-        version: { isCurrent: false, label: "old-version" },
-      },
-      title: "Page 5",
-      sourceName: "SourceE",
-      updated: new Date(),
-      format: "html",
-      action: "updated",
-      sourceType: "tech-docs",
     },
   ];
 
@@ -105,14 +72,21 @@ describe("loadPagesDataset", () => {
     await pageStore.close();
   });
 
-  it("should only return pages matching a sourceType", async () => {
+  it("should only return pages matching a regex", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
+      dataSourceRegex: /SourceA/,
       forbiddenUrls: [],
-      dataSourceTypes: ["tech-docs"],
     });
-    // page1 should be returned as it matches the sourceType, page 4 does not match
+    // page1 should be returned, page2 as well if not forbidden & not deleted, page3 is deleted.
+    // page4 is filtered out because dataSource doesn't match.
     expect(dataset.map((p) => p.url)).toContain("https://example.com/page1");
+    expect(dataset.map((p) => p.url)).not.toContain(
+      "https://example.com/page2"
+    );
+    expect(dataset.map((p) => p.url)).not.toContain(
+      "https://example.com/page3"
+    );
     expect(dataset.map((p) => p.url)).not.toContain(
       "https://example.com/page4"
     );
@@ -121,8 +95,8 @@ describe("loadPagesDataset", () => {
   it("should exclude forbidden urls", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
+      dataSourceRegex: /SourceA|SourceB/,
       forbiddenUrls: [samplePages[1].url],
-      dataSourceTypes: ["devcenter", "tech-docs"],
     });
     expect(dataset.map((p) => p.url)).toContain("https://example.com/page1");
     expect(dataset.map((p) => p.url)).not.toContain(
@@ -133,32 +107,19 @@ describe("loadPagesDataset", () => {
   it("should not include pages with action 'deleted'", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
+      dataSourceRegex: /foo/,
       forbiddenUrls: [],
-      dataSourceTypes: ["devcenter", "tech-docs"],
     });
     const urls = dataset.map((p) => p.url);
-    expect(urls.length).toBeGreaterThan(0);
     expect(urls).not.toContain("https://example.com/page3");
   });
 
-  it("should not include pages representing a non current version", async () => {
-    const dataset = await loadPagesDataset({
-      pageStore,
-      forbiddenUrls: [],
-      dataSourceTypes: ["tech-docs"],
-    });
-    const urls = dataset.map((p) => p.url);
-    expect(urls).toContain("https://example.com/page5");
-    expect(urls).not.toContain("https://example.com/old-version/page5");
-  });
-
   it("should only return the projected fields", async () => {
     const dataset = await loadPagesDataset({
       pageStore,
+      dataSourceRegex: /foo/,
       forbiddenUrls: [],
-      dataSourceTypes: ["devcenter", "tech-docs"],
     });
-    expect(dataset.length).toBeGreaterThan(0);
     for (const page of dataset) {
       const pageKeys = Object.keys(page);
 

packages/datasets/src/pageDataset/loadPageDataset.ts

@@ -10,7 +10,7 @@ export interface LoadPagesDatasetParams {
   /**
     Regular expression to filter pages by `Page.dataSource`
    */
-  dataSourceTypes: string[];
+  dataSourceRegex: RegExp;
   /**
     Set of urls to exclude from the dataset
    */
@@ -19,19 +19,15 @@ export interface LoadPagesDatasetParams {
 
 export async function loadPagesDataset({
   pageStore,
-  dataSourceTypes,
+  dataSourceRegex,
   forbiddenUrls,
 }: LoadPagesDatasetParams): Promise<PageDatasetEntry[]> {
   return pageStore.aggregatePages<PageDatasetEntry>([
     {
       $match: {
-        sourceType: { $in: dataSourceTypes },
+        sourceName: dataSourceRegex,
         url: { $nin: forbiddenUrls },
         action: { $ne: "deleted" },
-        $or: [
-          { "metadata.version.isCurrent": { $exists: false } },
-          { "metadata.version.isCurrent": true },
-        ],
       },
     },
     {

packages/ingest-mongodb-public/src/meta.config.ts

@@ -71,10 +71,12 @@ export const standardConfig = {
   }),
   dataSources: async () => {
     const source = makeSnootyDataSource({
-      name: metaSnootyProject.name,
+      name: `snooty-${metaSnootyProject.name}`,
       project: {
         ...metaSnootyProject,
+        currentBranch: metaSnootyProject.currentBranch,
         type: "snooty",
+        baseUrl: metaSnootyProject.baseUrl?.replace(/\/?$/, "/"),
       },
       snootyDataApiBaseUrl,
     });

packages/ingest-mongodb-public/src/sources/DevCenterDataSource.test.ts

@@ -57,7 +57,6 @@ describe("makeDevCenterPage()", () => {
       }),
       format: "md",
       sourceName: "devcenter",
-      sourceType: "devcenter",
       metadata: {
         tags: ["Realm", "GitHub Actions", "JavaScript"],
         pageDescription: devCenterDoc.description,

packages/ingest-mongodb-public/src/sources/DevCenterDataSource.ts

@@ -79,7 +79,6 @@ export function makeDevCenterPage(
     }),
     format: "md",
     sourceName: name,
-    sourceType: "devcenter",
     metadata: {
       tags: extractTags(document.tags),
       pageDescription: document.description,

packages/ingest-mongodb-public/src/sources/mongoose.ts

@@ -26,7 +26,7 @@ export const mongooseSourceConstructor = async () => {
     metadata: {
       productName: "Mongoose ODM",
       tags: ["node.js", "community library", "mongoose", "odm"],
-      versionLabel: "v7.x (current)",
+      version: "v7.x (current)",
     },
   });
 };