(EAI-645) deprecate coachGTM (#657)

* script to process url list for coachgtm, should be removed

* rm all references to coachGTM

* rm data source

* clean up imports

* rm files related to url check

* set up script to be re-usable, read from and write to local file system

* moved urlCheck to scripts directory

* changes to copy

Co-authored-by: Nick Larew <nick.larew@mongodb.com>

---------

Co-authored-by: Nick Larew <nick.larew@mongodb.com>

Author: yakubova92

.drone.yml

@@ -496,98 +496,6 @@ steps:
       api_server: https://api.staging.corp.mongodb.com
       kubernetes_token:
         from_secret: staging_kubernetes_token
-# ingest for coach gtm
----
-depends_on: ["test-all"]
-kind: pipeline
-type: kubernetes
-name: staging-build-ingest-coach-gtm
-
-trigger:
-  branch:
-    - main
-  event:
-    - push
-  paths:
-    include:
-      - packages/mongodb-rag-ingest/**/*
-      - packages/ingest-mongodb-public/**/*
-      - packages/mongodb-rag-core/**/*
-      - package-lock.json
-      - package.json
-
-steps:
-  # Builds and publishes Docker image for staging
-  - name: publish-staging-ingest-coach-gtm
-    image: plugins/kaniko-ecr
-    environment:
-      LG_ARTIFACTORY_TOKEN:
-        from_secret: lg_artifactory_token
-      LG_ARTIFACTORY_EMAIL:
-        from_secret: lg_artifactory_email
-    settings:
-      dockerfile: ingest-service.dockerfile
-      create_repository: true
-      registry: 795250896452.dkr.ecr.us-east-1.amazonaws.com
-      repo: docs/${DRONE_REPO_NAME}-ingest-coach-gtm
-      build_args:
-        - LG_ARTIFACTORY_TOKEN
-        - LG_ARTIFACTORY_EMAIL
-      tags:
-        - git-${DRONE_COMMIT_SHA:0:7}-coach-gtm-staging
-        - latest-coach-gtm-staging
-      access_key:
-        from_secret: ecr_access_key
-      secret_key:
-        from_secret: ecr_secret_key
-
-  # Promotes current drone build to staging environment
-  - name: promote-staging-ingest-coach-gtm
-    image: drone/cli:1.4.0-alpine
-    commands:
-      - drone build promote mongodb/chatbot ${DRONE_BUILD_NUMBER} staging
-    environment:
-      DRONE_SERVER: ${DRONE_SYSTEM_PROTO}://${DRONE_SYSTEM_HOST}
-      DRONE_TOKEN:
-        from_secret: drone_token
-
----
-kind: pipeline
-type: kubernetes
-name: staging-deploy-ingest-coach-gtm
-
-trigger:
-  event:
-    - promote
-  target:
-    - staging
-  paths:
-    include:
-      - packages/mongodb-rag-ingest/**/*
-      - packages/ingest-mongodb-public/**/*
-      - packages/mongodb-rag-core/**/*
-      - package-lock.json
-      - package.json
-
-  branch:
-    - main
-
-steps:
-  # Deploys docker image associated with staging build that triggered promotion
-  - name: deploy-staging-ingest-coach-gtm
-    image: quay.io/mongodb/drone-helm:v3
-    settings:
-      chart: mongodb/cronjobs
-      chart_version: 1.10.0
-      add_repos: [mongodb=https://10gen.github.io/helm-charts]
-      namespace: docs
-      release: ingest-coach-gtm-staging
-      values: image.tag=git-${DRONE_COMMIT_SHA:0:7}-coach-gtm-staging,image.repository=795250896452.dkr.ecr.us-east-1.amazonaws.com/docs/${DRONE_REPO_NAME}-ingest-coach-gtm
-      values_files:
-        - packages/ingest-mongodb-public/environments/coachGtm.staging.yml
-      api_server: https://api.staging.corp.mongodb.com
-      kubernetes_token:
-        from_secret: staging_kubernetes_token
 
 ---
 depends_on: ["test-all"]

packages/chatbot-server-mongodb-public/src/eval/bin/generateEvalCasesYamlFromCSV.ts

@@ -93,20 +93,6 @@ function normalizeUrl(url: string): string {
   return url.replace(/^https?:\/\/(www\.)?/i, "");
 }
 
-const findMissingResources = async (
-  expectedUrls: string[]
-): Promise<string[]> => {
-  const results = await Promise.all(
-    expectedUrls.map(async (url) => {
-      const page = await pageStore.loadPage({
-        query: { url: { $regex: new RegExp(normalizeUrl(url)) } },
-      });
-      return !page ? url : null;
-    })
-  );
-  return results.filter((url) => url !== null) as string[];
-};
-
 /**
  Main function to read CSV file, transform evaluation cases, and write to YAML file.
  @param csvFilePath - Path to the input CSV file
@@ -130,7 +116,10 @@ async function main({
   const expectedUrls = Array.from(
     new Set(evalCases.flatMap((caseItem) => caseItem.expectedLinks ?? []))
   );
-  const urlsNotIngested = await findMissingResources(expectedUrls);
+  const urlsNotIngested = await pageStore.getMissingPagesByUrl({
+    expectedUrls,
+    urlTransformer: normalizeUrl,
+  });
   if (urlsNotIngested.length > 0) {
     console.warn(
       `Warning: ${urlsNotIngested.length}/${

packages/ingest-mongodb-public/.env.example

@@ -14,6 +14,5 @@ DEVCENTER_CONNECTION_URI="mongodb+srv://<username>:<password>@devhub-cluster.sew
 ## MongoDB Marketing CMS (mongodb.com)
 MONGODB_DOT_COM_CONNECTION_URI="mongodb+srv://..."
 MONGODB_DOT_COM_DB_NAME=<name of DB in dotcom cluster>
-MONGODB_COACH_GTM_DATABASE_NAME=<some DB name>
 ## Docs style guide, meta reference (mongodb.com/docs/meta)
 MONGODB_META_DATABASE_NAME="docs-meta"

packages/ingest-mongodb-public/environments/coachGtm.staging.yml

@@ -22,7 +22,6 @@
     "ingest:pages": "../../node_modules/mongodb-rag-ingest/build/main.js pages init --config ./build/config.js && ingest pages update --config ./build/config.js",
     "ingest:embed": "../../node_modules/mongodb-rag-ingest/build/main.js embed init --config ./build/config.js && ingest embed update --config ./build/config.js",
     "ingest:pages:meta": "../../node_modules/mongodb-rag-ingest/build/main.js pages update --config ./build/meta.config.js",
-    "ingest:k8s:coachGtmPages": "../../node_modules/mongodb-rag-ingest/build/main.js pages update --config ./build/coachGtm.config.js",
     "test": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js --forceExit"
   },
   "devDependencies": {

packages/ingest-mongodb-public/package.json

@@ -9,10 +9,6 @@ export const PUBLIC_INGEST_MONGODB_DOT_COM_ENV_VARS = {
   MONGODB_DOT_COM_DB_NAME: "",
 };
 
-export const PUBLIC_INGEST_COACHGTM_ENV_VARS = {
-  MONGODB_COACH_GTM_DATABASE_NAME: "",
-};
-
 export const PUBLIC_INGEST_MONGODB_DOCS_META_ENV_VARS = {
   MONGODB_META_DATABASE_NAME: "",
 };