[DNM] tempest scraper

Author: dsariel

data_scraper/common/constants.py

@@ -3,6 +3,7 @@
 JIRA_COLLECTION_NAME = "rca-knowledge-base"
 OSP_DOCS_COLLECTION_NAME = "rca-osp-docs-knowledge-base"
 ERRATA_COLLECTION_NAME = "rca-errata"
+TEMPEST_COLLECTION_NAME = "rca-tempest"
 DEFAULT_EMBEDDING_MODEL = "BAAI/bge-m3"
 DEFAULT_JIRA_URL = "https://issues.redhat.com"
 DEFAULT_JIRA_PROJECTS = {

data_scraper/core/tempest_scraper.py

@@ -0,0 +1,66 @@
+"""Code for scraping Tempest data"""
+import logging
+import multiprocessing as mp
+import subprocess
+import sys
+from datetime import datetime
+from typing import TypedDict
+import regex as re
+
+import pandas as pd
+
+from data_scraper.core.scraper import Scraper
+from data_scraper.processors.tempest_provider import TempestProvider
+
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+class TempestRecord(TypedDict):
+    """Represents a record extracted from Tempest report.
+
+    Attributes:
+         test_name: of the failed test
+         traceback: of the failed test
+         report_url: URL of the tempest report
+
+    """
+    test_name: str
+    traceback: str
+    report_url: str
+
+
+
+class TempestScraper(Scraper):
+    """Main class for Tempest scraping and processing."""
+
+    def __init__(self, config: dict):
+        super().__init__(config=config)
+        self.config = config
+        self.tempest_provider = TempestProvider(self.config["zuul_url"])
+
+
+    def get_documents(self) -> list[dict]:
+
+
+        return results
+
+    def get_records(self, documents: list[dict]) -> list[TempestRecord]:
+        tempest_records: list[TempestRecord] = []
+
+
+
+        return errata_records
+
+    def get_chunks(self, record: dict) -> list[str]:
+        chunks = []
+
+
+        return chunks
+
+
+    def cleanup_records(
+        self, records: list, backup_path: str = "tempest_all_data.pickle"
+    ) -> list:
+       pass

data_scraper/main.py

@@ -6,6 +6,7 @@
 from data_scraper.common import constants
 from data_scraper.core.scraper import JiraScraper, OSPDocScraper
 from data_scraper.core.errata_scraper import ErrataScraper
+from data_scraper.core.tempest_scraper import TempestScraper
 
 logging.basicConfig(
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -178,3 +179,57 @@ def errata_scraper() -> None:
 
     scraper = ErrataScraper(config_args)
     scraper.run()
+
+
+
+
+def tempest_scraper() -> None:
+    """Entry point for command line execution."""
+    parser = ArgumentParser("tempest_scraper")
+
+    # Required arguments
+    parser.add_argument("--database_client_url", type=str, required=True)
+    parser.add_argument("--llm_server_url", type=str, required=True)
+    parser.add_argument("--llm_api_key", type=str, required=True)
+    parser.add_argument("--database_api_key", type=str, required=True)
+    parser.add_argument("--zuul-url", type=str, required=True)
+
+   
+
+    # Optional arguments
+    parser.add_argument("--chunk_size", type=int,
+                        default=constants.DEFAULT_CHUNK_SIZE)
+    parser.add_argument("--embedding_model", type=str,
+                        default=constants.DEFAULT_EMBEDDING_MODEL)
+    parser.add_argument("--db_collection_name", type=str,
+                        default=constants.TEMPEST_COLLECTION_NAME)
+    parser.add_argument("--scraper-processes", type=int,
+                        default=constants.DEFAULT_NUM_SCRAPER_PROCESSES)
+    parser.add_argument("--date_cutoff", type=datetime.fromisoformat,
+                        default=datetime.fromisoformat(constants.DEFAULT_DATE_CUTOFF),
+                        help=(
+                            "No issues from before this date will be used. "
+                            "Date must follow ISO format 'YYYY-MM-DD'"
+                        )
+    )
+    parser.add_argument("--recreate_collection", type=bool, default=True,
+                        help="Recreate database collection from scratch.")
+    args = parser.parse_args()
+
+    config_args = {
+        "database_client_url": args.database_client_url,
+        "llm_server_url": args.llm_server_url,
+        "llm_api_key": args.llm_api_key,
+        "database_api_key": args.database_api_key,
+        "chunk_size": args.chunk_size,
+        "embedding_model": args.embedding_model,
+        "db_collection_name": args.db_collection_name,
+        "zuul_url": args.errata_url,
+        "scraper_processes": args.scraper_processes,
+        "date_cutoff": args.date_cutoff,
+        "recreate_collection": args.recreate_collection,
+    }
+
+    scraper = TempestScraper(config_args)
+    scraper.run()
+

data_scraper/processors/tempest_provider.py

@@ -0,0 +1,41 @@
+"""Client to fetch Tempest."""
+import logging
+import requests
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+class TempestProvider:
+    """Provider for failed tempest tests for unijobs."""
+
+    def __init__(self, query_url: str):
+        self.query_url = query_url
+
+    def search_tempest_repords_with_failure(self) -> list:
+        """
+            Returns:
+            list: of urls to report.html
+        """
+
+
+        return []
+
+    def get_testname_traceback_pairs(self, report_urls: list) -> dict:
+        """For every entry on a list of urls issue a call to parser endpoint 
+           Get a list of (testname,traceback) pairs and add it to dictionary
+
+        Args:
+            report_urls: [ <address_to>/tepest_report1.html, ...,  <address_to>/tepest_reportN.html ]
+
+        Returns:
+            dict:
+               {  <address_to>/tepest_report1.html: [ [test1, traceback1], ... [testN, tracebackN] ],
+                  ...,
+                  <address_to>/tepest_reportM.html: [ [test1, traceback1], ... [testK, tracebackK] ]
+               }
+        """
+        LOG.info("Requesting tempest reports -> %s", query)
+
+
+
+        return {}