Switch from GitHub API to direct download endpoints to avoid rate limits

Co-authored-by: fedorov <313942+fedorov@users.noreply.github.com>

Author: Copilot

idc_index/index.py

@@ -26,7 +26,6 @@
 aws_endpoint_url = "https://s3.amazonaws.com"
 gcp_endpoint_url = "https://storage.googleapis.com"
 asset_endpoint_url = f"https://github.com/ImagingDataCommons/idc-index-data/releases/download/{idc_index_data.__version__}"
-github_api_url = f"https://api.github.com/repos/ImagingDataCommons/idc-index-data/releases/tags/{idc_index_data.__version__}"
 
 logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -161,9 +160,8 @@ def __init__(self):
     def _discover_available_indices(self, refresh: bool = False) -> dict:
         """Discover available index tables from the idc-index-data GitHub release assets.
 
-        This method attempts to discover available index parquet files by first trying
-        the GitHub releases API, and falling back to a known list of indices if the API
-        is unavailable. In either case, it populates descriptions from the accompanying
+        This method discovers available index parquet files using direct download URLs
+        from the GitHub releases. It populates descriptions from the accompanying
         JSON schema files.
 
         Args:
@@ -207,120 +205,40 @@ def _discover_available_indices(self, refresh: bool = False) -> dict:
             "analysis_results_index",
         ]
 
-        # Try to discover additional indices from the GitHub release API
-        discovered_from_api = False
-        try:
-            response = requests.get(github_api_url, timeout=30)
-            if response.status_code == 200:
-                release_data = response.json()
-                assets = release_data.get("assets", [])
-
-                # Find all parquet files in the release assets
-                parquet_assets = {
-                    a["name"]: a["browser_download_url"]
-                    for a in assets
-                    if a["name"].endswith(".parquet")
-                }
-
-                # Find all json schema files in the release assets
-                json_assets = {
-                    a["name"]: a["browser_download_url"]
-                    for a in assets
-                    if a["name"].endswith(".json")
-                }
-
-                discovered_from_api = True
-
-                # Update descriptions for bundled indices if available
-                for bundled_name, schema_filename in [
-                    ("index", "idc_index.json"),
-                    ("prior_versions_index", "prior_versions_index.json"),
-                ]:
-                    if schema_filename in json_assets:
-                        schema = self._fetch_index_schema_from_url(
-                            json_assets[schema_filename]
-                        )
-                        if schema and "table_description" in schema:
-                            indices[bundled_name]["description"] = schema[
-                                "table_description"
-                            ]
-
-                # Process discovered parquet files
-                for parquet_name, parquet_url in parquet_assets.items():
-                    # Extract index name from filename
-                    index_name = parquet_name.replace(".parquet", "")
-
-                    # Skip bundled indices
-                    if index_name in ("idc_index", "prior_versions_index"):
-                        continue
-
-                    # Determine description from schema file
-                    description = ""
-                    schema_name = f"{index_name}.json"
-                    if schema_name in json_assets:
-                        schema = self._fetch_index_schema_from_url(
-                            json_assets[schema_name]
-                        )
-                        if schema:
-                            description = schema.get("table_description", "")
+        # Discover indices using direct download URLs (no API rate limits)
+        for index_name in known_remote_indices:
+            # Try to fetch the schema directly
+            schema_url = f"{asset_endpoint_url}/{index_name}.json"
+            parquet_url = f"{asset_endpoint_url}/{index_name}.parquet"
 
-                    # Check if the index is already installed locally
-                    local_path = os.path.join(
-                        self.indices_data_dir, f"{index_name}.parquet"
-                    )
-                    installed = os.path.exists(local_path)
-                    file_path = local_path if installed else None
-
-                    indices[index_name] = {
-                        "description": description,
-                        "installed": installed,
-                        "url": parquet_url,
-                        "file_path": file_path,
-                    }
+            description = ""
+            schema = self._fetch_index_schema_from_url(schema_url)
+            if schema:
+                description = schema.get("table_description", "")
 
-            else:
-                logger.debug(
-                    f"GitHub API returned status {response.status_code}. "
-                    "Using known index list with schema discovery."
-                )
-        except requests.exceptions.RequestException as e:
-            logger.debug(f"GitHub API request failed: {e}. Using known index list.")
-
-        # If API discovery failed, use known list and try to fetch schemas directly
-        if not discovered_from_api:
-            for index_name in known_remote_indices:
-                # Try to fetch the schema directly
-                schema_url = f"{asset_endpoint_url}/{index_name}.json"
-                parquet_url = f"{asset_endpoint_url}/{index_name}.parquet"
-
-                description = ""
-                schema = self._fetch_index_schema_from_url(schema_url)
-                if schema:
-                    description = schema.get("table_description", "")
-
-                # Check if the index is already installed locally
-                local_path = os.path.join(
-                    self.indices_data_dir, f"{index_name}.parquet"
-                )
-                installed = os.path.exists(local_path)
-                file_path = local_path if installed else None
-
-                indices[index_name] = {
-                    "description": description,
-                    "installed": installed,
-                    "url": parquet_url,
-                    "file_path": file_path,
-                }
-
-            # Also try to update bundled index descriptions
-            for bundled_name, schema_filename in [
-                ("index", "idc_index.json"),
-                ("prior_versions_index", "prior_versions_index.json"),
-            ]:
-                schema_url = f"{asset_endpoint_url}/{schema_filename}"
-                schema = self._fetch_index_schema_from_url(schema_url)
-                if schema and "table_description" in schema:
-                    indices[bundled_name]["description"] = schema["table_description"]
+            # Check if the index is already installed locally
+            local_path = os.path.join(
+                self.indices_data_dir, f"{index_name}.parquet"
+            )
+            installed = os.path.exists(local_path)
+            file_path = local_path if installed else None
+
+            indices[index_name] = {
+                "description": description,
+                "installed": installed,
+                "url": parquet_url,
+                "file_path": file_path,
+            }
+
+        # Also try to update bundled index descriptions from schema files
+        for bundled_name, schema_filename in [
+            ("index", "idc_index.json"),
+            ("prior_versions_index", "prior_versions_index.json"),
+        ]:
+            schema_url = f"{asset_endpoint_url}/{schema_filename}"
+            schema = self._fetch_index_schema_from_url(schema_url)
+            if schema and "table_description" in schema:
+                indices[bundled_name]["description"] = schema["table_description"]
 
         return indices