fix: harden verification — pymatgen degradation, leaderboard TTL, retraction detection

backend/routes/discovery.py

@@ -726,7 +726,7 @@
    - metabolomics: compound identification (HMDB/PubChem), pathway mapping (KEGG), spectral matching (MassBank) (70% weight)
 
 2. **Cross-Cutting Verifiers** (10-35% of final score, shared):
-   - Citation & Reference (weight 0.15): DOI resolution, metadata matching, abstract similarity, freshness
+   - Citation & Reference (weight 0.15): DOI resolution + retraction detection, metadata matching, abstract similarity, freshness
    - Statistical Forensics (weight 0.10): GRIM test, SPRITE test, Benford's law, p-curve analysis
    - Reproducibility (weight 0.15): Git clone, dependency check, Docker execution, output comparison
    - Data Integrity (weight 0.10): Schema consistency, duplicate detection, outlier flagging, hash verification

backend/verification/citation_verifier.py

@@ -179,23 +179,64 @@ def _extract_doi_from_url(url: str) -> str:
         return match.group(0).rstrip(".,;)") if match else ""
 
     async def _resolve_doi(self, doi: str) -> dict:
-        """Resolve DOI via CrossRef API."""
+        """Resolve DOI via CrossRef API; check for retraction notices."""
         try:
             async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client:
                 resp = await client.get(f"{CROSSREF_API}/{doi}")
                 if resp.status_code == 200:
                     data = resp.json().get("message", {})
-                    return {
-                        "score": 1.0,
+
+                    # Check for retraction / correction / withdrawal via
+                    # CrossRef's "update-to" field (includes Retraction Watch data)
+                    retraction_info = self._check_retraction_status(data)
+
+                    score = 1.0
+                    if retraction_info.get("retracted"):
+                        score = 0.1  # severe penalty — paper is retracted
+                    elif retraction_info.get("has_correction"):
+                        score = 0.7  # mild penalty — paper has correction/erratum
+
+                    result = {
+                        "score": score,
                         "resolved": True,
                         "title": data.get("title", [""])[0] if data.get("title") else "",
                         "doi": doi,
                     }
+                    if retraction_info.get("retracted") or retraction_info.get("has_correction"):
+                        result["retraction_status"] = retraction_info
+                    return result
                 return {"score": 0.0, "resolved": False, "status": resp.status_code}
         except Exception as e:
             logger.warning("doi_resolution_failed", doi=doi, error=str(e))
             return {"score": 0.0, "resolved": False, "error": str(e)}
 
+    @staticmethod
+    def _check_retraction_status(crossref_data: dict) -> dict:
+        """Inspect CrossRef metadata for retraction/correction notices."""
+        update_to = crossref_data.get("update-to", [])
+        if not update_to:
+            return {"retracted": False, "has_correction": False}
+
+        retracted = False
+        has_correction = False
+        notices: list[str] = []
+
+        for update in update_to:
+            update_type = (update.get("type", "") or "").lower()
+            label = update.get("label", "") or update_type
+            if any(kw in update_type for kw in ("retraction", "withdrawal")):
+                retracted = True
+                notices.append(label)
+            elif any(kw in update_type for kw in ("correction", "erratum")):
+                has_correction = True
+                notices.append(label)
+
+        return {
+            "retracted": retracted,
+            "has_correction": has_correction,
+            "notices": notices,
+        }
+
     async def _check_metadata_match(self, citation: dict) -> dict:
         """Check title/author/year match via OpenAlex and Semantic Scholar."""
         title = citation.get("title", "")

backend/verification/materials_adapter.py

@@ -16,6 +16,34 @@
 
 logger = get_logger(__name__)
 
+# ------------------------------------------------------------------
+# Optional dependencies — degrade gracefully when unavailable
+# ------------------------------------------------------------------
+try:
+    from pymatgen.core import Structure, Element
+    from pymatgen.symmetry.analyzer import SpacegroupAnalyzer
+    PYMATGEN_AVAILABLE = True
+except ImportError:
+    PYMATGEN_AVAILABLE = False
+    Structure = None  # type: ignore[assignment,misc]
+    Element = None  # type: ignore[assignment,misc]
+    SpacegroupAnalyzer = None  # type: ignore[assignment,misc]
+    logger.warning(
+        "pymatgen_not_available",
+        note="CIF parsing / symmetry / composition checks will degrade to score=0.5",
+    )
+
+try:
+    from mp_api.client import MPRester
+    MP_API_AVAILABLE = True
+except ImportError:
+    MPRester = None  # type: ignore[assignment,misc]
+    MP_API_AVAILABLE = False
+    logger.warning(
+        "mp_api_not_available",
+        note="Materials Project client unavailable — REST fallback will be used",
+    )
+
 MP_API_KEY = os.getenv("MP_API_KEY", "")
 AFLOW_BASE = "http://aflowlib.org/API/aflux/"
 TIMEOUT = 30  # seconds total
@@ -209,8 +237,9 @@ async def _verify_property(self, result: dict) -> VerificationResult:
 
     @staticmethod
     def _check_cif_parse(cif_data: str) -> dict:
+        if not PYMATGEN_AVAILABLE:
+            return {"score": 0.5, "structure": None, "metrics": {"note": "pymatgen unavailable — skipped"}}
         try:
-            from pymatgen.core import Structure
             structure = Structure.from_str(cif_data, fmt="cif")
             if len(structure) == 0:
                 return {"score": 0.0, "metrics": {"error": "Empty structure"}}
@@ -253,8 +282,9 @@ def _check_overlapping_atoms(structure) -> dict:
     def _check_symmetry(structure) -> dict:
         if structure is None:
             return {"score": 0.0, "metrics": {}}
+        if not PYMATGEN_AVAILABLE:
+            return {"score": 0.5, "metrics": {"note": "pymatgen unavailable — skipped"}}
         try:
-            from pymatgen.symmetry.analyzer import SpacegroupAnalyzer
             analyzer = SpacegroupAnalyzer(structure)
             sg_symbol = analyzer.get_space_group_symbol()
             sg_number = analyzer.get_space_group_number()
@@ -274,8 +304,9 @@ def _check_symmetry(structure) -> dict:
     def _check_composition(structure) -> dict:
         if structure is None:
             return {"score": 0.0, "metrics": {}}
+        if not PYMATGEN_AVAILABLE:
+            return {"score": 0.5, "metrics": {"note": "pymatgen unavailable — skipped"}}
         try:
-            from pymatgen.core import Element
             comp = structure.composition
             elements = comp.elements
 
@@ -416,9 +447,8 @@ async def _check_materials_project(
 
     async def _query_mp_api(self, formula: str) -> dict | None:
         """Query Materials Project API via mp-api client or REST fallback."""
-        if MP_API_KEY:
+        if MP_API_KEY and MP_API_AVAILABLE:
             try:
-                from mp_api.client import MPRester
                 result = await asyncio.to_thread(self._mp_rester_query, formula)
                 return result
             except Exception as e:
@@ -453,7 +483,6 @@ async def _query_mp_api(self, formula: str) -> dict | None:
 
     @staticmethod
     def _mp_rester_query(formula: str) -> dict | None:
-        from mp_api.client import MPRester
         with MPRester(MP_API_KEY) as mpr:
             docs = mpr.materials.summary.search(
                 formula=formula,
@@ -503,9 +532,8 @@ async def _lookup_mp_properties(
         self, mp_id: str | None, formula: str | None,
     ) -> dict:
         """Look up material by MP ID or formula for property verification."""
-        if mp_id and MP_API_KEY:
+        if mp_id and MP_API_KEY and MP_API_AVAILABLE:
             try:
-                from mp_api.client import MPRester
                 result = await asyncio.to_thread(self._mp_rester_by_id, mp_id)
                 if result:
                     return {"found": True, "properties": result}
@@ -521,7 +549,6 @@ async def _lookup_mp_properties(
 
     @staticmethod
     def _mp_rester_by_id(mp_id: str) -> dict | None:
-        from mp_api.client import MPRester
         with MPRester(MP_API_KEY) as mpr:
             docs = mpr.materials.summary.search(
                 material_ids=[mp_id],

backend/verification/ml_repro_adapter.py

@@ -30,9 +30,10 @@
     "truthfulqa", "gsm8k", "humaneval", "mbpp", "piqa", "boolq",
 }
 
-# Cached leaderboard dataframe
+# Cached leaderboard dataframe with TTL
 _leaderboard_df = None
-_leaderboard_loaded = False
+_leaderboard_loaded_at: float = 0.0
+_LEADERBOARD_TTL = 3600 * 6  # 6 hours
 
 
 class MLReproAdapter(VerificationAdapter):
@@ -502,11 +503,17 @@ async def _check_model_exists(self, model_id: str) -> dict:
     async def _check_leaderboard(
         self, model_id: str, benchmark: str, claimed_metrics: dict,
     ) -> dict:
-        global _leaderboard_df, _leaderboard_loaded
-
-        if not _leaderboard_loaded:
-            _leaderboard_df = await self._load_leaderboard()
-            _leaderboard_loaded = True
+        global _leaderboard_df, _leaderboard_loaded_at
+
+        stale = (time.monotonic() - _leaderboard_loaded_at) > _LEADERBOARD_TTL
+        if _leaderboard_df is None or stale:
+            fresh_df = await self._load_leaderboard()
+            if fresh_df is not None:
+                _leaderboard_df = fresh_df
+                _leaderboard_loaded_at = time.monotonic()
+            elif _leaderboard_df is not None:
+                logger.warning("leaderboard_refresh_failed_using_stale")
+            # else: both None — first load failed, will fall through below
 
         if _leaderboard_df is None:
             return {