feat(agents:AI agents for title/abstract screening and workload reduc…

backend/api/core/cit_utils.py

@@ -14,6 +14,7 @@
 from fastapi.concurrency import run_in_threadpool
 
 from .config import settings
+from ..services.cit_db_service import cits_dp_service
 
 
 def _is_postgres_configured() -> bool:
@@ -94,5 +95,15 @@ async def load_sr_and_check(
         if not screening:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No screening database configured for this systematic review")
 
+        # Best-effort runtime schema evolution for agentic screening.
+        # CAN-SR uses per-upload screening tables, so we may need to add the
+        # validation columns to the specific table referenced by the SR.
+        try:
+            table_name = (screening or {}).get("table_name") or "citations"
+            await run_in_threadpool(cits_dp_service.ensure_step_validation_columns, table_name)
+        except Exception:
+            # Don't block requests if the DB isn't ready/configured.
+            pass
+
 
     return sr, screening

backend/api/screen/agentic_utils.py

@@ -0,0 +1,95 @@
+"""backend.api.screen.agentic_utils
+
+Utilities for the GREP-Agent style "screening + critical" workflow.
+
+We keep this module small and dependency-free so routers can reuse the helpers
+for title/abstract and fulltext pipelines.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class ParsedAgentXML:
+    answer: str
+    confidence: float
+    rationale: str
+    parse_ok: bool
+
+
+_TAG_RE_CACHE: dict[str, re.Pattern[str]] = {}
+
+
+def _tag_re(tag: str) -> re.Pattern[str]:
+    if tag not in _TAG_RE_CACHE:
+        _TAG_RE_CACHE[tag] = re.compile(rf"<{tag}>(.*?)</{tag}>", re.IGNORECASE | re.DOTALL)
+    return _TAG_RE_CACHE[tag]
+
+
+def parse_agent_xml(text: str) -> ParsedAgentXML:
+    """Parse <answer>, <confidence>, <rationale> tags from model output."""
+
+    raw = (text or "").strip()
+    ans_m = _tag_re("answer").search(raw)
+    conf_m = _tag_re("confidence").search(raw)
+    rat_m = _tag_re("rationale").search(raw)
+
+    answer = (ans_m.group(1).strip() if ans_m else "")
+    rationale = (rat_m.group(1).strip() if rat_m else "")
+
+    conf_val = 0.0
+    if conf_m:
+        try:
+            conf_val = float(conf_m.group(1).strip())
+        except Exception:
+            conf_val = 0.0
+    conf_val = max(0.0, min(1.0, conf_val))
+
+    parse_ok = bool(ans_m and conf_m)
+    return ParsedAgentXML(answer=answer, confidence=conf_val, rationale=rationale, parse_ok=parse_ok)
+
+
+def resolve_option(raw_answer: str, options: list[str]) -> str:
+    """Resolve a model answer to one of the provided options (best-effort)."""
+    ans = (raw_answer or "").strip()
+    if not ans:
+        return ans
+
+    # Exact match first
+    for opt in options or []:
+        if ans == opt:
+            return opt
+
+    # Case-insensitive exact
+    ans_l = ans.lower()
+    for opt in options or []:
+        if ans_l == (opt or "").lower():
+            return opt
+
+    # Substring containment (mirrors existing CAN-SR JSON screening logic)
+    for opt in options or []:
+        if (opt or "").lower() in ans_l:
+            return opt
+
+    return ans
+
+
+def build_critical_options(*, all_options: list[str], screening_answer: str) -> list[str]:
+    """Forced alternatives: (all_options - {screening_answer}) + ["None of the above"]."""
+    base = [o for o in (all_options or []) if (o or "").strip()]
+    sa = (screening_answer or "").strip()
+    if sa:
+        base = [o for o in base if o.strip() != sa]
+    base.append("None of the above")
+    # stable unique
+    seen = set()
+    out = []
+    for o in base:
+        if o not in seen:
+            seen.add(o)
+            out.append(o)
+    return out

backend/api/screen/prompts.py

@@ -72,4 +72,149 @@
 - Use sentence indices from the numbered full text for "evidence_sentences"
 - Use table numbers from the Tables section for "evidence_tables"
 - Use figure numbers from the Figures section for "evidence_figures"
+"""
+
+
+# ---------------------------------------------------------------------------
+# Agentic screening (GREP-Agent style) prompt contracts
+# ---------------------------------------------------------------------------
+
+# NOTE:
+# CAN-SR historically used JSON output for screening. The agentic plan expects
+# XML-tag parsing (<answer>, <confidence>, <rationale>) so we can reuse a stable
+# parsing contract across screening + critical steps.
+
+PROMPT_XML_TEMPLATE_TA = """
+You are a highly critical, helpful scientific evaluator completing an academic review.
+
+Task:
+Answer the question "{question}" for the following citation.
+
+Citation:
+{cit}
+
+Choose EXACTLY ONE of these options (exact text):
+{options}
+
+Additional guidance:
+{xtra}
+
+Output requirement:
+Return ONLY the following XML tags (no Markdown, no extra prose):
+<answer>...</answer>
+<confidence>...</confidence>
+<rationale>...</rationale>
+
+Confidence requirements:
+- confidence is a float between 0 and 1
+- be conservative; do not overestimate confidence
+"""
+
+
+PROMPT_XML_TEMPLATE_TA_CRITICAL = """
+You are a critical reviewer double-checking another model's screening answer.
+
+Original question:
+"{question}"
+
+Citation:
+{cit}
+
+The first model answered:
+"{screening_answer}"
+
+Now, you MUST choose from the following forced alternatives.
+Rules:
+- You are NOT allowed to choose the original answer.
+- If you agree with the original answer, choose "None of the above".
+
+Forced alternatives (choose exactly one; exact text):
+{options}
+
+Additional guidance:
+{xtra}
+
+Output requirement:
+Return ONLY the following XML tags (no Markdown, no extra prose):
+<answer>...</answer>
+<confidence>...</confidence>
+<rationale>...</rationale>
+
+Confidence requirements:
+- confidence is a float between 0 and 1
+- be conservative; do not overestimate confidence
+"""
+
+
+PROMPT_XML_TEMPLATE_FULLTEXT = """
+You are assisting with a scientific full-text screening task.
+
+Task:
+Evaluate the question "{question}" against the paper content provided as numbered sentences (e.g., "[0] ...", "[1] ...").
+
+Choose EXACTLY ONE of these options (exact text):
+{options}
+
+Additional guidance:
+{xtra}
+
+Full text (numbered sentences):
+{fulltext}
+
+Tables (numbered):
+{tables}
+
+Figures (numbered; captions correspond to images provided alongside this message):
+{figures}
+
+Output requirement:
+Return ONLY the following XML tags (no Markdown, no extra prose):
+<answer>...</answer>
+<confidence>...</confidence>
+<rationale>...</rationale>
+
+Confidence requirements:
+- confidence is a float between 0 and 1
+- be conservative; do not overestimate confidence
+"""
+
+
+PROMPT_XML_TEMPLATE_FULLTEXT_CRITICAL = """
+You are a critical reviewer double-checking another model's full-text screening answer.
+
+Original question:
+"{question}"
+
+The first model answered:
+"{screening_answer}"
+
+Now, you MUST choose from the following forced alternatives.
+Rules:
+- You are NOT allowed to choose the original answer.
+- If you agree with the original answer, choose "None of the above".
+
+Forced alternatives (choose exactly one; exact text):
+{options}
+
+Additional guidance:
+{xtra}
+
+Full text (numbered sentences):
+{fulltext}
+
+Tables (numbered):
+{tables}
+
+Figures (numbered; captions correspond to images provided alongside this message):
+{figures}
+
+Output requirement:
+Return ONLY the following XML tags (no Markdown, no extra prose):
+<answer>...</answer>
+<confidence>...</confidence>
+<rationale>...</rationale>
+
+Confidence requirements:
+- confidence is a float between 0 and 1
+- be conservative; do not overestimate confidence
 """