v0.2.1 - Frontend live data wiring and scanner updates

.github/workflows/deploy.yml

@@ -94,7 +94,7 @@ jobs:
         if: steps.check_config.outputs.is_configured == 'true' || github.event_name == 'workflow_dispatch'
         env:
           API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }}
-          JWT_SECRET: ${{ secrets.JWT_SECRET || 'change-me-in-production' }}
+          JWT_SECRET: ${{ secrets.JWT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}

.gitignore

@@ -216,3 +216,4 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+ai/vectorstore/

README.md

@@ -76,7 +76,11 @@ The OpenShield API is deployed to the Render free tier and is accessible at:
 > **Note:** As this is hosted on the Render free tier, the service may spin down after 15 minutes of inactivity. The first request after a spin-down can take 30-60 seconds to complete.
 
 > [!IMPORTANT]
-> **Security Requirement:** For absolute security, any production deployment **must** override the default `JWT_SECRET` with a strong, unique value in the environment variables.
+> **Security Requirement:** Production deployments **fail at startup** if `JWT_SECRET` is missing, set to the insecure default, or shorter than 32 characters. Generate a strong secret with:
+> ```
+> python -c "import secrets; print(secrets.token_urlsafe(32))"
+> ```
+> Set `OPENSHIELD_ENV=production` (or rely on Render's automatic `RENDER=true`) to enable this enforcement. Local development runs without these signals are allowed to use the default with a warning.
 
 ---
 

ai/README.md

@@ -0,0 +1,34 @@
+# OpenShield RAG Pipeline
+
+Document loader and chunker for OpenShield rules and compliance frameworks.
+Loads all scanner rules and CIS, NIST, ISO 27001 and SOC2 controls
+into structured documents for the RAG vector store.
+
+## Files
+
+- `ai/loader.py` — loads OpenShield rules and compliance frameworks as structured documents
+- `ai/chunker.py` — splits documents into overlapping chunks for embedding
+- `ai/embed.py` — builds the ChromaDB vector store (from PR 97)
+- `ai/retriever.py` — queries the vector store (from PR 97)
+
+## Vector Store
+
+The vector store is persisted at `ai/vectorstore/` using ChromaDB.
+
+## How loader.py works
+
+Reads all `scanner/rules/az_*.py` files and extracts:
+- Rule ID, name, severity, category
+- Description and remediation text
+
+Also reads all four compliance framework JSON files:
+- CIS Azure Benchmark
+- NIST CSF
+- ISO 27001
+- SOC2
+
+## How chunker.py works
+
+Splits documents into 512-character overlapping chunks with 64-character
+overlap. Tries to split on newlines to avoid breaking mid-sentence.
+Each chunk inherits the metadata of its parent document.

ai/__init__.py

@@ -0,0 +1 @@
+

ai/chunker.py

@@ -0,0 +1,47 @@
+"""Chunking pipeline for OpenShield documents."""
+import logging
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHUNK_SIZE = 512
+DEFAULT_CHUNK_OVERLAP = 64
+
+
+def chunk_documents(documents, chunk_size=DEFAULT_CHUNK_SIZE, chunk_overlap=DEFAULT_CHUNK_OVERLAP):
+    chunks = []
+    for doc in documents:
+        doc_id = doc.get("id", "unknown")
+        content = doc.get("content", "")
+        metadata = doc.get("metadata", {})
+        doc_chunks = _split_text(content, chunk_size, chunk_overlap)
+        for idx, chunk_text in enumerate(doc_chunks):
+            chunks.append({
+                "id": f"{doc_id}_chunk_{idx}",
+                "content": chunk_text,
+                "metadata": {**metadata, "parent_doc_id": doc_id, "chunk_index": idx, "total_chunks": len(doc_chunks)},
+            })
+    logger.info("Chunked %d documents into %d chunks", len(documents), len(chunks))
+    return chunks
+
+
+def _split_text(text, chunk_size, chunk_overlap):
+    if len(text) <= chunk_size:
+        return [text]
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        if end >= len(text):
+            chunks.append(text[start:].strip())
+            break
+        split_pos = text.rfind("
+", start, end)
+        if split_pos == -1 or split_pos <= start:
+            split_pos = end
+        chunk = text[start:split_pos].strip()
+        if chunk:
+            chunks.append(chunk)
+        start = split_pos - chunk_overlap
+        if start < 0:
+            start = 0
+    return [c for c in chunks if c]

ai/embed.py

@@ -0,0 +1,138 @@
+"""Build the OpenShield knowledge base vector store for RAG AI insights"""
+
+
+import importlib.util
+import json
+import logging
+from pathlib import Path
+
+import chromadb
+
+logger = logging.getLogger(__name__)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+RULES_DIR = REPO_ROOT / "scanner" / "rules"
+FRAMEWORKS_DIR = REPO_ROOT / "compliance" / "frameworks"
+SKILLS_DIR = REPO_ROOT / "ai" / "knowledge" / "skills"
+VECTORSTORE_DIR = REPO_ROOT / "ai" / "vectorstore"
+COLLECTION_NAME = "openshield"
+
+
+def _load_rule_module(path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _collect_skill_documents():
+    documents = []
+    if not SKILLS_DIR.exists():
+        logger.warning("Skills directory not found, skipping: %s", SKILLS_DIR)
+        return documents
+    for path in sorted(SKILLS_DIR.rglob("SKILL.md")):
+        try:
+            text = path.read_text(encoding="utf-8")
+        except Exception as exc:
+            logger.warning("Skipping %s: %s", path.name, exc)
+            continue
+        if not text.strip():
+            continue
+        skill_name = path.parent.name
+        documents.append({
+            "id": f"skill-{skill_name}",
+            "text": text,
+            "source": skill_name,
+            "type": "skill",
+        })
+    return documents
+
+
+def _collect_rule_documents():
+    documents = []
+    for path in sorted(RULES_DIR.glob("az_*.py")):
+        try:
+            module = _load_rule_module(path)
+        except Exception as exc:
+            logger.warning("Skipping %s: %s", path.name, exc)
+            continue
+        rule_id = getattr(module, "RULE_ID", None)
+        if not rule_id:
+            continue
+        text = (
+            f"OpenShield rule {rule_id}: {getattr(module, 'RULE_NAME', '')}\n"
+            f"Category: {getattr(module, 'CATEGORY', '')}\n"
+            f"Severity: {getattr(module, 'SEVERITY', '')}\n"
+            f"Description: {getattr(module, 'DESCRIPTION', '')}\n"
+            f"Remediation: {getattr(module, 'REMEDIATION', '')}"
+        )
+        documents.append({
+            "id": f"rule-{rule_id}",
+            "text": text,
+            "source": rule_id,
+            "type": "rule",
+        })
+    return documents
+
+
+def _collect_compliance_documents():
+    documents = []
+    for path in sorted(FRAMEWORKS_DIR.glob("*.json")):
+        framework = path.stem
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Skipping %s: %s", path.name, exc)
+            continue
+        for control_id, control in data.get("controls", {}).items():
+            description = control.get("description", "")
+            if not description:
+                continue
+            text = (
+                f"{framework} control {control_id}: "
+                f"{control.get('control_name', '')}\n{description}"
+            )
+            documents.append({
+                "id": f"{framework}-{control_id}",
+                "text": text,
+                "source": f"{framework} {control_id}",
+                "type": "control",
+            })
+    return documents
+
+
+def build_vectorstore():
+    VECTORSTORE_DIR.mkdir(parents=True, exist_ok=True)
+    client = chromadb.PersistentClient(path=str(VECTORSTORE_DIR))
+
+    try:
+        client.delete_collection(COLLECTION_NAME)
+    except Exception as exc:
+        logger.info("Could not delete collection '%s' before rebuild: %s", COLLECTION_NAME, exc)
+    collection = client.create_collection(COLLECTION_NAME)
+
+    documents = (
+        _collect_skill_documents()
+        + _collect_rule_documents()
+        + _collect_compliance_documents()
+    )
+    if not documents:
+        raise RuntimeError("No documents found to embed. Check repo paths.")
+
+    collection.add(
+        ids=[d["id"] for d in documents],
+        documents=[d["text"] for d in documents],
+        metadatas=[
+            {"source": d["source"], "type": d["type"]} for d in documents
+        ],
+    )
+    logger.info(
+        "Embedded %d documents into '%s'.", len(documents), COLLECTION_NAME
+    )
+    return len(documents)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    count = build_vectorstore()
+    print(f"Done. Vector store built with {count} documents at {VECTORSTORE_DIR}")