Metrics Dashboard

docker-compose.yml

@@ -81,7 +81,7 @@ services:
       retries: 3
       start_period: 40s
 
-  # Sentinel - Input Security Agent (internal only)
+  # Sentinel - Input Security Agent (now publicly accessible for testing)
   sentinel:
     build:
       context: ./services/security-agent
@@ -100,18 +100,17 @@ services:
       - DD_RUNTIME_METRICS_ENABLED=true
       - GEMINI_API_KEY=${GEMINI_API_KEY}
       - TELEMETRY_ENABLED=true
-      # Security Settings
-      - SECURITY_SANITIZATION_ENABLED=true
-      - SECURITY_PII_REDACTION_ENABLED=true
-      - SECURITY_XSS_PROTECTION_ENABLED=true
-      - SECURITY_SQL_INJECTION_DETECTION_ENABLED=true
-      - SECURITY_COMMAND_INJECTION_DETECTION_ENABLED=true
-      - SECURITY_LLM_CHECK_THRESHOLD=0.85
-      # TOON Conversion
-      - TOON_CONVERSION_ENABLED=true
-      # LLM Settings
-      - LLM_FORWARD_ENABLED=true
-      - LLM_MODEL_NAME=gemini-2.5-flash
+      # Security Settings (defaults to False - opt-in via request)
+      - SECURITY_SANITIZATION_ENABLED=false
+      - SECURITY_PII_REDACTION_ENABLED=false
+      - SECURITY_XSS_PROTECTION_ENABLED=false
+      - SECURITY_SQL_INJECTION_DETECTION_ENABLED=false
+      - SECURITY_COMMAND_INJECTION_DETECTION_ENABLED=false
+      # TOON Conversion (default False - opt-in via request)
+      - TOON_CONVERSION_ENABLED=false
+      # LLM Settings (default False - opt-in via request)
+      - LLM_FORWARD_ENABLED=false
+      - LLM_MODEL_NAME=gemini-3-flash-preview
       - LLM_MAX_TOKENS=8192
       # Guardian Service URL
       - GUARDIAN_SERVICE_URL=http://guardian:8002
@@ -157,8 +156,9 @@ services:
       - DEFAULT_MODERATION_MODE=moderate
       - HARMFUL_CONTENT_THRESHOLD=0.7
       - INAPPROPRIATE_CONTENT_THRESHOLD=0.6
-      - OUTPUT_PII_DETECTION_ENABLED=true
-      - AUTO_CONVERT_TOON_TO_JSON=true
+      # PII & TOON (defaults to False - opt-in via request)
+      - OUTPUT_PII_DETECTION_ENABLED=false
+      - AUTO_CONVERT_TOON_TO_JSON=false
     depends_on:
       - datadog-agent
     volumes:

services/eagle-eye/app/api/v1/endpoints/api_keys.py

@@ -9,6 +9,7 @@
 from app.schemas import ApiKeyCreate, ApiKeyResponse, ApiKeySecret
 import structlog
 from typing import List
+from app.core.telemetry import telemetry
 
 router = APIRouter()
 logger = structlog.get_logger()
@@ -53,7 +54,7 @@ async def create_api_key(
     await db.refresh(new_key)
 
     # Return Schema with SECRET plain key
-    return ApiKeySecret(
+    response_obj = ApiKeySecret(
         id=new_key.id,
         key_prefix=new_key.key_prefix,
         name=new_key.name,
@@ -63,6 +64,10 @@ async def create_api_key(
         api_key=plain_key,  # IMPORTANT: Shown only once
     )
 
+    telemetry.increment("clestiq.eagleeye.api_keys.created", tags=[f"app:{app.name}", f"user:{app.owner_id}"])
+
+    return response_obj
+
 
 @router.get("/apps/{app_id}/keys", response_model=List[ApiKeyResponse])
 async def list_api_keys(
@@ -111,4 +116,5 @@ async def revoke_api_key(
 
     await db.delete(key)  # Or set is_active = False for soft delete
     await db.commit()
+    telemetry.increment("clestiq.eagleeye.api_keys.revoked", tags=[f"app:{app.name}", f"user:{app.owner_id}"])
     return {"message": "API Key revoked"}

services/eagle-eye/app/api/v1/endpoints/apps.py

@@ -8,6 +8,7 @@
 import structlog
 from typing import List
 from app.api.deps import get_current_user
+from app.core.telemetry import telemetry
 
 router = APIRouter()
 logger = structlog.get_logger()
@@ -37,6 +38,7 @@ async def create_app(
         )
     await db.refresh(new_app)
     logger.info("Application created", app_id=str(new_app.id))
+    telemetry.increment("clestiq.eagleeye.apps.created", tags=[f"user:{user_id}"])
     return new_app
 
 
@@ -122,4 +124,5 @@ async def delete_app(
 
     await db.delete(app)
     await db.commit()
+    telemetry.increment("clestiq.eagleeye.apps.deleted", tags=[f"user:{user_id}"])
     return {"message": "Application deleted"}

services/eagle-eye/app/api/v1/endpoints/auth.py

@@ -8,6 +8,7 @@
 from app.schemas import UserCreate, UserResponse, TokenWithUser
 from datetime import timedelta
 import structlog
+from app.core.telemetry import telemetry
 
 router = APIRouter()
 logger = structlog.get_logger()
@@ -38,6 +39,7 @@ async def register(user_in: UserCreate, db: AsyncSession = Depends(get_db)):
     await db.refresh(new_user)
 
     logger.info("User registered", user_id=str(new_user.id), email=new_user.email)
+    telemetry.increment("clestiq.eagleeye.users.created")
     return new_user
 
 

services/eagle-eye/app/core/config.py

@@ -21,6 +21,9 @@ class Settings(BaseSettings):
     DD_SERVICE: str = "clestiq-shield-eagle-eye"
     DD_ENV: str = "production"
     DD_VERSION: str = "1.0.0"
+    DD_AGENT_HOST: str = "datadog-agent"
+    DD_DOGSTATSD_PORT: int = 8125
+    DD_DOGSTATSD_SOCKET: str = ""
 
     class Config:
         case_sensitive = True

services/eagle-eye/app/core/telemetry.py

@@ -2,12 +2,96 @@
 import sys
 import structlog
 from ddtrace import tracer
+from datadog import initialize, statsd
 
 from app.core.config import get_settings
 
 settings = get_settings()
 
 
+class TelemetryClient:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(TelemetryClient, cls).__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+
+        try:
+            # Initialize Datadog client
+            options = {
+                "statsd_host": settings.DD_AGENT_HOST,
+                "statsd_port": settings.DD_DOGSTATSD_PORT,
+            }
+
+            # Prefer Socket if configured (Docker/K8s standard)
+            if settings.DD_DOGSTATSD_SOCKET:
+                options = {"statsd_socket_path": settings.DD_DOGSTATSD_SOCKET}
+
+            initialize(**options)
+            self._initialized = True
+
+            # Use standard logger here to avoid circular deps or complex structlog init issues early on
+            logging.getLogger("uvicorn").info(
+                f"Telemetry initialized mode={'socket' if settings.DD_DOGSTATSD_SOCKET else 'udp'} "
+                f"target={settings.DD_DOGSTATSD_SOCKET or f'{settings.DD_AGENT_HOST}:{settings.DD_DOGSTATSD_PORT}'}"
+            )
+        except Exception as e:
+            logging.getLogger("uvicorn").error(
+                f"Failed to initialize telemetry: {str(e)}"
+            )
+
+    def increment(self, metric: str, value: int = 1, tags: list[str] = None):
+        """Increment a counter metric."""
+        if not settings.TELEMETRY_ENABLED:
+            return
+
+        try:
+            all_tags = self._get_default_tags() + (tags or [])
+            statsd.increment(metric, tags=all_tags, value=value)
+        except Exception as e:
+            # Squelch errors to prevent app crash, but log warning
+            logging.getLogger("uvicorn").warning(f"Failed to send metric {metric}: {e}")
+
+    def gauge(self, metric: str, value: float, tags: list[str] = None):
+        """Record a gauge metric."""
+        if not settings.TELEMETRY_ENABLED:
+            return
+
+        try:
+            all_tags = self._get_default_tags() + (tags or [])
+            statsd.gauge(metric, value, tags=all_tags)
+        except Exception as e:
+            logging.getLogger("uvicorn").warning(f"Failed to send metric {metric}: {e}")
+
+    def histogram(self, metric: str, value: float, tags: list[str] = None):
+        """Record a histogram metric."""
+        if not settings.TELEMETRY_ENABLED:
+            return
+
+        try:
+            all_tags = self._get_default_tags() + (tags or [])
+            statsd.histogram(metric, value, tags=all_tags)
+        except Exception as e:
+            logging.getLogger("uvicorn").warning(f"Failed to send metric {metric}: {e}")
+
+    def _get_default_tags(self) -> list[str]:
+        return [
+            f"service:{settings.DD_SERVICE}",
+            f"env:{settings.DD_ENV}",
+            f"version:{settings.DD_VERSION}",
+        ]
+
+
+# Global instance
+telemetry = TelemetryClient()
+
+
 def add_datadog_trace_context(_, __, event_dict):
     """Add Datadog trace context to logs for correlation."""
     span = tracer.current_span()

services/eagle-eye/app/models/api_key.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, String, ForeignKey, DateTime, Boolean
+from sqlalchemy import Column, String, ForeignKey, DateTime, Boolean, Integer, JSON
 from sqlalchemy.sql import func
 from app.core.db import Base
 import uuid
@@ -21,5 +21,9 @@ class ApiKey(Base):
     expires_at = Column(DateTime(timezone=True), nullable=True)
     last_used_at = Column(DateTime(timezone=True), nullable=True)
 
+    # Usage Stats
+    request_count = Column(Integer, default=0)
+    usage_data = Column(JSON, default=dict)
+
     # Relationships
     application = relationship("Application", back_populates="api_keys")

services/eagle-eye/pyproject.toml

@@ -21,6 +21,7 @@ pydantic-settings = "^2.1.0"
 structlog = "^24.1.0"
 passlib = {extras = ["bcrypt"], version = "^1.7.4"}
 python-multipart = "^0.0.9"
+datadog = "^0.48.0"
 
 [build-system]
 requires = ["poetry-core"]

services/gateway/app/api/deps.py

@@ -7,17 +7,16 @@
 import structlog
 
 from app.core.db import get_db
-from app.models.application import Application
 from app.models.api_key import ApiKey
 
 logger = structlog.get_logger()
 
 api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
 
 
-async def get_current_app(
+async def get_api_key(
     api_key: str = Security(api_key_header), db: AsyncSession = Depends(get_db)
-) -> Application:
+) -> ApiKey:
     if not api_key:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
@@ -42,8 +41,4 @@ async def get_current_app(
             detail="Invalid API Key",
         )
 
-    # Update last_used_at (optional, can be done async or skipped for performance)
-    # api_key_obj.last_used = func.now()
-    # await db.commit()
-
-    return api_key_obj.application
+    return api_key_obj