fix: auth resilience -- graceful JWT fallback + placeholder detection (production outage postmortem)

backend/config/startup_checks.py

@@ -53,4 +53,15 @@ def validate_environment() -> None:
         if not os.getenv(var_name):
             logger.warning(f"{var_name} not set ({description}). {fallback_msg}")
 
+    # Validate JWT secret looks real (not a placeholder)
+    jwt_secret = os.getenv("SUPABASE_JWT_SECRET", "")
+    if jwt_secret and len(jwt_secret) < 32:
+        logger.error(
+            "SUPABASE_JWT_SECRET is too short -- likely a placeholder. "
+            "Real Supabase JWT secrets are 40+ characters. "
+            "Auth will fall back to slow API verification until this is fixed. "
+            "Get the real secret: Supabase dashboard -> Settings -> API -> JWT Secret",
+            secret_length=len(jwt_secret),
+        )
+
     logger.info("Environment validation passed")

backend/services/auth.py

@@ -30,6 +30,19 @@ def __init__(self):
 
         if not self.jwt_secret:
             logger.warning("SUPABASE_JWT_SECRET not set -- falling back to API-based verification")
+        elif len(self.jwt_secret) < 32 or self.jwt_secret in (
+            "dev-secret-key", "secret", "your-jwt-secret", "change-me",
+            "test-secret-key", "super-secret-jwt-token-with-at-least-32-characters",
+        ):
+            logger.error(
+                "SUPABASE_JWT_SECRET looks like a placeholder. "
+                "Local JWT verification will fail for real Supabase tokens. "
+                "Get the real secret from Supabase dashboard -> Settings -> API -> JWT Secret. "
+                "Falling back to API-based verification until fixed.",
+                secret_length=len(self.jwt_secret),
+            )
+            # Null out the secret so verify_jwt takes the API path directly
+            self.jwt_secret = None
 
         self.client: Client = create_client(self.supabase_url, self.supabase_key)
 
@@ -38,16 +51,33 @@ def verify_jwt(self, token: str) -> Dict[str, Any]:
         Verify Supabase JWT token locally using the signing secret.
 
         No network call required -- instant verification using HS256.
-        Falls back to Supabase API call if JWT_SECRET is not configured.
+        Falls back to Supabase API call if JWT_SECRET is not configured,
+        or if local decode fails (wrong secret, config mismatch).
         """
         if token.lower().startswith("bearer "):
             token = token[7:]
 
-        # local decode when secret is available (fast path, no network)
+        # Local decode when secret is available (fast path, no network)
         if self.jwt_secret:
-            return self._verify_local(token)
+            try:
+                return self._verify_local(token)
+            except TokenExpiredError:
+                # Expired is expired -- no point retrying via API
+                raise
+            except TokenMissingClaimError:
+                # Token decoded but missing required claims -- won't help to retry
+                raise
+            except InvalidTokenError:
+                # Could be wrong secret, config mismatch, or genuinely bad token.
+                # Try API verification before giving up -- a wrong JWT_SECRET
+                # should degrade to slow auth, not broken auth.
+                logger.warning(
+                    "Local JWT decode failed, falling back to API verification. "
+                    "Check SUPABASE_JWT_SECRET if this persists."
+                )
+                return self._verify_via_api(token)
 
-        # fallback: API call to Supabase (slow path, requires network)
+        # No secret configured -- API call to Supabase (slow path)
         return self._verify_via_api(token)
 
     def _verify_local(self, token: str) -> Dict[str, Any]:

backend/tests/test_jwt_local_decode.py

@@ -5,7 +5,7 @@
 from unittest.mock import patch, MagicMock
 
 
-JWT_SECRET = "test-jwt-secret-for-unit-tests"
+JWT_SECRET = "test-jwt-secret-for-unit-tests-must-be-32-chars-or-longer"
 
 
 def _make_token(payload: dict, secret: str = JWT_SECRET) -> str:
@@ -57,12 +57,22 @@ def test_expired_token_raises_error(self, auth_service):
         with pytest.raises(TokenExpiredError):
             auth_service.verify_jwt(token)
 
-    def test_wrong_secret_raises_error(self, auth_service):
+    def test_wrong_secret_falls_back_not_raises(self, auth_service):
+        """Wrong secret should trigger API fallback, not raise."""
         token = _make_token({"sub": "user-000"}, secret="wrong-secret")
 
-        from services.exceptions import InvalidTokenError
-        with pytest.raises(InvalidTokenError):
-            auth_service.verify_jwt(token)
+        # Set up API fallback mock
+        user = MagicMock()
+        user.id = "user-000"
+        user.email = "fallback@test.com"
+        user.user_metadata = {}
+        response = MagicMock()
+        response.user = user
+        auth_service.client.auth.get_user.return_value = response
+
+        result = auth_service.verify_jwt(token)
+        assert result["user_id"] == "user-000"
+        auth_service.client.auth.get_user.assert_called_once()
 
     def test_missing_sub_claim_raises_error(self, auth_service):
         token = _make_token({"email": "no-sub@test.com"})
@@ -71,13 +81,23 @@ def test_missing_sub_claim_raises_error(self, auth_service):
         with pytest.raises(TokenMissingClaimError):
             auth_service.verify_jwt(token)
 
-    def test_wrong_audience_raises_error(self, auth_service):
+    def test_wrong_audience_falls_back_not_raises(self, auth_service):
+        """Wrong audience should trigger API fallback, not raise."""
         payload = {"sub": "user-aud", "aud": "wrong-audience", "exp": int(time.time()) + 3600}
         token = pyjwt.encode(payload, JWT_SECRET, algorithm="HS256")
 
-        from services.exceptions import InvalidTokenError
-        with pytest.raises(InvalidTokenError):
-            auth_service.verify_jwt(token)
+        # Set up API fallback mock
+        user = MagicMock()
+        user.id = "user-aud"
+        user.email = "aud@test.com"
+        user.user_metadata = {}
+        response = MagicMock()
+        response.user = user
+        auth_service.client.auth.get_user.return_value = response
+
+        result = auth_service.verify_jwt(token)
+        assert result["user_id"] == "user-aud"
+        auth_service.client.auth.get_user.assert_called_once()
 
     def test_no_network_call_made(self, auth_service):
         """The whole point of OPE-75: verify_jwt should NOT hit the network."""
@@ -134,3 +154,106 @@ def test_falls_back_to_api_when_no_secret(self):
 
                 assert result["user_id"] == "api-user-123"
                 client.auth.get_user.assert_called_once_with("some-token")
+
+    def test_wrong_secret_falls_back_to_api(self):
+        """
+        POSTMORTEM TEST -- Feb 2026 production auth outage.
+
+        Scenario: SUPABASE_JWT_SECRET is set but WRONG (doesn't match
+        what Supabase uses to sign tokens). Local decode fails.
+        System MUST fall back to API verification instead of returning 401.
+
+        Before this fix, wrong secret = broken auth for all users.
+        After this fix, wrong secret = slow auth (API call) but working.
+        """
+        with patch("services.auth.create_client") as mock_client:
+            client = MagicMock()
+            user = MagicMock()
+            user.id = "fallback-user-456"
+            user.email = "fallback@test.com"
+            user.user_metadata = {"tier": "pro"}
+            response = MagicMock()
+            response.user = user
+            client.auth.get_user.return_value = response
+            mock_client.return_value = client
+
+            with patch.dict("os.environ", {
+                "SUPABASE_URL": "https://test.supabase.co",
+                "SUPABASE_ANON_KEY": "test-key",
+                # Real-length secret but WRONG -- simulates production mismatch
+                "SUPABASE_JWT_SECRET": "a" * 64,
+            }):
+                from services.auth import SupabaseAuthService
+                service = SupabaseAuthService()
+
+                # Token signed with a DIFFERENT secret (like Supabase would)
+                token = _make_token(
+                    {"sub": "fallback-user-456", "email": "fallback@test.com"},
+                    secret="the-real-supabase-secret-that-we-dont-have",
+                )
+
+                result = service.verify_jwt(token)
+
+                # Should succeed via API fallback, not 401
+                assert result["user_id"] == "fallback-user-456"
+                assert result["email"] == "fallback@test.com"
+                # Verify it actually used the API path
+                client.auth.get_user.assert_called_once()
+
+    def test_expired_token_does_not_fallback(self):
+        """Expired tokens should NOT try API fallback -- expired is expired."""
+        with patch("services.auth.create_client") as mock_client:
+            mock_client.return_value = MagicMock()
+
+            with patch.dict("os.environ", {
+                "SUPABASE_URL": "https://test.supabase.co",
+                "SUPABASE_ANON_KEY": "test-key",
+                "SUPABASE_JWT_SECRET": JWT_SECRET,
+            }):
+                from services.auth import SupabaseAuthService
+                service = SupabaseAuthService()
+
+                expired_token = pyjwt.encode(
+                    {"sub": "expired-user", "aud": "authenticated",
+                     "exp": int(time.time()) - 120, "iat": int(time.time()) - 3720},
+                    JWT_SECRET, algorithm="HS256",
+                )
+
+                from services.exceptions import TokenExpiredError
+                with pytest.raises(TokenExpiredError):
+                    service.verify_jwt(expired_token)
+
+                # Should NOT have tried API fallback
+                service.client.auth.get_user.assert_not_called()
+
+    def test_placeholder_secret_nulled_at_startup(self):
+        """
+        Placeholder secrets like 'dev-secret-key' should be detected
+        at init time and nulled out, forcing API verification path.
+        """
+        with patch("services.auth.create_client") as mock_client:
+            client = MagicMock()
+            user = MagicMock()
+            user.id = "placeholder-user"
+            user.email = "ph@test.com"
+            user.user_metadata = {}
+            response = MagicMock()
+            response.user = user
+            client.auth.get_user.return_value = response
+            mock_client.return_value = client
+
+            with patch.dict("os.environ", {
+                "SUPABASE_URL": "https://test.supabase.co",
+                "SUPABASE_ANON_KEY": "test-key",
+                "SUPABASE_JWT_SECRET": "dev-secret-key",
+            }):
+                from services.auth import SupabaseAuthService
+                service = SupabaseAuthService()
+
+                # Secret should have been nulled out
+                assert service.jwt_secret is None
+
+                # Should work via API path directly
+                result = service.verify_jwt("any-token")
+                assert result["user_id"] == "placeholder-user"
+                client.auth.get_user.assert_called_once()