From 6980bbea1b90c850477b1028740f617ad8dcba51 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Mon, 15 Dec 2025 16:33:20 +0530 Subject: [PATCH 01/10] Add Kaapi LLM parameters and completion config; implement transformation to native provider format --- backend/app/models/llm/__init__.py | 3 + backend/app/models/llm/request.py | 75 +++++++++++++++++-- backend/app/services/llm/jobs.py | 18 ++++- backend/app/services/llm/mappers.py | 75 +++++++++++++++++++ .../app/services/llm/providers/registry.py | 24 +++--- 5 files changed, 177 insertions(+), 18 deletions(-) create mode 100644 backend/app/services/llm/mappers.py diff --git a/backend/app/models/llm/__init__.py b/backend/app/models/llm/__init__.py index f06954de5..8738e2126 100644 --- a/backend/app/models/llm/__init__.py +++ b/backend/app/models/llm/__init__.py @@ -3,5 +3,8 @@ CompletionConfig, QueryParams, ConfigBlob, + KaapiLLMParams, + KaapiCompletionConfig, + NativeCompletionConfig, ) from app.models.llm.response import LLMCallResponse, LLMResponse, LLMOutput, Usage diff --git a/backend/app/models/llm/request.py b/backend/app/models/llm/request.py index a63de1ebc..9f82ea565 100644 --- a/backend/app/models/llm/request.py +++ b/backend/app/models/llm/request.py @@ -1,8 +1,45 @@ -from typing import Any, Literal +from typing import Annotated, Any, Literal, Union from uuid import UUID from sqlmodel import Field, SQLModel -from pydantic import model_validator, HttpUrl +from pydantic import Discriminator, model_validator, HttpUrl + + +class KaapiLLMParams(SQLModel): + """ + Kaapi-abstracted parameters for LLM providers. + These parameters are mapped internally to provider-specific API parameters. + Provides a unified contract across all LLM providers (OpenAI, Claude, Gemini, etc.). + Provider-specific mappings are handled at the mapper level. + """ + + model: str = Field( + default=None, + description="Model identifier to use for completion (e.g., 'gpt-4o', 'gpt-5')", + ) + instructions: str | None = Field( + default=None, + description="System instructions to guide the model's behavior", + ) + knowledge_base_ids: list[str] | None = Field( + default=None, + description="List of vector store IDs to use for knowledge retrieval", + ) + reasoning: str | None = Field( + default=None, + description="Reasoning configuration or instructions", + ) + temperature: float | None = Field( + default=None, + ge=0.0, + le=2.0, + description="Sampling temperature between 0 and 2", + ) + max_num_results: int | None = Field( + default=None, + ge=1, + description="Maximum number of results to return", + ) class ConversationConfig(SQLModel): @@ -46,11 +83,16 @@ class QueryParams(SQLModel): ) -class CompletionConfig(SQLModel): - """Completion configuration with provider and parameters.""" +class NativeCompletionConfig(SQLModel): + """ + Native provider configuration (pass-through). + All parameters are forwarded as-is to the provider's API without transformation. + Supports any LLM provider's native API format. + """ - provider: Literal["openai"] = Field( - default="openai", description="LLM provider to use" + provider: Literal["openai-native"] = Field( + default="openai-native", + description="Native provider type (e.g., openai-native)", ) params: dict[str, Any] = Field( ..., @@ -58,6 +100,27 @@ class CompletionConfig(SQLModel): ) +class KaapiCompletionConfig(SQLModel): + """ + Kaapi abstraction for LLM completion providers. + Uses standardized Kaapi parameters that are mapped to provider-specific APIs internally. + Supports multiple providers: OpenAI, Claude, Gemini, etc. + """ + + provider: Literal["openai"] = Field(..., description="LLM provider (openai)") + params: KaapiLLMParams = Field( + ..., + description="Kaapi-standardized parameters mapped to provider-specific API", + ) + + +# Discriminated union for completion configs based on provider field +CompletionConfig = Annotated[ + Union[NativeCompletionConfig, KaapiCompletionConfig], + Field(discriminator="provider"), +] + + class ConfigBlob(SQLModel): """Raw JSON blob of config.""" diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index 831f3cca4..a8ee8cdf2 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -10,11 +10,12 @@ from app.crud.credentials import get_provider_credential from app.crud.jobs import JobCrud from app.models import JobStatus, JobType, JobUpdate, LLMCallRequest -from app.models.llm.request import ConfigBlob, LLMCallConfig +from app.models.llm.request import ConfigBlob, LLMCallConfig, KaapiCompletionConfig from app.utils import APIResponse, send_callback from app.celery.utils import start_high_priority_job from app.core.langfuse.langfuse import observe_llm_execution from app.services.llm.providers.registry import get_llm_provider +from app.services.llm.mappers import transform_kaapi_config_to_native logger = logging.getLogger(__name__) @@ -170,10 +171,23 @@ def execute_job( else: config_blob = config.blob + try: + # Transform Kaapi config to native config if needed (before getting provider) + completion_config = config_blob.completion + if isinstance(completion_config, KaapiCompletionConfig): + completion_config = transform_kaapi_config_to_native(completion_config) + except Exception as e: + callback_response = APIResponse.failure_response( + error=f"Error processing configuration: {str(e)}", + metadata=request.request_metadata, + ) + return handle_job_error(job_id, request.callback_url, callback_response) + + try: provider_instance = get_llm_provider( session=session, - provider_type=config_blob.completion.provider, + provider_type=completion_config.provider, # Now always native provider type project_id=project_id, organization_id=organization_id, ) diff --git a/backend/app/services/llm/mappers.py b/backend/app/services/llm/mappers.py new file mode 100644 index 000000000..e53302e5a --- /dev/null +++ b/backend/app/services/llm/mappers.py @@ -0,0 +1,75 @@ +"""Parameter mappers for converting Kaapi-abstracted parameters to provider-specific formats.""" + +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig + + +def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: + """Map Kaapi-abstracted parameters to OpenAI API parameters. + + This mapper transforms standardized Kaapi parameters into OpenAI-specific + parameter format, enabling provider-agnostic interface design. + + Args: + kaapi_params: KaapiLLMParams instance with standardized parameters + + Returns: + Dictionary of OpenAI API parameters ready to be passed to the API + + Supported Mapping: + - model → model + - instructions → instructions + - knowledge_base_ids → tools[file_search].vector_store_ids + - max_num_results → tools[file_search].max_num_results (fallback default) + - reasoning → reasoning.effort + - temperature → temperature + """ + openai_params = {} + + if kaapi_params.temperature and kaapi_params.reasoning: + raise ValueError( + "Cannot set both 'temperature' and 'reasoning' parameters together for OpenAI." + ) + + + if kaapi_params.model: + openai_params["model"] = kaapi_params.model + + if kaapi_params.instructions: + openai_params["instructions"] = kaapi_params.instructions + + if kaapi_params.knowledge_base_ids: + openai_params["tools"] = [{ + "type": "file_search", + "vector_store_ids": kaapi_params.knowledge_base_ids, + "max_num_results": kaapi_params.max_num_results or 20, + }] + + if kaapi_params.reasoning: + openai_params["reasoning"] = { + "effort": kaapi_params.reasoning + } + + if kaapi_params.temperature is not None: + openai_params["temperature"] = kaapi_params.temperature + + return openai_params + + +def transform_kaapi_config_to_native( + kaapi_config: KaapiCompletionConfig, +) -> NativeCompletionConfig: + """Transform Kaapi completion config to native provider config with mapped parameters. + + Currently supports OpenAI. Future: Claude, Gemini mappers. + + Args: + kaapi_config: KaapiCompletionConfig with abstracted parameters + + Returns: + NativeCompletionConfig with provider-native parameters ready for API + """ + if kaapi_config.provider == "openai": + mapped_params = map_kaapi_to_openai_params(kaapi_config.params) + return NativeCompletionConfig(provider="openai-native", params=mapped_params) + + raise ValueError(f"Unsupported provider: {kaapi_config.provider}") diff --git a/backend/app/services/llm/providers/registry.py b/backend/app/services/llm/providers/registry.py index 64b32c436..a5cfb4bb8 100644 --- a/backend/app/services/llm/providers/registry.py +++ b/backend/app/services/llm/providers/registry.py @@ -12,15 +12,16 @@ class LLMProvider: - OPENAI = "openai" - # Future constants: - # ANTHROPIC = "anthropic" - # GOOGLE = "google" + OPENAI_NATIVE = "openai-native" + # Future constants for native providers: + # CLAUDE_NATIVE = "claude-native" + # GEMINI_NATIVE = "gemini-native" _registry: dict[str, type[BaseProvider]] = { - OPENAI: OpenAIProvider, - # ANTHROPIC: AnthropicProvider, - # GOOGLE: GoogleProvider, + OPENAI_NATIVE: OpenAIProvider, + # Future native providers: + # CLAUDE_NATIVE: ClaudeProvider, + # GEMINI_NATIVE: GeminiProvider, } @classmethod @@ -45,19 +46,22 @@ def get_llm_provider( ) -> BaseProvider: provider_class = LLMProvider.get(provider_type) + # e.g., "openai-native" → "openai", "claude-native" → "claude" + credential_provider = provider_type.replace("-native", "") + credentials = get_provider_credential( session=session, - provider=provider_type, + provider=credential_provider, project_id=project_id, org_id=organization_id, ) if not credentials: raise ValueError( - f"Credentials for provider '{provider_type}' not configured for this project." + f"Credentials for provider '{credential_provider}' not configured for this project." ) - if provider_type == LLMProvider.OPENAI: + if provider_type == LLMProvider.OPENAI_NATIVE: if "api_key" not in credentials: raise ValueError("OpenAI credentials not configured for this project.") client = OpenAI(api_key=credentials["api_key"]) From 2847d50a07dab3be4cd53ba911fa0a32b502c9a1 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Tue, 16 Dec 2025 10:56:04 +0530 Subject: [PATCH 02/10] Refine LLM API documentation and improve code formatting for clarity; enhance configuration handling for OpenAI provider --- backend/app/api/docs/llm/llm_call.md | 10 ++++++---- backend/app/services/llm/jobs.py | 5 +++-- backend/app/services/llm/mappers.py | 17 ++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/backend/app/api/docs/llm/llm_call.md b/backend/app/api/docs/llm/llm_call.md index a284ab6a1..b379e35cc 100644 --- a/backend/app/api/docs/llm/llm_call.md +++ b/backend/app/api/docs/llm/llm_call.md @@ -21,11 +21,13 @@ for processing, and results are delivered via the callback URL when complete. - **Note**: When using stored configuration, do not include the `blob` field in the request body - **Mode 2: Ad-hoc Configuration** - - `blob` (object): Complete configuration object (see Create Config endpoint documentation for examples) - - `completion` (required): - - `provider` (required, string): Currently only "openai" - - `params` (required, object): Provider-specific parameters (flexible JSON) + - `blob` (object): Complete configuration object + - `completion` (required, object): Completion configuration + - `provider` (required, string): Provider type - either `"openai"` (Kaapi abstraction) or `"openai-native"` (pass-through) + - `params` (required, object): Parameters structure depends on provider type (see schema for detailed structure) - **Note**: When using ad-hoc configuration, do not include `id` and `version` fields + - **Recommendation**: Use stored configs (Mode 1) for production; use ad-hoc configs only for testing/validation + - **Schema**: Check the API schema or examples below for the complete parameter structure for each provider type **`callback_url`** (optional, HTTPS URL): - Webhook endpoint to receive the response diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index a8ee8cdf2..3f0225feb 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -175,7 +175,9 @@ def execute_job( # Transform Kaapi config to native config if needed (before getting provider) completion_config = config_blob.completion if isinstance(completion_config, KaapiCompletionConfig): - completion_config = transform_kaapi_config_to_native(completion_config) + completion_config = transform_kaapi_config_to_native( + completion_config + ) except Exception as e: callback_response = APIResponse.failure_response( error=f"Error processing configuration: {str(e)}", @@ -183,7 +185,6 @@ def execute_job( ) return handle_job_error(job_id, request.callback_url, callback_response) - try: provider_instance = get_llm_provider( session=session, diff --git a/backend/app/services/llm/mappers.py b/backend/app/services/llm/mappers.py index e53302e5a..f0bad7cb7 100644 --- a/backend/app/services/llm/mappers.py +++ b/backend/app/services/llm/mappers.py @@ -30,7 +30,6 @@ def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: "Cannot set both 'temperature' and 'reasoning' parameters together for OpenAI." ) - if kaapi_params.model: openai_params["model"] = kaapi_params.model @@ -38,16 +37,16 @@ def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: openai_params["instructions"] = kaapi_params.instructions if kaapi_params.knowledge_base_ids: - openai_params["tools"] = [{ - "type": "file_search", - "vector_store_ids": kaapi_params.knowledge_base_ids, - "max_num_results": kaapi_params.max_num_results or 20, - }] + openai_params["tools"] = [ + { + "type": "file_search", + "vector_store_ids": kaapi_params.knowledge_base_ids, + "max_num_results": kaapi_params.max_num_results or 20, + } + ] if kaapi_params.reasoning: - openai_params["reasoning"] = { - "effort": kaapi_params.reasoning - } + openai_params["reasoning"] = {"effort": kaapi_params.reasoning} if kaapi_params.temperature is not None: openai_params["temperature"] = kaapi_params.temperature From 63a3bce6b22bdd104d9e6285fd354cae2f31ec65 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:56:30 +0530 Subject: [PATCH 03/10] add/fix tests --- .../tests/api/routes/configs/test_config.py | 2 +- .../tests/api/routes/configs/test_version.py | 9 +- backend/app/tests/api/routes/test_llm.py | 121 +++++++- backend/app/tests/crud/config/test_config.py | 5 +- backend/app/tests/crud/config/test_version.py | 7 +- .../services/llm/providers/test_openai.py | 12 +- .../services/llm/providers/test_registry.py | 12 +- backend/app/tests/services/llm/test_jobs.py | 230 +++++++++++++-- .../app/tests/services/llm/test_mappers.py | 262 ++++++++++++++++++ backend/app/tests/utils/test_data.py | 46 ++- 10 files changed, 649 insertions(+), 57 deletions(-) create mode 100644 backend/app/tests/services/llm/test_mappers.py diff --git a/backend/app/tests/api/routes/configs/test_config.py b/backend/app/tests/api/routes/configs/test_config.py index 8f094f538..a30d162a0 100644 --- a/backend/app/tests/api/routes/configs/test_config.py +++ b/backend/app/tests/api/routes/configs/test_config.py @@ -19,7 +19,7 @@ def test_create_config_success( "description": "A test LLM configuration", "config_blob": { "completion": { - "provider": "openai", + "provider": "openai-native", "params": { "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/api/routes/configs/test_version.py b/backend/app/tests/api/routes/configs/test_version.py index acb9f2526..27fb14ba2 100644 --- a/backend/app/tests/api/routes/configs/test_version.py +++ b/backend/app/tests/api/routes/configs/test_version.py @@ -10,7 +10,8 @@ create_test_project, create_test_version, ) -from app.models import ConfigBlob, CompletionConfig +from app.models import ConfigBlob +from app.models.llm.request import NativeCompletionConfig def test_create_version_success( @@ -28,7 +29,7 @@ def test_create_version_success( version_data = { "config_blob": { "completion": { - "provider": "openai", + "provider": "openai-native", "params": { "model": "gpt-4-turbo", "temperature": 0.9, @@ -303,8 +304,8 @@ def test_get_version_by_number( config_id=config.id, project_id=user_api_key.project_id, config_blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4-turbo", "temperature": 0.5}, ) ), diff --git a/backend/app/tests/api/routes/test_llm.py b/backend/app/tests/api/routes/test_llm.py index 430ca77c0..78adffebf 100644 --- a/backend/app/tests/api/routes/test_llm.py +++ b/backend/app/tests/api/routes/test_llm.py @@ -6,6 +6,9 @@ LLMCallConfig, CompletionConfig, ConfigBlob, + KaapiLLMParams, + KaapiCompletionConfig, + NativeCompletionConfig, ) @@ -18,8 +21,8 @@ def test_llm_call_success(client: TestClient, user_api_key_header: dict[str, str query=QueryParams(input="What is the capital of France?"), config=LLMCallConfig( blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.7, @@ -43,3 +46,117 @@ def test_llm_call_success(client: TestClient, user_api_key_header: dict[str, str assert "response is being generated" in response_data["data"]["message"] mock_start_job.assert_called_once() + + +def test_llm_call_with_kaapi_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with Kaapi abstracted config.""" + with patch("app.services.llm.jobs.start_high_priority_job") as mock_start_job: + mock_start_job.return_value = "test-task-id" + + payload = LLMCallRequest( + query=QueryParams(input="Explain quantum computing"), + config=LLMCallConfig( + blob=ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4o", + instructions="You are a physics expert", + temperature=0.5, + ), + ) + ) + ), + ) + + response = client.post( + "api/v1/llm/call", + json=payload.model_dump(mode="json"), + headers=user_api_key_header, + ) + + assert response.status_code == 200 + response_data = response.json() + assert response_data["success"] is True + mock_start_job.assert_called_once() + + +def test_llm_call_with_native_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with native OpenAI config (pass-through mode).""" + with patch("app.services.llm.jobs.start_high_priority_job") as mock_start_job: + mock_start_job.return_value = "test-task-id" + + payload = LLMCallRequest( + query=QueryParams(input="Native API call test"), + config=LLMCallConfig( + blob=ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={ + "model": "gpt-4", + "temperature": 0.9, + "max_tokens": 500, + "top_p": 1.0, + }, + ) + ) + ), + ) + + response = client.post( + "api/v1/llm/call", + json=payload.model_dump(mode="json"), + headers=user_api_key_header, + ) + + assert response.status_code == 200 + response_data = response.json() + assert response_data["success"] is True + mock_start_job.assert_called_once() + + +def test_llm_call_missing_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with missing config fails validation.""" + payload = { + "query": {"input": "Test query"}, + # Missing config field + } + + response = client.post( + "api/v1/llm/call", + json=payload, + headers=user_api_key_header, + ) + + assert response.status_code == 422 # Validation error + + +def test_llm_call_invalid_provider( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with invalid provider type.""" + payload = { + "query": {"input": "Test query"}, + "config": { + "blob": { + "completion": { + "provider": "invalid-provider", + "params": {"model": "gpt-4"}, + } + } + }, + } + + response = client.post( + "api/v1/llm/call", + json=payload, + headers=user_api_key_header, + ) + + assert response.status_code == 422 # Validation error diff --git a/backend/app/tests/crud/config/test_config.py b/backend/app/tests/crud/config/test_config.py index e7837b98a..e4f90e25e 100644 --- a/backend/app/tests/crud/config/test_config.py +++ b/backend/app/tests/crud/config/test_config.py @@ -10,6 +10,7 @@ ConfigCreate, ConfigUpdate, ) +from app.models.llm.request import NativeCompletionConfig from app.crud.config import ConfigCrud from app.tests.utils.test_data import create_test_project, create_test_config from app.tests.utils.utils import random_lower_string @@ -18,8 +19,8 @@ @pytest.fixture def example_config_blob(): return ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/crud/config/test_version.py b/backend/app/tests/crud/config/test_version.py index c3c4bd582..a24940b7d 100644 --- a/backend/app/tests/crud/config/test_version.py +++ b/backend/app/tests/crud/config/test_version.py @@ -3,7 +3,8 @@ from sqlmodel import Session from fastapi import HTTPException -from app.models import ConfigVersionCreate, ConfigBlob, CompletionConfig +from app.models import ConfigVersionCreate, ConfigBlob +from app.models.llm.request import NativeCompletionConfig from app.crud.config import ConfigVersionCrud from app.tests.utils.test_data import ( create_test_project, @@ -15,8 +16,8 @@ @pytest.fixture def example_config_blob(): return ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/services/llm/providers/test_openai.py b/backend/app/tests/services/llm/providers/test_openai.py index c216ca540..745dd00b8 100644 --- a/backend/app/tests/services/llm/providers/test_openai.py +++ b/backend/app/tests/services/llm/providers/test_openai.py @@ -7,7 +7,7 @@ import openai from app.models.llm import ( - CompletionConfig, + NativeCompletionConfig, QueryParams, ) from app.models.llm.request import ConversationConfig @@ -31,8 +31,8 @@ def provider(self, mock_client): @pytest.fixture def completion_config(self): """Create a basic completion config.""" - return CompletionConfig( - provider="openai", + return NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) @@ -59,7 +59,7 @@ def test_execute_success_without_conversation( assert result is not None assert result.response.output.text == mock_response.output_text assert result.response.model == mock_response.model - assert result.response.provider == "openai" + assert result.response.provider == "openai-native" assert result.response.conversation_id is None assert result.usage.input_tokens == mock_response.usage.input_tokens assert result.usage.output_tokens == mock_response.usage.output_tokens @@ -233,8 +233,8 @@ def test_execute_with_conversation_parameter_removed_when_no_config( ): """Test that conversation param is removed if it exists in config but no conversation config.""" # Create a config with conversation in params (should be removed) - completion_config = CompletionConfig( - provider="openai", + completion_config = NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "conversation": {"id": "old_conv"}}, ) diff --git a/backend/app/tests/services/llm/providers/test_registry.py b/backend/app/tests/services/llm/providers/test_registry.py index f9c595c0d..c05222747 100644 --- a/backend/app/tests/services/llm/providers/test_registry.py +++ b/backend/app/tests/services/llm/providers/test_registry.py @@ -21,8 +21,8 @@ class TestProviderRegistry: def test_registry_contains_openai(self): """Test that registry contains OpenAI provider.""" - assert "openai" in LLMProvider._registry - assert LLMProvider._registry["openai"] == OpenAIProvider + assert "openai-native" in LLMProvider._registry + assert LLMProvider._registry["openai-native"] == OpenAIProvider def test_registry_values_are_provider_classes(self): """Test that all registry values are BaseProvider subclasses.""" @@ -46,7 +46,7 @@ def test_get_llm_provider_with_openai(self, db: Session): provider = get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) @@ -64,7 +64,7 @@ def test_get_llm_provider_with_openai(self, db: Session): with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) @@ -87,7 +87,7 @@ def test_get_llm_provider_with_invalid_provider(self, db: Session): error_message = str(exc_info.value) assert "invalid_provider" in error_message assert "is not supported" in error_message - assert "openai" in error_message + assert "openai-native" in error_message def test_get_llm_provider_with_missing_credentials(self, db: Session): """Test handling of errors when credentials are not found.""" @@ -101,7 +101,7 @@ def test_get_llm_provider_with_missing_credentials(self, db: Session): with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index 2f08b40c0..8c87efea1 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -13,12 +13,14 @@ from app.models import ConfigVersion, JobStatus, JobType from app.models.llm import ( LLMCallRequest, - CompletionConfig, + NativeCompletionConfig, QueryParams, LLMCallResponse, LLMResponse, LLMOutput, Usage, + KaapiLLMParams, + KaapiCompletionConfig, ) from app.models.llm.request import ConfigBlob, LLMCallConfig from app.services.llm.jobs import ( @@ -40,8 +42,8 @@ def llm_call_request(self): query=QueryParams(input="Test query"), config=LLMCallConfig( blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -225,7 +227,10 @@ def request_data(self): "query": {"input": "Test query"}, "config": { "blob": { - "completion": {"provider": "openai", "params": {"model": "gpt-4"}} + "completion": { + "provider": "openai-native", + "params": {"model": "gpt-4"}, + } } }, "include_provider_raw_response": False, @@ -396,8 +401,8 @@ def test_stored_config_success(self, db, job_for_execution, mock_llm_response): # Create a real config in the database config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.7}, ) ) @@ -445,8 +450,8 @@ def test_stored_config_with_callback( project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-3.5-turbo", "temperature": 0.5}, ) ) @@ -493,8 +498,8 @@ def test_stored_config_version_not_found(self, db, job_for_execution): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -523,6 +528,100 @@ def test_stored_config_version_not_found(self, db, job_for_execution): db.refresh(job_for_execution) assert job_for_execution.status == JobStatus.FAILED + def test_kaapi_config_success(self, db, job_for_execution, mock_llm_response): + """Test successful execution with Kaapi abstracted config.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + instructions="You are a helpful assistant", + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query with Kaapi config"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + mock_get_provider.assert_called_once() + mock_provider.execute.assert_called_once() + assert result["success"] + db.refresh(job_for_execution) + assert job_for_execution.status == JobStatus.SUCCESS + + def test_kaapi_config_with_callback(self, db, job_for_execution, mock_llm_response): + """Test successful execution with Kaapi config and callback.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-3.5-turbo", + temperature=0.5, + max_tokens=500, + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query with Kaapi config and callback"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": "https://example.com/callback", + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + patch("app.services.llm.jobs.send_callback") as mock_send_callback, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + mock_send_callback.assert_called_once() + callback_data = mock_send_callback.call_args[1]["data"] + assert callback_data["success"] + assert result["success"] + db.refresh(job_for_execution) + assert job_for_execution.status == JobStatus.SUCCESS + class TestResolveConfigBlob: """Test suite for resolve_config_blob function.""" @@ -532,8 +631,8 @@ def test_resolve_config_blob_success(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.8}, ) ) @@ -549,7 +648,7 @@ def test_resolve_config_blob_success(self, db: Session): assert error is None assert resolved_blob is not None - assert resolved_blob.completion.provider == "openai" + assert resolved_blob.completion.provider == "openai-native" assert resolved_blob.completion.params["model"] == "gpt-4" assert resolved_blob.completion.params["temperature"] == 0.8 @@ -558,8 +657,8 @@ def test_resolve_config_blob_version_not_found(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -583,8 +682,8 @@ def test_resolve_config_blob_invalid_blob_data(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -620,8 +719,8 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): # Create a config with version 1 config_blob_v1 = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-3.5-turbo", "temperature": 0.5}, ) ) @@ -635,8 +734,8 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): session=db, project_id=project.id, config_id=config.id ) config_blob_v2 = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.9}, ) ) @@ -668,3 +767,92 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): assert resolved_blob_v2 is not None assert resolved_blob_v2.completion.params["model"] == "gpt-4" assert resolved_blob_v2.completion.params["temperature"] == 0.9 + + def test_resolve_kaapi_config_blob_success(self, db: Session): + """Test successful resolution of stored Kaapi config blob.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.8, + instructions="You are a helpful assistant", + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + config_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=config.id + ) + llm_call_config = LLMCallConfig(id=str(config.id), version=1) + + resolved_blob, error = resolve_config_blob(config_crud, llm_call_config) + + assert error is None + assert resolved_blob is not None + assert isinstance(resolved_blob.completion, KaapiCompletionConfig) + assert resolved_blob.completion.provider == "openai" + assert resolved_blob.completion.params.model == "gpt-4" + assert resolved_blob.completion.params.temperature == 0.8 + assert ( + resolved_blob.completion.params.instructions + == "You are a helpful assistant" + ) + + def test_resolve_both_native_and_kaapi_configs(self, db: Session): + """Test that both native and Kaapi configs can be resolved correctly.""" + project = get_project(db) + + # Create native config + native_blob = ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={"model": "gpt-3.5-turbo", "temperature": 0.5}, + ) + ) + native_config = create_test_config( + db, project_id=project.id, config_blob=native_blob, use_kaapi_schema=False + ) + + # Create Kaapi config + kaapi_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + ) + kaapi_config = create_test_config( + db, project_id=project.id, config_blob=kaapi_blob, use_kaapi_schema=True + ) + db.commit() + + # Test native config resolution + native_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=native_config.id + ) + native_call_config = LLMCallConfig(id=str(native_config.id), version=1) + resolved_native, error_native = resolve_config_blob( + native_crud, native_call_config + ) + + assert error_native is None + assert isinstance(resolved_native.completion, NativeCompletionConfig) + assert resolved_native.completion.provider == "openai-native" + + # Test Kaapi config resolution + kaapi_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=kaapi_config.id + ) + kaapi_call_config = LLMCallConfig(id=str(kaapi_config.id), version=1) + resolved_kaapi, error_kaapi = resolve_config_blob(kaapi_crud, kaapi_call_config) + + assert error_kaapi is None + assert isinstance(resolved_kaapi.completion, KaapiCompletionConfig) + assert resolved_kaapi.completion.provider == "openai" diff --git a/backend/app/tests/services/llm/test_mappers.py b/backend/app/tests/services/llm/test_mappers.py new file mode 100644 index 000000000..4f1b0670b --- /dev/null +++ b/backend/app/tests/services/llm/test_mappers.py @@ -0,0 +1,262 @@ +""" +Unit tests for LLM parameter mapping functions. + +Tests the transformation of Kaapi-abstracted parameters to provider-native formats. +""" +import pytest + +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig +from app.services.llm.mappers import ( + map_kaapi_to_openai_params, + transform_kaapi_config_to_native, +) + + +class TestMapKaapiToOpenAIParams: + """Test cases for map_kaapi_to_openai_params function.""" + + def test_basic_model_mapping(self): + """Test basic model parameter mapping.""" + kaapi_params = KaapiLLMParams(model="gpt-4o") + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result == {"model": "gpt-4o"} + + def test_instructions_mapping(self): + """Test instructions parameter mapping.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + instructions="You are a helpful assistant.", + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert result["instructions"] == "You are a helpful assistant." + + def test_temperature_mapping(self): + """Test temperature parameter mapping.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert result["temperature"] == 0.7 + + def test_temperature_zero_mapping(self): + """Test that temperature=0 is correctly mapped (edge case).""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + temperature=0.0, + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["temperature"] == 0.0 + + def test_reasoning_mapping(self): + """Test reasoning parameter mapping to OpenAI format.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + reasoning="high", + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert result["reasoning"] == {"effort": "high"} + + def test_knowledge_base_ids_mapping(self): + """Test knowledge_base_ids mapping to OpenAI tools format.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_abc123", "vs_def456"], + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "tools" in result + assert len(result["tools"]) == 1 + assert result["tools"][0]["type"] == "file_search" + assert result["tools"][0]["vector_store_ids"] == ["vs_abc123", "vs_def456"] + assert result["tools"][0]["max_num_results"] == 20 # default + + def test_knowledge_base_with_max_num_results(self): + """Test knowledge_base_ids with custom max_num_results.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_abc123"], + max_num_results=50, + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["tools"][0]["max_num_results"] == 50 + + def test_complete_parameter_mapping(self): + """Test mapping all compatible parameters together.""" + kaapi_params = KaapiLLMParams( + model="gpt-4o", + instructions="You are an expert assistant.", + temperature=0.8, + knowledge_base_ids=["vs_123"], + max_num_results=30, + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4o" + assert result["instructions"] == "You are an expert assistant." + assert result["temperature"] == 0.8 + assert result["tools"][0]["type"] == "file_search" + assert result["tools"][0]["vector_store_ids"] == ["vs_123"] + assert result["tools"][0]["max_num_results"] == 30 + + def test_temperature_and_reasoning_conflict(self): + """Test that providing both temperature and reasoning raises ValueError.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + temperature=0.7, + reasoning="high", + ) + + with pytest.raises(ValueError) as exc_info: + map_kaapi_to_openai_params(kaapi_params) + + assert "Cannot set both 'temperature' and 'reasoning'" in str(exc_info.value) + + def test_minimal_params(self): + """Test mapping with minimal parameters (only model).""" + kaapi_params = KaapiLLMParams(model="gpt-4") + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result == {"model": "gpt-4"} + + def test_only_knowledge_base_ids(self): + """Test mapping with only knowledge_base_ids and model.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_xyz"], + ) + + result = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "tools" in result + assert result["tools"][0]["vector_store_ids"] == ["vs_xyz"] + + +class TestTransformKaapiConfigToNative: + """Test cases for transform_kaapi_config_to_native function.""" + + def test_transform_openai_config(self): + """Test transformation of Kaapi OpenAI config to native format.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + + result = transform_kaapi_config_to_native(kaapi_config) + + assert isinstance(result, NativeCompletionConfig) + assert result.provider == "openai-native" + assert result.params["model"] == "gpt-4" + assert result.params["temperature"] == 0.7 + + def test_transform_with_all_params(self): + """Test transformation with all Kaapi parameters.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4o", + instructions="System prompt here", + temperature=0.5, + knowledge_base_ids=["vs_abc"], + max_num_results=25, + ), + ) + + result = transform_kaapi_config_to_native(kaapi_config) + + assert result.provider == "openai-native" + assert result.params["model"] == "gpt-4o" + assert result.params["instructions"] == "System prompt here" + assert result.params["temperature"] == 0.5 + assert result.params["tools"][0]["type"] == "file_search" + assert result.params["tools"][0]["max_num_results"] == 25 + + def test_transform_with_reasoning(self): + """Test transformation with reasoning parameter.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="o1-preview", + reasoning="medium", + ), + ) + + result = transform_kaapi_config_to_native(kaapi_config) + + assert result.provider == "openai-native" + assert result.params["model"] == "o1-preview" + assert result.params["reasoning"] == {"effort": "medium"} + + def test_transform_validates_temperature_reasoning_conflict(self): + """Test that transformation validates temperature + reasoning conflict.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + reasoning="high", + ), + ) + + with pytest.raises(ValueError) as exc_info: + transform_kaapi_config_to_native(kaapi_config) + + assert "Cannot set both 'temperature' and 'reasoning'" in str(exc_info.value) + + def test_unsupported_provider_raises_error(self): + """Test that unsupported providers raise ValueError.""" + # Note: This would require modifying KaapiCompletionConfig to accept other providers + # For now, this tests the error handling in the mapper + # We'll create a mock config that bypasses validation + from unittest.mock import MagicMock + + mock_config = MagicMock() + mock_config.provider = "unsupported-provider" + mock_config.params = KaapiLLMParams(model="some-model") + + with pytest.raises(ValueError) as exc_info: + transform_kaapi_config_to_native(mock_config) + + assert "Unsupported provider" in str(exc_info.value) + + def test_transform_preserves_param_structure(self): + """Test that transformation correctly structures nested parameters.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_1", "vs_2", "vs_3"], + max_num_results=15, + ), + ) + + result = transform_kaapi_config_to_native(kaapi_config) + + # Verify the nested structure is correct + assert isinstance(result.params["tools"], list) + assert isinstance(result.params["tools"][0], dict) + assert isinstance(result.params["tools"][0]["vector_store_ids"], list) + assert len(result.params["tools"][0]["vector_store_ids"]) == 3 diff --git a/backend/app/tests/utils/test_data.py b/backend/app/tests/utils/test_data.py index b33656b22..bc871dbcc 100644 --- a/backend/app/tests/utils/test_data.py +++ b/backend/app/tests/utils/test_data.py @@ -9,7 +9,6 @@ OrganizationCreate, ProjectCreate, ConfigBlob, - CompletionConfig, CredsCreate, FineTuningJobCreate, Fine_Tuning, @@ -22,6 +21,7 @@ ConfigVersionCreate, ConfigBase, ) +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig from app.crud import ( create_organization, create_project, @@ -242,11 +242,20 @@ def create_test_config( name: str | None = None, description: str | None = None, config_blob: ConfigBlob | None = None, + use_kaapi_schema: bool = False, ) -> Config: """ Creates and returns a test configuration with an initial version. Persists the config and version to the database. + + Args: + db: Database session + project_id: Project ID (creates new project if None) + name: Config name (generates random if None) + description: Config description + config_blob: Config blob (creates default if None) + use_kaapi_schema: If True, creates Kaapi-format config; if False, creates native format """ if project_id is None: project = create_test_project(db) @@ -256,16 +265,29 @@ def create_test_config( name = f"test-config-{random_lower_string()}" if config_blob is None: - config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", - params={ - "model": "gpt-4", - "temperature": 0.7, - "max_tokens": 1000, - }, + if use_kaapi_schema: + # Create Kaapi-format config + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + ) + else: + # Create native-format config + config_blob = ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={ + "model": "gpt-4", + "temperature": 0.7, + "max_tokens": 1000, + }, + ) ) - ) config_create = ConfigCreate( name=name, @@ -294,8 +316,8 @@ def create_test_version( """ if config_blob is None: config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8, From b3393d8e9baf13bd213c72ebf44ff27e9bae47a8 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Tue, 16 Dec 2025 12:09:00 +0530 Subject: [PATCH 04/10] Fix validation logic in map_kaapi_to_openai_params to prevent simultaneous setting of 'temperature' and 'reasoning' parameters --- backend/app/services/llm/mappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/services/llm/mappers.py b/backend/app/services/llm/mappers.py index f0bad7cb7..1bf58adbb 100644 --- a/backend/app/services/llm/mappers.py +++ b/backend/app/services/llm/mappers.py @@ -25,7 +25,7 @@ def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: """ openai_params = {} - if kaapi_params.temperature and kaapi_params.reasoning: + if kaapi_params.temperature is not None and kaapi_params.reasoning is not None: raise ValueError( "Cannot set both 'temperature' and 'reasoning' parameters together for OpenAI." ) From f248495768f16476ba104485f7c9047f9d6fcfe7 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Tue, 16 Dec 2025 12:09:41 +0530 Subject: [PATCH 05/10] Remove default value for 'model' in KaapiLLMParams to enforce explicit assignment --- backend/app/models/llm/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/models/llm/request.py b/backend/app/models/llm/request.py index 9f82ea565..638fcc46d 100644 --- a/backend/app/models/llm/request.py +++ b/backend/app/models/llm/request.py @@ -14,7 +14,6 @@ class KaapiLLMParams(SQLModel): """ model: str = Field( - default=None, description="Model identifier to use for completion (e.g., 'gpt-4o', 'gpt-5')", ) instructions: str | None = Field( From 0840c5161d1bd49e7efb0aae88f85391ed45e8c1 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 18 Dec 2025 12:09:28 +0530 Subject: [PATCH 06/10] Refactor KaapiLLMParams to enforce explicit reasoning levels; update mapping logic to handle reasoning and temperature conflicts with warnings --- backend/app/models/llm/request.py | 2 +- backend/app/services/llm/jobs.py | 5 +- backend/app/services/llm/mappers.py | 62 ++++++--- backend/app/tests/services/llm/test_jobs.py | 102 ++++++++++++++- .../app/tests/services/llm/test_mappers.py | 120 +++++++++++++----- 5 files changed, 234 insertions(+), 57 deletions(-) diff --git a/backend/app/models/llm/request.py b/backend/app/models/llm/request.py index 638fcc46d..fc44235f9 100644 --- a/backend/app/models/llm/request.py +++ b/backend/app/models/llm/request.py @@ -24,7 +24,7 @@ class KaapiLLMParams(SQLModel): default=None, description="List of vector store IDs to use for knowledge retrieval", ) - reasoning: str | None = Field( + reasoning: Literal["low", "medium", "high"] | None = Field( default=None, description="Reasoning configuration or instructions", ) diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index 3f0225feb..d76c29907 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -175,9 +175,12 @@ def execute_job( # Transform Kaapi config to native config if needed (before getting provider) completion_config = config_blob.completion if isinstance(completion_config, KaapiCompletionConfig): - completion_config = transform_kaapi_config_to_native( + completion_config, warnings = transform_kaapi_config_to_native( completion_config ) + if request.request_metadata is None: + request.request_metadata = {} + request.request_metadata.setdefault("warnings", []).extend(warnings) except Exception as e: callback_response = APIResponse.failure_response( error=f"Error processing configuration: {str(e)}", diff --git a/backend/app/services/llm/mappers.py b/backend/app/services/llm/mappers.py index 1bf58adbb..9e076aa9a 100644 --- a/backend/app/services/llm/mappers.py +++ b/backend/app/services/llm/mappers.py @@ -1,9 +1,10 @@ """Parameter mappers for converting Kaapi-abstracted parameters to provider-specific formats.""" +import litellm from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig -def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: +def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> tuple[dict, list[str]]: """Map Kaapi-abstracted parameters to OpenAI API parameters. This mapper transforms standardized Kaapi parameters into OpenAI-specific @@ -12,23 +13,45 @@ def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: Args: kaapi_params: KaapiLLMParams instance with standardized parameters - Returns: - Dictionary of OpenAI API parameters ready to be passed to the API - Supported Mapping: - model → model - instructions → instructions - knowledge_base_ids → tools[file_search].vector_store_ids - max_num_results → tools[file_search].max_num_results (fallback default) - - reasoning → reasoning.effort - - temperature → temperature + - reasoning → reasoning.effort (if reasoning supported by model else suppressed) + - temperature → temperature (if reasoning not supported by model else suppressed) + + Returns: + Tuple of: + - Dictionary of OpenAI API parameters ready to be passed to the API + - List of warnings describing suppressed or ignored parameters """ openai_params = {} - - if kaapi_params.temperature is not None and kaapi_params.reasoning is not None: - raise ValueError( - "Cannot set both 'temperature' and 'reasoning' parameters together for OpenAI." - ) + warnings = [] + + support_reasoning = litellm.supports_reasoning( + model="openai/" + f"{kaapi_params.model}" + ) + + # Handle reasoning vs temperature mutual exclusivity + if support_reasoning: + if kaapi_params.reasoning is not None: + openai_params["reasoning"] = {"effort": kaapi_params.reasoning} + + if kaapi_params.temperature is not None: + warnings.append( + "Parameter 'temperature' was suppressed because the selected model " + "supports reasoning, and temperature is ignored when reasoning is enabled." + ) + else: + if kaapi_params.reasoning is not None: + warnings.append( + "Parameter 'reasoning' was suppressed because the selected model " + "does not support reasoning." + ) + + if kaapi_params.temperature is not None: + openai_params["temperature"] = kaapi_params.temperature if kaapi_params.model: openai_params["model"] = kaapi_params.model @@ -45,18 +68,12 @@ def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> dict: } ] - if kaapi_params.reasoning: - openai_params["reasoning"] = {"effort": kaapi_params.reasoning} - - if kaapi_params.temperature is not None: - openai_params["temperature"] = kaapi_params.temperature - - return openai_params + return openai_params, warnings def transform_kaapi_config_to_native( kaapi_config: KaapiCompletionConfig, -) -> NativeCompletionConfig: +) -> tuple[NativeCompletionConfig, list[str]]: """Transform Kaapi completion config to native provider config with mapped parameters. Currently supports OpenAI. Future: Claude, Gemini mappers. @@ -68,7 +85,10 @@ def transform_kaapi_config_to_native( NativeCompletionConfig with provider-native parameters ready for API """ if kaapi_config.provider == "openai": - mapped_params = map_kaapi_to_openai_params(kaapi_config.params) - return NativeCompletionConfig(provider="openai-native", params=mapped_params) + mapped_params, warnings = map_kaapi_to_openai_params(kaapi_config.params) + return ( + NativeCompletionConfig(provider="openai-native", params=mapped_params), + warnings, + ) raise ValueError(f"Unsupported provider: {kaapi_config.provider}") diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index 8c87efea1..c71179c30 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -584,7 +584,6 @@ def test_kaapi_config_with_callback(self, db, job_for_execution, mock_llm_respon params=KaapiLLMParams( model="gpt-3.5-turbo", temperature=0.5, - max_tokens=500, ), ) ) @@ -622,6 +621,107 @@ def test_kaapi_config_with_callback(self, db, job_for_execution, mock_llm_respon db.refresh(job_for_execution) assert job_for_execution.status == JobStatus.SUCCESS + def test_kaapi_config_warnings_passed_through_metadata( + self, db, job_for_execution, mock_llm_response + ): + """Test that warnings from Kaapi config transformation are passed through in metadata.""" + project = get_project(db) + + # Use a config that will generate warnings (temperature on reasoning model) + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="o1", # Reasoning model + temperature=0.7, # This will be suppressed with warning + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + # Verify the result includes warnings in metadata + assert result["success"] + assert "metadata" in result + assert "warnings" in result["metadata"] + assert len(result["metadata"]["warnings"]) == 1 + assert "temperature" in result["metadata"]["warnings"][0].lower() + assert "suppressed" in result["metadata"]["warnings"][0] + + def test_kaapi_config_warnings_merged_with_existing_metadata( + self, db, job_for_execution, mock_llm_response + ): + """Test that warnings are merged with existing request metadata.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", # Non-reasoning model + reasoning="high", # This will be suppressed with warning + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + "request_metadata": {"tracking_id": "test-123"}, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + # Verify warnings are added to existing metadata + assert result["success"] + assert "metadata" in result + assert result["metadata"]["tracking_id"] == "test-123" + assert "warnings" in result["metadata"] + assert len(result["metadata"]["warnings"]) == 1 + assert "reasoning" in result["metadata"]["warnings"][0].lower() + assert "does not support reasoning" in result["metadata"]["warnings"][0] + class TestResolveConfigBlob: """Test suite for resolve_config_blob function.""" diff --git a/backend/app/tests/services/llm/test_mappers.py b/backend/app/tests/services/llm/test_mappers.py index 4f1b0670b..c020753d2 100644 --- a/backend/app/tests/services/llm/test_mappers.py +++ b/backend/app/tests/services/llm/test_mappers.py @@ -19,9 +19,10 @@ def test_basic_model_mapping(self): """Test basic model parameter mapping.""" kaapi_params = KaapiLLMParams(model="gpt-4o") - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result == {"model": "gpt-4o"} + assert warnings == [] def test_instructions_mapping(self): """Test instructions parameter mapping.""" @@ -30,22 +31,24 @@ def test_instructions_mapping(self): instructions="You are a helpful assistant.", ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["model"] == "gpt-4" assert result["instructions"] == "You are a helpful assistant." + assert warnings == [] def test_temperature_mapping(self): - """Test temperature parameter mapping.""" + """Test temperature parameter mapping for non-reasoning models.""" kaapi_params = KaapiLLMParams( model="gpt-4", temperature=0.7, ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["model"] == "gpt-4" assert result["temperature"] == 0.7 + assert warnings == [] def test_temperature_zero_mapping(self): """Test that temperature=0 is correctly mapped (edge case).""" @@ -54,21 +57,23 @@ def test_temperature_zero_mapping(self): temperature=0.0, ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["temperature"] == 0.0 + assert warnings == [] - def test_reasoning_mapping(self): - """Test reasoning parameter mapping to OpenAI format.""" + def test_reasoning_mapping_for_reasoning_models(self): + """Test reasoning parameter mapping to OpenAI format for reasoning-capable models.""" kaapi_params = KaapiLLMParams( - model="gpt-4", + model="o1", reasoning="high", ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) - assert result["model"] == "gpt-4" + assert result["model"] == "o1" assert result["reasoning"] == {"effort": "high"} + assert warnings == [] def test_knowledge_base_ids_mapping(self): """Test knowledge_base_ids mapping to OpenAI tools format.""" @@ -77,7 +82,7 @@ def test_knowledge_base_ids_mapping(self): knowledge_base_ids=["vs_abc123", "vs_def456"], ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["model"] == "gpt-4" assert "tools" in result @@ -85,6 +90,7 @@ def test_knowledge_base_ids_mapping(self): assert result["tools"][0]["type"] == "file_search" assert result["tools"][0]["vector_store_ids"] == ["vs_abc123", "vs_def456"] assert result["tools"][0]["max_num_results"] == 20 # default + assert warnings == [] def test_knowledge_base_with_max_num_results(self): """Test knowledge_base_ids with custom max_num_results.""" @@ -94,9 +100,10 @@ def test_knowledge_base_with_max_num_results(self): max_num_results=50, ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["tools"][0]["max_num_results"] == 50 + assert warnings == [] def test_complete_parameter_mapping(self): """Test mapping all compatible parameters together.""" @@ -108,7 +115,7 @@ def test_complete_parameter_mapping(self): max_num_results=30, ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["model"] == "gpt-4o" assert result["instructions"] == "You are an expert assistant." @@ -116,27 +123,64 @@ def test_complete_parameter_mapping(self): assert result["tools"][0]["type"] == "file_search" assert result["tools"][0]["vector_store_ids"] == ["vs_123"] assert result["tools"][0]["max_num_results"] == 30 + assert warnings == [] - def test_temperature_and_reasoning_conflict(self): - """Test that providing both temperature and reasoning raises ValueError.""" + def test_reasoning_suppressed_for_non_reasoning_models(self): + """Test that reasoning is suppressed with warning for non-reasoning models.""" kaapi_params = KaapiLLMParams( model="gpt-4", + reasoning="high", + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "reasoning" not in result + assert len(warnings) == 1 + assert "reasoning" in warnings[0].lower() + assert "does not support reasoning" in warnings[0] + + def test_temperature_suppressed_for_reasoning_models(self): + """Test that temperature is suppressed with warning for reasoning models when reasoning is set.""" + kaapi_params = KaapiLLMParams( + model="o1", temperature=0.7, reasoning="high", ) - with pytest.raises(ValueError) as exc_info: - map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) - assert "Cannot set both 'temperature' and 'reasoning'" in str(exc_info.value) + assert result["model"] == "o1" + assert result["reasoning"] == {"effort": "high"} + assert "temperature" not in result + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] + + def test_temperature_without_reasoning_for_reasoning_models(self): + """Test that temperature is suppressed for reasoning models even without explicit reasoning parameter.""" + kaapi_params = KaapiLLMParams( + model="o1", + temperature=0.7, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "o1" + assert "temperature" not in result + assert "reasoning" not in result + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] def test_minimal_params(self): """Test mapping with minimal parameters (only model).""" kaapi_params = KaapiLLMParams(model="gpt-4") - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result == {"model": "gpt-4"} + assert warnings == [] def test_only_knowledge_base_ids(self): """Test mapping with only knowledge_base_ids and model.""" @@ -145,11 +189,12 @@ def test_only_knowledge_base_ids(self): knowledge_base_ids=["vs_xyz"], ) - result = map_kaapi_to_openai_params(kaapi_params) + result, warnings = map_kaapi_to_openai_params(kaapi_params) assert result["model"] == "gpt-4" assert "tools" in result assert result["tools"][0]["vector_store_ids"] == ["vs_xyz"] + assert warnings == [] class TestTransformKaapiConfigToNative: @@ -165,12 +210,13 @@ def test_transform_openai_config(self): ), ) - result = transform_kaapi_config_to_native(kaapi_config) + result, warnings = transform_kaapi_config_to_native(kaapi_config) assert isinstance(result, NativeCompletionConfig) assert result.provider == "openai-native" assert result.params["model"] == "gpt-4" assert result.params["temperature"] == 0.7 + assert warnings == [] def test_transform_with_all_params(self): """Test transformation with all Kaapi parameters.""" @@ -185,7 +231,7 @@ def test_transform_with_all_params(self): ), ) - result = transform_kaapi_config_to_native(kaapi_config) + result, warnings = transform_kaapi_config_to_native(kaapi_config) assert result.provider == "openai-native" assert result.params["model"] == "gpt-4o" @@ -193,38 +239,45 @@ def test_transform_with_all_params(self): assert result.params["temperature"] == 0.5 assert result.params["tools"][0]["type"] == "file_search" assert result.params["tools"][0]["max_num_results"] == 25 + assert warnings == [] def test_transform_with_reasoning(self): - """Test transformation with reasoning parameter.""" + """Test transformation with reasoning parameter for reasoning-capable models.""" kaapi_config = KaapiCompletionConfig( provider="openai", params=KaapiLLMParams( - model="o1-preview", + model="o1", reasoning="medium", ), ) - result = transform_kaapi_config_to_native(kaapi_config) + result, warnings = transform_kaapi_config_to_native(kaapi_config) assert result.provider == "openai-native" - assert result.params["model"] == "o1-preview" + assert result.params["model"] == "o1" assert result.params["reasoning"] == {"effort": "medium"} + assert warnings == [] - def test_transform_validates_temperature_reasoning_conflict(self): - """Test that transformation validates temperature + reasoning conflict.""" + def test_transform_with_both_temperature_and_reasoning(self): + """Test that transformation handles temperature + reasoning intelligently for reasoning models.""" kaapi_config = KaapiCompletionConfig( provider="openai", params=KaapiLLMParams( - model="gpt-4", + model="o1", temperature=0.7, reasoning="high", ), ) - with pytest.raises(ValueError) as exc_info: - transform_kaapi_config_to_native(kaapi_config) + result, warnings = transform_kaapi_config_to_native(kaapi_config) - assert "Cannot set both 'temperature' and 'reasoning'" in str(exc_info.value) + assert result.provider == "openai-native" + assert result.params["model"] == "o1" + assert result.params["reasoning"] == {"effort": "high"} + assert "temperature" not in result.params + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] def test_unsupported_provider_raises_error(self): """Test that unsupported providers raise ValueError.""" @@ -253,10 +306,11 @@ def test_transform_preserves_param_structure(self): ), ) - result = transform_kaapi_config_to_native(kaapi_config) + result, warnings = transform_kaapi_config_to_native(kaapi_config) # Verify the nested structure is correct assert isinstance(result.params["tools"], list) assert isinstance(result.params["tools"][0], dict) assert isinstance(result.params["tools"][0]["vector_store_ids"], list) assert len(result.params["tools"][0]["vector_store_ids"]) == 3 + assert warnings == [] From 0a21c4eaa6af9ec41b1ff643736463c7443e0d99 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 18 Dec 2025 12:22:12 +0530 Subject: [PATCH 07/10] Enhance LLM API documentation to clarify ad-hoc configuration parameters and warning handling for unsupported settings --- backend/app/api/docs/llm/llm_call.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/app/api/docs/llm/llm_call.md b/backend/app/api/docs/llm/llm_call.md index b379e35cc..fec4fbc49 100644 --- a/backend/app/api/docs/llm/llm_call.md +++ b/backend/app/api/docs/llm/llm_call.md @@ -25,7 +25,9 @@ for processing, and results are delivered via the callback URL when complete. - `completion` (required, object): Completion configuration - `provider` (required, string): Provider type - either `"openai"` (Kaapi abstraction) or `"openai-native"` (pass-through) - `params` (required, object): Parameters structure depends on provider type (see schema for detailed structure) - - **Note**: When using ad-hoc configuration, do not include `id` and `version` fields + - **Note** + - When using ad-hoc configuration, do not include `id` and `version` fields + - When using the Kaapi abstraction, parameters that are not supported by the selected provider or model are automatically suppressed. If any parameters are ignored, a list of warnings is included in the metadata.warnings. For example, the GPT-5 model does not support the temperature parameter, so Kaapi will neither throw an error nor pass this parameter to the model; instead, it will return a warning in the metadata.warnings response. - **Recommendation**: Use stored configs (Mode 1) for production; use ad-hoc configs only for testing/validation - **Schema**: Check the API schema or examples below for the complete parameter structure for each provider type @@ -41,4 +43,7 @@ for processing, and results are delivered via the callback URL when complete. - Custom JSON metadata - Passed through unchanged in the response +### Note +- `warnings` list is automatically added in response metadata when using Kaapi configs if any parameters are suppressed or adjusted (e.g., temperature on reasoning models) + --- From f5ab6855fd8deeb5430a2428b020f1f857a7ea6e Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 18 Dec 2025 12:46:12 +0530 Subject: [PATCH 08/10] Refactor execute_job to use completion_config directly instead of config_blob.completion --- backend/app/services/llm/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index d76c29907..773fe7fc3 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -221,7 +221,7 @@ def execute_job( )(provider_instance.execute) response, error = decorated_execute( - completion_config=config_blob.completion, + completion_config=completion_config, query=request.query, include_provider_raw_response=request.include_provider_raw_response, ) From 1399f22e1684a1dc52c2e4ac75d1ed6e6b45c1e3 Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 18 Dec 2025 14:55:24 +0530 Subject: [PATCH 09/10] Refactor LLM provider interfaces to use NativeCompletionConfig instead of CompletionConfig --- backend/app/core/langfuse/langfuse.py | 4 ++-- backend/app/services/llm/providers/base.py | 6 +++--- backend/app/services/llm/providers/openai.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/app/core/langfuse/langfuse.py b/backend/app/core/langfuse/langfuse.py index 5c20f5038..496a68772 100644 --- a/backend/app/core/langfuse/langfuse.py +++ b/backend/app/core/langfuse/langfuse.py @@ -6,7 +6,7 @@ from asgi_correlation_id import correlation_id from langfuse import Langfuse from langfuse.client import StatefulGenerationClient, StatefulTraceClient -from app.models.llm import CompletionConfig, QueryParams, LLMCallResponse +from app.models.llm import NativeCompletionConfig, QueryParams, LLMCallResponse logger = logging.getLogger(__name__) @@ -130,7 +130,7 @@ def observe_llm_execution( def decorator(func: Callable) -> Callable: @wraps(func) - def wrapper(completion_config: CompletionConfig, query: QueryParams, **kwargs): + def wrapper(completion_config: NativeCompletionConfig, query: QueryParams, **kwargs): # Skip observability if no credentials provided if not credentials: logger.info("[Langfuse] No credentials - skipping observability") diff --git a/backend/app/services/llm/providers/base.py b/backend/app/services/llm/providers/base.py index ecca36fcc..827f25910 100644 --- a/backend/app/services/llm/providers/base.py +++ b/backend/app/services/llm/providers/base.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from typing import Any -from app.models.llm import CompletionConfig, LLMCallResponse, QueryParams +from app.models.llm import NativeCompletionConfig, LLMCallResponse, QueryParams class BaseProvider(ABC): @@ -34,7 +34,7 @@ def __init__(self, client: Any): @abstractmethod def execute( self, - completion_config: CompletionConfig, + completion_config: NativeCompletionConfig, query: QueryParams, include_provider_raw_response: bool = False, ) -> tuple[LLMCallResponse | None, str | None]: @@ -43,7 +43,7 @@ def execute( Directly passes the user's config params to provider API along with input. Args: - completion_config: LLM completion configuration + completion_config: LLM completion configuration, pass params as-is to provider API query: Query parameters including input and conversation_id include_provider_raw_response: Whether to include the raw LLM provider response in the output diff --git a/backend/app/services/llm/providers/openai.py b/backend/app/services/llm/providers/openai.py index f24094a86..34e35e17e 100644 --- a/backend/app/services/llm/providers/openai.py +++ b/backend/app/services/llm/providers/openai.py @@ -5,7 +5,7 @@ from openai.types.responses.response import Response from app.models.llm import ( - CompletionConfig, + NativeCompletionConfig, LLMCallResponse, QueryParams, LLMOutput, @@ -30,7 +30,7 @@ def __init__(self, client: OpenAI): def execute( self, - completion_config: CompletionConfig, + completion_config: NativeCompletionConfig, query: QueryParams, include_provider_raw_response: bool = False, ) -> tuple[LLMCallResponse | None, str | None]: From 78c20ad4d319a60920394b83a524fe463ee1b57d Mon Sep 17 00:00:00 2001 From: Aviraj Gour <100823015+avirajsingh7@users.noreply.github.com> Date: Thu, 18 Dec 2025 15:04:29 +0530 Subject: [PATCH 10/10] precommit --- backend/app/core/langfuse/langfuse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/app/core/langfuse/langfuse.py b/backend/app/core/langfuse/langfuse.py index 496a68772..287790131 100644 --- a/backend/app/core/langfuse/langfuse.py +++ b/backend/app/core/langfuse/langfuse.py @@ -130,7 +130,9 @@ def observe_llm_execution( def decorator(func: Callable) -> Callable: @wraps(func) - def wrapper(completion_config: NativeCompletionConfig, query: QueryParams, **kwargs): + def wrapper( + completion_config: NativeCompletionConfig, query: QueryParams, **kwargs + ): # Skip observability if no credentials provided if not credentials: logger.info("[Langfuse] No credentials - skipping observability")