diff --git a/backend/app/api/docs/llm/llm_call.md b/backend/app/api/docs/llm/llm_call.md index a284ab6a1..fec4fbc49 100644 --- a/backend/app/api/docs/llm/llm_call.md +++ b/backend/app/api/docs/llm/llm_call.md @@ -21,11 +21,15 @@ for processing, and results are delivered via the callback URL when complete. - **Note**: When using stored configuration, do not include the `blob` field in the request body - **Mode 2: Ad-hoc Configuration** - - `blob` (object): Complete configuration object (see Create Config endpoint documentation for examples) - - `completion` (required): - - `provider` (required, string): Currently only "openai" - - `params` (required, object): Provider-specific parameters (flexible JSON) - - **Note**: When using ad-hoc configuration, do not include `id` and `version` fields + - `blob` (object): Complete configuration object + - `completion` (required, object): Completion configuration + - `provider` (required, string): Provider type - either `"openai"` (Kaapi abstraction) or `"openai-native"` (pass-through) + - `params` (required, object): Parameters structure depends on provider type (see schema for detailed structure) + - **Note** + - When using ad-hoc configuration, do not include `id` and `version` fields + - When using the Kaapi abstraction, parameters that are not supported by the selected provider or model are automatically suppressed. If any parameters are ignored, a list of warnings is included in the metadata.warnings. For example, the GPT-5 model does not support the temperature parameter, so Kaapi will neither throw an error nor pass this parameter to the model; instead, it will return a warning in the metadata.warnings response. + - **Recommendation**: Use stored configs (Mode 1) for production; use ad-hoc configs only for testing/validation + - **Schema**: Check the API schema or examples below for the complete parameter structure for each provider type **`callback_url`** (optional, HTTPS URL): - Webhook endpoint to receive the response @@ -39,4 +43,7 @@ for processing, and results are delivered via the callback URL when complete. - Custom JSON metadata - Passed through unchanged in the response +### Note +- `warnings` list is automatically added in response metadata when using Kaapi configs if any parameters are suppressed or adjusted (e.g., temperature on reasoning models) + --- diff --git a/backend/app/core/langfuse/langfuse.py b/backend/app/core/langfuse/langfuse.py index 5c20f5038..287790131 100644 --- a/backend/app/core/langfuse/langfuse.py +++ b/backend/app/core/langfuse/langfuse.py @@ -6,7 +6,7 @@ from asgi_correlation_id import correlation_id from langfuse import Langfuse from langfuse.client import StatefulGenerationClient, StatefulTraceClient -from app.models.llm import CompletionConfig, QueryParams, LLMCallResponse +from app.models.llm import NativeCompletionConfig, QueryParams, LLMCallResponse logger = logging.getLogger(__name__) @@ -130,7 +130,9 @@ def observe_llm_execution( def decorator(func: Callable) -> Callable: @wraps(func) - def wrapper(completion_config: CompletionConfig, query: QueryParams, **kwargs): + def wrapper( + completion_config: NativeCompletionConfig, query: QueryParams, **kwargs + ): # Skip observability if no credentials provided if not credentials: logger.info("[Langfuse] No credentials - skipping observability") diff --git a/backend/app/models/llm/__init__.py b/backend/app/models/llm/__init__.py index f06954de5..8738e2126 100644 --- a/backend/app/models/llm/__init__.py +++ b/backend/app/models/llm/__init__.py @@ -3,5 +3,8 @@ CompletionConfig, QueryParams, ConfigBlob, + KaapiLLMParams, + KaapiCompletionConfig, + NativeCompletionConfig, ) from app.models.llm.response import LLMCallResponse, LLMResponse, LLMOutput, Usage diff --git a/backend/app/models/llm/request.py b/backend/app/models/llm/request.py index a63de1ebc..fc44235f9 100644 --- a/backend/app/models/llm/request.py +++ b/backend/app/models/llm/request.py @@ -1,8 +1,44 @@ -from typing import Any, Literal +from typing import Annotated, Any, Literal, Union from uuid import UUID from sqlmodel import Field, SQLModel -from pydantic import model_validator, HttpUrl +from pydantic import Discriminator, model_validator, HttpUrl + + +class KaapiLLMParams(SQLModel): + """ + Kaapi-abstracted parameters for LLM providers. + These parameters are mapped internally to provider-specific API parameters. + Provides a unified contract across all LLM providers (OpenAI, Claude, Gemini, etc.). + Provider-specific mappings are handled at the mapper level. + """ + + model: str = Field( + description="Model identifier to use for completion (e.g., 'gpt-4o', 'gpt-5')", + ) + instructions: str | None = Field( + default=None, + description="System instructions to guide the model's behavior", + ) + knowledge_base_ids: list[str] | None = Field( + default=None, + description="List of vector store IDs to use for knowledge retrieval", + ) + reasoning: Literal["low", "medium", "high"] | None = Field( + default=None, + description="Reasoning configuration or instructions", + ) + temperature: float | None = Field( + default=None, + ge=0.0, + le=2.0, + description="Sampling temperature between 0 and 2", + ) + max_num_results: int | None = Field( + default=None, + ge=1, + description="Maximum number of results to return", + ) class ConversationConfig(SQLModel): @@ -46,11 +82,16 @@ class QueryParams(SQLModel): ) -class CompletionConfig(SQLModel): - """Completion configuration with provider and parameters.""" +class NativeCompletionConfig(SQLModel): + """ + Native provider configuration (pass-through). + All parameters are forwarded as-is to the provider's API without transformation. + Supports any LLM provider's native API format. + """ - provider: Literal["openai"] = Field( - default="openai", description="LLM provider to use" + provider: Literal["openai-native"] = Field( + default="openai-native", + description="Native provider type (e.g., openai-native)", ) params: dict[str, Any] = Field( ..., @@ -58,6 +99,27 @@ class CompletionConfig(SQLModel): ) +class KaapiCompletionConfig(SQLModel): + """ + Kaapi abstraction for LLM completion providers. + Uses standardized Kaapi parameters that are mapped to provider-specific APIs internally. + Supports multiple providers: OpenAI, Claude, Gemini, etc. + """ + + provider: Literal["openai"] = Field(..., description="LLM provider (openai)") + params: KaapiLLMParams = Field( + ..., + description="Kaapi-standardized parameters mapped to provider-specific API", + ) + + +# Discriminated union for completion configs based on provider field +CompletionConfig = Annotated[ + Union[NativeCompletionConfig, KaapiCompletionConfig], + Field(discriminator="provider"), +] + + class ConfigBlob(SQLModel): """Raw JSON blob of config.""" diff --git a/backend/app/services/llm/jobs.py b/backend/app/services/llm/jobs.py index 831f3cca4..773fe7fc3 100644 --- a/backend/app/services/llm/jobs.py +++ b/backend/app/services/llm/jobs.py @@ -10,11 +10,12 @@ from app.crud.credentials import get_provider_credential from app.crud.jobs import JobCrud from app.models import JobStatus, JobType, JobUpdate, LLMCallRequest -from app.models.llm.request import ConfigBlob, LLMCallConfig +from app.models.llm.request import ConfigBlob, LLMCallConfig, KaapiCompletionConfig from app.utils import APIResponse, send_callback from app.celery.utils import start_high_priority_job from app.core.langfuse.langfuse import observe_llm_execution from app.services.llm.providers.registry import get_llm_provider +from app.services.llm.mappers import transform_kaapi_config_to_native logger = logging.getLogger(__name__) @@ -170,10 +171,27 @@ def execute_job( else: config_blob = config.blob + try: + # Transform Kaapi config to native config if needed (before getting provider) + completion_config = config_blob.completion + if isinstance(completion_config, KaapiCompletionConfig): + completion_config, warnings = transform_kaapi_config_to_native( + completion_config + ) + if request.request_metadata is None: + request.request_metadata = {} + request.request_metadata.setdefault("warnings", []).extend(warnings) + except Exception as e: + callback_response = APIResponse.failure_response( + error=f"Error processing configuration: {str(e)}", + metadata=request.request_metadata, + ) + return handle_job_error(job_id, request.callback_url, callback_response) + try: provider_instance = get_llm_provider( session=session, - provider_type=config_blob.completion.provider, + provider_type=completion_config.provider, # Now always native provider type project_id=project_id, organization_id=organization_id, ) @@ -203,7 +221,7 @@ def execute_job( )(provider_instance.execute) response, error = decorated_execute( - completion_config=config_blob.completion, + completion_config=completion_config, query=request.query, include_provider_raw_response=request.include_provider_raw_response, ) diff --git a/backend/app/services/llm/mappers.py b/backend/app/services/llm/mappers.py new file mode 100644 index 000000000..9e076aa9a --- /dev/null +++ b/backend/app/services/llm/mappers.py @@ -0,0 +1,94 @@ +"""Parameter mappers for converting Kaapi-abstracted parameters to provider-specific formats.""" + +import litellm +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig + + +def map_kaapi_to_openai_params(kaapi_params: KaapiLLMParams) -> tuple[dict, list[str]]: + """Map Kaapi-abstracted parameters to OpenAI API parameters. + + This mapper transforms standardized Kaapi parameters into OpenAI-specific + parameter format, enabling provider-agnostic interface design. + + Args: + kaapi_params: KaapiLLMParams instance with standardized parameters + + Supported Mapping: + - model → model + - instructions → instructions + - knowledge_base_ids → tools[file_search].vector_store_ids + - max_num_results → tools[file_search].max_num_results (fallback default) + - reasoning → reasoning.effort (if reasoning supported by model else suppressed) + - temperature → temperature (if reasoning not supported by model else suppressed) + + Returns: + Tuple of: + - Dictionary of OpenAI API parameters ready to be passed to the API + - List of warnings describing suppressed or ignored parameters + """ + openai_params = {} + warnings = [] + + support_reasoning = litellm.supports_reasoning( + model="openai/" + f"{kaapi_params.model}" + ) + + # Handle reasoning vs temperature mutual exclusivity + if support_reasoning: + if kaapi_params.reasoning is not None: + openai_params["reasoning"] = {"effort": kaapi_params.reasoning} + + if kaapi_params.temperature is not None: + warnings.append( + "Parameter 'temperature' was suppressed because the selected model " + "supports reasoning, and temperature is ignored when reasoning is enabled." + ) + else: + if kaapi_params.reasoning is not None: + warnings.append( + "Parameter 'reasoning' was suppressed because the selected model " + "does not support reasoning." + ) + + if kaapi_params.temperature is not None: + openai_params["temperature"] = kaapi_params.temperature + + if kaapi_params.model: + openai_params["model"] = kaapi_params.model + + if kaapi_params.instructions: + openai_params["instructions"] = kaapi_params.instructions + + if kaapi_params.knowledge_base_ids: + openai_params["tools"] = [ + { + "type": "file_search", + "vector_store_ids": kaapi_params.knowledge_base_ids, + "max_num_results": kaapi_params.max_num_results or 20, + } + ] + + return openai_params, warnings + + +def transform_kaapi_config_to_native( + kaapi_config: KaapiCompletionConfig, +) -> tuple[NativeCompletionConfig, list[str]]: + """Transform Kaapi completion config to native provider config with mapped parameters. + + Currently supports OpenAI. Future: Claude, Gemini mappers. + + Args: + kaapi_config: KaapiCompletionConfig with abstracted parameters + + Returns: + NativeCompletionConfig with provider-native parameters ready for API + """ + if kaapi_config.provider == "openai": + mapped_params, warnings = map_kaapi_to_openai_params(kaapi_config.params) + return ( + NativeCompletionConfig(provider="openai-native", params=mapped_params), + warnings, + ) + + raise ValueError(f"Unsupported provider: {kaapi_config.provider}") diff --git a/backend/app/services/llm/providers/base.py b/backend/app/services/llm/providers/base.py index ecca36fcc..827f25910 100644 --- a/backend/app/services/llm/providers/base.py +++ b/backend/app/services/llm/providers/base.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from typing import Any -from app.models.llm import CompletionConfig, LLMCallResponse, QueryParams +from app.models.llm import NativeCompletionConfig, LLMCallResponse, QueryParams class BaseProvider(ABC): @@ -34,7 +34,7 @@ def __init__(self, client: Any): @abstractmethod def execute( self, - completion_config: CompletionConfig, + completion_config: NativeCompletionConfig, query: QueryParams, include_provider_raw_response: bool = False, ) -> tuple[LLMCallResponse | None, str | None]: @@ -43,7 +43,7 @@ def execute( Directly passes the user's config params to provider API along with input. Args: - completion_config: LLM completion configuration + completion_config: LLM completion configuration, pass params as-is to provider API query: Query parameters including input and conversation_id include_provider_raw_response: Whether to include the raw LLM provider response in the output diff --git a/backend/app/services/llm/providers/openai.py b/backend/app/services/llm/providers/openai.py index f24094a86..34e35e17e 100644 --- a/backend/app/services/llm/providers/openai.py +++ b/backend/app/services/llm/providers/openai.py @@ -5,7 +5,7 @@ from openai.types.responses.response import Response from app.models.llm import ( - CompletionConfig, + NativeCompletionConfig, LLMCallResponse, QueryParams, LLMOutput, @@ -30,7 +30,7 @@ def __init__(self, client: OpenAI): def execute( self, - completion_config: CompletionConfig, + completion_config: NativeCompletionConfig, query: QueryParams, include_provider_raw_response: bool = False, ) -> tuple[LLMCallResponse | None, str | None]: diff --git a/backend/app/services/llm/providers/registry.py b/backend/app/services/llm/providers/registry.py index 64b32c436..a5cfb4bb8 100644 --- a/backend/app/services/llm/providers/registry.py +++ b/backend/app/services/llm/providers/registry.py @@ -12,15 +12,16 @@ class LLMProvider: - OPENAI = "openai" - # Future constants: - # ANTHROPIC = "anthropic" - # GOOGLE = "google" + OPENAI_NATIVE = "openai-native" + # Future constants for native providers: + # CLAUDE_NATIVE = "claude-native" + # GEMINI_NATIVE = "gemini-native" _registry: dict[str, type[BaseProvider]] = { - OPENAI: OpenAIProvider, - # ANTHROPIC: AnthropicProvider, - # GOOGLE: GoogleProvider, + OPENAI_NATIVE: OpenAIProvider, + # Future native providers: + # CLAUDE_NATIVE: ClaudeProvider, + # GEMINI_NATIVE: GeminiProvider, } @classmethod @@ -45,19 +46,22 @@ def get_llm_provider( ) -> BaseProvider: provider_class = LLMProvider.get(provider_type) + # e.g., "openai-native" → "openai", "claude-native" → "claude" + credential_provider = provider_type.replace("-native", "") + credentials = get_provider_credential( session=session, - provider=provider_type, + provider=credential_provider, project_id=project_id, org_id=organization_id, ) if not credentials: raise ValueError( - f"Credentials for provider '{provider_type}' not configured for this project." + f"Credentials for provider '{credential_provider}' not configured for this project." ) - if provider_type == LLMProvider.OPENAI: + if provider_type == LLMProvider.OPENAI_NATIVE: if "api_key" not in credentials: raise ValueError("OpenAI credentials not configured for this project.") client = OpenAI(api_key=credentials["api_key"]) diff --git a/backend/app/tests/api/routes/configs/test_config.py b/backend/app/tests/api/routes/configs/test_config.py index 8f094f538..a30d162a0 100644 --- a/backend/app/tests/api/routes/configs/test_config.py +++ b/backend/app/tests/api/routes/configs/test_config.py @@ -19,7 +19,7 @@ def test_create_config_success( "description": "A test LLM configuration", "config_blob": { "completion": { - "provider": "openai", + "provider": "openai-native", "params": { "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/api/routes/configs/test_version.py b/backend/app/tests/api/routes/configs/test_version.py index acb9f2526..27fb14ba2 100644 --- a/backend/app/tests/api/routes/configs/test_version.py +++ b/backend/app/tests/api/routes/configs/test_version.py @@ -10,7 +10,8 @@ create_test_project, create_test_version, ) -from app.models import ConfigBlob, CompletionConfig +from app.models import ConfigBlob +from app.models.llm.request import NativeCompletionConfig def test_create_version_success( @@ -28,7 +29,7 @@ def test_create_version_success( version_data = { "config_blob": { "completion": { - "provider": "openai", + "provider": "openai-native", "params": { "model": "gpt-4-turbo", "temperature": 0.9, @@ -303,8 +304,8 @@ def test_get_version_by_number( config_id=config.id, project_id=user_api_key.project_id, config_blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4-turbo", "temperature": 0.5}, ) ), diff --git a/backend/app/tests/api/routes/test_llm.py b/backend/app/tests/api/routes/test_llm.py index 430ca77c0..78adffebf 100644 --- a/backend/app/tests/api/routes/test_llm.py +++ b/backend/app/tests/api/routes/test_llm.py @@ -6,6 +6,9 @@ LLMCallConfig, CompletionConfig, ConfigBlob, + KaapiLLMParams, + KaapiCompletionConfig, + NativeCompletionConfig, ) @@ -18,8 +21,8 @@ def test_llm_call_success(client: TestClient, user_api_key_header: dict[str, str query=QueryParams(input="What is the capital of France?"), config=LLMCallConfig( blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.7, @@ -43,3 +46,117 @@ def test_llm_call_success(client: TestClient, user_api_key_header: dict[str, str assert "response is being generated" in response_data["data"]["message"] mock_start_job.assert_called_once() + + +def test_llm_call_with_kaapi_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with Kaapi abstracted config.""" + with patch("app.services.llm.jobs.start_high_priority_job") as mock_start_job: + mock_start_job.return_value = "test-task-id" + + payload = LLMCallRequest( + query=QueryParams(input="Explain quantum computing"), + config=LLMCallConfig( + blob=ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4o", + instructions="You are a physics expert", + temperature=0.5, + ), + ) + ) + ), + ) + + response = client.post( + "api/v1/llm/call", + json=payload.model_dump(mode="json"), + headers=user_api_key_header, + ) + + assert response.status_code == 200 + response_data = response.json() + assert response_data["success"] is True + mock_start_job.assert_called_once() + + +def test_llm_call_with_native_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with native OpenAI config (pass-through mode).""" + with patch("app.services.llm.jobs.start_high_priority_job") as mock_start_job: + mock_start_job.return_value = "test-task-id" + + payload = LLMCallRequest( + query=QueryParams(input="Native API call test"), + config=LLMCallConfig( + blob=ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={ + "model": "gpt-4", + "temperature": 0.9, + "max_tokens": 500, + "top_p": 1.0, + }, + ) + ) + ), + ) + + response = client.post( + "api/v1/llm/call", + json=payload.model_dump(mode="json"), + headers=user_api_key_header, + ) + + assert response.status_code == 200 + response_data = response.json() + assert response_data["success"] is True + mock_start_job.assert_called_once() + + +def test_llm_call_missing_config( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with missing config fails validation.""" + payload = { + "query": {"input": "Test query"}, + # Missing config field + } + + response = client.post( + "api/v1/llm/call", + json=payload, + headers=user_api_key_header, + ) + + assert response.status_code == 422 # Validation error + + +def test_llm_call_invalid_provider( + client: TestClient, user_api_key_header: dict[str, str] +): + """Test LLM call with invalid provider type.""" + payload = { + "query": {"input": "Test query"}, + "config": { + "blob": { + "completion": { + "provider": "invalid-provider", + "params": {"model": "gpt-4"}, + } + } + }, + } + + response = client.post( + "api/v1/llm/call", + json=payload, + headers=user_api_key_header, + ) + + assert response.status_code == 422 # Validation error diff --git a/backend/app/tests/crud/config/test_config.py b/backend/app/tests/crud/config/test_config.py index e7837b98a..e4f90e25e 100644 --- a/backend/app/tests/crud/config/test_config.py +++ b/backend/app/tests/crud/config/test_config.py @@ -10,6 +10,7 @@ ConfigCreate, ConfigUpdate, ) +from app.models.llm.request import NativeCompletionConfig from app.crud.config import ConfigCrud from app.tests.utils.test_data import create_test_project, create_test_config from app.tests.utils.utils import random_lower_string @@ -18,8 +19,8 @@ @pytest.fixture def example_config_blob(): return ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/crud/config/test_version.py b/backend/app/tests/crud/config/test_version.py index c3c4bd582..a24940b7d 100644 --- a/backend/app/tests/crud/config/test_version.py +++ b/backend/app/tests/crud/config/test_version.py @@ -3,7 +3,8 @@ from sqlmodel import Session from fastapi import HTTPException -from app.models import ConfigVersionCreate, ConfigBlob, CompletionConfig +from app.models import ConfigVersionCreate, ConfigBlob +from app.models.llm.request import NativeCompletionConfig from app.crud.config import ConfigVersionCrud from app.tests.utils.test_data import ( create_test_project, @@ -15,8 +16,8 @@ @pytest.fixture def example_config_blob(): return ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8, diff --git a/backend/app/tests/services/llm/providers/test_openai.py b/backend/app/tests/services/llm/providers/test_openai.py index c216ca540..745dd00b8 100644 --- a/backend/app/tests/services/llm/providers/test_openai.py +++ b/backend/app/tests/services/llm/providers/test_openai.py @@ -7,7 +7,7 @@ import openai from app.models.llm import ( - CompletionConfig, + NativeCompletionConfig, QueryParams, ) from app.models.llm.request import ConversationConfig @@ -31,8 +31,8 @@ def provider(self, mock_client): @pytest.fixture def completion_config(self): """Create a basic completion config.""" - return CompletionConfig( - provider="openai", + return NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) @@ -59,7 +59,7 @@ def test_execute_success_without_conversation( assert result is not None assert result.response.output.text == mock_response.output_text assert result.response.model == mock_response.model - assert result.response.provider == "openai" + assert result.response.provider == "openai-native" assert result.response.conversation_id is None assert result.usage.input_tokens == mock_response.usage.input_tokens assert result.usage.output_tokens == mock_response.usage.output_tokens @@ -233,8 +233,8 @@ def test_execute_with_conversation_parameter_removed_when_no_config( ): """Test that conversation param is removed if it exists in config but no conversation config.""" # Create a config with conversation in params (should be removed) - completion_config = CompletionConfig( - provider="openai", + completion_config = NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "conversation": {"id": "old_conv"}}, ) diff --git a/backend/app/tests/services/llm/providers/test_registry.py b/backend/app/tests/services/llm/providers/test_registry.py index f9c595c0d..c05222747 100644 --- a/backend/app/tests/services/llm/providers/test_registry.py +++ b/backend/app/tests/services/llm/providers/test_registry.py @@ -21,8 +21,8 @@ class TestProviderRegistry: def test_registry_contains_openai(self): """Test that registry contains OpenAI provider.""" - assert "openai" in LLMProvider._registry - assert LLMProvider._registry["openai"] == OpenAIProvider + assert "openai-native" in LLMProvider._registry + assert LLMProvider._registry["openai-native"] == OpenAIProvider def test_registry_values_are_provider_classes(self): """Test that all registry values are BaseProvider subclasses.""" @@ -46,7 +46,7 @@ def test_get_llm_provider_with_openai(self, db: Session): provider = get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) @@ -64,7 +64,7 @@ def test_get_llm_provider_with_openai(self, db: Session): with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) @@ -87,7 +87,7 @@ def test_get_llm_provider_with_invalid_provider(self, db: Session): error_message = str(exc_info.value) assert "invalid_provider" in error_message assert "is not supported" in error_message - assert "openai" in error_message + assert "openai-native" in error_message def test_get_llm_provider_with_missing_credentials(self, db: Session): """Test handling of errors when credentials are not found.""" @@ -101,7 +101,7 @@ def test_get_llm_provider_with_missing_credentials(self, db: Session): with pytest.raises(ValueError) as exc_info: get_llm_provider( session=db, - provider_type="openai", + provider_type="openai-native", project_id=project.id, organization_id=project.organization_id, ) diff --git a/backend/app/tests/services/llm/test_jobs.py b/backend/app/tests/services/llm/test_jobs.py index 2f08b40c0..c71179c30 100644 --- a/backend/app/tests/services/llm/test_jobs.py +++ b/backend/app/tests/services/llm/test_jobs.py @@ -13,12 +13,14 @@ from app.models import ConfigVersion, JobStatus, JobType from app.models.llm import ( LLMCallRequest, - CompletionConfig, + NativeCompletionConfig, QueryParams, LLMCallResponse, LLMResponse, LLMOutput, Usage, + KaapiLLMParams, + KaapiCompletionConfig, ) from app.models.llm.request import ConfigBlob, LLMCallConfig from app.services.llm.jobs import ( @@ -40,8 +42,8 @@ def llm_call_request(self): query=QueryParams(input="Test query"), config=LLMCallConfig( blob=ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -225,7 +227,10 @@ def request_data(self): "query": {"input": "Test query"}, "config": { "blob": { - "completion": {"provider": "openai", "params": {"model": "gpt-4"}} + "completion": { + "provider": "openai-native", + "params": {"model": "gpt-4"}, + } } }, "include_provider_raw_response": False, @@ -396,8 +401,8 @@ def test_stored_config_success(self, db, job_for_execution, mock_llm_response): # Create a real config in the database config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.7}, ) ) @@ -445,8 +450,8 @@ def test_stored_config_with_callback( project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-3.5-turbo", "temperature": 0.5}, ) ) @@ -493,8 +498,8 @@ def test_stored_config_version_not_found(self, db, job_for_execution): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -523,6 +528,200 @@ def test_stored_config_version_not_found(self, db, job_for_execution): db.refresh(job_for_execution) assert job_for_execution.status == JobStatus.FAILED + def test_kaapi_config_success(self, db, job_for_execution, mock_llm_response): + """Test successful execution with Kaapi abstracted config.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + instructions="You are a helpful assistant", + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query with Kaapi config"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + mock_get_provider.assert_called_once() + mock_provider.execute.assert_called_once() + assert result["success"] + db.refresh(job_for_execution) + assert job_for_execution.status == JobStatus.SUCCESS + + def test_kaapi_config_with_callback(self, db, job_for_execution, mock_llm_response): + """Test successful execution with Kaapi config and callback.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-3.5-turbo", + temperature=0.5, + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query with Kaapi config and callback"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": "https://example.com/callback", + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + patch("app.services.llm.jobs.send_callback") as mock_send_callback, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + mock_send_callback.assert_called_once() + callback_data = mock_send_callback.call_args[1]["data"] + assert callback_data["success"] + assert result["success"] + db.refresh(job_for_execution) + assert job_for_execution.status == JobStatus.SUCCESS + + def test_kaapi_config_warnings_passed_through_metadata( + self, db, job_for_execution, mock_llm_response + ): + """Test that warnings from Kaapi config transformation are passed through in metadata.""" + project = get_project(db) + + # Use a config that will generate warnings (temperature on reasoning model) + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="o1", # Reasoning model + temperature=0.7, # This will be suppressed with warning + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + # Verify the result includes warnings in metadata + assert result["success"] + assert "metadata" in result + assert "warnings" in result["metadata"] + assert len(result["metadata"]["warnings"]) == 1 + assert "temperature" in result["metadata"]["warnings"][0].lower() + assert "suppressed" in result["metadata"]["warnings"][0] + + def test_kaapi_config_warnings_merged_with_existing_metadata( + self, db, job_for_execution, mock_llm_response + ): + """Test that warnings are merged with existing request metadata.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", # Non-reasoning model + reasoning="high", # This will be suppressed with warning + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + kaapi_request_data = { + "query": {"input": "Test query"}, + "config": { + "id": str(config.id), + "version": 1, + }, + "include_provider_raw_response": False, + "callback_url": None, + "request_metadata": {"tracking_id": "test-123"}, + } + + with ( + patch("app.services.llm.jobs.Session") as mock_session_class, + patch("app.services.llm.jobs.get_llm_provider") as mock_get_provider, + ): + mock_session_class.return_value.__enter__.return_value = db + mock_session_class.return_value.__exit__.return_value = None + + mock_provider = MagicMock() + mock_provider.execute.return_value = (mock_llm_response, None) + mock_get_provider.return_value = mock_provider + + result = self._execute_job(job_for_execution, db, kaapi_request_data) + + # Verify warnings are added to existing metadata + assert result["success"] + assert "metadata" in result + assert result["metadata"]["tracking_id"] == "test-123" + assert "warnings" in result["metadata"] + assert len(result["metadata"]["warnings"]) == 1 + assert "reasoning" in result["metadata"]["warnings"][0].lower() + assert "does not support reasoning" in result["metadata"]["warnings"][0] + class TestResolveConfigBlob: """Test suite for resolve_config_blob function.""" @@ -532,8 +731,8 @@ def test_resolve_config_blob_success(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.8}, ) ) @@ -549,7 +748,7 @@ def test_resolve_config_blob_success(self, db: Session): assert error is None assert resolved_blob is not None - assert resolved_blob.completion.provider == "openai" + assert resolved_blob.completion.provider == "openai-native" assert resolved_blob.completion.params["model"] == "gpt-4" assert resolved_blob.completion.params["temperature"] == 0.8 @@ -558,8 +757,8 @@ def test_resolve_config_blob_version_not_found(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -583,8 +782,8 @@ def test_resolve_config_blob_invalid_blob_data(self, db: Session): project = get_project(db) config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4"}, ) ) @@ -620,8 +819,8 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): # Create a config with version 1 config_blob_v1 = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-3.5-turbo", "temperature": 0.5}, ) ) @@ -635,8 +834,8 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): session=db, project_id=project.id, config_id=config.id ) config_blob_v2 = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={"model": "gpt-4", "temperature": 0.9}, ) ) @@ -668,3 +867,92 @@ def test_resolve_config_blob_with_multiple_versions(self, db: Session): assert resolved_blob_v2 is not None assert resolved_blob_v2.completion.params["model"] == "gpt-4" assert resolved_blob_v2.completion.params["temperature"] == 0.9 + + def test_resolve_kaapi_config_blob_success(self, db: Session): + """Test successful resolution of stored Kaapi config blob.""" + project = get_project(db) + + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.8, + instructions="You are a helpful assistant", + ), + ) + ) + config = create_test_config(db, project_id=project.id, config_blob=config_blob) + db.commit() + + config_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=config.id + ) + llm_call_config = LLMCallConfig(id=str(config.id), version=1) + + resolved_blob, error = resolve_config_blob(config_crud, llm_call_config) + + assert error is None + assert resolved_blob is not None + assert isinstance(resolved_blob.completion, KaapiCompletionConfig) + assert resolved_blob.completion.provider == "openai" + assert resolved_blob.completion.params.model == "gpt-4" + assert resolved_blob.completion.params.temperature == 0.8 + assert ( + resolved_blob.completion.params.instructions + == "You are a helpful assistant" + ) + + def test_resolve_both_native_and_kaapi_configs(self, db: Session): + """Test that both native and Kaapi configs can be resolved correctly.""" + project = get_project(db) + + # Create native config + native_blob = ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={"model": "gpt-3.5-turbo", "temperature": 0.5}, + ) + ) + native_config = create_test_config( + db, project_id=project.id, config_blob=native_blob, use_kaapi_schema=False + ) + + # Create Kaapi config + kaapi_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + ) + kaapi_config = create_test_config( + db, project_id=project.id, config_blob=kaapi_blob, use_kaapi_schema=True + ) + db.commit() + + # Test native config resolution + native_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=native_config.id + ) + native_call_config = LLMCallConfig(id=str(native_config.id), version=1) + resolved_native, error_native = resolve_config_blob( + native_crud, native_call_config + ) + + assert error_native is None + assert isinstance(resolved_native.completion, NativeCompletionConfig) + assert resolved_native.completion.provider == "openai-native" + + # Test Kaapi config resolution + kaapi_crud = ConfigVersionCrud( + session=db, project_id=project.id, config_id=kaapi_config.id + ) + kaapi_call_config = LLMCallConfig(id=str(kaapi_config.id), version=1) + resolved_kaapi, error_kaapi = resolve_config_blob(kaapi_crud, kaapi_call_config) + + assert error_kaapi is None + assert isinstance(resolved_kaapi.completion, KaapiCompletionConfig) + assert resolved_kaapi.completion.provider == "openai" diff --git a/backend/app/tests/services/llm/test_mappers.py b/backend/app/tests/services/llm/test_mappers.py new file mode 100644 index 000000000..c020753d2 --- /dev/null +++ b/backend/app/tests/services/llm/test_mappers.py @@ -0,0 +1,316 @@ +""" +Unit tests for LLM parameter mapping functions. + +Tests the transformation of Kaapi-abstracted parameters to provider-native formats. +""" +import pytest + +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig +from app.services.llm.mappers import ( + map_kaapi_to_openai_params, + transform_kaapi_config_to_native, +) + + +class TestMapKaapiToOpenAIParams: + """Test cases for map_kaapi_to_openai_params function.""" + + def test_basic_model_mapping(self): + """Test basic model parameter mapping.""" + kaapi_params = KaapiLLMParams(model="gpt-4o") + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result == {"model": "gpt-4o"} + assert warnings == [] + + def test_instructions_mapping(self): + """Test instructions parameter mapping.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + instructions="You are a helpful assistant.", + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert result["instructions"] == "You are a helpful assistant." + assert warnings == [] + + def test_temperature_mapping(self): + """Test temperature parameter mapping for non-reasoning models.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert result["temperature"] == 0.7 + assert warnings == [] + + def test_temperature_zero_mapping(self): + """Test that temperature=0 is correctly mapped (edge case).""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + temperature=0.0, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["temperature"] == 0.0 + assert warnings == [] + + def test_reasoning_mapping_for_reasoning_models(self): + """Test reasoning parameter mapping to OpenAI format for reasoning-capable models.""" + kaapi_params = KaapiLLMParams( + model="o1", + reasoning="high", + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "o1" + assert result["reasoning"] == {"effort": "high"} + assert warnings == [] + + def test_knowledge_base_ids_mapping(self): + """Test knowledge_base_ids mapping to OpenAI tools format.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_abc123", "vs_def456"], + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "tools" in result + assert len(result["tools"]) == 1 + assert result["tools"][0]["type"] == "file_search" + assert result["tools"][0]["vector_store_ids"] == ["vs_abc123", "vs_def456"] + assert result["tools"][0]["max_num_results"] == 20 # default + assert warnings == [] + + def test_knowledge_base_with_max_num_results(self): + """Test knowledge_base_ids with custom max_num_results.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_abc123"], + max_num_results=50, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["tools"][0]["max_num_results"] == 50 + assert warnings == [] + + def test_complete_parameter_mapping(self): + """Test mapping all compatible parameters together.""" + kaapi_params = KaapiLLMParams( + model="gpt-4o", + instructions="You are an expert assistant.", + temperature=0.8, + knowledge_base_ids=["vs_123"], + max_num_results=30, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4o" + assert result["instructions"] == "You are an expert assistant." + assert result["temperature"] == 0.8 + assert result["tools"][0]["type"] == "file_search" + assert result["tools"][0]["vector_store_ids"] == ["vs_123"] + assert result["tools"][0]["max_num_results"] == 30 + assert warnings == [] + + def test_reasoning_suppressed_for_non_reasoning_models(self): + """Test that reasoning is suppressed with warning for non-reasoning models.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + reasoning="high", + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "reasoning" not in result + assert len(warnings) == 1 + assert "reasoning" in warnings[0].lower() + assert "does not support reasoning" in warnings[0] + + def test_temperature_suppressed_for_reasoning_models(self): + """Test that temperature is suppressed with warning for reasoning models when reasoning is set.""" + kaapi_params = KaapiLLMParams( + model="o1", + temperature=0.7, + reasoning="high", + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "o1" + assert result["reasoning"] == {"effort": "high"} + assert "temperature" not in result + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] + + def test_temperature_without_reasoning_for_reasoning_models(self): + """Test that temperature is suppressed for reasoning models even without explicit reasoning parameter.""" + kaapi_params = KaapiLLMParams( + model="o1", + temperature=0.7, + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "o1" + assert "temperature" not in result + assert "reasoning" not in result + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] + + def test_minimal_params(self): + """Test mapping with minimal parameters (only model).""" + kaapi_params = KaapiLLMParams(model="gpt-4") + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result == {"model": "gpt-4"} + assert warnings == [] + + def test_only_knowledge_base_ids(self): + """Test mapping with only knowledge_base_ids and model.""" + kaapi_params = KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_xyz"], + ) + + result, warnings = map_kaapi_to_openai_params(kaapi_params) + + assert result["model"] == "gpt-4" + assert "tools" in result + assert result["tools"][0]["vector_store_ids"] == ["vs_xyz"] + assert warnings == [] + + +class TestTransformKaapiConfigToNative: + """Test cases for transform_kaapi_config_to_native function.""" + + def test_transform_openai_config(self): + """Test transformation of Kaapi OpenAI config to native format.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + + result, warnings = transform_kaapi_config_to_native(kaapi_config) + + assert isinstance(result, NativeCompletionConfig) + assert result.provider == "openai-native" + assert result.params["model"] == "gpt-4" + assert result.params["temperature"] == 0.7 + assert warnings == [] + + def test_transform_with_all_params(self): + """Test transformation with all Kaapi parameters.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4o", + instructions="System prompt here", + temperature=0.5, + knowledge_base_ids=["vs_abc"], + max_num_results=25, + ), + ) + + result, warnings = transform_kaapi_config_to_native(kaapi_config) + + assert result.provider == "openai-native" + assert result.params["model"] == "gpt-4o" + assert result.params["instructions"] == "System prompt here" + assert result.params["temperature"] == 0.5 + assert result.params["tools"][0]["type"] == "file_search" + assert result.params["tools"][0]["max_num_results"] == 25 + assert warnings == [] + + def test_transform_with_reasoning(self): + """Test transformation with reasoning parameter for reasoning-capable models.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="o1", + reasoning="medium", + ), + ) + + result, warnings = transform_kaapi_config_to_native(kaapi_config) + + assert result.provider == "openai-native" + assert result.params["model"] == "o1" + assert result.params["reasoning"] == {"effort": "medium"} + assert warnings == [] + + def test_transform_with_both_temperature_and_reasoning(self): + """Test that transformation handles temperature + reasoning intelligently for reasoning models.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="o1", + temperature=0.7, + reasoning="high", + ), + ) + + result, warnings = transform_kaapi_config_to_native(kaapi_config) + + assert result.provider == "openai-native" + assert result.params["model"] == "o1" + assert result.params["reasoning"] == {"effort": "high"} + assert "temperature" not in result.params + assert len(warnings) == 1 + assert "temperature" in warnings[0].lower() + assert "suppressed" in warnings[0] + + def test_unsupported_provider_raises_error(self): + """Test that unsupported providers raise ValueError.""" + # Note: This would require modifying KaapiCompletionConfig to accept other providers + # For now, this tests the error handling in the mapper + # We'll create a mock config that bypasses validation + from unittest.mock import MagicMock + + mock_config = MagicMock() + mock_config.provider = "unsupported-provider" + mock_config.params = KaapiLLMParams(model="some-model") + + with pytest.raises(ValueError) as exc_info: + transform_kaapi_config_to_native(mock_config) + + assert "Unsupported provider" in str(exc_info.value) + + def test_transform_preserves_param_structure(self): + """Test that transformation correctly structures nested parameters.""" + kaapi_config = KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + knowledge_base_ids=["vs_1", "vs_2", "vs_3"], + max_num_results=15, + ), + ) + + result, warnings = transform_kaapi_config_to_native(kaapi_config) + + # Verify the nested structure is correct + assert isinstance(result.params["tools"], list) + assert isinstance(result.params["tools"][0], dict) + assert isinstance(result.params["tools"][0]["vector_store_ids"], list) + assert len(result.params["tools"][0]["vector_store_ids"]) == 3 + assert warnings == [] diff --git a/backend/app/tests/utils/test_data.py b/backend/app/tests/utils/test_data.py index b33656b22..bc871dbcc 100644 --- a/backend/app/tests/utils/test_data.py +++ b/backend/app/tests/utils/test_data.py @@ -9,7 +9,6 @@ OrganizationCreate, ProjectCreate, ConfigBlob, - CompletionConfig, CredsCreate, FineTuningJobCreate, Fine_Tuning, @@ -22,6 +21,7 @@ ConfigVersionCreate, ConfigBase, ) +from app.models.llm import KaapiLLMParams, KaapiCompletionConfig, NativeCompletionConfig from app.crud import ( create_organization, create_project, @@ -242,11 +242,20 @@ def create_test_config( name: str | None = None, description: str | None = None, config_blob: ConfigBlob | None = None, + use_kaapi_schema: bool = False, ) -> Config: """ Creates and returns a test configuration with an initial version. Persists the config and version to the database. + + Args: + db: Database session + project_id: Project ID (creates new project if None) + name: Config name (generates random if None) + description: Config description + config_blob: Config blob (creates default if None) + use_kaapi_schema: If True, creates Kaapi-format config; if False, creates native format """ if project_id is None: project = create_test_project(db) @@ -256,16 +265,29 @@ def create_test_config( name = f"test-config-{random_lower_string()}" if config_blob is None: - config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", - params={ - "model": "gpt-4", - "temperature": 0.7, - "max_tokens": 1000, - }, + if use_kaapi_schema: + # Create Kaapi-format config + config_blob = ConfigBlob( + completion=KaapiCompletionConfig( + provider="openai", + params=KaapiLLMParams( + model="gpt-4", + temperature=0.7, + ), + ) + ) + else: + # Create native-format config + config_blob = ConfigBlob( + completion=NativeCompletionConfig( + provider="openai-native", + params={ + "model": "gpt-4", + "temperature": 0.7, + "max_tokens": 1000, + }, + ) ) - ) config_create = ConfigCreate( name=name, @@ -294,8 +316,8 @@ def create_test_version( """ if config_blob is None: config_blob = ConfigBlob( - completion=CompletionConfig( - provider="openai", + completion=NativeCompletionConfig( + provider="openai-native", params={ "model": "gpt-4", "temperature": 0.8,