Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
### Analyzer
#### Fixed
- `BasicLangExtractRecognizer` now honours values under `langextract.model.provider.language_model_params` (including `timeout` and `num_ctx`). Previously these were silently dropped because `langextract.extract()` ignores its `language_model_params` argument when a pre-built `ModelConfig` is passed via `config=`, causing Ollama-backed recognizers to fall back to langextract's 120s default regardless of the configured timeout. The recognizer now merges `language_model_params` into `ModelConfig.provider_kwargs`, which is the path that reaches the provider constructor. Explicit entries under `provider.kwargs:` still take precedence. Also fixed a `TypeError` when `kwargs:` or `language_model_params:` is `null` in the YAML. (#1943, Thanks @lsternlicht)
- Preserved LangExtract recognizer YAML options such as `config_path` and Azure OpenAI constructor parameters through recognizer registry validation.

### Anonymizer
### General
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
from .language_validation import validate_language_codes
from .schemas import ConfigurationValidator
from .yaml_recognizer_models import (
AzureOpenAILangExtractRecognizerConfig,
BaseRecognizerConfig,
BasicLangExtractRecognizerConfig,
CustomRecognizerConfig,
GLiNERRecognizerConfig,
HuggingFaceRecognizerConfig,
Expand All @@ -15,7 +17,9 @@
__all__ = [
"validate_language_codes",
"ConfigurationValidator",
"AzureOpenAILangExtractRecognizerConfig",
"BaseRecognizerConfig",
"BasicLangExtractRecognizerConfig",
"CustomRecognizerConfig",
"GLiNERRecognizerConfig",
"HuggingFaceRecognizerConfig",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,32 @@ def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
return super().model_dump(*args, **kwargs)


class BasicLangExtractRecognizerConfig(PredefinedRecognizerConfig):
"""Configuration specifically for Basic LangExtract recognizers."""

config_path: Optional[str] = Field(None, description="LangExtract config path")

def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
"""Serialize the config without None values by default."""
kwargs.setdefault("exclude_none", True)
return super().model_dump(*args, **kwargs)


class AzureOpenAILangExtractRecognizerConfig(PredefinedRecognizerConfig):
"""Configuration specifically for Azure OpenAI LangExtract recognizers."""

model_id: Optional[str] = Field(None, description="Azure OpenAI deployment name")
config_path: Optional[str] = Field(None, description="LangExtract config path")
azure_endpoint: Optional[str] = Field(None, description="Azure OpenAI endpoint")
api_key: Optional[str] = Field(None, description="Azure OpenAI API key")
api_version: Optional[str] = Field(None, description="Azure OpenAI API version")

def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
"""Serialize the config without None values by default."""
kwargs.setdefault("exclude_none", True)
return super().model_dump(*args, **kwargs)


class CustomRecognizerConfig(BaseRecognizerConfig):
"""Configuration for custom pattern-based recognizers."""

Expand Down Expand Up @@ -534,4 +560,6 @@ def validate_language_presence(self):
CONFIG_MODEL_MAP: Dict[str, Type[BaseModel]] = {
"HuggingFaceNerRecognizer": HuggingFaceRecognizerConfig,
"GLiNERRecognizer": GLiNERRecognizerConfig,
"BasicLangExtractRecognizer": BasicLangExtractRecognizerConfig,
"AzureOpenAILangExtractRecognizer": AzureOpenAILangExtractRecognizerConfig,
}
105 changes: 105 additions & 0 deletions presidio-analyzer/tests/test_yaml_recognizer_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -829,6 +829,111 @@ def test_huggingface_recognizer_config_model_dump_excludes_none():
assert "label_mapping" not in dumped


def test_basic_langextract_recognizer_config_path():
"""Test that BasicLangExtractRecognizer config_path is preserved."""
from presidio_analyzer.input_validation.yaml_recognizer_models import (
BasicLangExtractRecognizerConfig,
RecognizerRegistryConfig,
)

registry_config = {
"recognizers": [
{
"name": "e2eollama",
"class_name": "BasicLangExtractRecognizer",
"type": "predefined",
"supported_language": "en",
"config_path": "custom-basic.yaml",
}
]
}

config = RecognizerRegistryConfig(**registry_config)
recognizer = config.recognizers[0]

assert isinstance(recognizer, BasicLangExtractRecognizerConfig)
assert recognizer.name == "e2eollama"
assert recognizer.class_name == "BasicLangExtractRecognizer"
assert recognizer.config_path == "custom-basic.yaml"


def test_azure_openai_langextract_recognizer_config_fields():
"""Test that AzureOpenAILangExtractRecognizer fields are preserved."""
from presidio_analyzer.input_validation.yaml_recognizer_models import (
AzureOpenAILangExtractRecognizerConfig,
RecognizerRegistryConfig,
)

registry_config = {
"recognizers": [
{
"name": "custom_azure_langextract",
"class_name": "AzureOpenAILangExtractRecognizer",
"type": "predefined",
"supported_language": "en",
"config_path": "custom-azure.yaml",
"model_id": "gpt-4o-deployment",
"azure_endpoint": "https://example.openai.azure.com/",
"api_key": "PLACEHOLDER_NOT_A_REAL_KEY",
"api_version": "2024-02-01",
}
]
}

config = RecognizerRegistryConfig(**registry_config)
recognizer = config.recognizers[0]

assert isinstance(recognizer, AzureOpenAILangExtractRecognizerConfig)
assert recognizer.config_path == "custom-azure.yaml"
assert recognizer.model_id == "gpt-4o-deployment"
assert recognizer.azure_endpoint == "https://example.openai.azure.com/"
assert recognizer.api_key == "PLACEHOLDER_NOT_A_REAL_KEY"
assert recognizer.api_version == "2024-02-01"


def test_configuration_validator_preserves_langextract_recognizer_fields():
"""Validated YAML should preserve LangExtract constructor kwargs."""
from presidio_analyzer.input_validation.schemas import ConfigurationValidator

raw_config = {
"supported_languages": ["en"],
"recognizers": [
{
"name": "e2eollama",
"class_name": "BasicLangExtractRecognizer",
"type": "predefined",
"supported_language": "en",
"config_path": "custom-basic.yaml",
},
{
"name": "custom_azure_langextract",
"class_name": "AzureOpenAILangExtractRecognizer",
"type": "predefined",
"supported_language": "en",
"config_path": "custom-azure.yaml",
"model_id": "gpt-4o-deployment",
"azure_endpoint": "https://example.openai.azure.com/",
"api_key": "PLACEHOLDER_NOT_A_REAL_KEY",
"api_version": "2024-02-01",
},
],
}

validated = ConfigurationValidator.validate_recognizer_registry_configuration(
raw_config
)
basic_recognizer = validated["recognizers"][0]
azure_recognizer = validated["recognizers"][1]

assert basic_recognizer["config_path"] == "custom-basic.yaml"
assert "model_id" not in basic_recognizer
assert azure_recognizer["config_path"] == "custom-azure.yaml"
assert azure_recognizer["model_id"] == "gpt-4o-deployment"
assert azure_recognizer["azure_endpoint"] == "https://example.openai.azure.com/"
assert azure_recognizer["api_key"] == "PLACEHOLDER_NOT_A_REAL_KEY"
assert azure_recognizer["api_version"] == "2024-02-01"


def test_config_model_map_fallback_to_predefined():
"""Test CONFIG_MODEL_MAP falls back to Predefined for unknown class_name."""
from presidio_analyzer.input_validation.yaml_recognizer_models import (
Expand Down
Loading