Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ All notable changes to this project will be documented in this file.

### Analyzer
#### Fixed
- `BasicLangExtractRecognizer` now honours values under `langextract.model.provider.language_model_params` (including `timeout` and `num_ctx`). Previously these were silently dropped because `langextract.extract()` ignores its `language_model_params` argument when a pre-built `ModelConfig` is passed via `config=`, causing Ollama-backed recognizers to fall back to langextract's 120s default regardless of the configured timeout. The recognizer now merges `language_model_params` into `ModelConfig.provider_kwargs`, which is the path that reaches the provider constructor. Explicit entries under `provider.kwargs:` still take precedence. Also fixed a `TypeError` when `kwargs:` or `language_model_params:` is `null` in the YAML. (#1943, Thanks @lsternlicht)
- Language model recognizers (`BasicLangExtractRecognizer`, `AzureOpenAILangExtractRecognizer`) configured in a recognizer registry YAML now honour `config_path` (and other recognizer-specific kwargs). Previously these entries were validated by the strict `PredefinedRecognizerConfig` schema, which has no `config_path` field and does not allow extra keys, so `config_path` was silently dropped and the recognizer fell back to its bundled default model configuration. Added a `LangExtractRecognizerConfig` model (`extra="allow"`) and registered both recognizer class names in `CONFIG_MODEL_MAP`.
- `BasicLangExtractRecognizer` now honours values under `langextract.model.provider.language_model_params` (including `timeout` and `num_ctx`). Previously these were silently dropped because `langextract.extract()` ignores its `language_model_params` argument when a pre-built `ModelConfig` is passed via `config=`, causing Ollama-backed recognizers to fall back to langextract's 120s default timeout regardless of the configured timeout. The recognizer now merges `language_model_params` into `ModelConfig.provider_kwargs`, which is the path that reaches the provider constructor. Explicit entries under `provider.kwargs:` still take precedence. Also fixed a `TypeError` when `kwargs:` or `language_model_params:` is `null` in the YAML. (#1943, Thanks @lsternlicht)

### Anonymizer
### General
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,34 @@ def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
return super().model_dump(*args, **kwargs)


class LangExtractRecognizerConfig(PredefinedRecognizerConfig):
"""Configuration for language model (LangExtract) recognizers.

Covers ``BasicLangExtractRecognizer`` and ``AzureOpenAILangExtractRecognizer``.
``extra="allow"`` lets recognizer-specific kwargs (most importantly
``config_path``, pointing at the langextract model YAML) survive validation
and reach the recognizer constructor. Without this, the strict
``PredefinedRecognizerConfig`` schema drops ``config_path`` and the recognizer
silently falls back to its bundled default model configuration.
"""

model_config = ConfigDict(extra="allow")

config_path: Optional[str] = Field(
None, description="Path to the langextract model configuration YAML"
)

def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
"""Serialize the config without None values by default.

LangExtract recognizer kwargs are passed directly to the recognizer
constructor. Excluding None values preserves constructor defaults for
omitted YAML fields instead of overriding them with explicit None.
"""
kwargs.setdefault("exclude_none", True)
return super().model_dump(*args, **kwargs)


class CustomRecognizerConfig(BaseRecognizerConfig):
"""Configuration for custom pattern-based recognizers."""

Expand Down Expand Up @@ -534,4 +562,6 @@ def validate_language_presence(self):
CONFIG_MODEL_MAP: Dict[str, Type[BaseModel]] = {
"HuggingFaceNerRecognizer": HuggingFaceRecognizerConfig,
"GLiNERRecognizer": GLiNERRecognizerConfig,
"BasicLangExtractRecognizer": LangExtractRecognizerConfig,
"AzureOpenAILangExtractRecognizer": LangExtractRecognizerConfig,
}
64 changes: 64 additions & 0 deletions presidio-analyzer/tests/test_yaml_recognizer_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from presidio_analyzer.input_validation.yaml_recognizer_models import (
BaseRecognizerConfig,
CustomRecognizerConfig,
LangExtractRecognizerConfig,
LanguageContextConfig,
PredefinedRecognizerConfig,
RecognizerRegistryConfig,
Expand Down Expand Up @@ -306,6 +307,69 @@ def test_configuration_validator_uses_recognizer_specific_dump_rules():
assert predefined_recognizer["supported_language"] is None


def test_langextract_config_preserves_config_path():
"""A BasicLangExtractRecognizer YAML entry must keep ``config_path``.

Regression: without a dedicated config model (extra="allow"), the strict
``PredefinedRecognizerConfig`` schema drops ``config_path``, so the recognizer
silently falls back to its bundled default model config.
"""
config = LangExtractRecognizerConfig(
name="SmLlama32_3b",
class_name="BasicLangExtractRecognizer",
supported_languages=["en"],
config_path="/path/to/langextract_config.yml",
)
assert config.config_path == "/path/to/langextract_config.yml"
assert config.model_dump()["config_path"] == "/path/to/langextract_config.yml"


def test_langextract_config_selected_via_class_name():
"""``class_name: BasicLangExtractRecognizer`` selects the LangExtract model
and preserves ``config_path`` (plus arbitrary extra kwargs) through the full
registry validation used by AnalyzerEngineProvider."""
from presidio_analyzer.input_validation.schemas import ConfigurationValidator

raw_config = {
"supported_languages": ["en"],
"recognizers": [
{
"name": "SmLlama32_3b",
"type": "predefined",
"class_name": "BasicLangExtractRecognizer",
"enabled": True,
"supported_languages": ["en"],
"config_path": "/path/to/langextract_config.yml",
}
],
}

validated = ConfigurationValidator.validate_recognizer_registry_configuration(
raw_config
)
lm_recognizer = validated["recognizers"][0]
assert lm_recognizer["class_name"] == "BasicLangExtractRecognizer"
assert lm_recognizer["config_path"] == "/path/to/langextract_config.yml"


def test_langextract_config_azure_variant_selected():
"""AzureOpenAILangExtractRecognizer also maps to the LangExtract config model."""
config = RecognizerRegistryConfig(
supported_languages=["en"],
recognizers=[
{
"name": "AzureLM",
"type": "predefined",
"class_name": "AzureOpenAILangExtractRecognizer",
"config_path": "/path/to/azure_config.yml",
}
],
)
recognizer = config.recognizers[0]
assert isinstance(recognizer, LangExtractRecognizerConfig)
assert recognizer.config_path == "/path/to/azure_config.yml"


def test_custom_recognizer_config_with_deny_list():
"""Test custom recognizer with deny list only."""
config = CustomRecognizerConfig(
Expand Down