From 0dae57c40a6a0ffeaf23452d1e79784cec5a209f Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 29 Jun 2026 09:35:31 -0300 Subject: [PATCH 1/4] Wire Foundry IQ defaults into deployment parameters Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 ++++++ README.md | 9 ++++++--- infra | 2 +- main.parameters.json | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca0918a..6de9702c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [Unreleased] + +### Fixed + +- **Fresh Foundry IQ deployments now carry native Blob settings into infrastructure.** `main.parameters.json` passes `RETRIEVAL_BACKEND=foundry_iq` and the native `FOUNDRY_IQ_*` defaults through azd/Bicep so fresh Zero Trust provisions stamp App Configuration for Foundry IQ `azureBlob` instead of silently falling back to direct Azure AI Search. + ## [v3.0.6] - 2026-06-28 ### User and operator impact diff --git a/README.md b/README.md index 7dd72b18..bba9a58e 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,12 @@ For unattended provisioning, set `AZURE_SKIP_NETWORK_ISOLATION_WARNING=true` to ## Retrieval backend configuration -GPT-RAG supports Azure AI Search as the direct retrieval backend and is adding -Foundry IQ as a first-class backend for v3 deployments. Existing deployments -should keep `RETRIEVAL_BACKEND=ai_search` until they intentionally migrate. +GPT-RAG supports Azure AI Search as the direct retrieval backend and Foundry IQ +as the default backend for fresh v3 deployments. Native Foundry IQ Blob +Knowledge Sources use `RETRIEVAL_BACKEND=foundry_iq`, +`FOUNDRY_IQ_PATTERN=azureBlob`, and `FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND=azureBlob`. +Existing deployments can keep `RETRIEVAL_BACKEND=ai_search` until they +intentionally migrate. For the operator guide, including Pattern A vs Pattern B, security modes, `knowledgeRetrieval` billing, rollback, and known limitations, see the diff --git a/infra b/infra index 6e687dd6..d9a06e24 160000 --- a/infra +++ b/infra @@ -1 +1 @@ -Subproject commit 6e687dd67b18443c9b565ebcb497b802e03a434e +Subproject commit d9a06e24461775dc6b9ee2467289b2402424ab4c diff --git a/main.parameters.json b/main.parameters.json index 42f1e1a5..d9645f00 100644 --- a/main.parameters.json +++ b/main.parameters.json @@ -23,6 +23,20 @@ "useCAppAPIKey": { "value": "${USE_CAPP_API_KEY}" }, "useZoneRedundancy": { "value": "false" }, "enableAgenticRetrieval": { "value": "${ENABLE_AGENTIC_RETRIEVAL}" }, + "retrievalBackend": { "value": "${RETRIEVAL_BACKEND=foundry_iq}" }, + "foundryIqPattern": { "value": "${FOUNDRY_IQ_PATTERN=azureBlob}" }, + "foundryIqKnowledgeSourceKind": { "value": "${FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND=azureBlob}" }, + "foundryIqApiVersion": { "value": "${FOUNDRY_IQ_API_VERSION=2026-05-01-preview}" }, + "foundryIqKnowledgeRetrievalBillingPlan": { "value": "${FOUNDRY_IQ_KNOWLEDGE_RETRIEVAL_BILLING_PLAN=free}" }, + "knowledgeBaseName": { "value": "${KNOWLEDGE_BASE_NAME=knowledge-base}" }, + "knowledgeBaseConnectionName": { "value": "${KNOWLEDGE_BASE_CONNECTION_NAME=knowledge-base-connection}" }, + "foundryIqKnowledgeSourceName": { "value": "${FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME=knowledge-base-blob-ks}" }, + "foundryIqStorageContainerName": { "value": "${FOUNDRY_IQ_STORAGE_CONTAINER_NAME=documents}" }, + "foundryIqStorageFolderPath": { "value": "${FOUNDRY_IQ_STORAGE_FOLDER_PATH=}" }, + "foundryIqIsAdlsGen2": { "value": "${FOUNDRY_IQ_IS_ADLS_GEN2=false}" }, + "foundryIqContentExtractionMode": { "value": "${FOUNDRY_IQ_CONTENT_EXTRACTION_MODE=standard}" }, + "foundryIqIngestionPermissionOptionsJson": { "value": "${FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS=[\"rbacScope\"]}" }, + "foundryIqFilterAddOnEnabled": { "value": "${FOUNDRY_IQ_FILTER_ADD_ON_ENABLED=false}" }, "aiSearchResourceId": { "value": null }, "aiFoundryStorageAccountResourceId": { "value": null }, From f3e9c2bb0a9bab93d4071d9a16bfa57fbbcac7f6 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 29 Jun 2026 09:48:49 -0300 Subject: [PATCH 2/4] Fix Foundry IQ standard extraction setup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + config/search/search.j2 | 15 +++- config/search/search.settings.j2 | 24 ++++-- config/search/setup.py | 61 +++++++++++++- .../search/tests/test_foundry_iq_templates.py | 81 +++++++++++++++++++ infra | 2 +- main.parameters.json | 1 + scripts/postProvision.ps1 | 2 + 8 files changed, 173 insertions(+), 14 deletions(-) create mode 100644 config/search/tests/test_foundry_iq_templates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6de9702c..6414bb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - **Fresh Foundry IQ deployments now carry native Blob settings into infrastructure.** `main.parameters.json` passes `RETRIEVAL_BACKEND=foundry_iq` and the native `FOUNDRY_IQ_*` defaults through azd/Bicep so fresh Zero Trust provisions stamp App Configuration for Foundry IQ `azureBlob` instead of silently falling back to direct Azure AI Search. +- **Fresh Foundry IQ standard extraction setup now seeds required search and AI Services settings.** Post-provision search setup derives `SEARCH_RAG_INDEX_NAME` before rendering Foundry IQ knowledge resources and supplies the `aiServices.uri` / chat model resource URI required by Azure AI Search when `FOUNDRY_IQ_CONTENT_EXTRACTION_MODE=standard`. ## [v3.0.6] - 2026-06-28 diff --git a/config/search/search.j2 b/config/search/search.j2 index c8d4bc8d..c34e71bd 100644 --- a/config/search/search.j2 +++ b/config/search/search.j2 @@ -1,3 +1,4 @@ +{% set is_foundry_iq_standard_blob = RETRIEVAL_BACKEND == 'foundry_iq' and FOUNDRY_IQ_PATTERN != 'searchIndex' and FOUNDRY_IQ_CONTENT_EXTRACTION_MODE == 'standard' %} { "indexes": [ { @@ -357,7 +358,14 @@ "ingestionParameters": { "identity": null, "disableImageVerbalization": false, - "chatCompletionModel": null, + "chatCompletionModel": {% if is_foundry_iq_standard_blob and GPT_MODEL_INFO %}{ + "kind": "azureOpenAI", + "azureOpenAIParameters": { + "resourceUri": "{{FOUNDRY_IQ_AI_SERVICES_ENDPOINT}}", + "deploymentId": "{{GPT_MODEL_INFO.deployment_name}}", + "modelName": "{{GPT_MODEL_INFO.model_name}}" + } + }{% else %}null{% endif %}, "embeddingModel": {% if EMBEDDING_MODEL_INFO %}{ "kind": "azureOpenAI", "azureOpenAIParameters": { @@ -368,7 +376,10 @@ }{% else %}null{% endif %}, "contentExtractionMode": "{{FOUNDRY_IQ_CONTENT_EXTRACTION_MODE}}", "ingestionSchedule": null, - "ingestionPermissionOptions": {{FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS | tojson}} + "ingestionPermissionOptions": {{FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS | tojson}}{% if is_foundry_iq_standard_blob %}, + "aiServices": { + "uri": "{{FOUNDRY_IQ_AI_SERVICES_ENDPOINT}}" + }{% endif %} } } } diff --git a/config/search/search.settings.j2 b/config/search/search.settings.j2 index ca443697..fe2e5389 100644 --- a/config/search/search.settings.j2 +++ b/config/search/search.settings.j2 @@ -1,30 +1,40 @@ +{% set search_rag_index_name = SEARCH_RAG_INDEX_NAME | default('ragindex-' ~ RESOURCE_TOKEN, true) %} +{% set foundry_iq_pattern = FOUNDRY_IQ_PATTERN | default('azureBlob', true) %} +{% set foundry_iq_ai_services_endpoint = FOUNDRY_IQ_AI_SERVICES_ENDPOINT | default('', true) %} +{% if not foundry_iq_ai_services_endpoint and AI_FOUNDRY_PROJECT_ENDPOINT is defined and AI_FOUNDRY_PROJECT_ENDPOINT %} +{% set foundry_iq_ai_services_endpoint = AI_FOUNDRY_PROJECT_ENDPOINT.split('/api/projects/')[0].rstrip('/') ~ '/' %} +{% endif %} +{% if not foundry_iq_ai_services_endpoint and AI_FOUNDRY_ACCOUNT_NAME is defined and AI_FOUNDRY_ACCOUNT_NAME %} +{% set foundry_iq_ai_services_endpoint = 'https://' ~ AI_FOUNDRY_ACCOUNT_NAME ~ '.services.ai.azure.com/' %} +{% endif %} { "SEARCH_API_VERSION": "2025-05-01-preview", "SEARCH_ANALYZER_NAME": "standard.lucene", "EMBEDDINGS_VECTOR_DIMENSIONS": "3072", - "SEARCH_RAG_INDEX_NAME": "ragindex-{{RESOURCE_TOKEN}}", + "SEARCH_RAG_INDEX_NAME": "{{search_rag_index_name}}", "SEARCH_QUERIES_INDEX_NAME": "nl2sql-{{RESOURCE_TOKEN}}-queries", "SEARCH_TABLES_INDEX_NAME": "nl2sql-{{RESOURCE_TOKEN}}-tables", "SEARCH_MEASURES_INDEX_NAME": "nl2sql-{{RESOURCE_TOKEN}}-measures", "ENABLE_AGENTIC_RETRIEVAL": "{{ENABLE_AGENTIC_RETRIEVAL | default('false')}}", "RETRIEVAL_BACKEND": "{{RETRIEVAL_BACKEND | default('ai_search')}}", - "FOUNDRY_IQ_PATTERN": "{{FOUNDRY_IQ_PATTERN | default('azureBlob')}}", - "KNOWLEDGE_BASE_NAME": "{{KNOWLEDGE_BASE_NAME | default(SEARCH_RAG_INDEX_NAME ~ '-rag-kb')}}", + "FOUNDRY_IQ_PATTERN": "{{foundry_iq_pattern}}", + "KNOWLEDGE_BASE_NAME": "{{KNOWLEDGE_BASE_NAME | default(search_rag_index_name ~ '-rag-kb', true)}}", "KNOWLEDGE_BASE_ENDPOINT": "{{KNOWLEDGE_BASE_ENDPOINT | default(SEARCH_SERVICE_QUERY_ENDPOINT)}}", "KNOWLEDGE_BASE_CONNECTION_ID": "{{KNOWLEDGE_BASE_CONNECTION_ID | default('')}}", "FOUNDRY_IQ_API_VERSION": "{{FOUNDRY_IQ_API_VERSION | default('2026-05-01-preview')}}", "FOUNDRY_IQ_KNOWLEDGE_RETRIEVAL_BILLING_PLAN": "{{FOUNDRY_IQ_KNOWLEDGE_RETRIEVAL_BILLING_PLAN | default('free')}}", - "FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME": "{{FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME | default(SEARCH_RAG_INDEX_NAME ~ ('-rag-ks' if (FOUNDRY_IQ_PATTERN | default('azureBlob')) == 'searchIndex' else '-blob-ks'))}}", - "FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND": "{{FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND | default('searchIndex' if (FOUNDRY_IQ_PATTERN | default('azureBlob')) == 'searchIndex' else 'azureBlob')}}", + "FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME": "{{FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME | default(search_rag_index_name ~ ('-rag-ks' if foundry_iq_pattern == 'searchIndex' else '-blob-ks'), true)}}", + "FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND": "{{FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND | default('searchIndex' if foundry_iq_pattern == 'searchIndex' else 'azureBlob', true)}}", "FOUNDRY_IQ_STORAGE_CONTAINER_NAME": "{{FOUNDRY_IQ_STORAGE_CONTAINER_NAME | default('documents')}}", "FOUNDRY_IQ_STORAGE_FOLDER_PATH": "{{FOUNDRY_IQ_STORAGE_FOLDER_PATH | default('')}}", "FOUNDRY_IQ_IS_ADLS_GEN2": "{{FOUNDRY_IQ_IS_ADLS_GEN2 | default('false')}}", - "FOUNDRY_IQ_CONTENT_EXTRACTION_MODE": "{{FOUNDRY_IQ_CONTENT_EXTRACTION_MODE | default('minimal')}}", + "FOUNDRY_IQ_CONTENT_EXTRACTION_MODE": "{{FOUNDRY_IQ_CONTENT_EXTRACTION_MODE | default('standard')}}", + "FOUNDRY_IQ_AI_SERVICES_ENDPOINT": "{{foundry_iq_ai_services_endpoint}}", "FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS": {{FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS | default(['rbacScope']) | tojson}}, "FOUNDRY_IQ_FILTER_ADD_ON_ENABLED": "{{FOUNDRY_IQ_FILTER_ADD_ON_ENABLED | default('false')}}", "FOUNDRY_IQ_SECURITY_FIELD_NAME": "{{FOUNDRY_IQ_SECURITY_FIELD_NAME | default('metadata_security_id')}}", "FOUNDRY_IQ_MAX_OUTPUT_DOCUMENTS": "{{FOUNDRY_IQ_MAX_OUTPUT_DOCUMENTS | default('')}}", - {% if GPT_MODEL_INFO %} + {% if GPT_MODEL_INFO | default({}) %} "GPT_MODEL_DEPLOYMENT_NAME": "{{GPT_MODEL_INFO.deployment_name}}", "GPT_MODEL_NAME": "{{GPT_MODEL_INFO.model_name}}" {% else %} diff --git a/config/search/setup.py b/config/search/setup.py index fc61e427..ddad1368 100644 --- a/config/search/setup.py +++ b/config/search/setup.py @@ -89,6 +89,51 @@ def normalize_foundry_iq_settings(settings: Dict[str, Any]) -> Dict[str, Any]: return settings +def normalize_endpoint_uri(value: Any) -> str: + endpoint = str(value or "").strip() + if not endpoint: + return "" + return endpoint.rstrip("/") + "/" + + +def derive_foundry_iq_ai_services_endpoint(settings: Dict[str, Any]) -> str: + endpoint = normalize_endpoint_uri(settings.get("FOUNDRY_IQ_AI_SERVICES_ENDPOINT")) + if endpoint: + return endpoint + + project_endpoint = str(settings.get("AI_FOUNDRY_PROJECT_ENDPOINT") or "").strip() + if "/api/projects/" in project_endpoint: + return normalize_endpoint_uri(project_endpoint.split("/api/projects/", 1)[0]) + + account_name = str(settings.get("AI_FOUNDRY_ACCOUNT_NAME") or "").strip() + if account_name: + return f"https://{account_name}.services.ai.azure.com/" + + return "" + + +def is_foundry_iq_standard_blob(settings: Dict[str, Any]) -> bool: + retrieval_backend = str(settings.get("RETRIEVAL_BACKEND") or "").lower() + pattern = str(settings.get("FOUNDRY_IQ_PATTERN") or "").lower() + kind = str(settings.get("FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND") or "").lower() + mode = str(settings.get("FOUNDRY_IQ_CONTENT_EXTRACTION_MODE") or "").lower() + return ( + retrieval_backend == "foundry_iq" + and pattern != "searchindex" + and kind == "azureblob" + and mode == "standard" + ) + + +def validate_foundry_iq_settings(settings: Dict[str, Any]) -> None: + if is_foundry_iq_standard_blob(settings) and not settings.get("FOUNDRY_IQ_AI_SERVICES_ENDPOINT"): + raise ValueError( + "FOUNDRY_IQ_CONTENT_EXTRACTION_MODE is set to 'standard', but no AI Services endpoint " + "could be derived. Set FOUNDRY_IQ_AI_SERVICES_ENDPOINT to the Foundry resource endpoint " + "on services.ai.azure.com, for example 'https://.services.ai.azure.com/'." + ) + + def load_appconfig_settings(ac_client: AzureAppConfigurationClient, label_filter: Optional[str] = None) -> Dict[str, Any]: """ Reads all settings from App Configuration under given label_filter (or None for no label). @@ -171,10 +216,10 @@ def extract_gpt_model_info(ctx: dict) -> dict: model_obj = model.get("model") if isinstance(model_obj, dict): model_name = model_obj.get("name") - model_format = model_obj.get("format") + model_format = model_obj.get("format") or model.get("modelFormat") else: model_name = model_obj - model_format = None + model_format = model.get("modelFormat") gpt_info = { "deployment_name": model.get("name"), @@ -254,6 +299,15 @@ def render_and_parse_json(template_name_inner: str, ctx: dict) -> Optional[dict] if vars_dict: vars_dict = normalize_foundry_iq_settings(normalize_json_like_settings(vars_dict)) context.update(vars_dict) + ai_services_endpoint = derive_foundry_iq_ai_services_endpoint(context) + if ai_services_endpoint: + context["FOUNDRY_IQ_AI_SERVICES_ENDPOINT"] = ai_services_endpoint + vars_dict["FOUNDRY_IQ_AI_SERVICES_ENDPOINT"] = ai_services_endpoint + try: + validate_foundry_iq_settings(context) + except ValueError as ve: + logging.error(str(ve)) + return None, context for key, val in vars_dict.items(): if isinstance(val, (dict, list)): final_val = json.dumps(val) @@ -445,8 +499,7 @@ def provision_knowledge_bases(defs: dict, context: dict, cred: ChainedTokenCrede # ── Main Provisioning to AI Search elements (datasources, indexes, skillset and indexers) ───────────────────── def execute_setup(defs: Optional[dict], context: dict): if defs is None: - logging.error("No search definitions to provision. Skipping setup.") - return + raise RuntimeError("No search definitions were rendered; aborting Azure Search setup") cred = ChainedTokenCredential(AzureCliCredential(),ManagedIdentityCredential()) indexers = defs.get("indexers", []) ds_to_indexers = {} diff --git a/config/search/tests/test_foundry_iq_templates.py b/config/search/tests/test_foundry_iq_templates.py new file mode 100644 index 00000000..e977bc2d --- /dev/null +++ b/config/search/tests/test_foundry_iq_templates.py @@ -0,0 +1,81 @@ +import json +import unittest +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + + +TEMPLATE_DIR = Path(__file__).resolve().parents[1] + + +def render_json_template(template_name, context): + env = Environment( + loader=FileSystemLoader(TEMPLATE_DIR), + undefined=StrictUndefined, + keep_trailing_newline=True, + ) + rendered = env.get_template(template_name).render(**context) + return json.loads(rendered) + + +class FoundryIqTemplateTests(unittest.TestCase): + def test_settings_template_derives_search_index_before_dependent_defaults(self): + settings = render_json_template( + "search.settings.j2", + { + "RESOURCE_TOKEN": "abc123", + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "AI_FOUNDRY_PROJECT_ENDPOINT": "https://aif-abc123.services.ai.azure.com/api/projects/proj", + }, + ) + + self.assertEqual(settings["SEARCH_RAG_INDEX_NAME"], "ragindex-abc123") + self.assertEqual(settings["KNOWLEDGE_BASE_NAME"], "ragindex-abc123-rag-kb") + self.assertEqual(settings["FOUNDRY_IQ_KNOWLEDGE_SOURCE_NAME"], "ragindex-abc123-blob-ks") + self.assertEqual( + settings["FOUNDRY_IQ_AI_SERVICES_ENDPOINT"], + "https://aif-abc123.services.ai.azure.com/", + ) + + def test_standard_blob_knowledge_source_includes_ai_services_endpoint(self): + settings = render_json_template( + "search.settings.j2", + { + "RESOURCE_TOKEN": "abc123", + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "AI_FOUNDRY_ACCOUNT_NAME": "aif-abc123", + "RETRIEVAL_BACKEND": "foundry_iq", + "FOUNDRY_IQ_CONTENT_EXTRACTION_MODE": "standard", + }, + ) + context = { + **settings, + "STORAGE_ACCOUNT_RESOURCE_ID": "/subscriptions/s/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/st", + "EMBEDDING_MODEL_INFO": { + "endpoint": "https://aif-abc123.openai.azure.com/", + "deployment_name": "text-embedding", + "model_name": "text-embedding-3-large", + }, + "GPT_MODEL_INFO": { + "deployment_name": "chat", + "model_name": "gpt-5-nano", + }, + } + + search_definitions = render_json_template("search.j2", context) + knowledge_source = search_definitions["knowledgeSources"][0] + ingestion_parameters = knowledge_source["azureBlobParameters"]["ingestionParameters"] + + self.assertEqual(ingestion_parameters["contentExtractionMode"], "standard") + self.assertEqual( + ingestion_parameters["aiServices"]["uri"], + "https://aif-abc123.services.ai.azure.com/", + ) + self.assertEqual( + ingestion_parameters["chatCompletionModel"]["azureOpenAIParameters"]["resourceUri"], + ingestion_parameters["aiServices"]["uri"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra b/infra index d9a06e24..55dc46cf 160000 --- a/infra +++ b/infra @@ -1 +1 @@ -Subproject commit d9a06e24461775dc6b9ee2467289b2402424ab4c +Subproject commit 55dc46cfed5269212cb79c999586aec9548e1155 diff --git a/main.parameters.json b/main.parameters.json index d9645f00..7e593d69 100644 --- a/main.parameters.json +++ b/main.parameters.json @@ -35,6 +35,7 @@ "foundryIqStorageFolderPath": { "value": "${FOUNDRY_IQ_STORAGE_FOLDER_PATH=}" }, "foundryIqIsAdlsGen2": { "value": "${FOUNDRY_IQ_IS_ADLS_GEN2=false}" }, "foundryIqContentExtractionMode": { "value": "${FOUNDRY_IQ_CONTENT_EXTRACTION_MODE=standard}" }, + "foundryIqAiServicesEndpoint": { "value": "${FOUNDRY_IQ_AI_SERVICES_ENDPOINT=}" }, "foundryIqIngestionPermissionOptionsJson": { "value": "${FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS=[\"rbacScope\"]}" }, "foundryIqFilterAddOnEnabled": { "value": "${FOUNDRY_IQ_FILTER_ADD_ON_ENABLED=false}" }, diff --git a/scripts/postProvision.ps1 b/scripts/postProvision.ps1 index 8b54120c..ba72843d 100755 --- a/scripts/postProvision.ps1 +++ b/scripts/postProvision.ps1 @@ -278,6 +278,7 @@ function Set-GptRagAppConfiguration { ENVIRONMENT_NAME = $environmentName DEPLOYMENT_NAME = $deploymentName RESOURCE_TOKEN = $resourceToken + SEARCH_RAG_INDEX_NAME = $ragIndexName ENABLE_AGENTIC_RETRIEVAL = (Get-OptionalEnvValue 'ENABLE_AGENTIC_RETRIEVAL' 'false') RETRIEVAL_BACKEND = $retrievalBackend FOUNDRY_IQ_PATTERN = $foundryIqPattern @@ -289,6 +290,7 @@ function Set-GptRagAppConfiguration { FOUNDRY_IQ_STORAGE_FOLDER_PATH = (Get-OptionalEnvValue 'FOUNDRY_IQ_STORAGE_FOLDER_PATH') FOUNDRY_IQ_IS_ADLS_GEN2 = (Get-OptionalEnvValue 'FOUNDRY_IQ_IS_ADLS_GEN2' 'false') FOUNDRY_IQ_CONTENT_EXTRACTION_MODE = (Get-OptionalEnvValue 'FOUNDRY_IQ_CONTENT_EXTRACTION_MODE' 'standard') + FOUNDRY_IQ_AI_SERVICES_ENDPOINT = (Get-OptionalEnvValue 'FOUNDRY_IQ_AI_SERVICES_ENDPOINT' "https://$foundryName.services.ai.azure.com/") FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS = (Get-OptionalEnvValue 'FOUNDRY_IQ_INGESTION_PERMISSION_OPTIONS' '["rbacScope"]') FOUNDRY_IQ_SEARCH_INDEX_NAME = (Get-OptionalEnvValue 'FOUNDRY_IQ_SEARCH_INDEX_NAME' $ragIndexName) FOUNDRY_IQ_SEMANTIC_CONFIGURATION_NAME = (Get-OptionalEnvValue 'FOUNDRY_IQ_SEMANTIC_CONFIGURATION_NAME' 'semantic-config') From dc0afd97a3eef64cea0a9a433394b680c6c79574 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 29 Jun 2026 14:03:35 -0300 Subject: [PATCH 3/4] release: v3.0.7 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ manifest.json | 4 ++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca0918a..7112b4e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [v3.0.7] - 2026-06-29 + +### User and operator impact + +NL2SQL deployments now consume the orchestrator patch that strengthens the +read-only enforcement before SQL execution. This umbrella release pins +`gpt-rag-orchestrator` to `v3.0.4`, which rejects stacked SQL batches and +non-read-only SQL before datasource access. Operators should still keep every +SQL Server, Azure SQL, or Fabric SQL datasource principal least-privilege and +read-only. + +### Changed + +- **Orchestrator pin bumped to [`v3.0.4`](https://github.com/Azure/gpt-rag-orchestrator/releases/tag/v3.0.4):** consumes the NL2SQL read-only enforcement hardening from [Azure/gpt-rag-orchestrator#256](https://github.com/Azure/gpt-rag-orchestrator/pull/256). UI, ingestion, and AI Landing Zone pins are unchanged from `v3.0.6`. + +### Validation + +- Release metadata validation passed for `manifest.json` and `CHANGELOG.md`. + +The following component versions are pinned for this release: + +| Component | Version | +| --- | --- | +| gpt-rag-ui | v2.3.13 | +| gpt-rag-orchestrator | v3.0.4 | +| gpt-rag-ingestion | v2.4.14 | +| bicep-ptn-aiml-landing-zone | v2.1.4 | + ## [v3.0.6] - 2026-06-28 ### User and operator impact diff --git a/manifest.json b/manifest.json index 171f83e6..63a52191 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { - "tag": "v3.0.6", + "tag": "v3.0.7", "repo": "https://github.com/azure/gpt-rag.git", "ailz_tag": "v2.1.4", "components": [ @@ -11,7 +11,7 @@ { "name": "gpt-rag-orchestrator", "repo": "https://github.com/azure/gpt-rag-orchestrator.git", - "tag": "v3.0.3" + "tag": "v3.0.4" }, { "name": "gpt-rag-ingestion", From ebd89abf6b6b79932cb5ca27b145178b0fccba7e Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 29 Jun 2026 18:13:10 -0300 Subject: [PATCH 4/4] Fix Foundry IQ Zero Trust provisioning Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- config/search/search.j2 | 4 +- config/search/setup.py | 54 +++++++++ .../search/tests/test_foundry_iq_templates.py | 104 +++++++++++++++++- infra | 2 +- scripts/Invoke-RegionalPreflight.ps1 | 33 ++++++ 5 files changed, 194 insertions(+), 3 deletions(-) diff --git a/config/search/search.j2 b/config/search/search.j2 index c34e71bd..1588e67e 100644 --- a/config/search/search.j2 +++ b/config/search/search.j2 @@ -1,4 +1,6 @@ {% set is_foundry_iq_standard_blob = RETRIEVAL_BACKEND == 'foundry_iq' and FOUNDRY_IQ_PATTERN != 'searchIndex' and FOUNDRY_IQ_CONTENT_EXTRACTION_MODE == 'standard' %} +{% set content_understanding_chat_models = ['gpt-4.1', 'gpt-4.1-mini', 'gpt-4.1-nano', 'gpt-4o', 'gpt-4o-mini', 'gpt-5.2'] %} +{% set is_content_understanding_chat_model_supported = GPT_MODEL_INFO and GPT_MODEL_INFO.model_name in content_understanding_chat_models %} { "indexes": [ { @@ -358,7 +360,7 @@ "ingestionParameters": { "identity": null, "disableImageVerbalization": false, - "chatCompletionModel": {% if is_foundry_iq_standard_blob and GPT_MODEL_INFO %}{ + "chatCompletionModel": {% if is_foundry_iq_standard_blob and is_content_understanding_chat_model_supported %}{ "kind": "azureOpenAI", "azureOpenAIParameters": { "resourceUri": "{{FOUNDRY_IQ_AI_SERVICES_ENDPOINT}}", diff --git a/config/search/setup.py b/config/search/setup.py index ddad1368..9bdcb297 100644 --- a/config/search/setup.py +++ b/config/search/setup.py @@ -76,6 +76,18 @@ def is_truthy_setting(value: Any) -> bool: return str(value).strip().lower() in {"1", "true", "t", "yes", "y"} +def strip_odata_metadata(value: Any) -> Any: + if isinstance(value, dict): + return { + key: strip_odata_metadata(item) + for key, item in value.items() + if not key.startswith("@odata.") + } + if isinstance(value, list): + return [strip_odata_metadata(item) for item in value] + return value + + def normalize_foundry_iq_settings(settings: Dict[str, Any]) -> Dict[str, Any]: kind = str(settings.get("FOUNDRY_IQ_KNOWLEDGE_SOURCE_KIND") or "").lower() is_adls_gen2 = is_truthy_setting(settings.get("FOUNDRY_IQ_IS_ADLS_GEN2")) @@ -470,6 +482,47 @@ def provision_knowledge_sources(defs: dict, context: dict, cred: ChainedTokenCre return success_count == len(knowledge_sources) +def enforce_private_execution_for_generated_indexers(defs: dict, context: dict, cred: ChainedTokenCredential, search_endpoint: str, api_version: str): + if not is_truthy_setting(context.get("NETWORK_ISOLATION")): + return + + generated_indexers = [] + for ks in defs.get("knowledgeSources", []): + if ks.get("kind") != "azureBlob" or not ks.get("name"): + continue + generated_indexers.append( + ks.get("azureBlobParameters", {}).get("createdResources", {}).get("indexer") + or f"{ks['name']}-indexer" + ) + generated_indexers = [name for name in generated_indexers if name] + if not generated_indexers: + return + + token = cred.get_token("https://search.azure.com/.default").token + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + for indexer_name in generated_indexers: + url = f"{search_endpoint}/indexers/{indexer_name}?api-version={api_version}" + resp = requests.get(url, headers=headers) + if resp.status_code == 404: + logging.info(f"ℹ️ Generated indexer '{indexer_name}' not found yet; skipping private execution update.") + continue + if resp.status_code >= 400: + logging.warning(f"❗️ GET indexers/{indexer_name} failed {resp.status_code}: {resp.text}") + continue + + body = strip_odata_metadata(resp.json()) + if not isinstance(body.get("parameters"), dict): + body["parameters"] = {} + if not isinstance(body["parameters"].get("configuration"), dict): + body["parameters"]["configuration"] = {} + body["parameters"]["configuration"]["executionEnvironment"] = "Private" + update_resp = requests.put(url, headers=headers, json=body) + if update_resp.status_code >= 400: + logging.warning(f"❗️ PUT indexers/{indexer_name} failed {update_resp.status_code}: {update_resp.text}") + continue + logging.info(f"✅ Set generated indexer '{indexer_name}' executionEnvironment to Private") + + def provision_knowledge_bases(defs: dict, context: dict, cred: ChainedTokenCredential, search_endpoint: str): """Create or update Foundry IQ knowledge bases. @@ -526,6 +579,7 @@ def execute_setup(defs: Optional[dict], context: dict): # Step 3: Provision knowledge base resources (KS -> KB) knowledge_sources_ok = provision_knowledge_sources(defs, context, cred, search_endpoint) + enforce_private_execution_for_generated_indexers(defs, context, cred, search_endpoint, api_version) knowledge_bases_ok = provision_knowledge_bases(defs, context, cred, search_endpoint) if context.get("RETRIEVAL_BACKEND") == "foundry_iq" and (not knowledge_sources_ok or not knowledge_bases_ok): raise RuntimeError("Foundry IQ knowledge source/base provisioning failed") diff --git a/config/search/tests/test_foundry_iq_templates.py b/config/search/tests/test_foundry_iq_templates.py index e977bc2d..1362c4da 100644 --- a/config/search/tests/test_foundry_iq_templates.py +++ b/config/search/tests/test_foundry_iq_templates.py @@ -1,9 +1,12 @@ import json import unittest from pathlib import Path +from unittest.mock import Mock, patch from jinja2 import Environment, FileSystemLoader, StrictUndefined +from config.search import setup + TEMPLATE_DIR = Path(__file__).resolve().parents[1] @@ -37,7 +40,7 @@ def test_settings_template_derives_search_index_before_dependent_defaults(self): "https://aif-abc123.services.ai.azure.com/", ) - def test_standard_blob_knowledge_source_includes_ai_services_endpoint(self): + def test_standard_blob_knowledge_source_includes_ai_services_endpoint_without_unsupported_chat_model(self): settings = render_json_template( "search.settings.j2", { @@ -71,10 +74,109 @@ def test_standard_blob_knowledge_source_includes_ai_services_endpoint(self): ingestion_parameters["aiServices"]["uri"], "https://aif-abc123.services.ai.azure.com/", ) + self.assertIsNone(ingestion_parameters["chatCompletionModel"]) + + def test_standard_blob_knowledge_source_includes_supported_chat_model(self): + settings = render_json_template( + "search.settings.j2", + { + "RESOURCE_TOKEN": "abc123", + "SEARCH_SERVICE_QUERY_ENDPOINT": "https://search.search.windows.net", + "AI_FOUNDRY_ACCOUNT_NAME": "aif-abc123", + "RETRIEVAL_BACKEND": "foundry_iq", + "FOUNDRY_IQ_CONTENT_EXTRACTION_MODE": "standard", + }, + ) + context = { + **settings, + "STORAGE_ACCOUNT_RESOURCE_ID": "/subscriptions/s/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/st", + "EMBEDDING_MODEL_INFO": { + "endpoint": "https://aif-abc123.openai.azure.com/", + "deployment_name": "text-embedding", + "model_name": "text-embedding-3-large", + }, + "GPT_MODEL_INFO": { + "deployment_name": "chat", + "model_name": "gpt-5.2", + }, + } + + search_definitions = render_json_template("search.j2", context) + knowledge_source = search_definitions["knowledgeSources"][0] + ingestion_parameters = knowledge_source["azureBlobParameters"]["ingestionParameters"] + self.assertEqual( ingestion_parameters["chatCompletionModel"]["azureOpenAIParameters"]["resourceUri"], ingestion_parameters["aiServices"]["uri"], ) + self.assertEqual( + ingestion_parameters["chatCompletionModel"]["azureOpenAIParameters"]["modelName"], + "gpt-5.2", + ) + + def test_network_isolated_blob_knowledge_source_sets_generated_indexer_private(self): + credential = Mock() + credential.get_token.return_value.token = "token" + get_response = Mock() + get_response.status_code = 200 + get_response.json.return_value = { + "@odata.context": "https://search/$metadata#indexers/$entity", + "name": "blob-ks-indexer", + "dataSourceName": "blob-ks-datasource", + "targetIndexName": "blob-ks-index", + "parameters": { + "configuration": { + "dataToExtract": "contentAndMetadata", + } + }, + } + put_response = Mock() + put_response.status_code = 200 + + with patch.object(setup.requests, "get", return_value=get_response), patch.object( + setup.requests, "put", return_value=put_response + ) as put: + setup.enforce_private_execution_for_generated_indexers( + { + "knowledgeSources": [ + { + "name": "blob-ks", + "kind": "azureBlob", + "azureBlobParameters": {}, + } + ] + }, + {"NETWORK_ISOLATION": "true"}, + credential, + "https://search.search.windows.net", + "2025-05-01-preview", + ) + + body = put.call_args.kwargs["json"] + self.assertNotIn("@odata.context", body) + self.assertEqual(body["parameters"]["configuration"]["executionEnvironment"], "Private") + + def test_non_network_isolated_setup_does_not_update_generated_indexer(self): + credential = Mock() + + with patch.object(setup.requests, "get") as get: + setup.enforce_private_execution_for_generated_indexers( + { + "knowledgeSources": [ + { + "name": "blob-ks", + "kind": "azureBlob", + "azureBlobParameters": {}, + } + ] + }, + {"NETWORK_ISOLATION": "false"}, + credential, + "https://search.search.windows.net", + "2025-05-01-preview", + ) + + get.assert_not_called() if __name__ == "__main__": diff --git a/infra b/infra index 55dc46cf..10b679e5 160000 --- a/infra +++ b/infra @@ -1 +1 @@ -Subproject commit 55dc46cfed5269212cb79c999586aec9548e1155 +Subproject commit 10b679e5d34327a116431500514e3bb7fd7955ba diff --git a/scripts/Invoke-RegionalPreflight.ps1 b/scripts/Invoke-RegionalPreflight.ps1 index a2ec3be5..110a652e 100644 --- a/scripts/Invoke-RegionalPreflight.ps1 +++ b/scripts/Invoke-RegionalPreflight.ps1 @@ -394,6 +394,32 @@ function Test-ContainerAppsQuota { Test-QuotaRemaining -CheckName 'quota:container-apps:managed-environments' -QuotaData $quota -Names @('ManagedEnvironmentCount') -Required 1 } +function Test-ContentUnderstandingRegion { + param([Parameter(Mandatory = $true)][string]$Location) + + $supportedRegions = @( + 'australiaeast', + 'eastus', + 'eastus2', + 'japaneast', + 'northeurope', + 'southcentralus', + 'southeastasia', + 'swedencentral', + 'uksouth', + 'westeurope', + 'westus', + 'westus3' + ) + + $normalized = Get-NormalizedLocation $Location + if ($supportedRegions -contains $normalized) { + Write-PreflightCheck -Status PASS -Name 'foundry-iq:content-understanding-region' -Message "Content Understanding supports Foundry resource location $Location." + } else { + Write-PreflightCheck -Status FAIL -Name 'foundry-iq:content-understanding-region' -Message "Content Understanding does not support Foundry resource location $Location. Use one of: $($supportedRegions -join ', ')." + } +} + function Test-ModelReadiness { param( [Parameter(Mandatory = $true)][string]$Location, @@ -519,6 +545,9 @@ $deployJumpbox = Get-DeploymentFlag -Parameters $parameters -Name 'deployJumpbox $deployBastion = Get-DeploymentFlag -Parameters $parameters -Name 'deployBastion' -DefaultValue $false $deployNatGateway = Get-DeploymentFlag -Parameters $parameters -Name 'deployNatGateway' -DefaultValue $false $networkIsolation = Get-DeploymentFlag -Parameters $parameters -Name 'networkIsolation' -DefaultValue $false +$retrievalBackend = Get-ParameterValue -Parameters $parameters -Name 'retrievalBackend' -DefaultValue 'foundry_iq' +$foundryIqPattern = Get-ParameterValue -Parameters $parameters -Name 'foundryIqPattern' -DefaultValue 'azureBlob' +$foundryIqContentExtractionMode = Get-ParameterValue -Parameters $parameters -Name 'foundryIqContentExtractionMode' -DefaultValue 'standard' $providerNamespaces = [System.Collections.Generic.HashSet[string]]::new() if ($deployAiFoundry -or $deploySpeech) { [void]$providerNamespaces.Add('Microsoft.CognitiveServices') } @@ -580,6 +609,10 @@ if ($deployCosmos) { } if ($deployAiFoundry) { + if ($retrievalBackend -eq 'foundry_iq' -and $foundryIqPattern -ne 'searchIndex' -and $foundryIqContentExtractionMode -eq 'standard') { + Test-ContentUnderstandingRegion -Location $aiFoundryLocation + } + $models = Get-ParameterValue -Parameters $parameters -Name 'modelDeploymentList' -DefaultValue @() if (@($models).Count -gt 0) { Test-ModelReadiness -Location $aiFoundryLocation -Models $models