From 0b9e68de40dbdf6fa2b33eeef9960eab4700df1f Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Thu, 23 Apr 2026 13:41:54 +0200 Subject: [PATCH 01/10] Add draft documents --- app/_includes/ai-gateway/forward-proxy.md | 5 + app/_kong_plugins/ai-aws-guardrails/index.md | 4 + .../ai-azure-content-safety/index.md | 4 + .../ai-custom-guardrail/index.md | 6 +- app/_kong_plugins/ai-gcp-model-armor/index.md | 6 +- app/_kong_plugins/ai-lakera-guard/index.md | 6 +- .../ai-prompt-compressor/index.md | 4 + .../examples/forward-proxy.yaml | 45 ++++ app/_kong_plugins/ai-proxy-advanced/index.md | 6 + app/_kong_plugins/ai-rag-injector/index.md | 4 + app/_kong_plugins/ai-sanitizer/index.md | 4 + app/_kong_plugins/ai-semantic-cache/index.md | 4 + .../ai-semantic-prompt-guard/index.md | 4 + .../ai-semantic-response-guard/index.md | 4 + app/_kong_plugins/forward-proxy/index.md | 4 + app/ai-gateway/forward-proxy.md | 228 ++++++++++++++++++ 16 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 app/_includes/ai-gateway/forward-proxy.md create mode 100644 app/_kong_plugins/ai-proxy-advanced/examples/forward-proxy.yaml create mode 100644 app/ai-gateway/forward-proxy.md diff --git a/app/_includes/ai-gateway/forward-proxy.md b/app/_includes/ai-gateway/forward-proxy.md new file mode 100644 index 0000000000..f47cdade1f --- /dev/null +++ b/app/_includes/ai-gateway/forward-proxy.md @@ -0,0 +1,5 @@ +Set `config.proxy_config` on this plugin to route its outbound requests through an HTTP forward proxy. Use this in network-isolated deployments where {{site.ai_gateway}} cannot open direct connections to LLM providers or auxiliary services. + +The `proxy_config` record is identical across all {{site.ai_gateway}} plugins that support forward proxying. Existing capabilities such as load balancing, health checking, streaming, WebSocket, and HTTP/2 continue to work when the proxy is active. + +For the full field reference, traffic flow, and limitations, see [Forward proxy support](/ai-gateway/forward-proxy/). diff --git a/app/_kong_plugins/ai-aws-guardrails/index.md b/app/_kong_plugins/ai-aws-guardrails/index.md index 66a2f97d29..dfbd3a8a60 100644 --- a/app/_kong_plugins/ai-aws-guardrails/index.md +++ b/app/_kong_plugins/ai-aws-guardrails/index.md @@ -93,3 +93,7 @@ The AI AWS Guardrails plugin emits structured log data for every inspected reque To log the raw content of blocked requests and responses, enable [`config.log_blocked_content`](/plugins/ai-aws-guardrails/reference/#schema--config-log-blocked-content). {% new_in 3.14 %} When enabled, the blocked prompt or response body appears under `ai.proxy.aws-guardrails.input_faulty_prompt` and `ai.proxy.aws-guardrails.output_faulty_response` in each log entry. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-azure-content-safety/index.md b/app/_kong_plugins/ai-azure-content-safety/index.md index 52fdf0371b..b0fd72e775 100644 --- a/app/_kong_plugins/ai-azure-content-safety/index.md +++ b/app/_kong_plugins/ai-azure-content-safety/index.md @@ -115,3 +115,7 @@ To log the raw content of blocked requests and responses, enable [`config.log_bl This plugin works with all of the AI Proxy plugin's `route_type` settings (excluding the `preserve` mode), and is able to compose an Azure Content Safety text check by compiling all chat history, or just the `'user'` content. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-custom-guardrail/index.md b/app/_kong_plugins/ai-custom-guardrail/index.md index cde1e34450..6cf7117ece 100644 --- a/app/_kong_plugins/ai-custom-guardrail/index.md +++ b/app/_kong_plugins/ai-custom-guardrail/index.md @@ -130,4 +130,8 @@ The [`config.metrics`](./reference/#schema--config-metrics) field allows you to * `block_details`: Additional details about the blocked request or response. * `masked`: Whether content was masked in the request or response. -The values can be set to Lua expressions. You can also use the [`config.custom_metrics`](./reference/#schema--config-custom-metrics) field to define additional metrics. \ No newline at end of file +The values can be set to Lua expressions. You can also use the [`config.custom_metrics`](./reference/#schema--config-custom-metrics) field to define additional metrics. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-gcp-model-armor/index.md b/app/_kong_plugins/ai-gcp-model-armor/index.md index d6e39a1b02..664e0d52ea 100644 --- a/app/_kong_plugins/ai-gcp-model-armor/index.md +++ b/app/_kong_plugins/ai-gcp-model-armor/index.md @@ -171,4 +171,8 @@ To log the raw content of blocked requests and responses, enable [`config.log_bl * Only chat prompts and chat responses are inspected; embeddings and other modalities are not checked. * Inspects one chat message or one response body at a time. Combining multiple messages reduces accuracy. * For SSE streaming, unsafe content may appear briefly before termination with `"stop_reason: blocked by content safety"`. -* Only one `template_id` can be configured per plugin instance. \ No newline at end of file +* Only one `template_id` can be configured per plugin instance. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-lakera-guard/index.md b/app/_kong_plugins/ai-lakera-guard/index.md index aadba36986..4187b1f49e 100644 --- a/app/_kong_plugins/ai-lakera-guard/index.md +++ b/app/_kong_plugins/ai-lakera-guard/index.md @@ -170,4 +170,8 @@ When a request is blocked, the log captures the violation reason, detector detai } } } -``` \ No newline at end of file +``` + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-prompt-compressor/index.md b/app/_kong_plugins/ai-prompt-compressor/index.md index 8db73c546c..44bd36f6bc 100644 --- a/app/_kong_plugins/ai-prompt-compressor/index.md +++ b/app/_kong_plugins/ai-prompt-compressor/index.md @@ -215,3 +215,7 @@ sequenceDiagram The AI Prompt Compressor plugin applies structured compression to preserve essential context of prompts sent by users, rather than trimming prompts arbitrarily or risking token overflows. This ensures the LLM receives a well-formed, focused prompt keeping token usage under control. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-proxy-advanced/examples/forward-proxy.yaml b/app/_kong_plugins/ai-proxy-advanced/examples/forward-proxy.yaml new file mode 100644 index 0000000000..cdf7c548a5 --- /dev/null +++ b/app/_kong_plugins/ai-proxy-advanced/examples/forward-proxy.yaml @@ -0,0 +1,45 @@ +--- + +title: 'Route inference traffic through a forward proxy' +description: 'Configure AI Proxy Advanced to send outbound inference requests through an HTTP forward proxy for network-isolated deployments.' + +weight: 950 + +group: network + +config: + targets: + - route_type: llm/v1/chat + auth: + header_name: Authorization + header_value: Bearer ${openai_key} + model: + provider: openai + name: gpt-5.1 + proxy_config: + http_proxy_host: proxy.internal + http_proxy_port: 3128 + https_proxy_host: proxy.internal + https_proxy_port: 3128 + proxy_scheme: http + auth_username: ${proxy_user} + auth_password: ${proxy_password} + https_verify: false + +variables: + openai_key: + value: $OPENAI_API_KEY + description: The API key to use to connect to OpenAI. + proxy_user: + value: $FORWARD_PROXY_USER + description: Username for the corporate forward proxy. + proxy_password: + value: $FORWARD_PROXY_PASSWORD + description: Password for the corporate forward proxy. + +tools: + - deck + - admin-api + - konnect-api + - kic + - terraform diff --git a/app/_kong_plugins/ai-proxy-advanced/index.md b/app/_kong_plugins/ai-proxy-advanced/index.md index 43425d1ea8..46d10f2af5 100644 --- a/app/_kong_plugins/ai-proxy-advanced/index.md +++ b/app/_kong_plugins/ai-proxy-advanced/index.md @@ -71,6 +71,8 @@ examples_groups: text: Native routes - slug: claude-code text: claude-code + - slug: network + text: Network configuration faqs: - q: Can I override `config.model.name` by specifying a different model name in the request? @@ -197,3 +199,7 @@ For setup instructions, see [AI plugin Partials](/gateway/entities/partial/#ai-p {% include_cached /plugins/redis/redis-cloud-auth.md tier=page.tier %} {% include_cached /plugins/redis/enterprise.md name=page.name heading_level=3 %} + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-rag-injector/index.md b/app/_kong_plugins/ai-rag-injector/index.md index 499925a083..2882483c24 100644 --- a/app/_kong_plugins/ai-rag-injector/index.md +++ b/app/_kong_plugins/ai-rag-injector/index.md @@ -498,3 +498,7 @@ Use the [Admin API](/plugins/ai-rag-injector/api/) to ingest content with metada {% include_cached /plugins/ai-partials-vectordb-embeddings.md %} {% include_cached /plugins/redis/redis-cloud-auth.md tier=page.tier %} + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-sanitizer/index.md b/app/_kong_plugins/ai-sanitizer/index.md index 2c57cca44b..1e682f0db1 100644 --- a/app/_kong_plugins/ai-sanitizer/index.md +++ b/app/_kong_plugins/ai-sanitizer/index.md @@ -245,3 +245,7 @@ You can use the following fields in the `anonymize` array: * `custom`: Anonymizes user-defined custom PII patterns using regular expressions only when custom patterns are provided. * `credentials`: Anonymizes the credentials, similar to `/sanitize_credentials`. * `all`: Includes all the fields above, including custom ones. + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-semantic-cache/index.md b/app/_kong_plugins/ai-semantic-cache/index.md index 8efb5816bd..4220987878 100644 --- a/app/_kong_plugins/ai-semantic-cache/index.md +++ b/app/_kong_plugins/ai-semantic-cache/index.md @@ -182,3 +182,7 @@ The plugin respects cache control headers to determine if requests and responses {% include_cached /plugins/redis/redis-cloud-auth.md tier=page.tier %} {% include_cached /plugins/redis/enterprise.md name=page.name heading_level=3 %} + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-semantic-prompt-guard/index.md b/app/_kong_plugins/ai-semantic-prompt-guard/index.md index 3d4b4e1575..fb01c3c970 100644 --- a/app/_kong_plugins/ai-semantic-prompt-guard/index.md +++ b/app/_kong_plugins/ai-semantic-prompt-guard/index.md @@ -106,3 +106,7 @@ The matching behavior is as follows: {% include_cached /plugins/redis/redis-cloud-auth.md tier=page.tier %} {% include_cached /plugins/redis/enterprise.md name=page.name heading_level=3 %} + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/ai-semantic-response-guard/index.md b/app/_kong_plugins/ai-semantic-response-guard/index.md index 89507be12c..3d646d0941 100644 --- a/app/_kong_plugins/ai-semantic-response-guard/index.md +++ b/app/_kong_plugins/ai-semantic-response-guard/index.md @@ -101,3 +101,7 @@ To enforce these rules, the plugin: {% include_cached /plugins/redis/redis-cloud-auth.md tier=page.tier %} {% include_cached /plugins/redis/enterprise.md name=page.name heading_level=3 %} + +## Forward proxy support + +{% include ai-gateway/forward-proxy.md %} diff --git a/app/_kong_plugins/forward-proxy/index.md b/app/_kong_plugins/forward-proxy/index.md index ff327da3bc..aeef7b4834 100644 --- a/app/_kong_plugins/forward-proxy/index.md +++ b/app/_kong_plugins/forward-proxy/index.md @@ -38,6 +38,10 @@ search_aliases: min_version: gateway: '1.0' + +related_resources: + - text: Forward proxy support for {{site.ai_gateway}} + url: /ai-gateway/forward-proxy/ --- The Forward Proxy Advanced plugin allows {{site.base_gateway}} to connect to intermediary transparent HTTP proxies, instead of directly to the `upstream_url`, when forwarding requests upstream. diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md new file mode 100644 index 0000000000..290ddbf2e9 --- /dev/null +++ b/app/ai-gateway/forward-proxy.md @@ -0,0 +1,228 @@ +--- +title: "Forward proxy support" +content_type: reference +layout: reference + +description: "Route outbound traffic from {{site.ai_gateway}} plugins through a forward proxy to operate in network-isolated environments without breaking load balancing, streaming, WebSocket, or HTTP/2." + +breadcrumbs: + - /ai-gateway/ + +products: + - gateway + - ai-gateway + +works_on: + - on-prem + - konnect + +tools: + - deck + - admin-api + - konnect-api + - kic + - terraform + +# TODO: confirm the minimum Gateway version with engineering before publishing. +# min_version: +# gateway: '' + +plugins: + - ai-proxy-advanced + - ai-semantic-cache + - ai-semantic-prompt-guard + - ai-semantic-response-guard + - ai-rag-injector + - ai-prompt-compressor + - ai-sanitizer + - ai-aws-guardrails + - ai-azure-content-safety + - ai-lakera-guard + - ai-gcp-model-armor + - ai-custom-guardrail + +tags: + - ai + - network + - security + +related_resources: + - text: "{{site.ai_gateway}}" + url: /ai-gateway/ + - text: AI Proxy Advanced + url: /plugins/ai-proxy-advanced/ + - text: Forward Proxy Advanced plugin + url: /plugins/forward-proxy/ +--- + +## What is forward proxy support? + +In network-isolated deployments, {{site.ai_gateway}} cannot open direct outbound connections to LLM providers or auxiliary services. Forward proxy support lets {{site.ai_gateway}} plugins route their outbound requests through a controlled HTTP forward proxy so inference traffic, semantic operations, and guardrail checks continue to work behind strict egress policy. + +A shared `proxy_config` record, added to every affected {{site.ai_gateway}} plugin, names the proxy host, port, scheme, and optional credentials. When configured, all outbound requests issued by that plugin go through the proxy. Existing capabilities such as [load balancing](/ai-gateway/load-balancing/), health checking, [streaming](/ai-gateway/streaming/), WebSocket, and HTTP/2 continue to work. + +## How forward proxy support works + +{{site.ai_gateway}} plugins issue three categories of outbound request. `proxy_config` applies to all three, though the underlying mechanism differs depending on where the request originates. + + +{% mermaid %} +flowchart LR + Client --> Gateway[Kong AI Gateway] + subgraph AI Plugins + AIProxy[AI Proxy Advanced] + Semantic[Semantic plugins] + Guardrails[Guardrail plugins] + end + Gateway --> AIProxy + Gateway --> Semantic + Gateway --> Guardrails + AIProxy -- inference --> Proxy[Forward proxy] + Semantic -- embeddings --> Proxy + Guardrails -- guardrail checks --> Proxy + Proxy --> LLM[LLM providers] + Proxy --> Aux[Auxiliary services] +{% endmermaid %} +> _Figure 1: Outbound traffic from {{site.ai_gateway}} plugins routed through a forward proxy._ + + +The three request categories are: + +- **Inference traffic**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/). This is the majority of {{site.ai_gateway}} traffic. It runs through the native {{site.base_gateway}} upstream path, so load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. +- **Embeddings and semantic operations**: requests from semantic plugins to the embeddings service used for cache lookups, RAG retrieval, and prompt or response guarding. +- **Security and transformation calls**: requests from guardrail, sanitizer, and compressor plugins to their external services (AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a custom endpoint). + +When `proxy_config` is set on a plugin, every outbound request that plugin issues goes through the configured proxy. + +## Supported plugins + +The following plugins expose `proxy_config`. The record structure and behavior are identical across all of them. + + +{% table %} +columns: + - title: Traffic + key: traffic + - title: Plugins + key: plugins +rows: + - traffic: "Inference requests to LLM providers" + plugins: | + - [AI Proxy Advanced](/plugins/ai-proxy-advanced/) + - traffic: "Embeddings and semantic operations" + plugins: | + - [AI Semantic Cache](/plugins/ai-semantic-cache/) + - [AI Semantic Prompt Guard](/plugins/ai-semantic-prompt-guard/) + - [AI Semantic Response Guard](/plugins/ai-semantic-response-guard/) + - [AI RAG Injector](/plugins/ai-rag-injector/) + - traffic: "Prompt compression and sanitization" + plugins: | + - [AI Prompt Compressor](/plugins/ai-prompt-compressor/) + - [AI Sanitizer](/plugins/ai-sanitizer/) + - traffic: "Guardrail services" + plugins: | + - [AI AWS Guardrails](/plugins/ai-aws-guardrails/) + - [AI Azure Content Safety](/plugins/ai-azure-content-safety/) + - [AI Lakera Guard](/plugins/ai-lakera-guard/) + - [AI GCP Model Armor](/plugins/ai-gcp-model-armor/) + - [AI Custom Guardrail](/plugins/ai-custom-guardrail/) +{% endtable %} + + +## proxy_config fields + +Every plugin in the supported list accepts the same `proxy_config` record at the top level of its `config` block. + + +{% table %} +columns: + - title: Field + key: field + - title: Type + key: type + - title: Description + key: description +rows: + - field: "`http_proxy_host`" + type: "host" + description: "Hostname of the forward proxy used for HTTP upstreams. Must be set together with `http_proxy_port`." + - field: "`http_proxy_port`" + type: "port" + description: "Port of the forward proxy used for HTTP upstreams. Must be set together with `http_proxy_host`." + - field: "`https_proxy_host`" + type: "host" + description: "Hostname of the forward proxy used for HTTPS upstreams. Must be set together with `https_proxy_port`." + - field: "`https_proxy_port`" + type: "port" + description: "Port of the forward proxy used for HTTPS upstreams. Must be set together with `https_proxy_host`." + - field: "`proxy_scheme`" + type: "string" + description: "Scheme used to connect to the forward proxy itself. One of `http` or `https`. Defaults to `http`." + - field: "`auth_username`" + type: "string" + description: "Username for proxy authentication. Optional. Referenceable from a [Vault](/gateway/entities/vault/)." + - field: "`auth_password`" + type: "string" + description: "Password for proxy authentication. Optional. Encrypted at rest and referenceable from a [Vault](/gateway/entities/vault/)." + - field: "`https_verify`" + type: "boolean" + description: "Whether to verify the forward proxy's TLS certificate when `proxy_scheme` is `https`. Defaults to `false`." +{% endtable %} + + +Two validation rules apply to the record: + +- `http_proxy_host` and `http_proxy_port` must both be set or both be absent. +- `https_proxy_host` and `https_proxy_port` must both be set or both be absent. + +{:.warning} +> When `proxy_scheme` is `https` and the global `tls_certificate_verify` flag is enabled, `https_verify` cannot be set to `false`. + +## Configuration + +The minimal configuration adds a `proxy_config` block to any supported plugin. The same block applies unchanged across plugins: configure it once per plugin instance that needs to reach external services through the proxy. + +{% entity_example %} +type: plugin +data: + name: ai-proxy-advanced + config: + targets: + - model: + provider: openai + name: gpt-5.1 + auth: + header_name: Authorization + header_value: Bearer ${openai_key} + route_type: llm/v1/chat + proxy_config: + http_proxy_host: proxy.internal + http_proxy_port: 3128 + https_proxy_host: proxy.internal + https_proxy_port: 3128 + proxy_scheme: http + auth_username: ${proxy_user} + auth_password: ${proxy_password} + https_verify: false +variables: + openai_key: + value: $OPENAI_API_KEY + description: The API key to use to connect to OpenAI. + proxy_user: + value: $FORWARD_PROXY_USER + description: Username for the corporate forward proxy. + proxy_password: + value: $FORWARD_PROXY_PASSWORD + description: Password for the corporate forward proxy. +{% endentity_example %} + +## Relationship to the Forward Proxy Advanced plugin + +{{site.base_gateway}} also ships the [Forward Proxy Advanced plugin](/plugins/forward-proxy/) for routing non-AI upstream traffic through an intermediary HTTP proxy. That plugin takes over the request before the balancer phase runs, which works for standard Kong Services but breaks behavior that {{site.ai_gateway}} depends on: upstream load balancing, health check reporting, retries, WebSocket upgrades, and HTTP/2 request bodies. + +{{site.ai_gateway}} plugins use `proxy_config` instead so the balancer phase continues to run normally. Load balancing across LLM targets, streaming, real-time API traffic, and HTTP/2 inference requests all remain functional when the forward proxy is active. Apply the Forward Proxy Advanced plugin to non-AI Services only; use `proxy_config` on any Service that serves traffic through an {{site.ai_gateway}} plugin. + +## Limitations + +- Connections to vector databases (pgvector, Redis Vector, Pinecone) use native database protocols rather than HTTP and are not routed through the forward proxy. If these connections must traverse a proxy, handle it at the network layer. +- The [AI Request Transformer](/plugins/ai-request-transformer/), [AI Response Transformer](/plugins/ai-response-transformer/), and [AI LLM as a Judge](/plugins/ai-llm-as-judge/) plugins keep their existing flat proxy fields (`http_proxy_host`, `http_proxy_port`, `https_proxy_host`, `https_proxy_port`) and do not accept a `proxy_config` record. Authentication to the proxy and HTTPS proxy schemes are not available on these three plugins. From 376c3fe375bfa72f0bd6c57f7bf9e0d669e321ea Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Fri, 24 Apr 2026 09:15:17 +0200 Subject: [PATCH 02/10] tweak mermaid graph --- app/ai-gateway/forward-proxy.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 290ddbf2e9..8222727591 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -68,20 +68,25 @@ A shared `proxy_config` record, added to every affected {{site.ai_gateway}} plug {% mermaid %} flowchart LR - Client --> Gateway[Kong AI Gateway] - subgraph AI Plugins - AIProxy[AI Proxy Advanced] - Semantic[Semantic plugins] - Guardrails[Guardrail plugins] + subgraph Gateway_Group[Kong AI Gateway] + subgraph Plugins[AI Plugins] + AIProxy[AI Proxy Advanced] + Semantic[Semantic plugins] + Transform[Compressor and sanitizer] + Guardrails[Guardrail plugins] + end end - Gateway --> AIProxy - Gateway --> Semantic - Gateway --> Guardrails - AIProxy -- inference --> Proxy[Forward proxy] - Semantic -- embeddings --> Proxy + Client --> AIProxy + Client --> Semantic + Client --> Transform + Client --> Guardrails + AIProxy -- inference ---> Proxy[Forward proxy] + Semantic -- embeddings ---> Proxy + Transform -- compression and sanitization --> Proxy Guardrails -- guardrail checks --> Proxy - Proxy --> LLM[LLM providers] - Proxy --> Aux[Auxiliary services] + Proxy ---> LLM[LLM providers] + Proxy ---> Aux[Auxiliary services] + style Plugins stroke-dasharray: 5 5 {% endmermaid %} > _Figure 1: Outbound traffic from {{site.ai_gateway}} plugins routed through a forward proxy._ From b75f772d85f15cb0efef6eb881d3ad29375ceed3 Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Fri, 24 Apr 2026 12:11:41 +0200 Subject: [PATCH 03/10] add more details --- app/ai-gateway/forward-proxy.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 8222727591..020716767d 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -22,6 +22,7 @@ tools: - konnect-api - kic - terraform + - kongctl # TODO: confirm the minimum Gateway version with engineering before publishing. # min_version: @@ -93,10 +94,12 @@ flowchart LR The three request categories are: -- **Inference traffic**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/). This is the majority of {{site.ai_gateway}} traffic. It runs through the native {{site.base_gateway}} upstream path, so load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. +- **Inference traffic**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/). This is the majority of {{site.ai_gateway}} traffic. It runs through the native {{site.base_gateway}} upstream path, so load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. - **Embeddings and semantic operations**: requests from semantic plugins to the embeddings service used for cache lookups, RAG retrieval, and prompt or response guarding. - **Security and transformation calls**: requests from guardrail, sanitizer, and compressor plugins to their external services (AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a custom endpoint). +For cloud providers that require managed identity (AWS Bedrock's SigV4 signing, Azure and GCP managed identity token acquisition), the authentication subrequests issued by the provider SDK libraries also traverse the proxy. + When `proxy_config` is set on a plugin, every outbound request that plugin issues goes through the configured proxy. ## Supported plugins @@ -110,20 +113,25 @@ columns: key: traffic - title: Plugins key: plugins + - title: Proxied destination + key: service rows: - traffic: "Inference requests to LLM providers" plugins: | - [AI Proxy Advanced](/plugins/ai-proxy-advanced/) + service: "LLM provider APIs" - traffic: "Embeddings and semantic operations" plugins: | - [AI Semantic Cache](/plugins/ai-semantic-cache/) - [AI Semantic Prompt Guard](/plugins/ai-semantic-prompt-guard/) - [AI Semantic Response Guard](/plugins/ai-semantic-response-guard/) - [AI RAG Injector](/plugins/ai-rag-injector/) + service: "The configured embeddings service" - traffic: "Prompt compression and sanitization" plugins: | - [AI Prompt Compressor](/plugins/ai-prompt-compressor/) - [AI Sanitizer](/plugins/ai-sanitizer/) + service: "The configured `compressor_url` or `sanitizer_url`" - traffic: "Guardrail services" plugins: | - [AI AWS Guardrails](/plugins/ai-aws-guardrails/) @@ -131,9 +139,12 @@ rows: - [AI Lakera Guard](/plugins/ai-lakera-guard/) - [AI GCP Model Armor](/plugins/ai-gcp-model-armor/) - [AI Custom Guardrail](/plugins/ai-custom-guardrail/) + service: "Managed or custom guardrail service" {% endtable %} +`proxy_config` is per-plugin. Setting it on AI Proxy Advanced does not apply to other {{site.ai_gateway}} plugins attached to the same Route. Configure it on every plugin whose external calls need to traverse the proxy. + ## proxy_config fields Every plugin in the supported list accepts the same `proxy_config` record at the top level of its `config` block. @@ -180,8 +191,11 @@ Two validation rules apply to the record: - `http_proxy_host` and `http_proxy_port` must both be set or both be absent. - `https_proxy_host` and `https_proxy_port` must both be set or both be absent. +{:.info} +> `https_verify` defaults to `false`. When `proxy_scheme` is `https`, set `https_verify: true` in production so the proxy's TLS certificate is validated. + {:.warning} -> When `proxy_scheme` is `https` and the global `tls_certificate_verify` flag is enabled, `https_verify` cannot be set to `false`. +> When `proxy_scheme` is `https` and the global `tls_certificate_verify` flag is enabled, `https_verify` cannot be set to `false`. Config load fails. ## Configuration @@ -230,4 +244,4 @@ variables: ## Limitations - Connections to vector databases (pgvector, Redis Vector, Pinecone) use native database protocols rather than HTTP and are not routed through the forward proxy. If these connections must traverse a proxy, handle it at the network layer. -- The [AI Request Transformer](/plugins/ai-request-transformer/), [AI Response Transformer](/plugins/ai-response-transformer/), and [AI LLM as a Judge](/plugins/ai-llm-as-judge/) plugins keep their existing flat proxy fields (`http_proxy_host`, `http_proxy_port`, `https_proxy_host`, `https_proxy_port`) and do not accept a `proxy_config` record. Authentication to the proxy and HTTPS proxy schemes are not available on these three plugins. +- The [AI Request Transformer](/plugins/ai-request-transformer/), [AI Response Transformer](/plugins/ai-response-transformer/), and [AI LLM as a Judge](/plugins/ai-llm-as-judge/) plugins keep their existing flat proxy fields (`http_proxy_host`, `http_proxy_port`, `https_proxy_host`, `https_proxy_port`) and do not accept a `proxy_config` record. They do not expose `auth_username`, `auth_password`, `proxy_scheme`, or `https_verify`, so proxy authentication and HTTPS-scheme proxies are unavailable for their traffic. From f11222bf1ca1baed182c8c1e7c5d07e5ef74bf39 Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Fri, 24 Apr 2026 14:08:15 +0200 Subject: [PATCH 04/10] Update section --- app/ai-gateway/forward-proxy.md | 34 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 020716767d..bc2cdd9220 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -24,9 +24,9 @@ tools: - terraform - kongctl -# TODO: confirm the minimum Gateway version with engineering before publishing. -# min_version: -# gateway: '' +TODO: confirm the minimum Gateway version with engineering before publishing. +min_version: + gateway: '2.0' plugins: - ai-proxy-advanced @@ -69,24 +69,20 @@ A shared `proxy_config` record, added to every affected {{site.ai_gateway}} plug {% mermaid %} flowchart LR + Client --> AIProxy + Client --> Aux subgraph Gateway_Group[Kong AI Gateway] subgraph Plugins[AI Plugins] AIProxy[AI Proxy Advanced] - Semantic[Semantic plugins] - Transform[Compressor and sanitizer] - Guardrails[Guardrail plugins] + Aux[Auxiliary plugins] end end - Client --> AIProxy - Client --> Semantic - Client --> Transform - Client --> Guardrails - AIProxy -- inference ---> Proxy[Forward proxy] - Semantic -- embeddings ---> Proxy - Transform -- compression and sanitization --> Proxy - Guardrails -- guardrail checks --> Proxy + AIProxy -- inference --> Proxy[Forward proxy] + AIProxy -- "identity auth" --> Proxy + Aux -- "auxiliary calls" --> Proxy Proxy ---> LLM[LLM providers] - Proxy ---> Aux[Auxiliary services] + Proxy ---> CloudAPI[Cloud platform APIs] + Proxy ---> AuxSvc[Auxiliary services] style Plugins stroke-dasharray: 5 5 {% endmermaid %} > _Figure 1: Outbound traffic from {{site.ai_gateway}} plugins routed through a forward proxy._ @@ -94,11 +90,9 @@ flowchart LR The three request categories are: -- **Inference traffic**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/). This is the majority of {{site.ai_gateway}} traffic. It runs through the native {{site.base_gateway}} upstream path, so load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. -- **Embeddings and semantic operations**: requests from semantic plugins to the embeddings service used for cache lookups, RAG retrieval, and prompt or response guarding. -- **Security and transformation calls**: requests from guardrail, sanitizer, and compressor plugins to their external services (AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a custom endpoint). - -For cloud providers that require managed identity (AWS Bedrock's SigV4 signing, Azure and GCP managed identity token acquisition), the authentication subrequests issued by the provider SDK libraries also traverse the proxy. +- **Inference**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/) through the native {{site.base_gateway}} upstream path. This is the majority of {{site.ai_gateway}} traffic. Load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. +- **Identity auth**: cloud identity authentication issued by provider SDKs. AWS Bedrock SigV4 signing, Azure and GCP managed identity token acquisition, when targets require managed identity. +- **Auxiliary calls**: direct HTTP calls from semantic, RAG, guardrail, sanitizer, and compressor plugins to their external services (embeddings service, AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a configured custom endpoint). When `proxy_config` is set on a plugin, every outbound request that plugin issues goes through the configured proxy. From db4e61545184807f76f65a3220e39b9cc8162648 Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Fri, 24 Apr 2026 14:11:38 +0200 Subject: [PATCH 05/10] change min version --- app/ai-gateway/forward-proxy.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index bc2cdd9220..7161bce4f2 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -21,12 +21,10 @@ tools: - admin-api - konnect-api - kic - - terraform - - kongctl TODO: confirm the minimum Gateway version with engineering before publishing. min_version: - gateway: '2.0' + ai-gateway: '2.0' plugins: - ai-proxy-advanced From 595ddfca62982d436e2b0c7adc8647ab7389cd2b Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Fri, 24 Apr 2026 14:25:29 +0200 Subject: [PATCH 06/10] fixes --- app/ai-gateway/forward-proxy.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 7161bce4f2..72f9b567a2 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -50,8 +50,6 @@ related_resources: url: /ai-gateway/ - text: AI Proxy Advanced url: /plugins/ai-proxy-advanced/ - - text: Forward Proxy Advanced plugin - url: /plugins/forward-proxy/ --- ## What is forward proxy support? @@ -78,9 +76,9 @@ flowchart LR AIProxy -- inference --> Proxy[Forward proxy] AIProxy -- "identity auth" --> Proxy Aux -- "auxiliary calls" --> Proxy - Proxy ---> LLM[LLM providers] - Proxy ---> CloudAPI[Cloud platform APIs] - Proxy ---> AuxSvc[Auxiliary services] + Proxy --> LLM[LLM providers] + Proxy --> CloudAPI[Cloud platform APIs] + Proxy --> AuxSvc[Auxiliary services] style Plugins stroke-dasharray: 5 5 {% endmermaid %} > _Figure 1: Outbound traffic from {{site.ai_gateway}} plugins routed through a forward proxy._ From 65bb13d08d043063ddcb33e6f51e1ac98d40ffe9 Mon Sep 17 00:00:00 2001 From: tomek-labuk Date: Mon, 27 Apr 2026 10:30:52 +0200 Subject: [PATCH 07/10] updates --- app/_landing_pages/ai-gateway.yaml | 10 ++++++++++ app/ai-gateway/forward-proxy.md | 14 ++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app/_landing_pages/ai-gateway.yaml b/app/_landing_pages/ai-gateway.yaml index 49dfa32166..8ad700c244 100644 --- a/app/_landing_pages/ai-gateway.yaml +++ b/app/_landing_pages/ai-gateway.yaml @@ -167,6 +167,7 @@ rows: - "[{{site.ai_gateway}} resource sizing guidelines](/ai-gateway/resource-sizing-guidelines-ai/): Review recommended resource allocation guidelines for {{site.ai_gateway}}." - "[Deployment topologies](/gateway/deployment-topologies/): Learn about the different ways to deploy {{ site.base_gateway }}." - "[Hosting options](/gateway/topology-hosting-options/): Decide where you want to host your Data Plane nodes, and whether you want Kong to host them or host them yourself." + - "[Route AI Proxy Advanced traffic through a forward proxy](/how-to/route-ai-proxy-advanced-traffic-through-forward-proxy/): Configure a local forward proxy and validate that {{site.ai_gateway}} traffic traverses it." - header: type: h2 text: "Tools to manage {{site.ai_gateway}}" @@ -340,6 +341,15 @@ rows: cta: url: /ai-gateway/streaming/ align: end + - blocks: + - type: card + config: + title: Forward proxy support + description: Route outbound {{site.ai_gateway}} plugin traffic through a controlled HTTP forward proxy + icon: /assets/icons/network.svg + cta: + url: /ai-gateway/forward-proxy/ + align: end - blocks: - type: card config: diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 72f9b567a2..d2c8f42157 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -62,6 +62,12 @@ A shared `proxy_config` record, added to every affected {{site.ai_gateway}} plug {{site.ai_gateway}} plugins issue three categories of outbound request. `proxy_config` applies to all three, though the underlying mechanism differs depending on where the request originates. +The three request categories are: + +- **Inference**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/) through the native {{site.base_gateway}} upstream path. This is the majority of {{site.ai_gateway}} traffic. Load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. +- **Identity auth**: cloud identity authentication issued by provider SDKs. AWS Bedrock SigV4 signing, Azure and GCP managed identity token acquisition, when targets require managed identity. +- **Auxiliary calls**: direct HTTP calls from semantic, RAG, guardrail, sanitizer, and compressor plugins to their external services (embeddings service, AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a configured custom endpoint). + {% mermaid %} flowchart LR @@ -84,12 +90,6 @@ flowchart LR > _Figure 1: Outbound traffic from {{site.ai_gateway}} plugins routed through a forward proxy._ -The three request categories are: - -- **Inference**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/) through the native {{site.base_gateway}} upstream path. This is the majority of {{site.ai_gateway}} traffic. Load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. -- **Identity auth**: cloud identity authentication issued by provider SDKs. AWS Bedrock SigV4 signing, Azure and GCP managed identity token acquisition, when targets require managed identity. -- **Auxiliary calls**: direct HTTP calls from semantic, RAG, guardrail, sanitizer, and compressor plugins to their external services (embeddings service, AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a configured custom endpoint). - When `proxy_config` is set on a plugin, every outbound request that plugin issues goes through the configured proxy. ## Supported plugins @@ -191,6 +191,8 @@ Two validation rules apply to the record: The minimal configuration adds a `proxy_config` block to any supported plugin. The same block applies unchanged across plugins: configure it once per plugin instance that needs to reach external services through the proxy. +For a complete local setup with a running forward proxy, full decK configuration, and validation steps, see [Route AI Proxy Advanced traffic through a forward proxy](/how-to/route-ai-proxy-advanced-traffic-through-forward-proxy/). + {% entity_example %} type: plugin data: From d66d4864caa90202de65c25372ff0e935763f154 Mon Sep 17 00:00:00 2001 From: jbaross Date: Wed, 13 May 2026 09:11:22 +0100 Subject: [PATCH 08/10] minor restructure --- app/ai-gateway/forward-proxy.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index d2c8f42157..646ac7bb28 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -54,9 +54,9 @@ related_resources: ## What is forward proxy support? -In network-isolated deployments, {{site.ai_gateway}} cannot open direct outbound connections to LLM providers or auxiliary services. Forward proxy support lets {{site.ai_gateway}} plugins route their outbound requests through a controlled HTTP forward proxy so inference traffic, semantic operations, and guardrail checks continue to work behind strict egress policy. +In network-isolated deployments, {{site.ai_gateway}} cannot open direct outbound connections to LLM providers or auxiliary services. Forward proxy support lets {{site.ai_gateway}} plugins route their outbound requests through a controlled HTTP forward proxy so that inference traffic, semantic operations, and guardrail checks continue to work behind a strict egress policy. -A shared `proxy_config` record, added to every affected {{site.ai_gateway}} plugin, names the proxy host, port, scheme, and optional credentials. When configured, all outbound requests issued by that plugin go through the proxy. Existing capabilities such as [load balancing](/ai-gateway/load-balancing/), health checking, [streaming](/ai-gateway/streaming/), WebSocket, and HTTP/2 continue to work. +A shared `proxy_config` record, can be added to each {{site.ai_gateway}} plugin that names the proxy host, port, scheme, and optional credentials. When configured, all outbound requests issued by that plugin go through the proxy. Existing capabilities such as [load balancing](/ai-gateway/load-balancing/), health checking, [streaming](/ai-gateway/streaming/), WebSocket, and HTTP/2 continue to work. ## How forward proxy support works @@ -92,6 +92,12 @@ flowchart LR When `proxy_config` is set on a plugin, every outbound request that plugin issues goes through the configured proxy. +## Relationship to the Forward Proxy Advanced plugin + +{{site.base_gateway}} also provides the [Forward Proxy Advanced plugin](/plugins/forward-proxy/) for routing non-AI upstream traffic through an intermediary HTTP proxy. That plugin takes over the request before the balancer phase runs, which works for standard Kong Services but breaks behavior that {{site.ai_gateway}} depends on: upstream load balancing, health check reporting, retries, WebSocket upgrades, and HTTP/2 request bodies. + +{{site.ai_gateway}} plugins use `proxy_config` instead so the balancer phase continues to run normally. Load balancing across LLM targets, streaming, real-time API traffic, and HTTP/2 inference requests all remain functional when the forward proxy is active. Apply the Forward Proxy Advanced plugin to non-AI Services only; use `proxy_config` on any Service that serves traffic through an {{site.ai_gateway}} plugin. + ## Supported plugins The following plugins expose `proxy_config`. The record structure and behavior are identical across all of them. @@ -227,12 +233,6 @@ variables: description: Password for the corporate forward proxy. {% endentity_example %} -## Relationship to the Forward Proxy Advanced plugin - -{{site.base_gateway}} also ships the [Forward Proxy Advanced plugin](/plugins/forward-proxy/) for routing non-AI upstream traffic through an intermediary HTTP proxy. That plugin takes over the request before the balancer phase runs, which works for standard Kong Services but breaks behavior that {{site.ai_gateway}} depends on: upstream load balancing, health check reporting, retries, WebSocket upgrades, and HTTP/2 request bodies. - -{{site.ai_gateway}} plugins use `proxy_config` instead so the balancer phase continues to run normally. Load balancing across LLM targets, streaming, real-time API traffic, and HTTP/2 inference requests all remain functional when the forward proxy is active. Apply the Forward Proxy Advanced plugin to non-AI Services only; use `proxy_config` on any Service that serves traffic through an {{site.ai_gateway}} plugin. - ## Limitations - Connections to vector databases (pgvector, Redis Vector, Pinecone) use native database protocols rather than HTTP and are not routed through the forward proxy. If these connections must traverse a proxy, handle it at the network layer. From 683d0dc9aea9e02ac332d9571a02d6003f4a94f6 Mon Sep 17 00:00:00 2001 From: jbaross Date: Thu, 14 May 2026 11:12:47 +0100 Subject: [PATCH 09/10] disambiguate with frwd proxy advanced --- app/ai-gateway/forward-proxy.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index 646ac7bb28..c9bfcbb93f 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -56,17 +56,17 @@ related_resources: In network-isolated deployments, {{site.ai_gateway}} cannot open direct outbound connections to LLM providers or auxiliary services. Forward proxy support lets {{site.ai_gateway}} plugins route their outbound requests through a controlled HTTP forward proxy so that inference traffic, semantic operations, and guardrail checks continue to work behind a strict egress policy. -A shared `proxy_config` record, can be added to each {{site.ai_gateway}} plugin that names the proxy host, port, scheme, and optional credentials. When configured, all outbound requests issued by that plugin go through the proxy. Existing capabilities such as [load balancing](/ai-gateway/load-balancing/), health checking, [streaming](/ai-gateway/streaming/), WebSocket, and HTTP/2 continue to work. +A shared `proxy_config` record can be added to each {{site.ai_gateway}} plugin that names the proxy host, port, scheme, and optional credentials. When configured, all outbound requests issued by that plugin go through the proxy. Existing capabilities such as [load balancing](/ai-gateway/load-balancing/), health checking, [streaming](/ai-gateway/streaming/), WebSocket, and HTTP/2 continue to work. ## How forward proxy support works -{{site.ai_gateway}} plugins issue three categories of outbound request. `proxy_config` applies to all three, though the underlying mechanism differs depending on where the request originates. +{{site.ai_gateway}} plugins issue three categories of outbound request. `proxy_config` applies to all three, using a different mechanism depending on where the request originates. The three request categories are: -- **Inference**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/) through the native {{site.base_gateway}} upstream path. This is the majority of {{site.ai_gateway}} traffic. Load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when the proxy is active. Upstream keepalive is disabled while the proxy is active, so inference connections are not reused across requests targeting different upstream peers. +- **Inference**: requests from clients to LLM providers, proxied by [AI Proxy Advanced](/plugins/ai-proxy-advanced/) through the native {{site.base_gateway}} upstream path. This is the majority of {{site.ai_gateway}} traffic. Load balancing, health checks, retries, streaming, WebSocket, and HTTP/2 all continue to function when forward proxy support is active. Upstream keepalive is disabled while the forward proxy is active, so inference connections are not reused across requests targeting different upstream peers. - **Identity auth**: cloud identity authentication issued by provider SDKs. AWS Bedrock SigV4 signing, Azure and GCP managed identity token acquisition, when targets require managed identity. -- **Auxiliary calls**: direct HTTP calls from semantic, RAG, guardrail, sanitizer, and compressor plugins to their external services (embeddings service, AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a configured custom endpoint). +- **Auxiliary calls**: direct HTTP calls from semantic, RAG, guardrail, sanitizer, and compressor plugins to their external services. For example, an embeddings service, AWS Bedrock Guardrails, Azure Content Safety, Lakera, GCP Model Armor, or a configured custom endpoint. {% mermaid %} @@ -94,9 +94,11 @@ When `proxy_config` is set on a plugin, every outbound request that plugin issue ## Relationship to the Forward Proxy Advanced plugin -{{site.base_gateway}} also provides the [Forward Proxy Advanced plugin](/plugins/forward-proxy/) for routing non-AI upstream traffic through an intermediary HTTP proxy. That plugin takes over the request before the balancer phase runs, which works for standard Kong Services but breaks behavior that {{site.ai_gateway}} depends on: upstream load balancing, health check reporting, retries, WebSocket upgrades, and HTTP/2 request bodies. +{{site.base_gateway}} also provides the [Forward Proxy Advanced plugin](/plugins/forward-proxy/) for routing non-AI upstream traffic through an intermediary HTTP proxy. For non-AI services use the Forward Proxy Advanced plugin. -{{site.ai_gateway}} plugins use `proxy_config` instead so the balancer phase continues to run normally. Load balancing across LLM targets, streaming, real-time API traffic, and HTTP/2 inference requests all remain functional when the forward proxy is active. Apply the Forward Proxy Advanced plugin to non-AI Services only; use `proxy_config` on any Service that serves traffic through an {{site.ai_gateway}} plugin. +The Forward Proxy Advanced plugin plugin takes over the request before the balancer phase runs, which works for standard Kong Services but not with behavior that {{site.ai_gateway}} depends on: upstream load balancing, health check reporting, retries, WebSocket upgrades, and HTTP/2 request bodies. + +For any Service that serves traffic through a {{site.ai_gateway}} plugin you should use `proxy_config` instead, so the balancer phase continues to run normally. Load balancing across LLM targets, streaming, real-time API traffic, and HTTP/2 inference requests all remain functional when the forward proxy is active and you have configured `proxy_config`. ## Supported plugins From dbacd9a702dfbaf84ffc3809cf6e6d98e37e2144 Mon Sep 17 00:00:00 2001 From: jbaross Date: Thu, 14 May 2026 11:26:13 +0100 Subject: [PATCH 10/10] clarity improvements --- app/ai-gateway/forward-proxy.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/ai-gateway/forward-proxy.md b/app/ai-gateway/forward-proxy.md index c9bfcbb93f..92ee792769 100644 --- a/app/ai-gateway/forward-proxy.md +++ b/app/ai-gateway/forward-proxy.md @@ -102,6 +102,8 @@ For any Service that serves traffic through a {{site.ai_gateway}} plugin you sho ## Supported plugins +`proxy_config` is set per-plugin. Enabling forward proxy support on AI Proxy Advanced does not apply to other {{site.ai_gateway}} plugins attached to the same Route. You should configure it on every plugin whose external calls need to traverse the forward proxy. + The following plugins expose `proxy_config`. The record structure and behavior are identical across all of them. @@ -141,8 +143,6 @@ rows: {% endtable %} -`proxy_config` is per-plugin. Setting it on AI Proxy Advanced does not apply to other {{site.ai_gateway}} plugins attached to the same Route. Configure it on every plugin whose external calls need to traverse the proxy. - ## proxy_config fields Every plugin in the supported list accepts the same `proxy_config` record at the top level of its `config` block. @@ -237,5 +237,5 @@ variables: ## Limitations -- Connections to vector databases (pgvector, Redis Vector, Pinecone) use native database protocols rather than HTTP and are not routed through the forward proxy. If these connections must traverse a proxy, handle it at the network layer. +- Connections to vector databases (such as pgvector, Redis Vector, or Pinecone) use native database protocols rather than HTTP and are not routed through the forward proxy. If these connections must traverse a forward proxy, you should handle it at the network layer. - The [AI Request Transformer](/plugins/ai-request-transformer/), [AI Response Transformer](/plugins/ai-response-transformer/), and [AI LLM as a Judge](/plugins/ai-llm-as-judge/) plugins keep their existing flat proxy fields (`http_proxy_host`, `http_proxy_port`, `https_proxy_host`, `https_proxy_port`) and do not accept a `proxy_config` record. They do not expose `auth_username`, `auth_password`, `proxy_scheme`, or `https_verify`, so proxy authentication and HTTPS-scheme proxies are unavailable for their traffic.