diff --git a/docs/pages/features/destinations.mdx b/docs/pages/destinations.mdx similarity index 62% rename from docs/pages/features/destinations.mdx rename to docs/pages/destinations.mdx index 24b2b2cb..c5aaf2c5 100644 --- a/docs/pages/features/destinations.mdx +++ b/docs/pages/destinations.mdx @@ -2,45 +2,47 @@ title: Destinations --- -Outpost supports multiple event destination types. Each tenant can have multiple destinations, up to a maximum set by the `MAX_DESTINATIONS_PER_TENANT` environment variable (defaulting to `20`). Destinations can be registered either through the tenant-facing portal or with the API. The current supported destination types are: - -- Webhooks -- Hookdeck Event Gateway -- AWS Kinesis -- AWS SQS -- AWS S3 -- Azure Service Bus -- GCP Pub/Sub -- RabbitMQ (AMQP) - -Plans for additional event destination types include: - -- Amazon EventBridge -- Kafka +Outpost supports multiple event destination types. Each tenant can have multiple destinations, up to a maximum set by the `MAX_DESTINATIONS_PER_TENANT` environment variable (defaulting to `20`). > We recommend setting the `MAX_DESTINATIONS_PER_TENANT` value as low as is appropriate for your use case to prevent abuse and performance degradation. Updating the value to a lower value later will not delete existing destinations. -See the [roadmap](/docs/references/roadmap) for more information on planned destination types. To be eligible as a destination type, it must be asynchronous in nature and not run any business logic. +## Supported Destinations + +| Destination | Description | +| ----------- | ----------- | +| [Webhook](/docs/destinations/webhook) | Send events via HTTP POST to a URL | +| [Hookdeck](/docs/destinations/hookdeck) | Route events through Hookdeck Event Gateway | +| [AWS Kinesis](/docs/destinations/aws-kinesis) | Stream events to Amazon Kinesis | +| [AWS SQS](/docs/destinations/aws-sqs) | Send events to an Amazon SQS queue | +| [AWS S3](/docs/destinations/aws-s3) | Store events in an Amazon S3 bucket | +| [Azure Service Bus](/docs/destinations/azure-service-bus) | Send events to Azure Service Bus | +| [GCP Pub/Sub](/docs/destinations/gcp-pubsub) | Publish events to Google Cloud Pub/Sub | +| [RabbitMQ](/docs/destinations/rabbitmq) | Send events to a RabbitMQ exchange | -## Creating a destination +See the [roadmap](/docs/references/roadmap) for planned destination types. To be eligible as a destination type, it must be asynchronous in nature and not run any business logic. -Destinations can be registered either through the tenant-facing portal or with the API. Since each destination type uses its own credentials and configuration, the required fields are different. Refer to the [Create Destination API](/docs/api/destinations#create-destination) for the required `config` and `credentials` fields for each destination type. +## Creating a Destination -For example, creating a "Webhook" destination requires the `config.url` field. +Destinations can be registered through the tenant portal or via the API. Each destination type has its own configuration and credentials. Refer to the [Create Destination API](/docs/api/destinations#create-destination) for the required `config` and `credentials` fields for each destination type. ```sh -curl --location 'localhost:3333/api/v1//destinations' \ +curl --location 'https:///api/v1//destinations' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer ' \ --data '{ - "type": "", - "topics": [""], - "config": { - "url": "https://example.test/webhooks" - } + "type": "", + "topics": [""], + "config": { ... }, + "credentials": { ... } }' ``` +## Destination Filtering + +Destinations can be configured with filters to selectively receive only events matching specific criteria. This allows tenants to create fine-grained routing rules based on event properties. + +See the [Filters](/docs/features/filter) documentation for the complete filter syntax and examples. + ## Getting Destination Types & Schemas When using the API, you may want to build your own UI to capture user input on the destination configuration. Since each destination requires a specific configuration, the `GET /destination-types` endpoint provides a JSON schema for standardized input fields for each destination type. @@ -84,9 +86,9 @@ Some destinations will require instructions to configure. For instance, with Pub Some destinations may have OAuth flow or other managed setup flow that can be triggered with a link. If a `remote_setup_url` is set, then the user should be prompted to follow the link to configure the destination. -See the [building your own UI guide](../../guides/building-your-own-ui.mdx) for recommended UI patterns and wireframes for implementation in your own app. +See the [building your own UI guide](/docs/guides/building-your-own-ui) for recommended UI patterns and wireframes for implementation in your own app. -## Customizing destination type definitions & instructions +## Customizing Destination Type Definitions & Instructions The destination type definitions (label, description, icon, etc) and instructions can be customized by setting the `DESTINATIONS_METADATA_PATH` environment variable to a path on disk containing the destination type definitions and instructions. Outpost will load both the default destination type definitions and any custom destination type definitions and merge them. diff --git a/docs/pages/destinations/aws-kinesis.mdx b/docs/pages/destinations/aws-kinesis.mdx new file mode 100644 index 00000000..7ea72a76 --- /dev/null +++ b/docs/pages/destinations/aws-kinesis.mdx @@ -0,0 +1,124 @@ +--- +title: AWS Kinesis +--- + +Stream events to an Amazon Kinesis Data Stream. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.stream_name` | string | Yes | The Kinesis stream name | +| `config.region` | string | Yes | AWS region (e.g., `us-east-1`) | +| `config.endpoint` | string | No | Custom endpoint URL (for LocalStack, etc.) | +| `config.partition_key_template` | string | No | JMESPath expression for partition key | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.key` | string | Yes | AWS Access Key ID | +| `credentials.secret` | string | Yes | AWS Secret Access Key | +| `credentials.session` | string | No | AWS Session Token (for temporary credentials) | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "aws_kinesis", + "topics": ["orders"], + "config": { + "stream_name": "my-stream", + "region": "us-east-1" + }, + "credentials": { + "key": "", + "secret": "" + } +}' +``` + +## Partition Key + +By default, the event ID is used as the partition key. You can customize this using a JMESPath expression in `partition_key_template`: + +```json +{ + "config": { + "stream_name": "my-stream", + "region": "us-east-1", + "partition_key_template": "data.customer_id" + } +} +``` + +## Record Format + +If you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +By default, the Kinesis record includes both metadata and the event's `data` field. The `metadata` object contains system metadata (`event-id`, `topic`, `timestamp`) plus any event metadata from the published event: + +```json +{ + "metadata": { + "event-id": "evt_123", + "topic": "orders", + "timestamp": "1704067200", + "source": "checkout-service" + }, + "data": { + "order_id": "123", + "status": "created" + } +} +``` + +When `DESTINATIONS_AWS_KINESIS_METADATA_IN_PAYLOAD` is set to `false`, only the event's `data` field is sent: + +```json +{ + "order_id": "123", + "status": "created" +} +``` + +## IAM Permissions + +The IAM user or role needs the following permission: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "kinesis:PutRecord", + "Resource": "arn:aws:kinesis:*:*:stream/my-stream" + } + ] +} +``` + +## Operator Configuration + +| Environment Variable | Default | Description | +| -------------------- | ------- | ----------- | +| `DESTINATIONS_AWS_KINESIS_METADATA_IN_PAYLOAD` | `true` | Include Outpost metadata in the Kinesis record payload | diff --git a/docs/pages/destinations/aws-s3.mdx b/docs/pages/destinations/aws-s3.mdx new file mode 100644 index 00000000..94d18d4d --- /dev/null +++ b/docs/pages/destinations/aws-s3.mdx @@ -0,0 +1,126 @@ +--- +title: AWS S3 +--- + +Store events in an Amazon S3 bucket. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.bucket` | string | Yes | The S3 bucket name | +| `config.region` | string | Yes | AWS region (e.g., `us-east-1`) | +| `config.key_template` | string | No | JMESPath expression for S3 object key | +| `config.storage_class` | string | No | S3 storage class (e.g., `STANDARD`, `GLACIER`) | +| `config.endpoint` | string | No | Custom endpoint URL (for LocalStack, etc.) | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.key` | string | Yes | AWS Access Key ID | +| `credentials.secret` | string | Yes | AWS Secret Access Key | +| `credentials.session` | string | No | AWS Session Token (for temporary credentials) | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "aws_s3", + "topics": ["orders"], + "config": { + "bucket": "my-events-bucket", + "region": "us-east-1" + }, + "credentials": { + "key": "", + "secret": "" + } +}' +``` + +## Object Key + +By default, objects are stored with keys in the format: `{timestamp}_{event-id}.json` + +You can customize the key using a JMESPath expression in `key_template`: + +```json +{ + "config": { + "bucket": "my-events-bucket", + "region": "us-east-1", + "key_template": "join('/', [data.customer_id, metadata.\"event-id\", '.json'])" + } +} +``` + +## Object Format + +Events are stored as JSON files containing the event's `data` field. For example, if you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +The S3 object body will be: + +```json +{ + "order_id": "123", + "status": "created" +} +``` + +Event metadata is stored in the S3 object's metadata (not in the body). This includes system metadata and any event metadata from the published event: + +| Metadata Key | Source | Description | +| ------------ | ------ | ----------- | +| `event-id` | System | The unique event ID | +| `topic` | System | The event topic | +| `timestamp` | System | Event timestamp (Unix) | +| `*` | Event | Any additional metadata from the published event's `metadata` field | + +## IAM Permissions + +The IAM user or role needs the following permission: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::my-events-bucket/*" + } + ] +} +``` + +## Storage Classes + +Supported storage classes: + +- `STANDARD` (default) +- `REDUCED_REDUNDANCY` +- `STANDARD_IA` +- `ONEZONE_IA` +- `INTELLIGENT_TIERING` +- `GLACIER` +- `DEEP_ARCHIVE` +- `GLACIER_IR` diff --git a/docs/pages/destinations/aws-sqs.mdx b/docs/pages/destinations/aws-sqs.mdx new file mode 100644 index 00000000..c1831d0c --- /dev/null +++ b/docs/pages/destinations/aws-sqs.mdx @@ -0,0 +1,122 @@ +--- +title: AWS SQS +--- + +Send events to an Amazon SQS queue. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.queue_url` | string | Yes | The SQS queue URL | +| `config.endpoint` | string | No | Custom endpoint URL (for LocalStack, etc.) | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.key` | string | Yes | AWS Access Key ID | +| `credentials.secret` | string | Yes | AWS Secret Access Key | +| `credentials.session` | string | No | AWS Session Token (for temporary credentials) | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "aws_sqs", + "topics": ["orders"], + "config": { + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789012/my-queue" + }, + "credentials": { + "key": "", + "secret": "" + } +}' +``` + +## Message Format + +Events are sent as SQS messages with: + +- **Message Body**: The event's `data` field (JSON) +- **Message Attributes**: Event metadata + +### Example Message + +If you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +**Message Body:** +```json +{ + "order_id": "123", + "status": "created" +} +``` + +**Message Attributes:** + +The `metadata` attribute contains a JSON string with system metadata and any event metadata: + +| Attribute | Type | Description | +| --------- | ---- | ----------- | +| `metadata` | String | JSON containing `event-id`, `topic`, `timestamp`, plus any event metadata | + +Example value: +```json +{"event-id": "evt_123", "topic": "orders", "timestamp": "1704067200", "source": "checkout-service"} +``` + +## IAM Permissions + +The IAM user or role needs the following permission: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "sqs:SendMessage", + "Resource": "arn:aws:sqs:*:*:my-queue" + } + ] +} +``` + +## Using with LocalStack + +For local development with LocalStack, set the `config.endpoint` field: + +```json +{ + "type": "aws_sqs", + "topics": ["orders"], + "config": { + "queue_url": "http://localhost:4566/000000000000/my-queue", + "endpoint": "http://localhost:4566" + }, + "credentials": { + "key": "test", + "secret": "test" + } +} +``` diff --git a/docs/pages/destinations/azure-service-bus.mdx b/docs/pages/destinations/azure-service-bus.mdx new file mode 100644 index 00000000..53b9bed1 --- /dev/null +++ b/docs/pages/destinations/azure-service-bus.mdx @@ -0,0 +1,87 @@ +--- +title: Azure Service Bus +--- + +Send events to an Azure Service Bus queue or topic. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.name` | string | Yes | The queue or topic name | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.connection_string` | string | Yes | Azure Service Bus connection string | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "azure_servicebus", + "topics": ["orders"], + "config": { + "name": "my-queue" + }, + "credentials": { + "connection_string": "Endpoint=sb://my-namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=" + } +}' +``` + +## Message Format + +Events are sent as Service Bus messages with: + +- **Message Body**: The event's `data` field (JSON) +- **Application Properties**: Event metadata + +### Example Message + +If you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +**Body:** +```json +{ + "order_id": "123", + "status": "created" +} +``` + +**Application Properties:** + +Properties include system metadata and any event metadata from the published event: + +| Property | Source | Description | +| -------- | ------ | ----------- | +| `event-id` | System | The unique event ID | +| `topic` | System | The event topic | +| `timestamp` | System | Event timestamp (Unix) | +| `*` | Event | Any additional metadata from the published event's `metadata` field | + +## Getting a Connection String + +1. Navigate to your Service Bus namespace in the Azure Portal +2. Go to **Shared access policies** +3. Select or create a policy with **Send** permission +4. Copy the **Primary Connection String** diff --git a/docs/pages/destinations/gcp-pubsub.mdx b/docs/pages/destinations/gcp-pubsub.mdx new file mode 100644 index 00000000..d7be053a --- /dev/null +++ b/docs/pages/destinations/gcp-pubsub.mdx @@ -0,0 +1,121 @@ +--- +title: GCP Pub/Sub +--- + +Publish events to a Google Cloud Pub/Sub topic. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.project_id` | string | Yes | GCP project ID | +| `config.topic` | string | Yes | Pub/Sub topic name | +| `config.endpoint` | string | No | Custom endpoint (for emulator) | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.service_account_json` | string | Yes* | Service account JSON key | + +*Not required when using the Pub/Sub emulator. + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "gcp_pubsub", + "topics": ["orders"], + "config": { + "project_id": "my-project", + "topic": "my-topic" + }, + "credentials": { + "service_account_json": "{\"type\": \"service_account\", ...}" + } +}' +``` + +## Message Format + +Events are published as Pub/Sub messages with: + +- **Data**: The event's `data` field (JSON) +- **Attributes**: Event metadata + +### Example Message + +If you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +**Data:** +```json +{ + "order_id": "123", + "status": "created" +} +``` + +**Attributes:** + +Attributes include system metadata and any event metadata from the published event: + +| Attribute | Source | Description | +| --------- | ------ | ----------- | +| `event-id` | System | The unique event ID | +| `topic` | System | The event topic | +| `timestamp` | System | Event timestamp (Unix) | +| `*` | Event | Any additional metadata from the published event's `metadata` field | + +## IAM Permissions + +The service account needs the following role: + +- `roles/pubsub.publisher` on the topic + +Or the specific permission: + +- `pubsub.topics.publish` + +## Creating a Service Account + +1. Navigate to **IAM & Admin > Service Accounts** in the GCP Console +2. Click **Create Service Account** +3. Grant the **Pub/Sub Publisher** role +4. Create a JSON key and download it +5. Use the JSON content as `service_account_json` + +## Using with Emulator + +For local development with the Pub/Sub emulator, set the `config.endpoint` field: + +```json +{ + "type": "gcp_pubsub", + "topics": ["orders"], + "config": { + "project_id": "test-project", + "topic": "test-topic", + "endpoint": "localhost:8085" + } +} +``` + +When using the emulator, `service_account_json` is not required. diff --git a/docs/pages/destinations/hookdeck.mdx b/docs/pages/destinations/hookdeck.mdx new file mode 100644 index 00000000..ce375897 --- /dev/null +++ b/docs/pages/destinations/hookdeck.mdx @@ -0,0 +1,69 @@ +--- +title: Hookdeck +--- + +Route events through the [Hookdeck Event Gateway](https://hookdeck.com) for advanced webhook management, including automatic retries, rate limiting, transformations, and observability. + +## Overview + +The Hookdeck destination enables tenants to route their events through Hookdeck's Event Gateway. This provides tenants with additional delivery features managed by Hookdeck, such as configurable retries, request transformations, and detailed event logs. + +This destination uses a **managed source** flow. When tenants create a Hookdeck destination, they are guided through Hookdeck's dashboard to create a managed source, then paste the resulting token back into Outpost. + +## Configuration + +### Config + +This destination has no configuration fields. Routing and delivery settings are managed in the Hookdeck dashboard by tenants. + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.token` | string | Yes | Hookdeck source token | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "hookdeck", + "topics": ["orders"], + "credentials": { + "token": "" + } +}' +``` + +## Portal Setup Flow + +In the Outpost portal, tenants follow this flow: + +1. Select **Hookdeck** as the destination type +2. Open the Hookdeck dashboard via the portal link +3. Create a managed source in Hookdeck +4. Copy the source token +5. Paste the token into the Outpost portal and create the destination +6. Return to Hookdeck to create connections that route events to their endpoints + +## Event Flow + +1. Events published to Outpost are sent to the Hookdeck Event Gateway +2. Hookdeck buffers and processes the events +3. Based on tenant connection configuration, Hookdeck routes events to their endpoints +4. Hookdeck handles retries, rate limiting, and delivery guarantees + +## Hookdeck Features + +Tenants using Hookdeck as a destination get access to: + +- **Event buffering** - Handle traffic spikes without overwhelming endpoints +- **Automatic retries** - Configurable retry strategies for failed deliveries +- **Transformations** - Modify event payloads before delivery +- **Filtering & routing** - Route events to different endpoints based on content +- **Observability** - Event logs, metrics, and debugging tools +- **Alerts** - Notifications for delivery issues + +See the [Hookdeck documentation](https://hookdeck.com/docs) for more details. diff --git a/docs/pages/destinations/rabbitmq.mdx b/docs/pages/destinations/rabbitmq.mdx new file mode 100644 index 00000000..854f94d4 --- /dev/null +++ b/docs/pages/destinations/rabbitmq.mdx @@ -0,0 +1,121 @@ +--- +title: RabbitMQ +--- + +Send events to a RabbitMQ exchange via AMQP. + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.server_url` | string | Yes | RabbitMQ server URL (host:port) | +| `config.exchange` | string | Yes | Exchange name | +| `config.tls` | string | No | Enable TLS (`true` or `false`, default: `false`) | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.username` | string | Yes | RabbitMQ username | +| `credentials.password` | string | Yes | RabbitMQ password | + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "rabbitmq", + "topics": ["orders"], + "config": { + "server_url": "rabbitmq.example.com:5672", + "exchange": "events" + }, + "credentials": { + "username": "guest", + "password": "guest" + } +}' +``` + +## Message Format + +Events are published as AMQP messages with: + +- **Body**: The event's `data` field (JSON) +- **Headers**: Event metadata +- **Routing Key**: The event topic + +### Example Message + +If you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +**Body:** +```json +{ + "order_id": "123", + "status": "created" +} +``` + +**Headers:** + +Headers include system metadata and any event metadata from the published event: + +| Header | Source | Description | +| ------ | ------ | ----------- | +| `event-id` | System | The unique event ID | +| `topic` | System | The event topic | +| `timestamp` | System | Event timestamp (Unix) | +| `*` | Event | Any additional metadata from the published event's `metadata` field | + +## TLS Configuration + +To enable TLS, set `config.tls` to `true`: + +```json +{ + "type": "rabbitmq", + "topics": ["orders"], + "config": { + "server_url": "rabbitmq.example.com:5671", + "exchange": "events", + "tls": "true" + }, + "credentials": { + "username": "user", + "password": "password" + } +} +``` + +## Exchange Setup + +Before sending events, ensure the exchange exists in RabbitMQ. Outpost publishes messages with the event topic as the routing key, allowing you to bind queues based on topic patterns. + +Example RabbitMQ setup: + +```bash +# Create a topic exchange +rabbitmqadmin declare exchange name=events type=topic + +# Create a queue and bind it to receive all order events +rabbitmqadmin declare queue name=order-events +rabbitmqadmin declare binding source=events destination=order-events routing_key="orders.*" +``` diff --git a/docs/pages/destinations/webhook.mdx b/docs/pages/destinations/webhook.mdx new file mode 100644 index 00000000..464163ad --- /dev/null +++ b/docs/pages/destinations/webhook.mdx @@ -0,0 +1,261 @@ +--- +title: Webhook +--- + +Send events via HTTP POST requests to a URL endpoint. + +Outpost supports two webhook modes: +- **Default mode** - Customizable headers and signature format +- **Standard Webhooks mode** - Follows the [Standard Webhooks](https://www.standardwebhooks.com/) specification + +## Overview + +When you publish an event: + +```json +{ + "topic": "orders", + "data": { + "order_id": "123", + "status": "created" + }, + "metadata": { + "source": "checkout-service" + } +} +``` + +Outpost sends an HTTP POST request with: + +- **Body**: The event's `data` field as JSON +- **Headers**: System metadata (`event-id`, `topic`, `timestamp`) plus any event metadata, with a configurable prefix + +Example request: + +``` +POST /webhooks HTTP/1.1 +Content-Type: application/json +x-outpost-id: evt_123 +x-outpost-topic: orders +x-outpost-timestamp: 1704067200 +x-outpost-signature: t=1704067200,v0=abc123... +x-outpost-source: checkout-service + +{"order_id": "123", "status": "created"} +``` + +## Configuration + +### Config + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `config.url` | string | Yes | The URL to send events to | +| `config.custom_headers` | string | No | JSON object of custom headers to include | + +### Credentials + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `credentials.secret` | string | No | Signing secret (auto-generated if not provided) | +| `credentials.previous_secret` | string | No | Previous secret during rotation | +| `credentials.previous_secret_invalid_at` | string | No | RFC3339 timestamp when previous secret expires | + +If `secret` is not provided, one will be auto-generated. Tenants cannot modify secrets directly - they can only trigger rotation. + +### Example + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "webhook", + "topics": ["orders"], + "config": { + "url": "https://example.com/webhooks" + } +}' +``` + +## Secret Rotation + +Secret rotation allows you to change the signing secret without downtime. During the rotation window, both the old and new secrets are valid for signature verification. + +To rotate a secret, set `rotate_secret` to `true` when updating a destination: + +```sh +curl --location --request PATCH 'https:///api/v1//destinations/' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "credentials": { + "rotate_secret": "true" + } +}' +``` + +When rotation is triggered: +1. The current secret becomes `previous_secret` +2. A new secret is generated +3. The previous secret remains valid for 24 hours by default +4. Both secrets are included in the signature header during the rotation window + +To customize the rotation window, set `previous_secret_invalid_at`: + +```sh +curl --location --request PATCH 'https:///api/v1//destinations/' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "credentials": { + "rotate_secret": "true", + "previous_secret_invalid_at": "2025-01-15T00:00:00Z" + } +}' +``` + +--- + +## Default Mode + +In default mode, Outpost provides a customizable webhook implementation. + +### Headers + +Outpost adds headers using the configured prefix (default `x-outpost-`): + +| Header | Source | Description | +| ------ | ------ | ----------- | +| `x-outpost-id` | System | The unique event ID | +| `x-outpost-timestamp` | System | Event timestamp (Unix) | +| `x-outpost-topic` | System | The event topic | +| `x-outpost-signature` | System | HMAC-SHA256 signature | +| `x-outpost-{key}` | Event | Any metadata from the published event's `metadata` field | + +### Signature + +The signature is computed over the timestamp and request body: + +``` +signature = HMAC-SHA256(secret, "${timestamp}.${body}") +``` + +The signature header value follows the format: `t=${timestamp},v0=${signature}` + +### Verifying Signatures + +To verify webhook requests: + +1. Extract the timestamp and signature from the `x-outpost-signature` header +2. Compute the expected signature using your secret +3. Compare signatures using a constant-time comparison +4. Optionally, reject requests with old timestamps to prevent replay attacks + +### Operator Configuration + +Operators can customize webhook behavior via environment variables. + +#### Header Configuration + +| Environment Variable | Default | Description | +| -------------------- | ------- | ----------- | +| `DESTINATIONS_WEBHOOK_HEADER_PREFIX` | `x-outpost-` | Prefix for all webhook headers | +| `DESTINATIONS_WEBHOOK_DISABLE_EVENT_ID_HEADER` | `false` | Disable the event ID header | +| `DESTINATIONS_WEBHOOK_DISABLE_TIMESTAMP_HEADER` | `false` | Disable the timestamp header | +| `DESTINATIONS_WEBHOOK_DISABLE_TOPIC_HEADER` | `false` | Disable the topic header | +| `DESTINATIONS_WEBHOOK_DISABLE_SIGNATURE_HEADER` | `false` | Disable the signature header | + +#### Signature Configuration + +| Environment Variable | Default | Description | +| -------------------- | ------- | ----------- | +| `DESTINATIONS_WEBHOOK_SIGNATURE_ALGORITHM` | `hmac-sha256` | Signature algorithm | +| `DESTINATIONS_WEBHOOK_SIGNATURE_ENCODING` | `hex` | Signature encoding (`hex` or `base64`) | +| `DESTINATIONS_WEBHOOK_SIGNATURE_VALUE_TEMPLATE` | `{{.Timestamp.Unix}}.{{.Body}}` | Template for the value being signed | +| `DESTINATIONS_WEBHOOK_SIGNATURE_HEADER_TEMPLATE` | `t={{.Timestamp.Unix}},v0={{.Signatures \| join ","}}` | Template for signature header value | + +See the [migration guide](/docs/guides/migrate-to-outpost#webhook-customization) for common customization patterns. + +--- + +## Standard Webhooks Mode + +Standard Webhooks mode follows the [Standard Webhooks](https://www.standardwebhooks.com/) specification, providing a standardized format for webhook signatures and headers used by many webhook providers. + +To enable Standard Webhooks mode, set: + +``` +DESTINATIONS_WEBHOOK_MODE=standard +``` + +### Headers + +| Header | Source | Description | +| ------ | ------ | ----------- | +| `webhook-id` | System | The unique event ID | +| `webhook-timestamp` | System | Event timestamp (Unix) | +| `webhook-signature` | System | Signature in `v1,` format | +| `webhook-{key}` | Event | Any metadata from the published event's `metadata` field | + +### Signature + +The signature follows the Standard Webhooks specification: + +``` +signature = base64(HMAC-SHA256(secret, "${webhook-id}.${timestamp}.${body}")) +``` + +### Verifying Signatures + +Use the official [Standard Webhooks SDK](https://github.com/standard-webhooks/standard-webhooks/tree/main/libraries) to verify signatures. SDKs are available for multiple languages including JavaScript, Python, Go, Ruby, and more. + +### Secret Format + +Standard Webhooks secrets use the format `whsec_`, which is automatically generated when creating a destination. + +### Operator Configuration + +| Environment Variable | Default | Description | +| -------------------- | ------- | ----------- | +| `DESTINATIONS_WEBHOOK_MODE` | `default` | Set to `standard` to enable Standard Webhooks mode | +| `DESTINATIONS_WEBHOOK_HEADER_PREFIX` | `webhook-` | Prefix for Standard Webhooks headers | + +--- + +## Custom Headers + +Tenants can add custom HTTP headers to webhook requests. This is useful for authentication, routing, or passing additional metadata to their endpoint. + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "webhook", + "topics": ["orders"], + "config": { + "url": "https://example.com/webhooks", + "custom_headers": "{\"x-api-key\": \"secret123\", \"x-tenant-id\": \"tenant-abc\"}" + } +}' +``` + +### Header Name Requirements + +- Must start with a letter or digit +- Can contain letters, digits, underscores (`_`), and hyphens (`-`) + +### Reserved Headers + +The following headers cannot be overridden: + +- `content-type` +- `content-length` +- `host` +- `connection` +- `user-agent` + +### Operator Configuration + +Custom headers are disabled in the tenant portal by default. To enable, set `PORTAL_ENABLE_WEBHOOK_CUSTOM_HEADERS=true`. diff --git a/docs/pages/features.mdx b/docs/pages/features.mdx index 8902eba9..bc5c2959 100644 --- a/docs/pages/features.mdx +++ b/docs/pages/features.mdx @@ -5,8 +5,9 @@ title: Outpost Features Outpost offers a range of features designed to provide a robust and flexible event delivery system. Explore the details of each feature below: - **[Multi-Tenant Support](./features/multi-tenant-support.mdx)**: The Outpost API supports creating multiple tenants on a single Outpost deployment. -- **[Event Destinations](./features/destinations.mdx)**: Outpost supports a variety of event destination types, including Webhooks, Hookdeck Event Gateway, Amazon EventBridge, AWS SQS, AWS S3, GCP Pub/Sub, RabbitMQ, and Kafka. +- **[Destinations](/docs/destinations)**: Outpost supports a variety of event destination types, including Webhooks, Hookdeck Event Gateway, Amazon EventBridge, AWS SQS, AWS S3, GCP Pub/Sub, RabbitMQ, and Kafka. - **[Topics](./features/topics.mdx)**: Outpost supports event topics (also known as "event types") in order to segment, filter, and route events to appropriate destinations. +- **[Filters](./features/filter.mdx)**: Configure filters on destinations to selectively receive only events matching specific criteria. - **[Publishing Events](./features/publish-events.mdx)**: Learn how events are published, either to a configured message queue or via the publish API endpoint. - **[Delivery & Retries](./features/event-delivery.mdx)**: Understand how Outpost ensures reliable event delivery with support for various destination types, retries, and best practices. - **[Alerts](./features/alerts.mdx)**: Configure alerts and auto-disable destinations for event delivery failures based on consecutive failures. diff --git a/docs/pages/features/event-delivery.mdx b/docs/pages/features/event-delivery.mdx index 07d9f4d5..a0ebff20 100644 --- a/docs/pages/features/event-delivery.mdx +++ b/docs/pages/features/event-delivery.mdx @@ -17,28 +17,3 @@ Manual retries can be triggered for any given event via the [Event API](/docs/ap ## Disabled destinations If the destination for an event is disabled—through the API, user portal, or automatically because a [failure threshold](/docs/features/alerts) has been reached—the event will be discarded and cannot be retried. - -## Webhook signature & headers - -For the `webhook` destination type, Outpost will automatically add the following headers to the webhook request: - -- `X-Event-ID`: The unique ID of the event. -- `X-Event-Timestamp`: The timestamp of the event in ISO 8601 format. -- `X-Event-Topic`: The topic of the event. -- `X-Event-Signature`: A HMAC-SHA256 signature of the event. - -For backward compatibility with your existing webhook implementation, you can change the header prefix, disable headers, or customize the signature algorithm and logic. You can refer to the [migration guide](/docs/guides/migrate-to-outpost#webhook-customization) to learn more. - -The following environment variables can be used to customize the webhook behavior: - -| Environment Variable | Default Value | Required | -| ------------------------------------------------ | ------------------------------------------------------ | -------- | -| `DESTINATIONS_WEBHOOK_HEADER_PREFIX` | `x-outpost-` | No | -| `DESTINATIONS_WEBHOOK_DISABLE_EVENT_ID_HEADER` | `false` | No | -| `DESTINATIONS_WEBHOOK_DISABLE_SIGNATURE_HEADER` | `false` | No | -| `DESTINATIONS_WEBHOOK_DISABLE_TIMESTAMP_HEADER` | `false` | No | -| `DESTINATIONS_WEBHOOK_DISABLE_TOPIC_HEADER` | `false` | No | -| `DESTINATIONS_WEBHOOK_SIGNATURE_VALUE_TEMPLATE` | `{{.Timestamp.Unix}}.{{.Body}}` | No | -| `DESTINATIONS_WEBHOOK_SIGNATURE_HEADER_TEMPLATE` | `t={{.Timestamp.Unix}},v0={{.Signatures \| join ","}}` | No | -| `DESTINATIONS_WEBHOOK_SIGNATURE_ENCODING` | `hex` | No | -| `DESTINATIONS_WEBHOOK_SIGNATURE_ALGORITHM` | `hmac-sha256` | No | diff --git a/docs/pages/features/filter.mdx b/docs/pages/features/filter.mdx new file mode 100644 index 00000000..489e5ac3 --- /dev/null +++ b/docs/pages/features/filter.mdx @@ -0,0 +1,306 @@ +--- +title: Destination Filters +--- + +Destination filters allow tenants to selectively receive only events that match specific criteria. Instead of receiving all events for subscribed topics, a destination can define filters to route events based on their properties. + +## Overview + +When an event is published, Outpost evaluates each destination's filter against the event. A destination receives an event only if: + +1. The destination is enabled +2. The event's topic matches one of the destination's subscribed topics +3. The event matches the destination's filter (or no filter is defined) + +Filters are optional. Destinations without filters receive all events matching their subscribed topics. + +## Event Structure + +Events have five top-level properties that can be filtered: + +| Property | Description | +| -------- | ----------- | +| `id` | The event ID | +| `topic` | The event topic | +| `time` | Event timestamp (RFC 3339 format) | +| `metadata` | Additional event information | +| `data` | The event payload | + +Your filter should specify one or more of these at the top level: + +```json +{ + "topic": "order.created", + "data": { + "status": "paid" + }, + "metadata": { + "source": "api" + } +} +``` + +## Exact Match + +Specify the field path and value to match: + +```json +{ + "data": { + "status": "paid" + } +} +``` + +Multiple conditions are combined with AND logic: + +```json +{ + "data": { + "status": "paid", + "currency": "usd" + } +} +``` + +Nested objects are supported: + +```json +{ + "data": { + "customer": { + "tier": "premium" + } + } +} +``` + +## Arrays + +Arrays match if they contain the specified value: + +```json +{ + "data": { + "tags": "urgent" + } +} +``` + +This matches events like: `{ "tags": ["urgent", "support"] }` + +## Operators + +Use operators for complex matching. + +### Comparison Operators + +| Operator | Description | +| -------- | ----------- | +| `$eq` | Equals | +| `$neq` | Not equals | +| `$gt` | Greater than | +| `$gte` | Greater or equal | +| `$lt` | Less than | +| `$lte` | Less or equal | + +Match orders over $100: + +```json +{ + "data": { + "amount": { "$gte": 100 } + } +} +``` + +Multiple operators on the same field are combined with AND: + +```json +{ + "data": { + "amount": { + "$gte": 100, + "$lt": 500 + } + } +} +``` + +### Membership Operators + +| Operator | Description | +| -------- | ----------- | +| `$in` | Value in array, or substring match | +| `$nin` | Value not in array | + +Match specific statuses: + +```json +{ + "data": { + "status": { + "$in": ["pending", "processing", "completed"] + } + } +} +``` + +### String Operators + +| Operator | Description | +| -------- | ----------- | +| `$startsWith` | String starts with | +| `$endsWith` | String ends with | + +Match emails ending with a domain: + +```json +{ + "data": { + "email": { "$endsWith": "@example.com" } + } +} +``` + +### Field Existence Operator + +| Operator | Description | +| -------- | ----------- | +| `$exist` | Field exists (`true`) or doesn't (`false`) | + +Check if a field exists: + +```json +{ + "data": { + "discount_code": { "$exist": true } + } +} +``` + +Check if a field doesn't exist: + +```json +{ + "data": { + "deleted_at": { "$exist": false } + } +} +``` + +## Logical Operators + +### `$or` - Match any condition + +```json +{ + "$or": [ + { "data": { "status": "paid" } }, + { "data": { "status": "shipped" } } + ] +} +``` + +### `$and` - Match all conditions (explicit) + +While multiple conditions are implicitly ANDed, `$and` is useful when you need to apply multiple conditions that can't be nested together: + +```json +{ + "$and": [ + { "data": { "status": "active" } }, + { "data": { "amount": { "$gte": 100 } } } + ] +} +``` + +### `$not` - Negate a condition + +```json +{ + "data": { + "status": { + "$not": { + "$in": ["cancelled", "refunded"] + } + } + } +} +``` + +## Common Examples + +### Filter by Metadata + +Route events based on source: + +```json +{ + "metadata": { + "source": "api" + } +} +``` + +### Filter by Time Range + +Match events within a specific time window: + +```json +{ + "time": { + "$gte": "2025-01-01T00:00:00Z", + "$lt": "2025-02-01T00:00:00Z" + } +} +``` + +### Filter by Numeric Value + +Process only high-value orders: + +```json +{ + "data": { + "amount": { "$gte": 1000 } + } +} +``` + +## API Usage + +Filters are set when creating or updating a destination via the `filter` field: + +```sh +curl --location 'https:///api/v1//destinations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "type": "webhook", + "topics": ["orders"], + "config": { + "url": "https://example.test/webhooks" + }, + "filter": { + "data": { + "status": { "$in": ["paid", "shipped"] } + } + } +}' +``` + +To remove a filter from a destination, set the `filter` field to an empty object: + +```sh +curl --location --request PATCH 'https:///api/v1//destinations/' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "filter": {} +}' +``` + +## Portal Configuration + +To enable destination filters in the tenant portal UI, set the `PORTAL_ENABLE_DESTINATION_FILTER` environment variable to `true`. diff --git a/docs/zudoku.config.ts b/docs/zudoku.config.ts index 08961642..28de9c60 100644 --- a/docs/zudoku.config.ts +++ b/docs/zudoku.config.ts @@ -110,8 +110,8 @@ const config: ZudokuConfig = { collapsible: false, items: [ { type: "doc", id: "features/multi-tenant-support" }, - { type: "doc", id: "features/destinations" }, { type: "doc", id: "features/topics" }, + { type: "doc", id: "features/filter" }, { type: "doc", id: "features/publish-events" }, { type: "doc", id: "features/event-delivery" }, { type: "doc", id: "features/alerts" }, @@ -125,6 +125,23 @@ const config: ZudokuConfig = { }, ], }, + { + type: "category", + label: "Destinations", + link: "destinations", + collapsed: false, + collapsible: false, + items: [ + { type: "doc", id: "destinations/webhook" }, + { type: "doc", id: "destinations/hookdeck" }, + { type: "doc", id: "destinations/aws-kinesis", label: "AWS Kinesis" }, + { type: "doc", id: "destinations/aws-sqs", label: "AWS SQS" }, + { type: "doc", id: "destinations/aws-s3", label: "AWS S3" }, + { type: "doc", id: "destinations/azure-service-bus", label: "Azure Service Bus" }, + { type: "doc", id: "destinations/gcp-pubsub", label: "GCP Pub/Sub" }, + { type: "doc", id: "destinations/rabbitmq", label: "RabbitMQ" }, + ], + }, { type: "category", label: "Guides",