diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 266988b..ba25b50 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -19,7 +19,9 @@ ** xref:ROOT:astream-subscriptions-shared.adoc[] ** xref:ROOT:astream-subscriptions-failover.adoc[] ** xref:ROOT:astream-subscriptions-keyshared.adoc[] -* xref:developing:astream-functions.adoc[] +* {pulsar-short} functions +** xref:developing:astream-functions.adoc[] +** xref:developing:astream-functions-transform.adoc[] * xref:developing:starlight.adoc[] * xref:developing:astream-cdc.adoc[] * xref:developing:gpt-schema-translator.adoc[] diff --git a/modules/ROOT/partials/functions-ui-common-steps-pt1.adoc b/modules/ROOT/partials/functions-ui-common-steps-pt1.adoc new file mode 100644 index 0000000..3ad0d1f --- /dev/null +++ b/modules/ROOT/partials/functions-ui-common-steps-pt1.adoc @@ -0,0 +1,7 @@ +. In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. + +. Click the name of the tenant where you want to deploy a function. + +. On the *Functions* tab, click *Create Function*. + +. Enter a function name. \ No newline at end of file diff --git a/modules/ROOT/partials/functions-ui-common-steps-pt2.adoc b/modules/ROOT/partials/functions-ui-common-steps-pt2.adoc new file mode 100644 index 0000000..ab6fe4e --- /dev/null +++ b/modules/ROOT/partials/functions-ui-common-steps-pt2.adoc @@ -0,0 +1,37 @@ +. Select the namespace and one or more input topics for the function. + +. If needed, select an output namespace and topics. ++ +Whether you need to specify an output topic depends on the function's logic. +For example, the `cast` xref:developing:astream-functions-transform.adoc[transform function] needs an output topic to receive the transformed messages. + +. Optional: Select a log topic. ++ +The log topic is a separate output topic for messages containing additional `loglevel`, `fqn`, and `instance` properties. + +. Optional: Configure advanced settings: ++ +* *Instances*: Enter a number of sink instances to run. +* *Processing Guarantee*: Select one of the following: +** *ATLEAST_ONCE* (default): Each message sent to the function can be processed one or more times. +** *ATMOST_ONCE*: Each message sent to the function is processed 0 or 1 times. +This means there is a change that a message is not processed. +** *EFFECTIVELY_ONCE*: Each message sent to the function has only one output associated with it. +* *Timeout*: Set a timeout limit. +* *Auto Acknowledge*: Enable or disable automatic message acknowledgment. + +. If needed, provide a JSON-formatted config key for your function. +This configuration depends on your function code. + +. Click *Create*. + +. Confirm your function was created on the *Functions* tab. ++ +You can also verify the function deployment with the {pulsar-short} CLI: ++ +[source,shell] +---- +./bin/pulsar-admin functions list --tenant $TENANT +---- + +Once created, the function starts processing messages from the input topics according to its logic. \ No newline at end of file diff --git a/modules/apis/pages/api-operations.adoc b/modules/apis/pages/api-operations.adoc index 733cfd3..c04e56d 100644 --- a/modules/apis/pages/api-operations.adoc +++ b/modules/apis/pages/api-operations.adoc @@ -93,8 +93,6 @@ curl -sS --fail -L -X GET "https://$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT" ---- .Result -[%collapsible] -==== [source,json] ---- [ @@ -102,7 +100,6 @@ curl -sS --fail -L -X GET "https://$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT" "mytenant/mynamespace" ] ---- -==== === Create a namespace @@ -126,9 +123,8 @@ After you create a namespace, you can use targeted endpoints to get and set spec The following examples demonstrate some of these endpoints. For more information and all endpoints, see the xref:astra-streaming:apis:attachment$pulsar-admin.html[{product} {pulsar-short} Admin API specification reference]. -.Message retention policy -[%collapsible] -====== +==== Message retention policy + Get the message retention settings for a namespace: [source,curl] @@ -163,11 +159,9 @@ curl -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESPACE/retenti ---- include::ROOT:partial$curl-get-template-tip.adoc[] -====== -.Backlog quota policy -[%collapsible] -====== +==== Backlog quota policy + Get the backlog quota settings for a namespace: [source,curl] @@ -208,11 +202,9 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESP ---- include::ROOT:partial$curl-get-template-tip.adoc[] -====== -.Message TTL policy -[%collapsible] -====== +==== Message TTL policy + Get the message time-to-live (TTL) setting for a namespace: [source,curl] @@ -233,11 +225,9 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESP --header "Content-Type: application/json" \ --data 5000 ---- -====== -.Automatic topic creation policy -[%collapsible] -====== +==== Automatic topic creation policy + Edit the automatic topic creation settings for the specified namespace, passing the new settings in the request body: [source,curl] @@ -259,11 +249,9 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESP `topicType` indicates the type of topic that can be automatically created. It can be either `non-partitioned` or `partitioned`. If set to `non-partitioned`, then the `defaultNumPartitions` field is ignored. -====== -.Maximum consumers per topic policy -[%collapsible] -====== +==== Maximum consumers per topic policy + Get the maximum number of consumers allowed for each topic in a namespace: [source,curl] @@ -282,11 +270,9 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESP --header "Content-Type: application/json" \ --data 100 ---- -====== -.Maximum topics per namespace policy -[%collapsible] -====== +==== Maximum topics per namespace policy + Get the maximum number of topics allowed in a namespace: [source,curl] @@ -305,11 +291,10 @@ curl -sS --fail -L -X POST "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/$NAMESP --header "Authorization: Bearer $PULSAR_TOKEN" \ --data 1000 ---- -====== === Delete a namespace -[source,curl] +[source,curl,subs="+quotes"] ---- curl -sS --fail -L -X DELETE "$WEB_SERVICE_URL/admin/v2/namespaces/$TENANT/**NAMESPACE_TO_DELETE**" \ --header "Authorization: Bearer $PULSAR_TOKEN" @@ -443,9 +428,8 @@ There are multiple endpoints you can use to get information about topics, includ The following examples demonstrate some of these endpoints. For more information and all endpoints, see the xref:astra-streaming:apis:attachment$pulsar-admin.html[{product} {pulsar-short} Admin API specification reference]. -.Get internal statistics for a persistent, non-partitioned topic -[%collapsible] -====== +==== Get internal statistics for a persistent, non-partitioned topic + [source,curl] ---- curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPACE/$TOPIC/internalStats" \ @@ -487,11 +471,9 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPA } } ---- -====== -.Get statistics for a persistent, non-partitioned topic -[%collapsible] -====== +==== Get statistics for a persistent, non-partitioned topic + [source,curl] ---- curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPACE/$TOPIC/stats" \ @@ -537,11 +519,9 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPA ...TRUNCATED FOR READABILITY... } ---- -====== -.Get statistics for a persistent, partitioned topic -[%collapsible] -====== +==== Get statistics for a persistent, partitioned topic + [source,curl] ---- curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPACE/$TOPIC/partitioned-stats" \ @@ -658,11 +638,9 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPA ...TRUNCATED FOR READABILITY... } ---- -====== -.Get statistics for all topics in a namespace -[%collapsible] -====== +==== Get statistics for all topics in a namespace + [source,curl] ---- curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/stats/topics/$TENANT/$NAMESPACE" \ @@ -792,7 +770,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/stats/topics/$TENANT/$NAMES ...TRUNCATED FOR READABILITY... } ---- -====== == Subscription operations @@ -816,8 +793,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPA ---- .Result -[%collapsible] -==== [source,json] ---- [ @@ -825,7 +800,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v2/persistent/$TENANT/$NAMESPA "subscript2" ] ---- -==== === Create a subscription @@ -883,15 +857,12 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ---- .Result -[%collapsible] -==== [source,json] ---- [ "testfunction1" ] ---- -==== === Get function status @@ -903,8 +874,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -931,7 +900,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ] } ---- -==== === Get function statistics @@ -943,8 +911,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -984,7 +950,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ] } ---- -==== === Get function details @@ -996,8 +961,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -1073,7 +1036,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/functions/$TENANT/$NAMESPAC "subscriptionPosition": "Latest" } ---- -==== === Start a function @@ -1119,15 +1081,12 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE" \ ---- .Result -[%collapsible] -==== [source,json] ---- [ "mysink1" ] ---- -==== === Get the status of a sink @@ -1139,8 +1098,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -1166,7 +1123,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ ] } ---- -==== === Get sink details @@ -1178,8 +1134,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$S ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -1233,7 +1187,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sinks/$TENANT/$NAMESPACE/$S "transformFunctionConfig": null } ---- -==== === Create a sink @@ -1328,15 +1281,12 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE" ---- .Result -[%collapsible] -==== [source,json] ---- [ "mysource1" ] ---- -==== === Get the status of a source @@ -1348,8 +1298,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -1375,7 +1323,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ ] } ---- -==== === Get source details @@ -1387,8 +1334,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -1427,7 +1372,6 @@ curl -sS --fail -L -X GET "$WEB_SERVICE_URL/admin/v3/sources/$TENANT/$NAMESPACE/ "topicName": "persistent://testcreate/ns0/t1" } ---- -==== === Create a source diff --git a/modules/apis/pages/index.adoc b/modules/apis/pages/index.adoc index 7149748..fe78b56 100644 --- a/modules/apis/pages/index.adoc +++ b/modules/apis/pages/index.adoc @@ -63,8 +63,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/tenants" ---- .Result -[%collapsible] -==== [source,json] ---- [ @@ -116,7 +114,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/tenants" } ] ---- -==== [#get-cloud-providers-and-regions] === Get cloud providers and regions @@ -131,8 +128,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/providers ---- .Result -[%collapsible] -==== [source,json] ---- { @@ -155,7 +150,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/providers ] } ---- -==== [#create-a-tenant] === Create a tenant @@ -253,8 +247,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/replicati If georeplication is enabled, the response includes the replication configuration details. .Result -[%collapsible] -==== [source,json] ---- { @@ -298,7 +290,6 @@ If georeplication is enabled, the response includes the replication configuratio } } ---- -==== === Enable georeplication @@ -398,50 +389,37 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/streaming/tenants/$ You can get token IDs from <>, and you can get the cluster name from <>. .Result -[%collapsible] -==== [source,json] ---- Output: Raw string JWT eyJhbGciOiJSUzI1NiIsI... ---- -==== === Create a {pulsar-short} token Create a new {pulsar-short} token (JWT) for a given tenant and cluster. You can get the cluster name from <>. -[tabs] -====== v2 endpoint:: -+ --- The `v2/streaming/tenants/$TENANT/tokens` endpoint returns a raw string JWT with an unlimited lifetime: - ++ [source,curl] ---- curl -sS --fail -L -X POST "https://api.astra.datastax.com/v2/streaming/tenants/$TENANT/tokens" \ --header "Authorization: Bearer $ASTRA_TOKEN" \ --header "X-DataStax-Pulsar-Cluster: $CLUSTER" ---- - ++ .Result -[%collapsible] -==== [source,json] ---- Output: new raw string JWT eyJhbGciOiJSUzI1NiIsI... ---- -==== --- v3 endpoint:: -+ --- The `v3/streaming/tenants/$TENANT/tokens` endpoint returns a JSON response and accepts additional token parameters: - ++ [source,curl,subs="+quotes"] ---- curl -sS --fail -L -X POST "https://api.astra.datastax.com/v3/streaming/tenants/$TENANT/tokens" \ @@ -453,16 +431,15 @@ curl -sS --fail -L -X POST "https://api.astra.datastax.com/v3/streaming/tenants/ "type": "**TYPE**" }' ---- - ++ Replace the following: - ++ * **`EXPIRATION`**: The token lifetime expressed as a number and unit, such as `1d` for one day. * **`ROLE`**: A string describing the token's role. * **`TYPE`**: The token's privilege level, either `admin` or `non-admin`. ++ .Result -[%collapsible] -==== [source,json] ---- { @@ -471,9 +448,6 @@ Replace the following: "token": "eyJhbGciOiJSUzI1NiIsI..." } ---- -==== --- -====== === Delete a {pulsar-short} token diff --git a/modules/developing/pages/astream-cdc.adoc b/modules/developing/pages/astream-cdc.adoc index bf2c328..bddca5e 100644 --- a/modules/developing/pages/astream-cdc.adoc +++ b/modules/developing/pages/astream-cdc.adoc @@ -269,11 +269,8 @@ After you <> and <>, For <>, you must use the xref:astra-db-serverless:api-reference:devops-api.adoc[{product-short} {devops-api}] to enable CDC in secondary regions. -[tabs] -====== -{astra-ui}:: -+ --- +==== Enable CDC in the {astra-ui} + . In the {astra-ui}, click the name of the database where you want to enable CDC. + If you created the demo table in <>, select the database where you created that table. @@ -291,11 +288,9 @@ If you created the demo table in <>, select the database where you Enabling CDC on any table disables the *Add a region* functionality in the {astra-ui} for that database. You must use the {product-short} {devops-api} to add a region after enabling CDC. ==== --- -{product-short} {devops-api}:: -+ --- +==== Enable CDC with the {product-short} {devops-api} + [IMPORTANT] ==== CDC for multi-region {astra-db} {db-serverless-vector} databases is available only to qualified participants in this private preview release. @@ -307,12 +302,10 @@ If you're interested in this private preview feature, contact your {company} acc Use the {product-short} {devops-api} to enable CDC on one or more tables in the same database in the same request. -You can use these endpoints to enable CDC in single-region and multi-region databases. +You can use the following endpoints to enable CDC in single-region and multi-region databases. +To check the CDC configuration for a database or table, see <>. -[tabs] -==== Enable CDC after deploying a region:: -+ Use these steps to enable CDC in a single-region database or in previously-deployed regions of a multi-region database. You can also use this configuration to enable CDC on new tables. + @@ -387,7 +380,6 @@ Tenant must be deployed to the same region as the database. You can get these names with the xref:astra-streaming:apis:index.adoc#astra-streaming-devops-api-tenant-operations[{astra-stream} {devops-api}] or from the **{astra-stream}** dashboard in the {astra-ui} (click icon:grip[name="Applications"], and then select *Streaming*). Deploy a secondary region with CDC enabled:: -+ Use `{devops-api-ref-url}#tag/Database-Operations/operation/addDatacenters[POST /v2/databases/**DB_ID**/datacenters]` to add a region to a multi-region database and enable CDC in the same command: + [source,curl,subs="+quotes"] @@ -431,11 +423,8 @@ The tenant must be in the same region as specified in `region`. + All tables replicated to the new region automatically have CDC enabled. The `astracdc` namespace and CDC topics for each table are created in the specified regional tenant. -==== -To check the CDC configuration for a database or table, see <>. --- -====== +==== CDC topics When you enable CDC on a database for the first time, {astra-db} automatically creates an `astracdc` namespace in your streaming tenant. For each table where you enable CDC, {astra-db} creates two topics in the `astracdc` namespace: @@ -514,11 +503,9 @@ If your table has more than a few rows, use a more specific `select` statement t select * from default_keyspace.cdc_demo; ---- + -.Result -[%collapsible] -==== The demo table now has two rows: - ++ +.Result [source,console] ---- key | c1 @@ -528,7 +515,6 @@ The demo table now has two rows: (2 rows) ---- -==== . Verify that the change was passed from CDC to your sink by fetching the data from your sink service deployment. + @@ -695,8 +681,6 @@ bin/pulsar-admin topics partitioned-stats persistent://**DATA_TOPIC_STRING** Replace `**DATA_TOPIC_STRING**` with the `data-` topic string in the format of `**TENANT_NAME**/astracdc/data-**DB_ID**-**KEYSPACE_NAME**.**TABLE_NAME**`. + .Result -[%collapsible] -==== [source,json] ---- { @@ -736,7 +720,6 @@ Replace `**DATA_TOPIC_STRING**` with the `data-` topic string in the format of ` "partitions" : { } } ---- -==== [#multi-region-cdc] == Enable CDC for multi-region databases @@ -795,25 +778,17 @@ However, be aware of the following limitations: You can check your active CDC configurations in the {astra-ui} or with the {product-short} {devops-api}. -[tabs] -====== -{astra-ui}:: -+ --- +=== Check CDC status in the {astra-ui} + . In the {astra-ui}, click the name of the database that you want to inspect. . Click the *CDC* tab, and then review the list of tables where you have enabled CDC. . Click a table's name to inspect the table's CDC configuration. --- -{product-short} {devops-api}:: -+ --- -[tabs] -==== +=== Check CDC status with the {product-short} {devops-api} + By database:: -+ Use `{devops-api-ref-url}#tag/Database-Operations/operation/getCDC[GET /v3/databases/**DB_ID**/cdc]` to get CDC configuration details for an entire database: + [source,curl,subs="+quotes"] @@ -860,7 +835,6 @@ For multi-region databases with CDC enabled in multiple regions, the `regions` a ---- By table:: -+ Use `{devops-api-ref-url}#tag/Database-Operations/operation/getCDC[GET /v3/databases/**DB_ID**/keyspaces/**KEYSPACE_NAME**/tables/**TABLE_NAME**/cdc]` to get CDC configuration details for a specific table: + [source,curl,subs="+quotes"] @@ -895,9 +869,6 @@ For multi-region databases with CDC enabled in multiple regions, the `regions` a } ] ---- -==== --- -====== [#put-cdc] === Update a database's CDC configuration @@ -976,13 +947,13 @@ You can get these names with the xref:astra-streaming:apis:index.adoc#astra-stre . Send the updated configuration to `{devops-api-ref-url}#tag/Database-Operations/operation/updateCDC[PUT /v3/databases/**DB_ID**/cdc]`. + [IMPORTANT] -===== +==== This is a desired state list. Make sure that you include all existing tables and regions that you want to keep in the CDC configuration. If you omit any existing tables or regions from the request, CDC is disabled for those tables or regions. -===== +==== + [source,curl,subs="+quotes"] ---- @@ -1042,11 +1013,20 @@ CDC is automatically disabled if you drop a table, terminate a database, or remo You can remove a table's CDC configuration without deleting the table. For multi-region databases, this disables CDC for the table across all regions. -[tabs] -====== -{astra-ui}:: +Disabling CDC doesn't remove the associated namespace, topics, or {astra-stream} tenant: + +* If you reenable CDC for the same table, the existing topics are reused with the existing records. + +* If you want to discard a table's CDC records, you must manually remove the associated {astra-stream} artifacts after disabling CDC: + --- +** If you remove CDC from a table, then you can delete the table's `data-` and `log-` topics from the `astracdc` namespace in the {astra-stream} tenant. +For multi-region databases, make sure that you delete the topics in the tenant for each region where the database is deployed +** If you remove CDC from all tables in a region, and you no longer need CDC in that region, then you can delete the `astracdc` namespace from the {astra-stream} tenant in that region. +Deleting the namespace also deletes the topics within that namespace. +** If you remove CDC from all of a database's tables, and you no longer need an {astra-stream} tenant for any reason, you can delete the entire tenant. + +==== Disable CDC with the {astra-ui} + . In the {astra-ui}, click the name of the database where you want to disable CDC. . Click the *CDC* tab. @@ -1054,11 +1034,9 @@ For multi-region databases, this disables CDC for the table across all regions. . In the *Change Data Capture* list, click the table's name. . Click *Disable* to remove the table's CDC configuration. --- -{product-short} {devops-api}:: -+ --- +==== Disable CDC with the {product-short} {devops-api} + Use `{devops-api-ref-url}#tag/Database-Operations/operation/deleteCDC[DELETE /v3/databases/**DB_ID**/cdc]` to disable CDC for a table: [source,curl,subs="+quotes"] @@ -1089,20 +1067,6 @@ If you want to disable CDC for multiple tables at once, include an object for ea You can use `PUT /v3/databases/**DB_ID**/cdc` to make multiple changes to a database's CDC configuration in one request, including additions, changes, and removals of tables and regions. For more information, see <>. ==== --- -====== - -Disabling CDC doesn't remove the associated namespace, topics, or {astra-stream} tenant: - -* If you reenable CDC for the same table, the existing topics are reused with the existing records. - -* If you want to discard a table's CDC records, you must manually remove the associated {astra-stream} artifacts after disabling CDC: -+ -** If you remove CDC from a table, then you can delete the table's `data-` and `log-` topics from the `astracdc` namespace in the {astra-stream} tenant. -For multi-region databases, make sure that you delete the topics in the tenant for each region where the database is deployed -** If you remove CDC from all tables in a region, and you no longer need CDC in that region, then you can delete the `astracdc` namespace from the {astra-stream} tenant in that region. -Deleting the namespace also deletes the topics within that namespace. -** If you remove CDC from all of a database's tables, and you no longer need an {astra-stream} tenant for any reason, you can delete the entire tenant. == See also diff --git a/modules/developing/pages/astream-functions-transform.adoc b/modules/developing/pages/astream-functions-transform.adoc new file mode 100644 index 0000000..f8d3dba --- /dev/null +++ b/modules/developing/pages/astream-functions-transform.adoc @@ -0,0 +1,55 @@ += {product} {pulsar-short} transform functions configuration reference +:navtitle: Transform functions configuration reference + +include::common:streaming:partial$transform-functions/intro.adoc[] + +To deploy transform functions, see xref:developing:astream-functions.adoc#deploy-functions-in-the-astra-portal[Deploy functions in the {astra-ui}]. + +== Configure transform functions + +include::common:streaming:partial$transform-functions/config.adoc[] + +[#expression-language] +=== Expression language + +include::common:streaming:partial$transform-functions/expression-language.adoc[] + +[#conditional-steps] +=== Conditional steps + +include::common:streaming:partial$transform-functions/when.adoc[] + +[#cast] +== Cast + +include::common:streaming:partial$transform-functions/cast.adoc[] + +[#compute] +== Compute + +include::common:streaming:partial$transform-functions/compute.adoc[] + +[#drop] +== Drop + +include::common:streaming:partial$transform-functions/drop.adoc[] + +[#drop-fields] +== Drop fields + +include::common:streaming:partial$transform-functions/drop-fields.adoc[] + +[#flatten] +== Flatten + +include::common:streaming:partial$transform-functions/flatten.adoc[] + +[#merge-keyvalue] +== Merge KeyValue + +include::common:streaming:partial$transform-functions/merge-keyvalue.adoc[] + +[#unwrap-keyvalue] +== Unwrap KeyValue + +include::common:streaming:partial$transform-functions/unwrap-keyvalue.adoc[] \ No newline at end of file diff --git a/modules/developing/pages/astream-functions.adoc b/modules/developing/pages/astream-functions.adoc index fb0b33c..def758d 100644 --- a/modules/developing/pages/astream-functions.adoc +++ b/modules/developing/pages/astream-functions.adoc @@ -1,5 +1,5 @@ -= {product} functions -:navtitle: Use {pulsar-short} functions += Deploy and manage {product} {pulsar-short} functions +:navtitle: Deploy and manage {pulsar-short} functions Functions are lightweight compute processes that you can run on each message a topic receives. You can apply custom logic to a message, transforming or enriching it, and then output it to a different topic. @@ -16,7 +16,7 @@ Custom functions require a xref:operations:astream-pricing.adoc[paid {product} s ==== [#create-an-archive] -== Deploy a function archive +== Deploy a function ZIP or JAR You can write {pulsar-short} functions for {product} in Python, Java, or Go. @@ -26,11 +26,8 @@ You can deploy the same archive to either environment. This is recommended for complex functions with long scripts, multiple scripts, or many dependencies. You can also use this approach to deploy multiple function classes with a single package. -[tabs] -====== -Python functions:: -+ --- +=== Prepare a Python function for deployment + To create a Python function, the `.zip` file must have the correct structure. For example, assuming a project named `testpythonfunction`, the extracted archive has the following structure: @@ -44,7 +41,7 @@ For example, assuming a project named `testpythonfunction`, the extracted archiv . Prepare the directory structure: + -[source,bash] +[source,bash,subs="+quotes"] ---- mkdir **FUNCTION_NAME** mkdir **FUNCTION_NAME**/python-code @@ -85,11 +82,11 @@ pip install pulsar-client==2.10.0 cd deps zip -r ../my-python-function.zip . ---- --- -Java functions:: -+ --- +. <>. + +=== Prepare a Java function for deployment + To deploy a Java function, you must create a `.jar` file. . Create a Java project for your function. @@ -165,8 +162,6 @@ mvn package ---- + .Result -[%collapsible] -==== [source,console] ---- [INFO] ------------------------------------------------------------------------ @@ -176,20 +171,12 @@ mvn package [INFO] Finished at: 2023-05-16T16:19:05-04:00 [INFO] ------------------------------------------------------------------------ ---- -==== --- -//// -TODO: -Go functions:: -+ --- -Must have 4 steps to maintain numbering. --- -//// -====== +. <>. + +[#deploy-the-function-with-the-pulsar-cli] +=== Deploy the function with the {pulsar-short} CLI -[start=5] . If you haven't done so already, xref:configure-pulsar-env.adoc[set up your environment for the {pulsar-short} binaries]. . Create a deployment configuration YAML file that defines the function metadata and associated topics: @@ -242,13 +229,9 @@ Optionally, you can declare a `logTopic` in the same way: `persistent://**TENANT bin/pulsar-admin functions create --function-config-file **PATH_TO_FUNCTION_CONFIG_YAML** ---- + -.Result -[%collapsible] -==== A response of `Created Successfully!` indicates the function is deployed and ready to accept messages. - ++ If the response is `402 Payment Required` with `Reason: only qualified organizations can create functions`, then you must upgrade to a xref:operations:astream-pricing.adoc[paid {product} subscription plan]. -==== + [TIP] ==== @@ -282,33 +265,28 @@ You can also check the {astra-ui} to confirm the function is listed on the tenan + See <> for more information on testing and monitoring your function in {product}. +[#deploy-functions-in-the-astra-portal] == Deploy functions in the {astra-ui} -. In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. +In the {astra-ui}, you can deploy your own function code and built-in {company} transform functions. -. Click the name of the tenant where you want to deploy a function. +=== Deploy custom functions in the {astra-ui} -. On the *Functions* tab, click *Create Function*. +include::ROOT:partial$functions-ui-common-steps-pt1.adoc[] -. Enter a function name, and then select the namespace within the tenant. +. Select *Upload my own code*. -. Upload function code: -+ -[tabs] -====== -Upload your own code:: -+ --- -.. Select *Upload my own code*. - -.. Select your function file: +. Select your function file: + * `.py`: A single, independent Python script * `.zip`: A Python script with dependencies * `.jar`: A Java function * `.go`: A Go function -.. Based on the uploaded file, select the specific class (function) to deploy. ++ +For examples, see <> and <>. + +. Based on the uploaded file, select the specific class (function) to deploy. + {product-short} generates a list of acceptable classes detected in the code. A file can contain multiple classes, but only one is used per deployment. @@ -320,36 +298,53 @@ If there isn't a class in the Python file, the class name is the filename withou For Java scripts, the class name is the class to execute. + image::developing:astream-exclamation-function.png[Exclamation Function] --- -Use {company} transform function:: -+ --- -.. Select *Use {company} transform function*. --- -====== +include::ROOT:partial$functions-ui-common-steps-pt2.adoc[] -. Select input topics. +=== Deploy {company} transform functions in the {astra-ui} -. Optional: Select output and log topics. +To deploy built-in {company} transform functions, you select the function you want to deploy and provide a JSON-formatted config key, if you want to use a non-default configuration. +For more information, see the xref:developing:astream-functions-transform.adoc[transform functions configuration reference]. -. Optional: Configure advanced settings. -+ -* *Instances*: Enter a number of sink instances to run. -* *Processing Guarantee*: Select one of the following: -** *ATLEAST_ONCE* (default): Each message sent to the function can be processed one or more times. -** *ATMOST_ONCE*: Each message sent to the function is processed 0 or 1 times. -This means there is a change that a message is not processed. -** *EFFECTIVELY_ONCE*: Each message sent to the function has only one output associated with it. -* *Timeout*: Set a timeout limit. -* *Auto Acknowledge*: Enable or disable automatic message acknowledgment. +include::ROOT:partial$functions-ui-common-steps-pt1.adoc[] + +. Select *Use {company} transform function*. + +. Select the xref:developing:astream-functions-transform.adoc[transform function] that you want to use. + +include::ROOT:partial$functions-ui-common-steps-pt2.adoc[] + +=== Deploy functions in sinks + +By default, functions modify data either after it is written to a topic by a source connector, or before it is read from a topic by a sink connector. +This requires either a custom connector or an intermediate topic (with additional storage, IO, and latency). + +Alternatively, you can deploy functions in a sink to apply preprocessing to sink topic writes (outgoing messages from the sink). + +To deploy a function in a sink, xref:streaming-learning:pulsar-io:connectors/index.adoc[create a sink], and include the function in your sink configuration. +You can do this in the {astra-ui}, with the {pulsar-short} CLI, or with {product} {pulsar-short} Admin API. -. Optional: Provide a config key, if required. -For more information, see the https://pulsar.apache.org/functions-rest-api/#operation/registerFunction[{pulsar-short} documentation]. +For example, with the {astra-ui}: -. Click *Create*. +. In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. + +. Click the name of the tenant where you want to create the sink. + +. Click the **Sinks** tab, and then click **Create Sink**. + +. Configure the sink and select a function to deploy with the sink. + +. Click **Create**, and then wait for the sink to initialize. ++ +When the sink is ready, its status changes to **Running**. -. Confirm your function was created on the *Functions* tab. +. When the sink is running, inspect the sink connector's log to verify that the function was loaded at sink creation. +For example: ++ +[source,console] +---- +2022-11-14T15:01:02.398190413Z 2022-11-14T15:01:02,397+0000 [main] INFO org.apache.pulsar.functions.runtime.thread.ThreadRuntime - ThreadContainer starting function with instanceId 0 functionId f584ae69-2eda-449b-9759-2d19fd7c4da5 namespace astracdc +---- == Manage deployed functions diff --git a/modules/developing/pages/clients/nodejs-produce-consume.adoc b/modules/developing/pages/clients/nodejs-produce-consume.adoc index 722067e..647ee8f 100644 --- a/modules/developing/pages/clients/nodejs-produce-consume.adoc +++ b/modules/developing/pages/clients/nodejs-produce-consume.adoc @@ -18,13 +18,9 @@ For a complete source code example, see the https://github.com/datastax/astra-st Install the {cpp} {pulsar-short} library dependency required by the Node.js {pulsar-short} client npm package. -{pulsar-short} Node client versions 1.8 and later do not require installation of the {cpp} {pulsar-short} library dependency. +{pulsar-short} Node client versions 1.8 and later don't require installation of the {cpp} {pulsar-short} library dependency. -[tabs] -====== -Ubuntu-based Debian:: -+ --- +.Ubuntu-based Debian [source,shell] ---- wget https://archive.apache.org/dist/pulsar/pulsar-2.10.2/DEB/apache-pulsar-client.deb @@ -35,11 +31,8 @@ sudo apt install -y ./apache-pulsar-client*.deb sudo ldconfig ---- --- -Centos/RHEL-based rpm:: -+ --- +.Centos/RHEL-based rpm [source,shell] ---- wget https://archive.apache.org/dist/pulsar/pulsar-2.10.2/RPMS/apache-pulsar-client-2.10.2-1.x86_64.rpm @@ -50,8 +43,6 @@ sudo rpm -i ./apache-pulsar-client-devel-2.10.2-1.x86_64.rpm sudo ldconfig ---- --- -====== == Create a project diff --git a/modules/developing/pages/clients/spring-produce-consume.adoc b/modules/developing/pages/clients/spring-produce-consume.adoc index 51e4695..9ebeda8 100644 --- a/modules/developing/pages/clients/spring-produce-consume.adoc +++ b/modules/developing/pages/clients/spring-produce-consume.adoc @@ -31,8 +31,6 @@ image::developing:spring-initializr.png[Spring Initializr] This file contains the main method that will run your application with the specified dependencies. + .DemoApplication.java -[%collapsible] -==== [source,java] ---- package com.example.demo; @@ -99,7 +97,6 @@ public class DemoApplication } } ---- -==== . Replace the following values in `DemoApplication.java` with values from your tenant's *Connect* tab in the {astra-ui}: + @@ -115,10 +112,8 @@ public class DemoApplication + You can also modify the values in the `application.properties` file in the `/resources` directory, which is recommended for production applications. On your tenant's *Connect* tab, you can download a properties file with the values your application needs to connect to your {product} instance. +For example: + -.Spring application properties -[%collapsible] -==== [source,yaml] ---- spring: @@ -136,7 +131,6 @@ auth-plugin-class-name: org.apache.pulsar.client.impl.auth.AuthenticationToken authentication: token: *** ---- -==== . From your project's root directory, compile the Java application with the {pulsar-short} client dependency: + @@ -146,8 +140,6 @@ mvn clean compile ---- + .Result -[%collapsible] -==== [source,console] ---- [INFO] ------------------------------------------------------------------------ @@ -157,7 +149,6 @@ mvn clean compile [INFO] Finished at: 2023-05-09T15:36:33-04:00 [INFO] ------------------------------------------------------------------------ ---- -==== . Run the project to send a message to your {product} cluster: + @@ -167,13 +158,10 @@ mvn spring-boot:run ---- + .Result -[%collapsible] -==== [source,console] ---- Message received: Hello World ---- -==== . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. diff --git a/modules/developing/pages/gpt-schema-translator.adoc b/modules/developing/pages/gpt-schema-translator.adoc index b1eccb6..afa5ec2 100644 --- a/modules/developing/pages/gpt-schema-translator.adoc +++ b/modules/developing/pages/gpt-schema-translator.adoc @@ -38,8 +38,6 @@ If this button isn't available, the GPT schema translator doesn't have an availa image::developing:two-schemas.png[Schema mapping,320,240] + .{cass-short}-to-{pulsar-short} schema mapping example -[%collapsible] -==== [cols="1,1,1"] |=== | {cass-short} table schema | {pulsar-short} JSON schema | Generated mapping @@ -121,7 +119,6 @@ id=key, file1=value.file1, file2=value.file2, file3=value.file3 ---- |=== -==== . Save the mapping configuration. @@ -136,8 +133,6 @@ This example demonstrates how you can generates schema mapping in real time. . The xref:streaming-learning:pulsar-io:connectors/sources/data-generator.adoc[DataGenerator source connector] generates data for a {pulsar-short} topic with an AVRO schema. + .AVRO schema example -[%collapsible] -==== [source,avro] ---- "pulsar_topic_schema": { @@ -369,13 +364,10 @@ This example demonstrates how you can generates schema mapping in real time. }, } ---- -==== . The {astra-db} sink connector writes data to the {cass-short} table with a CQL schema. + .CQL schema example -[%collapsible] -==== [source,cql] ---- "cassandra_table_schemas": { @@ -410,7 +402,6 @@ This example demonstrates how you can generates schema mapping in real time. }, } ---- -==== . In the {astra-ui}, create a mapping for the tenant. When a topic schema is available to the {gpt-schema-translator}, click *Generate Mapping*. diff --git a/modules/getting-started/pages/real-time-data-pipelines-tutorial.adoc b/modules/getting-started/pages/real-time-data-pipelines-tutorial.adoc index 48799cc..3ff8df2 100644 --- a/modules/getting-started/pages/real-time-data-pipelines-tutorial.adoc +++ b/modules/getting-started/pages/real-time-data-pipelines-tutorial.adoc @@ -138,8 +138,6 @@ CREATE TABLE click_data.product_clicks ( The console prints create statements describing the keyspace itself and the two tables. + .Result -[%collapsible] -==== [source,console] ---- token@cqlsh> describe click_data; @@ -193,7 +191,6 @@ CREATE TABLE click_data.product_clicks ( AND read_repair = 'BLOCKING' AND speculative_retry = '99PERCENTILE'; ---- -==== === Connect the topics to the store diff --git a/modules/operations/pages/astream-georeplication.adoc b/modules/operations/pages/astream-georeplication.adoc index e28aec3..8ecec92 100644 --- a/modules/operations/pages/astream-georeplication.adoc +++ b/modules/operations/pages/astream-georeplication.adoc @@ -80,14 +80,11 @@ bin/pulsar-admin namespaces get-clusters **TENANT_NAME**/**NAMESPACE_NAME** ---- + .Result -[%collapsible] -==== [source,json] ---- pulsar-aws-useast1-staging pulsar-aws-useast2-staging ---- -==== . Create a {pulsar-short} consumer with a subscription to a specified topic: + @@ -161,11 +158,9 @@ Consumer consumer = pulsarClient.newConsumer(Schema.STRING) bin/pulsar-admin topics stats persistent://**TENANT_NAME**/**NAMESPACE_NAME**/**TOPIC_NAME** ---- + -In the configuration, `isReplicated` is now `true` for this subscription. +In the configuration, `isReplicated` is now `true` for this subscription: + .config.json -[%collapsible] -==== [source,json] ---- { @@ -252,7 +247,6 @@ In the configuration, `isReplicated` is now `true` for this subscription. } } ---- -==== [#monitor] == Monitor replicated clusters diff --git a/modules/operations/pages/astream-limits.adoc b/modules/operations/pages/astream-limits.adoc index d6562fe..66a8100 100644 --- a/modules/operations/pages/astream-limits.adoc +++ b/modules/operations/pages/astream-limits.adoc @@ -155,11 +155,8 @@ You can read more about Kubernetes pod naming restrictions https://kubernetes.io === Namespace policy configuration file -The following is an example of a default namespace policy with limits set. +The following is an example of a default namespace policy with limits set: -.Default namespace policy with limits set -[%collapsible] -==== [source,json] ---- { @@ -253,7 +250,6 @@ The following is an example of a default namespace policy with limits set. "schema_validation_enforced" : false } ---- -==== == {product} topic and namespace actions diff --git a/modules/operations/pages/astream-release-notes.adoc b/modules/operations/pages/astream-release-notes.adoc index 0e9127e..5c4426e 100644 --- a/modules/operations/pages/astream-release-notes.adoc +++ b/modules/operations/pages/astream-release-notes.adoc @@ -72,11 +72,8 @@ For more information, see xref:developing:clients/spring-produce-consume.adoc[Sp * Launched a refreshed {astra-ui}. -== Previous releases +== 2022 release notes -.2022 release notes -[%collapsible] -==== [discrete] === November 21, 2022 @@ -95,7 +92,7 @@ For more information, see xref:developing:clients/spring-produce-consume.adoc[Sp [discrete] ==== {new} -* {product} now supports xref:streaming-learning:functions:index.adoc[transform functions]. +* {product} now supports xref:developing:astream-functions-transform.adoc[transform functions]. [discrete] === November 16, 2022 @@ -159,11 +156,9 @@ For more information, see xref:developing:clients/spring-produce-consume.adoc[Sp * Added support for the Google Cloud `us-central1` (Council Bluffs, Iowa) region. * xref:operations:astream-token-gen.adoc[{pulsar-short} tokens] simplify connecting to your streaming instances. * Enabled xref:operations:astream-pricing.adoc[billing]. -==== -.2021 release notes -[%collapsible] -==== +== 2021 release notes + [discrete] === December 20, 2021 @@ -174,5 +169,4 @@ For more information, see xref:developing:clients/spring-produce-consume.adoc[Sp * Security upgrade to Log4J 2.17.0 to mitigate CVE-2021-45105. -* The public preview of {product} brings the ability to quickly create {pulsar-reg} instances, manage your clusters, scale across cloud regions, and manage {pulsar-short} resources such as topics, connectors, functions and subscriptions. -==== +* The public preview of {product} brings the ability to quickly create {pulsar-reg} instances, manage your clusters, scale across cloud regions, and manage {pulsar-short} resources such as topics, connectors, functions and subscriptions. \ No newline at end of file diff --git a/modules/operations/pages/astream-scrape-metrics.adoc b/modules/operations/pages/astream-scrape-metrics.adoc index a074f7d..93f9c66 100644 --- a/modules/operations/pages/astream-scrape-metrics.adoc +++ b/modules/operations/pages/astream-scrape-metrics.adoc @@ -97,11 +97,7 @@ image::operations:astream-prometheus-graph.png[Scraping {product} with Prometheu With your Prometheus scrape container running, you can use curl commands to decompress your Prometheus scrape data: -[tabs] -====== -Deflate:: -+ --- +.Deflate [source,bash] ---- curl -v \ @@ -110,11 +106,8 @@ curl -v \ https://prometheus-aws-useast1.streaming.datastax.com/pulsarmetrics/tenant-1 \ --compressed ---- --- -Gzip:: -+ --- +.Gzip [source,bash] ---- curl -v \ @@ -123,8 +116,6 @@ curl -v \ https://prometheus-aws-useast1.streaming.datastax.com/pulsarmetrics/tenant-1 \ | zcat ---- --- -====== Deflate or Gzip will extract your scraped metrics in a format such as the following: @@ -152,5 +143,4 @@ A tenant can only access its own metrics on the broker pod or function worker po * xref:monitoring/index.adoc[] * xref:getting-started:index.adoc[] -* https://prometheus.io/docs/introduction/overview/[Prometheus documentation] - +* https://prometheus.io/docs/introduction/overview/[Prometheus documentation] \ No newline at end of file diff --git a/modules/operations/pages/create-delete-tenants.adoc b/modules/operations/pages/create-delete-tenants.adoc index 76541f4..31159eb 100644 --- a/modules/operations/pages/create-delete-tenants.adoc +++ b/modules/operations/pages/create-delete-tenants.adoc @@ -12,38 +12,21 @@ If you're new to {product}, try the xref:getting-started:index.adoc[{product} qu == Create a tenant -[tabs] -====== {astra-ui}:: + --- . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. - . Click *Create tenant*. - . Enter a name for the streaming tenant. - . Select a xref:astra-streaming:operations:astream-regions.adoc[cloud provider and region that supports {astra-stream}]. -+ Not all {astra-db} regions support {product}. If your preferred region isn't available, contact your {company} account representative or {support_url}[{company} Support]. - . Click *Create tenant*. -+ --- API:: -+ --- Use `xref:apis:index.adoc#create-a-tenant[POST "https://api.astra.datastax.com/v2/streaming/tenants"]`. --- CLI:: -+ --- Use `xref:astra-cli:commands:astra-streaming-create.adoc[astra streaming create]`. --- -====== == View and edit a tenant, namespace, or topic @@ -52,15 +35,10 @@ After you create a tenant, you can inspect the tenant, create and manage namespa [#get-tenant-details] === Get tenant details -[tabs] -====== {astra-ui}:: + --- . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. - . Click the name of your tenant. - . Click any of the tabs to access tenant details and settings, such as **Connect** or **Sinks**. + [TIP] @@ -71,68 +49,41 @@ You can also create a xref:operations:astream-token-gen.adoc[{pulsar-short} toke image::developing:tenant-details.png[Tenant details in {product}] ==== --- API:: -+ --- Use `xref:apis:index.adoc#get-tenant-details[GET "https://api.astra.datastax.com/v2/streaming/tenants"]`. --- CLI:: -+ --- Use `xref:astra-cli:commands:astra-streaming-get.adoc[astra streaming get]`. --- -====== === Manage namespaces -[tabs] -====== {astra-ui}:: + --- . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. - . Click the name of your tenant, and then click the *Namespace And Topics* tab to manage namespaces: + * **Create a namespace**: Click *Create Namespace*, enter a namespace name, and then click *Create Namespace*. * **Configure replication**: See xref:operations:astream-georeplication.adoc[]. * **Delete a namespace**: Find the namespace you want to delete, click **Modify Namespace**, find the **Delete This Namespace** section, and then click *Delete Namespace*. * **Return to all namespaces**: When modifying an individual namespace, click **Back to Namespaces** to return to the list of all namespaces in the current tenant. --- API:: -+ --- See xref:apis:api-operations.adoc#pulsar-api-namespace-operations[Pulsar Admin API namespace operations]. --- -====== === Manage topics - -[tabs] -====== {astra-ui}:: + --- . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. - . Click the name of your tenant, and then click the *Namespace And Topics* tab to manage topics: + * **Create a topic**: Find the namespace where you want to create the topic, click *Add Topic*, enter a topic name, and then click *Add Topic*. * **Inspect and configure a topic**: Click icon:chevron-down[name="Expand"] to view the list of topics in a specific namespace, click the name of the topic you want to manage, and then use the tabs to access the topic's information and settings, such as **Subscriptions** and **Consumers**. * **Delete a topic**: Click icon:chevron-down[name="Expand"] to view the list of topics in a specific namespace, find the topic you want to delete, click icon:ellipsis[name="More"], and then select *Delete Topic*. --- API:: -+ --- See xref:apis:api-operations.adoc#pulsar-admin-api-topic-operations[Pulsar Admin API topic operations]. --- -====== == Terminate a tenant @@ -141,25 +92,13 @@ See xref:apis:api-operations.adoc#pulsar-admin-api-topic-operations[Pulsar Admin Terminating a tenant is a destructive operation that permanently deletes the tenant and all associated sinks, topics, messages, and namespaces. ==== -[tabs] -====== {astra-ui}:: + --- . In the {astra-ui-link} header, click icon:grip[name="Applications"], and then select *Streaming*. - . Find the tenant that you want to delete, click icon:ellipsis-vertical[name="More"], and then select **Terminate**. --- API:: -+ --- Use `xref:apis:index.adoc#delete-a-tenant[DELETE "https://api.astra.datastax.com/v2/streaming/tenants/**TENANT_NAME**/clusters/**CLUSTER_NAME**"]`. --- CLI:: -+ --- -Use `xref:astra-cli:commands:astra-streaming-delete.adoc[astra streaming delete]`. --- -====== \ No newline at end of file +Use `xref:astra-cli:commands:astra-streaming-delete.adoc[astra streaming delete]`. \ No newline at end of file diff --git a/modules/operations/pages/monitoring/index.adoc b/modules/operations/pages/monitoring/index.adoc index db66af4..3966dbe 100644 --- a/modules/operations/pages/monitoring/index.adoc +++ b/modules/operations/pages/monitoring/index.adoc @@ -299,59 +299,56 @@ For more information, see the https://prometheus.io/docs/prometheus/latest/query The following examples use the `pulsar_msg_backlog` raw metric to demonstrate the patterns. In accordance with the recommendations in <>, the example patterns aggregate messages at the parent topic level or higher and they exclude system topics. -.Filter system topics -[%collapsible] -==== +Filter system topics:: You can use the following expression to filter system topics: - ++ [source,pgsql] ---- {topic !~ ".*__.*"} ---- - ++ This expression excludes messages with topic labels that include two consecutive underscores. This works because {pulsar-short} system topics and namespaces are usually prefixed by two underscores, such as `+persistent://some_tenant/__kafka/__consumer_offsets_partition_0+`. - ++ To use this expression, your applications' namespace and topic names don't contain double underscores. If they do, they will also be excluded by this filter. -==== - -==== Get the total message backlog of a specific parent topic, excluding system topics +Get the total message backlog of a specific parent topic, excluding system topics:: `$ptopic` is a Grafana dashboard variable that represents a specific parent topic. - +For example: ++ [source,pgsql] ---- sum(pulsar_msg_backlog{topic=~$ptopic, topic !~ ".*__.*"}) ---- -==== Get the total message backlog of a specific namespace, excluding system topics - +Get the total message backlog of a specific namespace, excluding system topics:: `$namespace` is a Grafana dashboard variable that represents a specific namespace. - +For example: ++ [source,pgsql] ---- sum(pulsar_msg_backlog{namespace=~"$namespace", topic !~ ".*__.*"}) ---- -==== Get the total message backlog of a tenant, excluding system topics - +Get the total message backlog of a tenant, excluding system topics:: `$tenant` is a (Grafana dashboard) variable that represents a specific tenant. - +For example: ++ [source,pgsql] ---- sum(pulsar_msg_backlog{namespace=~"$tenant.+", topic !~ ".*__.*"}) ---- -==== Get the total message backlog of each topic within a specific namespace, excluding system topics - +Get the total message backlog of each topic within a specific namespace, excluding system topics:: ++ [source,pgsql] ---- sum by(topic) (pulsar_msg_backlog{namespace=~"$namespace", topic !~ ".*__.*"}) ---- -==== Get the top 10 message backlog by topic within a specific namespace, excluding system topics - +Get the top 10 message backlog by topic within a specific namespace, excluding system topics:: ++ [source,pgsql] ---- topk (10, sum by(topic) (pulsar_msg_backlog{namespace=~"$namespace", topic !~ ".*__.*"})) diff --git a/modules/operations/pages/monitoring/stream-audit-logs.adoc b/modules/operations/pages/monitoring/stream-audit-logs.adoc index 8579784..66bda8c 100644 --- a/modules/operations/pages/monitoring/stream-audit-logs.adoc +++ b/modules/operations/pages/monitoring/stream-audit-logs.adoc @@ -68,13 +68,10 @@ Replace the following: * The `auth_strategy` and other authentication details depend on your Pulsar configuration. + .Result -[%collapsible] -==== [source,json] ---- HTTP/1.1 202 Accepted ---- -==== . Retrieve and verify the audit log streaming configuration: + @@ -86,8 +83,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/organizations/**ORG ---- + .Result -[%collapsible] -==== [source,json] ---- { @@ -100,7 +95,6 @@ curl -sS --fail -L -X GET "https://api.astra.datastax.com/v2/organizations/**ORG } } ---- -==== == Delete an audit log streaming configuration