From 45b09e98c5f449d3c76e7e780841742105556c5b Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 1 Jun 2026 09:13:18 -0700 Subject: [PATCH 1/2] replace product with zdm attributes --- antora.yml | 10 +- local-preview-playbook.yml | 9 +- modules/ROOT/nav.adoc | 11 +- modules/ROOT/pages/astra-migration-paths.adoc | 2 +- .../ROOT/pages/cassandra-data-migrator.adoc | 317 ------------------ modules/ROOT/pages/change-read-routing.adoc | 34 +- modules/ROOT/pages/components.adoc | 84 ++--- .../ROOT/pages/connect-clients-to-proxy.adoc | 102 +++--- .../ROOT/pages/connect-clients-to-target.adoc | 4 +- modules/ROOT/pages/create-target.adoc | 18 +- .../ROOT/pages/deploy-proxy-monitoring.adoc | 92 ++--- .../ROOT/pages/deployment-infrastructure.adoc | 74 ++-- modules/ROOT/pages/dse-migration-paths.adoc | 10 +- .../ROOT/pages/enable-async-dual-reads.adoc | 14 +- modules/ROOT/pages/faqs.adoc | 104 +++--- .../ROOT/pages/feasibility-checklists.adoc | 150 ++++----- modules/ROOT/pages/hcd-migration-paths.adoc | 30 +- modules/ROOT/pages/index.adoc | 10 +- modules/ROOT/pages/introduction.adoc | 48 +-- .../ROOT/pages/manage-proxy-instances.adoc | 154 ++++----- modules/ROOT/pages/metrics.adoc | 62 ++-- .../ROOT/pages/migrate-and-validate-data.adoc | 24 +- modules/ROOT/pages/phase1.adoc | 10 +- modules/ROOT/pages/rollback.adoc | 6 +- .../ROOT/pages/setup-ansible-playbooks.adoc | 96 +++--- modules/ROOT/pages/troubleshooting-tips.adoc | 178 +++++----- modules/ROOT/pages/zdm-logs.adoc | 42 +-- .../ROOT/pages/zdm-proxy-migration-paths.adoc | 8 +- .../partials/cassandra-protocol-versions.adoc | 2 +- .../partials/dsbulk-migrator-deprecation.adoc | 2 +- .../ROOT/partials/migration-scenarios.adoc | 10 +- modules/ROOT/partials/zdm-logs-intro.adoc | 2 +- .../sideloader/pages/prepare-sideloader.adoc | 10 +- .../sideloader/pages/sideloader-overview.adoc | 12 +- modules/sideloader/partials/validate.adoc | 2 +- 35 files changed, 713 insertions(+), 1030 deletions(-) delete mode 100644 modules/ROOT/pages/cassandra-data-migrator.adoc diff --git a/antora.yml b/antora.yml index bf54a64c..f5686ba8 100644 --- a/antora.yml +++ b/antora.yml @@ -42,23 +42,19 @@ asciidoc: devops-api: 'DevOps API' devops-api-ref-url: 'xref:astra-api-docs:ROOT:attachment$devops-api/index.html' data-api: 'Data API' - product: 'Zero Downtime Migration' + product: 'Zero Downtime Migration (ZDM)' product-short: 'ZDM' product-proxy: 'ZDM Proxy' product-proxy-repo: 'https://github.com/datastax/zdm-proxy' product-utility: 'ZDM Utility' product-automation: 'ZDM Proxy Automation' product-automation-repo: 'https://github.com/datastax/zdm-proxy-automation' - product-automation-shield: 'image:https://img.shields.io/github/v/release/datastax/zdm-proxy-automation?label=latest[alt="Latest zdm-proxy-automation release on GitHub",link="{product-automation-repo}/releases"]' - product-demo: 'ZDM Demo Client' dsbulk: 'DataStax Bulk Loader (DSBulk)' dsbulk-short: 'DSBulk' - dsbulk-repo: 'https://github.com/datastax/dsbulk' - dsbulk-migrator: 'DSBulk Migrator' + dsbulk-repo-url: 'https://github.com/datastax/dsbulk' cass-migrator: 'Cassandra Data Migrator (CDM)' cass-migrator-short: 'CDM' - cass-migrator-repo: 'https://github.com/datastax/cassandra-data-migrator' - cass-migrator-shield: 'image:https://img.shields.io/github/v/release/datastax/cassandra-data-migrator?label=latest[alt="Latest cassandra-data-migrator release on GitHub",link="{cass-migrator-repo}/packages"]' + cass-migrator-repo-url: 'https://github.com/datastax/cassandra-data-migrator' sstable-sideloader: '{astra-db} Sideloader' # Astra role attributes (compare with astra-vector-docs antora.yml) diff --git a/local-preview-playbook.yml b/local-preview-playbook.yml index 811461b1..7b7722bb 100644 --- a/local-preview-playbook.yml +++ b/local-preview-playbook.yml @@ -66,7 +66,7 @@ asciidoc: support-url: 'https://www.ibm.com/mysupport/s/' dsbulk: 'DataStax Bulk Loader (DSBulk)' dsbulk-short: 'DSBulk' - dsbulk-repo: 'https://github.com/datastax/dsbulk' + dsbulk-repo-url: 'https://github.com/datastax/dsbulk' astra: 'Astra' astra-db: 'Astra DB' astra-ui: 'Astra Portal' @@ -85,11 +85,16 @@ asciidoc: starlight-kafka: 'Starlight for Kafka' starlight-rabbitmq: 'Starlight for RabbitMQ' sstable-sideloader: '{astra-db} Sideloader' - zdm: 'Zero Downtime Migration' + zdm: 'Zero Downtime Migration (ZDM)' zdm-short: 'ZDM' zdm-proxy: 'ZDM Proxy' + zdm-proxy-repo-url: 'https://github.com/datastax/zdm-proxy' + zdm-utility: 'ZDM Utility' + zdm-automation: 'ZDM Proxy Automation' + zdm-automation-repo-url: 'https://github.com/datastax/zdm-proxy-automation' cass-migrator: 'Cassandra Data Migrator (CDM)' cass-migrator-short: 'CDM' + cass-migrator-repo-url: 'https://github.com/datastax/cassandra-data-migrator' hcd: 'Hyper-Converged Database (HCD)' hcd-short: 'HCD' dse: 'DataStax Enterprise (DSE)' diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index cb350533..337967ea 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -6,14 +6,14 @@ * xref:ROOT:hcd-migration-paths.adoc[] * xref:ROOT:mc-migration-paths.adoc[] -.{product} +.{zdm} * xref:ROOT:introduction.adoc[] * Plan and prepare ** xref:ROOT:feasibility-checklists.adoc[] ** xref:ROOT:deployment-infrastructure.adoc[] ** xref:ROOT:create-target.adoc[] ** xref:ROOT:rollback.adoc[] -* Phase 1: Deploy {product-proxy} +* Phase 1: Deploy {zdm-proxy} ** xref:ROOT:phase1.adoc[] ** xref:ROOT:setup-ansible-playbooks.adoc[] ** xref:ROOT:deploy-proxy-monitoring.adoc[] @@ -28,8 +28,8 @@ * xref:ROOT:troubleshooting-tips.adoc[] * xref:ROOT:faqs.adoc[] * Release notes -** {product-proxy-repo}/releases[{product-proxy} release notes] -** {product-automation-repo}/releases[{product-automation} release notes] +** {zdm-proxy-repo-url}/releases[{zdm-proxy} release notes] +** {zdm-automation-repo-url}/releases[{zdm-automation} release notes] .{sstable-sideloader} * xref:sideloader:sideloader-overview.adoc[] @@ -40,8 +40,7 @@ * xref:sideloader:troubleshoot-sideloader.adoc[] .{cass-migrator} -* xref:ROOT:cassandra-data-migrator.adoc[] -* {cass-migrator-repo}/releases[{cass-migrator-short} release notes] +* {cass-migrator-repo-url}[About {cass-migrator-short}] .{dsbulk} * xref:dsbulk:overview:dsbulk-about.adoc[] \ No newline at end of file diff --git a/modules/ROOT/pages/astra-migration-paths.adoc b/modules/ROOT/pages/astra-migration-paths.adoc index 9b4b8f9f..5e0fc793 100644 --- a/modules/ROOT/pages/astra-migration-paths.adoc +++ b/modules/ROOT/pages/astra-migration-paths.adoc @@ -9,7 +9,7 @@ If you have questions about migrating from a specific source to {astra-db}, cont .Migration tool compatibility [cols="2,1,1,1,1"] |=== -|Origin |{sstable-sideloader} |{cass-migrator} |{product-proxy} |{dsbulk} +|Origin |{sstable-sideloader} |{cass-migrator} |{zdm-proxy} |{dsbulk} |Aiven for {cass-short} |icon:check[role="text-success",alt="Supported"] diff --git a/modules/ROOT/pages/cassandra-data-migrator.adoc b/modules/ROOT/pages/cassandra-data-migrator.adoc deleted file mode 100644 index 205d39e8..00000000 --- a/modules/ROOT/pages/cassandra-data-migrator.adoc +++ /dev/null @@ -1,317 +0,0 @@ -= Use {cass-migrator} with {product-proxy} -:navtitle: Use {cass-migrator-short} -:description: You can use {cass-migrator} for data migration and validation between {cass-reg}-based databases. -:page-aliases: cdm-parameters.adoc, ROOT:cdm-steps.adoc, ROOT:cdm-overview.adoc - -{description} - -[IMPORTANT] -==== -To use {cass-migrator-short} successfully, your origin and target clusters must be {cass-short}-based databases with matching schemas. -==== - -{cass-migrator-short} is best for large or complex migrations that benefit from advanced features and configuration options, such as the following: - -* Logging and run tracking -* Automatic reconciliation -* Performance tuning -* Record filtering -* Column renaming -* Support for advanced data types, including sets, lists, maps, and UDTs -* Support for SSL, including custom cipher algorithms -* Use `writetime` timestamps to maintain chronological write history -* Use Time To Live (TTL) values to maintain data lifecycles - -For more information and a complete list of features, see the {cass-migrator-repo}?tab=readme-ov-file#features[{cass-migrator-short} GitHub repository]. - -== {cass-migrator-short} last-write-wins with {product-proxy} - -You can use {cass-migrator-short} alone, with {product-proxy}, or for data validation after using another data migration tool. - -When using {cass-migrator-short} with {product-proxy}, {cass-short}'s last-write-wins semantics ensure that new, real-time writes accurately take precedence over historical writes. - -Last-write-wins compares the `writetime` of conflicting records, and then retains the most recent write. - -For example, if a new write occurs in your target cluster with a `writetime` of `2023-10-01T12:05:00Z`, and then {cass-migrator-short} migrates a record against the same row with a `writetime` of `2023-10-01T12:00:00Z`, the target cluster retains the data from the new write because it has the most recent `writetime`. - -== Install {cass-migrator-short} - -{company} recommends that you always install the latest version of {cass-migrator-short} to get the latest features, dependencies, and bug fixes. - -=== Install as a container - -Get the latest `cassandra-data-migrator` image that includes all dependencies from https://hub.docker.com/r/datastax/cassandra-data-migrator[DockerHub]. -The container's `assets` directory includes all required migration tools: `cassandra-data-migrator`, `dsbulk`, and `cqlsh`. - -=== Install as a JAR file on a single VM - -For one-off migrations, you can install the {spark-short} binary on a single VM where you will run the {cass-migrator-short} job: - -. Install Java 11 or later, which includes {spark-short} binaries. - -. Install https://spark.apache.org/downloads.html[{spark-reg}] version 3.5.x with Scala 2.13 and {hadoop-reg} 3.3 and later: -+ -.. Get the {spark-reg} tarball from the {spark} archive: -+ -[source,bash,subs="+quotes"] ----- -wget https://archive.apache.org/dist/spark/spark-3.5.**PATCH**/spark-3.5.**PATCH**-bin-hadoop3-scala2.13.tgz ----- -+ -Replace `**PATCH**` with your {spark-short} patch version. -+ -.. Change to the directory where you want install {spark-short}, and then extract the tarball: -+ -[source,bash,subs="+quotes"] ----- -tar -xvzf spark-3.5.**PATCH**-bin-hadoop3-scala2.13.tgz ----- -+ -Replace `**PATCH**` with your {spark-short} patch version. - -. Download the latest {cass-migrator-repo}/packages/1832128/versions[cassandra-data-migrator JAR file] {cass-migrator-shield}. - -. Add the `cassandra-data-migrator` dependency to `pom.xml`: -+ -[source,xml,subs="+quotes"] ----- - - datastax.cdm - cassandra-data-migrator - **VERSION** - ----- -+ -Replace `**VERSION**` with your {cass-migrator-short} version. - -. Run `mvn install`. - -=== Install as a JAR file on a {spark} cluster or {spark-short} Serverless platform - -For large (several terabytes) migrations, complex migrations, and use of {cass-migrator-short} as a long-term data transfer utility, {company} recommends that you use an {spark} cluster or {spark-short} Serverless platform. - -. Install Java 11 or later, which includes {spark-short} binaries. - -. Deploy a https://spark.apache.org/downloads.html[{spark-reg} cluster or {spark-short} Serverless instance] running version 3.5.x with Scala 2.13 and {hadoop-reg} 3.3 and later. -+ -[IMPORTANT] -==== -If you deploy {cass-migrator-short} on a {spark-short} cluster, you must modify your `spark-submit` commands as follows: - -* Replace `--master "local[*]"` with the host and port for your {spark-short} cluster, as in `--master "spark://**MASTER_HOST**:**PORT**"`. -* Remove parameters related to single-VM installations, such as `--driver-memory` and `--executor-memory`. -==== - -. Download the latest {cass-migrator-repo}/packages/1832128/versions[cassandra-data-migrator JAR file] {cass-migrator-shield}. - -. Add the `cassandra-data-migrator` dependency to `pom.xml`: -+ -[source,xml,subs="+quotes"] ----- - - datastax.cdm - cassandra-data-migrator - **VERSION** - ----- -+ -Replace `**VERSION**` with your {cass-migrator-short} version. - -. Run `mvn install`. - -=== Build for local development or Scala 2.12.x environments - -If you need to build the JAR for local development, or your environment only has Scala version 2.12.x, see the alternative installation instructions in the {cass-migrator-repo}?tab=readme-ov-file[{cass-migrator-short} README]. - -== Configure {cass-migrator-short} - -. Create a `cdm.properties` file. -+ -If you use a different name, make sure you specify the correct filename in your `spark-submit` commands. - -. Configure the properties for your environment. -+ -In the {cass-migrator-short} repository, you can find a {cass-migrator-repo}/blob/main/src/resources/cdm.properties[sample properties file with default values], as well as a {cass-migrator-repo}/blob/main/src/resources/cdm-detailed.properties[fully annotated properties file]. -+ -{cass-migrator-short} jobs process all uncommented parameters. -Any parameters that are commented out are ignored or use default values. -+ -If you want to reuse a properties file created for a previous {cass-migrator-short} version, make sure it is compatible with the version you are currently using. -Check the {cass-migrator-repo}/releases[{cass-migrator-short} release notes] for possible breaking changes in interim releases. -For example, the 4.x series of {cass-migrator-short} isn't backwards compatible with earlier properties files. - -. Store your properties file where it can be accessed while running {cass-migrator-short} jobs using `spark-submit`. - -[#migrate] -== Run a {cass-migrator-short} data migration job - -A data migration job copies data from a table in your origin cluster to a table with the same schema in your target cluster. - -To optimize large-scale migrations, {cass-migrator-short} can run multiple concurrent migration jobs on the same table. - -The following `spark-submit` command migrates one table from the origin to the target cluster, using the configuration in your properties file. -The migration job is specified in the `--class` argument. - -.Migration job using a local installation -[source,bash,subs="+quotes,+attributes"] ----- -./spark-submit --properties-file cdm.properties \ ---conf spark.cdm.schema.origin.keyspaceTable="**KEYSPACE_NAME**.**TABLE_NAME**" \ ---master "local[{asterisk}]" --driver-memory 25G --executor-memory 25G \ ---class com.datastax.cdm.job.Migrate cassandra-data-migrator-**VERSION**.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt ----- - -.Migration job using a {spark-reg} cluster -[source,bash,subs="+quotes"] ----- -./spark-submit --properties-file cdm.properties \ ---conf spark.cdm.schema.origin.keyspaceTable="**KEYSPACE_NAME**.**TABLE_NAME**" \ ---master "spark://**MASTER_HOST**:**PORT**" \ ---class com.datastax.cdm.job.Migrate cassandra-data-migrator-**VERSION**.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt ----- - -Replace or modify the following, if needed: - -* `--properties-file cdm.properties`: If your properties file has a different name, specify the actual name of your properties file. -+ -Depending on where your properties file is stored, you might need to specify the full or relative file path. - -* `**KEYSPACE_NAME**.**TABLE_NAME**`: Specify the name of the table that you want to migrate and the keyspace that it belongs to. -+ -You can also set `spark.cdm.schema.origin.keyspaceTable` in your properties file using the same format of `**KEYSPACE_NAME**.**TABLE_NAME**`. - -* `--driver-memory` and `--executor-memory` (local installations only): Specify the appropriate memory settings for your environment. - -* `--master` ({spark-short} cluster deployments only): Provide the URL of your {spark-short} cluster. - -* `**VERSION**`: Specify the full {cass-migrator-short} version that you installed, such as `5.2.1`. - -This command generates a log file (`logfile_name_**TIMESTAMP**.txt`) instead of logging output to the console. -For additional modifications to this command, see <>. - -[#cdm-validation-steps] -== Run a {cass-migrator-short} data validation job - -After migrating data, use {cass-migrator-short}'s data validation mode to identify any inconsistencies between the origin and target tables, such as missing or mismatched records. - -Optionally, {cass-migrator-short} can automatically correct discrepancies in the target cluster during validation. - -. Use the following `spark-submit` command to run a data validation job using the configuration in your properties file. -The data validation job is specified in the `--class` argument. -+ -.Validation job using a local installation -[source,bash,subs="+quotes,+attributes"] ----- -./spark-submit --properties-file cdm.properties \ ---conf spark.cdm.schema.origin.keyspaceTable="**KEYSPACE_NAME**.**TABLE_NAME**" \ ---master "local[{asterisk}]" --driver-memory 25G --executor-memory 25G \ ---class com.datastax.cdm.job.DiffData cassandra-data-migrator-**VERSION**.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt ----- -+ -.Validation job using a {spark-reg} cluster -[source,bash,subs="+quotes"] ----- -./spark-submit --properties-file cdm.properties \ ---conf spark.cdm.schema.origin.keyspaceTable="**KEYSPACE_NAME**.**TABLE_NAME**" \ ---master "spark://**MASTER_HOST**:**PORT**" \ ---class com.datastax.cdm.job.DiffData cassandra-data-migrator-**VERSION**.jar &> logfile_name_$(date +%Y%m%d_%H_%M).txt ----- -+ -Replace or modify the following, if needed: -+ --- -* `--properties-file cdm.properties`: If your properties file has a different name, specify the actual name of your properties file. -+ -Depending on where your properties file is stored, you might need to specify the full or relative file path. - -* `**KEYSPACE_NAME**.**TABLE_NAME**`: Specify the name of the table that you want to validate and the keyspace that it belongs to. -+ -You can also set `spark.cdm.schema.origin.keyspaceTable` in your properties file using the same format of `**KEYSPACE_NAME**.**TABLE_NAME**`. - -* `--driver-memory` and `--executor-memory` (local installations only): Specify the appropriate memory settings for your environment. - -* `--master` ({spark-short} cluster deployments only): Provide the URL of your {spark-short} cluster. - -* `**VERSION**`: Specify the full {cass-migrator-short} version that you installed, such as `5.2.1`. --- - -. Allow the command some time to run, and then open the log file (`logfile_name_**TIMESTAMP**.txt`) and look for `ERROR` entries. -+ -The {cass-migrator-short} validation job records differences as `ERROR` entries in the log file, listed by primary key values. -For example: -+ -[source,plaintext] ----- -23/04/06 08:43:06 ERROR DiffJobSession: Mismatch row found for key: [key3] Mismatch: Target Index: 1 Origin: valueC Target: value999) -23/04/06 08:43:06 ERROR DiffJobSession: Corrected mismatch row in target: [key3] -23/04/06 08:43:06 ERROR DiffJobSession: Missing target row found for key: [key2] -23/04/06 08:43:06 ERROR DiffJobSession: Inserted missing row in target: [key2] ----- -+ -When validating large datasets or multiple tables, you might want to extract the complete list of missing or mismatched records. -There are many ways to do this. -For example, you can grep for all `ERROR` entries in your {cass-migrator-short} log files or use the `log4j2` example provided in the {cass-migrator-repo}?tab=readme-ov-file#steps-for-data-validation[{cass-migrator-short} repository]. - -=== Run a validation job in AutoCorrect mode - -Optionally, you can run {cass-migrator-short} validation jobs in **AutoCorrect** mode, which offers the following functions: - -* `autocorrect.missing`: Add any missing records in the target with the value from the origin. - -* `autocorrect.mismatch`: Reconcile any mismatched records between the origin and target by replacing the target value with the origin value. -+ -[IMPORTANT] -==== -Timestamps have an effect on this function. - -If the `writetime` of the origin record (determined with `.writetime.names`) is before the `writetime` of the corresponding target record, then the original write won't appear in the target cluster. - -This comparative state can be challenging to troubleshoot if individual columns or cells were modified in the target cluster. -==== - -* `autocorrect.missing.counter`: By default, counter tables are not copied when missing, unless explicitly set. - -In your `cdm.properties` file, use the following properties to enable (`true`) or disable (`false`) autocorrect functions: - -[source,properties] ----- -spark.cdm.autocorrect.missing false|true -spark.cdm.autocorrect.mismatch false|true -spark.cdm.autocorrect.missing.counter false|true ----- - -The {cass-migrator-short} validation job never deletes records from either the origin or target. -Data validation only inserts or updates data on the target. - -For an initial data validation, consider disabling AutoCorrect so that you can generate a list of data discrepancies, investigate those discrepancies, and then decide whether you want to rerun the validation with AutoCorrect enabled. - -[#advanced] -== Additional {cass-migrator-short} options - -You can modify your properties file or append additional `--conf` arguments to your `spark-submit` commands to customize your {cass-migrator-short} jobs. -For example, you can do the following: - -* Check for large field guardrail violations before migrating. -* Use the `partition.min` and `partition.max` parameters to migrate or validate specific token ranges. -* Use the `track-run` feature to monitor progress and rerun a failed migration or validation job from point of failure. - -For all options, see the {cass-migrator-repo}[{cass-migrator-short} repository]. -Specifically, see the {cass-migrator-repo}/blob/main/src/resources/cdm-detailed.properties[fully annotated properties file]. - -== Troubleshoot {cass-migrator-short} - -Java NoSuchMethodError:: -If you installed {spark-short} as a JAR file, and your {spark-short} and Scala versions aren't compatible with your installed version of {cass-migrator-short}, {cass-migrator-short} jobs can throw exceptions such a the following: -+ -[source,console] ----- -Exception in thread "main" java.lang.NoSuchMethodError: 'void scala.runtime.Statics.releaseFence()' ----- -+ -Make sure that your {spark-short} binary is compatible with your {cass-migrator-short} version. -If you installed an earlier version of {cass-migrator-short}, you might need to install an earlier {spark-short} binary. - -Rerun a failed or partially completed job:: -You can use the `track-run` feature to track the progress of a migration or validation, and then, if necessary, use the `run-id` to rerun a failed job from the last successful migration or validation point. -+ -For more information, see the {cass-migrator-repo}[{cass-migrator-short} repository] and the {cass-migrator-repo}/blob/main/src/resources/cdm-detailed.properties[fully annotated properties file]. \ No newline at end of file diff --git a/modules/ROOT/pages/change-read-routing.adoc b/modules/ROOT/pages/change-read-routing.adoc index f104a5c9..cabe49d5 100644 --- a/modules/ROOT/pages/change-read-routing.adoc +++ b/modules/ROOT/pages/change-read-routing.adoc @@ -1,6 +1,6 @@ = Phase 4: Route reads to the target -After you migrate and validate your data in xref:ROOT:migrate-and-validate-data.adoc[Phase 2], and then test your target cluster's production readiness in xref:ROOT:enable-async-dual-reads.adoc[Phase 3], you can configure {product-proxy} to route _all_ read requests to the target cluster instead of the origin cluster. +After you migrate and validate your data in xref:ROOT:migrate-and-validate-data.adoc[Phase 2], and then test your target cluster's production readiness in xref:ROOT:enable-async-dual-reads.adoc[Phase 3], you can configure {zdm-proxy} to route _all_ read requests to the target cluster instead of the origin cluster. [IMPORTANT] ==== @@ -8,7 +8,7 @@ This phase routes production read requests to the target cluster exclusively. Make sure all data is present on the target cluster, and it is prepared to handle full-scale production workloads. ==== -image::ROOT:migration-phase4ra.svg[In migration Phase 4, {product-proxy}'s read routing switches to the target cluster] +image::ROOT:migration-phase4ra.svg[In migration Phase 4, {zdm-proxy}'s read routing switches to the target cluster] == Prerequisites @@ -20,7 +20,7 @@ If your migration was idle for some time after completing Phase 2, or you skippe * Complete xref:ROOT:enable-async-dual-reads.adoc[Phase 3], and then disable asynchronous dual reads by setting `read_mode` to `PRIMARY_ONLY`. + -If you don't disable asynchronous dual reads, {product-proxy} sends asynchronous, duplicate read requests to your origin cluster. +If you don't disable asynchronous dual reads, {zdm-proxy} sends asynchronous, duplicate read requests to your origin cluster. This is harmless but unnecessary. [#change-the-read-routing-configuration] @@ -28,26 +28,26 @@ This is harmless but unnecessary. Read routing is controlled by a xref:manage-proxy-instances.adoc#change-mutable-config-variable[mutable configuration variable]. -. Edit the {product-proxy} core configuration file `vars/zdm_proxy_core_config.yml`. +. Edit the {zdm-proxy} core configuration file `vars/zdm_proxy_core_config.yml`. . Change the `primary_cluster` variable to `TARGET`. -. xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[Perform a rolling restart] to apply the configuration change to your entire {product-proxy} deployment. +. xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[Perform a rolling restart] to apply the configuration change to your entire {zdm-proxy} deployment. Once the instances are restarted, all reads are routed to the target cluster instead of the origin cluster. -At this point, the target cluster is considered the primary cluster, but {product-proxy} still keeps the origin cluster synchronized through dual writes. +At this point, the target cluster is considered the primary cluster, but {zdm-proxy} still keeps the origin cluster synchronized through dual writes. == Verify the read routing change Once the read routing configuration change has been rolled out, you might want to verify that reads are being sent to the target cluster as expected. This isn't required, but it can provide confirmation that the change was applied successfully. -However, it is difficult to assess read routing because the purpose of {product-short} is to align the clusters and provide an invisible proxy layer between your client application and the database clusters. +However, it is difficult to assess read routing because the purpose of {zdm-short} is to align the clusters and provide an invisible proxy layer between your client application and the database clusters. By design, the data is expected to be identical on both clusters, and your client application has no awareness of which cluster is servicing its requests. For this reason, the only way to manually test read routing is to intentionally write mismatched test data to the clusters. -Then, you can send a read request to {product-proxy} and see which cluster-specific data is returned, which indicates the cluster that received the read request. +Then, you can send a read request to {zdm-proxy} and see which cluster-specific data is returned, which indicates the cluster that received the read request. There are two ways to do this: Manually create mismatched tables:: @@ -56,7 +56,7 @@ To manually create mismatched data, you can create a test table on each cluster, [IMPORTANT] ==== When you write the mismatched data to the tables, make sure you connect to each cluster directly. -Don't connect to {product-proxy}, because {product-proxy} will, by design, write the same data to both clusters through dual writes. +Don't connect to {zdm-proxy}, because {zdm-proxy} will, by design, write the same data to both clusters through dual writes. ==== + . Create a small test table on both clusters, such as a simple key/value table. @@ -84,7 +84,7 @@ For example: INSERT INTO test_keyspace.test_table(k, v) VALUES ('1', 'Hello from the target cluster!'); ---- + -. Use `cqlsh` to xref:connect-clients-to-proxy.adoc#_connecting_cqlsh_to_the_zdm_proxy[connect to {product-proxy}], and then issue a read request to your test table. +. Use `cqlsh` to xref:connect-clients-to-proxy.adoc#_connecting_cqlsh_to_the_zdm_proxy[connect to {zdm-proxy}], and then issue a read request to your test table. For example: + [source,cql] @@ -104,26 +104,26 @@ For example: If you created dedicated test keyspaces, drop the keyspaces as well. Use the Themis sample client application:: -The xref:connect-clients-to-proxy.adoc#_themis_client[Themis sample client application] connects directly to the origin cluster, the target cluster, and {product-proxy}. +The xref:connect-clients-to-proxy.adoc#_themis_client[Themis sample client application] connects directly to the origin cluster, the target cluster, and {zdm-proxy}. It inserts some test data in its own, dedicated table. Then, you can view the results of reads from each source. For more information, see the https://github.com/absurdfarce/themis/blob/main/README.md[Themis README]. === System tables cannot validate read routing -Issuing a `DESCRIBE` command or read request to any system table through {product-proxy} cannot sufficiently validate read routing. +Issuing a `DESCRIBE` command or read request to any system table through {zdm-proxy} cannot sufficiently validate read routing. -When {product-proxy} receives system reads, it intercepts them and always routes them to the origin, regardless of the `primary_cluster` variable. -In some cases, {product-proxy} partially populates these queries at the proxy level. +When {zdm-proxy} receives system reads, it intercepts them and always routes them to the origin, regardless of the `primary_cluster` variable. +In some cases, {zdm-proxy} partially populates these queries at the proxy level. -This means that system reads don't represent how {product-proxy} routes regular read requests. +This means that system reads don't represent how {zdm-proxy} routes regular read requests. Although `DESCRIBE` requests aren't system reads, they are also resolved differently than other `DESCRIBE` requests. Don't use `DESCRIBE` requests to verify read routing behavior. == Monitor and troubleshoot read performance -After changing read routing, monitor the performance of {product-proxy} and the target cluster to ensure reads are succeeding and meeting your performance expectations. +After changing read routing, monitor the performance of {zdm-proxy} and the target cluster to ensure reads are succeeding and meeting your performance expectations. If read requests fail or perform poorly, you can <> back to `ORIGIN` while you investigate the issue. @@ -137,6 +137,6 @@ If your target cluster performs poorly, or you skipped Phase 3 previously, go ba == Next steps You can stay at this phase as long as you like. -{product-proxy} continues to perform dual writes to both clusters, keeping the origin and target clusters synchronized. +{zdm-proxy} continues to perform dual writes to both clusters, keeping the origin and target clusters synchronized. When you're ready to complete the migration and stop using your origin cluster, proceed to xref:ROOT:connect-clients-to-target.adoc[Phase 5] to disable dual writes and cut over to the target cluster exclusively. \ No newline at end of file diff --git a/modules/ROOT/pages/components.adoc b/modules/ROOT/pages/components.adoc index 474b602a..a904184d 100644 --- a/modules/ROOT/pages/components.adoc +++ b/modules/ROOT/pages/components.adoc @@ -2,13 +2,13 @@ :navtitle: Compare migration tools :description: Learn about {company} migration tools. -The {company} {product} ({product-short}) toolkit includes {product-proxy}, {product-utility}, {product-automation}, and several data migration tools. +The {company} {zdm} toolkit includes {zdm-proxy}, {zdm-utility}, {zdm-automation}, and several data migration tools. -For live migrations, {product-proxy} orchestrates activity-in-transition on your clusters. -{product-utility} and {product-automation} facilitate the deployment and management of {product-proxy}. +For live migrations, {zdm-proxy} orchestrates activity-in-transition on your clusters. +{zdm-utility} and {zdm-automation} facilitate the deployment and management of {zdm-proxy}. To move and validate data, you use data migration tools. -You can use these tools alone or with {product-proxy}. +You can use these tools alone or with {zdm-proxy}. == When to use migration tools @@ -23,26 +23,26 @@ For example: * You want to consolidate client applications running on separate clusters onto one shared cluster to minimize sprawl and maintenance. -== {product-proxy} +== {zdm-proxy} -The main component of the {company} {product} toolkit is {product-proxy-repo}[{product-proxy}], which is designed to be a lightweight proxy that handles all real-time requests generated by your client applications during the migration process. +The main component of the {company} {zdm} toolkit is {zdm-proxy-repo-url}[{zdm-proxy}], which is designed to be a lightweight proxy that handles all real-time requests generated by your client applications during the migration process. This tool is open-source software. -{product-proxy} is an orchestrator for monitoring application activity and keeping multiple clusters (databases) in sync through dual writes. -{product-proxy} isn't linked to the actual migration process. +{zdm-proxy} is an orchestrator for monitoring application activity and keeping multiple clusters (databases) in sync through dual writes. +{zdm-proxy} isn't linked to the actual migration process. It doesn't perform data migrations and it doesn't have awareness of ongoing migrations. Instead, you use a <> to perform the data migration and validate migrated data. -{product-proxy} reduces risks to upgrades and migrations by decoupling the origin (source) cluster from the target (destination) cluster and maintaining consistency between both clusters. +{zdm-proxy} reduces risks to upgrades and migrations by decoupling the origin (source) cluster from the target (destination) cluster and maintaining consistency between both clusters. You decide when you want to switch permanently to the target cluster. After migrating your data, changes to your application code are usually minimal, depending on your client's compatibility with the origin and target clusters. Typically, you only need to update the connection string. [#how-zdm-proxy-handles-reads-and-writes] -=== How {product-proxy} handles reads and writes +=== How {zdm-proxy} handles reads and writes -{company} created {product-proxy} to orchestrate requests between a client application and both the origin and target clusters. +{company} created {zdm-proxy} to orchestrate requests between a client application and both the origin and target clusters. These clusters can be any data store that supports the xref:cql:ROOT:index.adoc[{cql}], such as {cass-reg}, {dse}, {hcd}, and {astra-db}. During the migration process, you designate one cluster as the _primary cluster_, which serves as the source of truth for reads. @@ -50,25 +50,25 @@ For the majority of the migration process, this is typically the origin cluster. Towards the end of the migration process, when you are ready to read exclusively from your target cluster, you set the target cluster as the primary cluster. The other cluster is referred to as the _secondary cluster_. -While {product-proxy} is active, write requests are sent to both clusters to ensure data consistency, but only the primary cluster serves read requests. +While {zdm-proxy} is active, write requests are sent to both clusters to ensure data consistency, but only the primary cluster serves read requests. ==== Writes (dual-write logic) -{product-proxy} sends every write operation (`INSERT`, `UPDATE`, `DELETE`) synchronously to both clusters at the client application's requested consistency level: +{zdm-proxy} sends every write operation (`INSERT`, `UPDATE`, `DELETE`) synchronously to both clusters at the client application's requested consistency level: * If the write is acknowledged in both clusters at the requested consistency level, then the operation returns a successful write acknowledgement to the client that issued the request. -* If the write fails in either cluster, then {product-proxy} passes a write failure, originating from the primary cluster, back to the client. +* If the write fails in either cluster, then {zdm-proxy} passes a write failure, originating from the primary cluster, back to the client. The client can then retry the request, if appropriate, based on the client's retry policy. This design ensures that new data is always written to both clusters, and that any failure on either cluster is always made visible to the client application. -For information about how {product-proxy} handles Lightweight Transactions (LWTs), see xref:feasibility-checklists.adoc#_lightweight_transactions_and_the_applied_flag[Lightweight Transactions and the applied flag]. +For information about how {zdm-proxy} handles Lightweight Transactions (LWTs), see xref:feasibility-checklists.adoc#_lightweight_transactions_and_the_applied_flag[Lightweight Transactions and the applied flag]. ==== Reads -By default, {product-proxy} sends all reads to the primary cluster, and then returns the result to the client application. +By default, {zdm-proxy} sends all reads to the primary cluster, and then returns the result to the client application. -If you enable _asynchronous dual reads_, {product-proxy} sends asynchronous read requests to the secondary cluster (typically the target cluster) in addition to the synchronous read requests that are sent to the primary cluster. +If you enable _asynchronous dual reads_, {zdm-proxy} sends asynchronous read requests to the secondary cluster (typically the target cluster) in addition to the synchronous read requests that are sent to the primary cluster. This feature is designed to test the target cluster's ability to handle a production workload before you permanently switch to the target cluster at the end of the migration process. @@ -79,7 +79,7 @@ For more information, see xref:ROOT:enable-async-dual-reads.adoc[]. ==== Consistency levels -{product-proxy} doesn't directly manage or track consistency levels. +{zdm-proxy} doesn't directly manage or track consistency levels. Instead, it passes the requested consistency level from the client application to each cluster (origin and target) when routing requests. For reads, the consistency level is always passed to the primary cluster, which always receives read requests. @@ -94,12 +94,12 @@ If either request fails to attain the required quorum, the failure is returned t If either cluster is an {astra-db} database, be aware that `CL.ONE` isn't supported by {astra}. Requests sent with `CL.ONE` to {astra-db} databases always fail. -{product-proxy} doesn't mute these failures because you need to be aware of them. +{zdm-proxy} doesn't mute these failures because you need to be aware of them. You must adapt your client application to use a consistency level that is supported by both clusters to ensure that the migration is seamless and error-free. ==== Timeouts and connection failures -When requests are routed through {product-proxy}, there is a proxy-side timeout and application-side timeout. +When requests are routed through {zdm-proxy}, there is a proxy-side timeout and application-side timeout. If a response isn't received within the timeout period (`xref:ROOT:manage-proxy-instances.adoc#zdm_proxy_request_timeout_ms[zdm_proxy_request_timeout_ms]`), nothing is returned to the request handling thread, and, by extension, no response is sent to the client. This inevitably results in a client-side timeout, which is an accurate representation of the fact that at least one cluster failed to respond to the request. @@ -107,47 +107,47 @@ The clusters that are required to respond depend on the type of request and whet See also xref:ROOT:feasibility-checklists.adoc#driver-retry-policy-and-query-idempotence[Driver retry policy and query idempotence] and `xref:manage-proxy-instances.adoc#zdm_proxy_max_stream_ids[zdm_proxy_max_stream_ids]`. -=== High availability and multiple {product-proxy} instances +=== High availability and multiple {zdm-proxy} instances -{product-proxy} is designed to be highly available and run a clustered fashion to avoid a single point of failure. +{zdm-proxy} is designed to be highly available and run a clustered fashion to avoid a single point of failure. -With the exception of local test environments, {company} recommends that all {product-proxy} deployments have multiple {product-proxy} instances. +With the exception of local test environments, {company} recommends that all {zdm-proxy} deployments have multiple {zdm-proxy} instances. Deployments typically consist of three or more instances. [TIP] ==== -Throughout the {product-short} documentation, the term _{product-proxy} deployment_ refers to the entire deployment, and _{product-proxy} instance_ refers to an individual proxy process in the deployment. +Throughout the {zdm-short} documentation, the term _{zdm-proxy} deployment_ refers to the entire deployment, and _{zdm-proxy} instance_ refers to an individual proxy process in the deployment. ==== -You can scale {product-proxy} instances horizontally and vertically. -To avoid downtime when applying configuration changes, you can xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[perform a rolling restart] of your {product-proxy} instances. +You can scale {zdm-proxy} instances horizontally and vertically. +To avoid downtime when applying configuration changes, you can xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[perform a rolling restart] of your {zdm-proxy} instances. -For simplicity, you can use {product-utility} and {product-automation} to set up and run Ansible playbooks that deploy and manage {product-proxy} and its monitoring stack. +For simplicity, you can use {zdm-utility} and {zdm-automation} to set up and run Ansible playbooks that deploy and manage {zdm-proxy} and its monitoring stack. -== {product-utility} and {product-automation} +== {zdm-utility} and {zdm-automation} -You can use {product-automation-repo}[{product-utility} and {product-automation}] to set up and run Ansible playbooks that deploy and manage multiple {product-proxy} instances and the associated monitoring stack (Prometheus metrics and associated Grafana visualizations). +You can use {zdm-automation-repo-url}[{zdm-utility} and {zdm-automation}] to set up and run Ansible playbooks that deploy and manage multiple {zdm-proxy} instances and the associated monitoring stack (Prometheus metrics and associated Grafana visualizations). https://www.ansible.com/[Ansible] is a suite of software tools that enable infrastructure as code. It is open source, and its capabilities include software provisioning, configuration management, and application deployment. -Ansible playbooks streamline and automate the deployment and management of {product-proxy} instances and their monitoring components. +Ansible playbooks streamline and automate the deployment and management of {zdm-proxy} instances and their monitoring components. Playbooks are YAML files that define a series of tasks to be executed on one or more remote machines, including installing software, configuring settings, and managing services. They are repeatable and reusable, and they simplify deployment and configuration management because each playbook focuses on a specific operation, such as rolling restarts. You run playbooks from a centralized machine known as the Ansible Control Host. -{product-utility}, which is included with {product-automation}, creates the Docker container that acts as the Ansible Control Host. +{zdm-utility}, which is included with {zdm-automation}, creates the Docker container that acts as the Ansible Control Host. -To use {product-utility} and {product-automation}, you must xref:deployment-infrastructure.adoc[prepare the recommended infrastructure]. +To use {zdm-utility} and {zdm-automation}, you must xref:deployment-infrastructure.adoc[prepare the recommended infrastructure]. -For more information about the role of Ansible and Ansible playbooks in the {product-short} process, see xref:setup-ansible-playbooks.adoc[] and xref:deploy-proxy-monitoring.adoc[]. +For more information about the role of Ansible and Ansible playbooks in the {zdm-short} process, see xref:setup-ansible-playbooks.adoc[] and xref:deploy-proxy-monitoring.adoc[]. [#data-migration-tools] == Data migration tools You use data migration tools to move data between clusters and validate the migrated data. -You can use these tools alone or with {product-proxy}. +You can use these tools alone or with {zdm-proxy}. === {sstable-sideloader} @@ -161,9 +161,9 @@ For more information, see xref:sideloader:sideloader-overview.adoc[]. You can use {cass-migrator-short} for data migration and validation between {cass-short}-based databases. It offers extensive functionality and configuration options to support large and complex migrations as well as post-migration data validation. -You can use {cass-migrator-short} by itself, with {product-proxy}, or for data validation after using another data migration tool. +You can use {cass-migrator-short} by itself, with {zdm-proxy}, or for data validation after using another data migration tool. -For more information, see xref:ROOT:cassandra-data-migrator.adoc[]. +For more information, see the {cass-migrator-repo-url}[{cass-migrator-short} repository]. === {dsbulk} @@ -172,7 +172,7 @@ You can use it to load, unload, and count records. Because {dsbulk-short} doesn't have the same data validation capabilities as {cass-migrator-short}, it is best for migrations that don't require extensive data validation, aside from post-migration row counts. -You can use {dsbulk-short} alone or with {product-proxy}. +You can use {dsbulk-short} alone or with {zdm-proxy}. For more information, see xref:dsbulk:overview:dsbulk-about.adoc[]. @@ -186,25 +186,25 @@ include::ROOT:partial$dsbulk-migrator-deprecation.adoc[] Depending on your origin and target databases, there might be other data migration tools available for your migration. For example, if you want to write your own custom data migration processes, you can use a tool like {spark-reg}. -To use a data migration tool with {product-proxy}, it must meet the following requirements: +To use a data migration tool with {zdm-proxy}, it must meet the following requirements: * Built-in data validation functionality or compatibility with another data validation tool, such as {cass-migrator-short}. * Avoids or minimizes changes to your data model, including column names and data types. + -Because {product-proxy} requires that both databases can successfully process the same read/write statements, migrations that perform significant data transformations might not be compatible with {product-proxy}. +Because {zdm-proxy} requires that both databases can successfully process the same read/write statements, migrations that perform significant data transformations might not be compatible with {zdm-proxy}. The impact of data transformations depends on your specific data model, database platforms, and the scale of your migration. For data-only migrations that aren't concerned with live application traffic or minimizing downtime, your chosen tool depends on your origin and target databases, the compatibility of the data models, and the scale of your migration. -Describing the full range of these tools is beyond the scope of this document, which focuses on full-scale platform migrations with the {product-short} tools and verified {product-short}-compatible data migration tools. +Describing the full range of these tools is beyond the scope of this document, which focuses on full-scale platform migrations with the {zdm-short} tools and verified {zdm-short}-compatible data migration tools. == In-place migrations [WARNING] ==== -In-place migrations carry a higher risk of data loss or corruption, require progressive manual reconfiguration of the cluster, and are more cumbersome to rollback compared to the {product-short} process. +In-place migrations carry a higher risk of data loss or corruption, require progressive manual reconfiguration of the cluster, and are more cumbersome to rollback compared to the {zdm-short} process. -Whenever possible, {company} recommends using the {product-short} process to orchestrate live migrations between separate clusters, which eliminates the need for progressive configuration changes, and allows you to seamlessly xref:ROOT:rollback.adoc[rollback to your origin cluster] if there is a problem during the migration. +Whenever possible, {company} recommends using the {zdm-short} process to orchestrate live migrations between separate clusters, which eliminates the need for progressive configuration changes, and allows you to seamlessly xref:ROOT:rollback.adoc[rollback to your origin cluster] if there is a problem during the migration. ==== For certain migration paths, it is possible to perform in-place database platform replacements on the same cluster where you data already exists. diff --git a/modules/ROOT/pages/connect-clients-to-proxy.adoc b/modules/ROOT/pages/connect-clients-to-proxy.adoc index 0bb34bc1..e942ee47 100644 --- a/modules/ROOT/pages/connect-clients-to-proxy.adoc +++ b/modules/ROOT/pages/connect-clients-to-proxy.adoc @@ -1,14 +1,14 @@ -= Connect client applications to {product-proxy} += Connect client applications to {zdm-proxy} -{product-proxy} is designed to mimic communication with a typical cluster based on {cass-reg}. -This means that your client applications connect to {product-proxy} in the same way that they already connect to your existing {cass-short}-based clusters. +{zdm-proxy} is designed to mimic communication with a typical cluster based on {cass-reg}. +This means that your client applications connect to {zdm-proxy} in the same way that they already connect to your existing {cass-short}-based clusters. -You can communicate with {product-proxy} using the same xref:cql:ROOT:index.adoc[{cql}] statements used in your existing client applications. +You can communicate with {zdm-proxy} using the same xref:cql:ROOT:index.adoc[{cql}] statements used in your existing client applications. It understands the same messaging protocols used by {cass-short}, {dse-short}, {hcd-short}, and {astra-db}. -As a result, most client applications cannot distinguish between connections to {product-proxy} and direct connections to a {cass-short}-based cluster. +As a result, most client applications cannot distinguish between connections to {zdm-proxy} and direct connections to a {cass-short}-based cluster. -This page explains how to connect client applications to a {cass-short}-based cluster using {cass-short} drivers, and then describes the how you need to modify your client applications to connect to {product-proxy}. -It also explains how to connect the {cql-shell} (`cqlsh`) to {product-proxy}, which is often used in conjunction with {cass-short} drivers and during the migration process for certain validation tasks. +This page explains how to connect client applications to a {cass-short}-based cluster using {cass-short} drivers, and then describes the how you need to modify your client applications to connect to {zdm-proxy}. +It also explains how to connect the {cql-shell} (`cqlsh`) to {zdm-proxy}, which is often used in conjunction with {cass-short} drivers and during the migration process for certain validation tasks. == Connect applications to {cass-short} clusters @@ -90,19 +90,19 @@ print(release_version) ---- Review your client application's code to understand how it connects to your existing {cass-short}-based clusters. -Then, proceed to <> to learn how to modify that code to connect to {product-proxy} instead. +Then, proceed to <> to learn how to modify that code to connect to {zdm-proxy} instead. [#connect-applications-to-zdm-proxy] -== Connect applications to {product-proxy} +== Connect applications to {zdm-proxy} -By design, {product-proxy} should be transparent to your client application. -This means that connecting your client application to {product-proxy} should be seamless and nearly identical to the direct cluster connection. -However, there are some considerations and adjustments you might need to make for {product-proxy}. +By design, {zdm-proxy} should be transparent to your client application. +This means that connecting your client application to {zdm-proxy} should be seamless and nearly identical to the direct cluster connection. +However, there are some considerations and adjustments you might need to make for {zdm-proxy}. -=== Understand how {product-proxy} handles certain client behaviors +=== Understand how {zdm-proxy} handles certain client behaviors -If you haven't done so already, thoroughly review the xref:ROOT:feasibility-checklists.adoc[compatibility requirements for {product-proxy}]. -This ensures that your client application's code is compatible with {product-proxy}, and you understand how certain operations and behaviors are handled by {product-proxy}. +If you haven't done so already, thoroughly review the xref:ROOT:feasibility-checklists.adoc[compatibility requirements for {zdm-proxy}]. +This ensures that your client application's code is compatible with {zdm-proxy}, and you understand how certain operations and behaviors are handled by {zdm-proxy}. For example: * xref:ROOT:feasibility-checklists.adoc#driver-retry-policy-and-query-idempotence[Driver retry policy and query idempotence] @@ -111,28 +111,28 @@ For example: === Set connection points -Connect your client application to {product-proxy} instead of directly to the origin cluster by modifying the client's connection points. +Connect your client application to {zdm-proxy} instead of directly to the origin cluster by modifying the client's connection points. -Typically, this involves setting the client's connection points to the IP addresses or hostnames of the local {product-proxy} instances. -The specific connection points for your migration depend on where and how you configured your xref:ROOT:deployment-infrastructure.adoc[{product-proxy} infrastructure]. +Typically, this involves setting the client's connection points to the IP addresses or hostnames of the local {zdm-proxy} instances. +The specific connection points for your migration depend on where and how you configured your xref:ROOT:deployment-infrastructure.adoc[{zdm-proxy} infrastructure]. [#provide-authentication-credentials] === Provide authentication credentials -Make sure {product-proxy} receives the correct authentication credentials from your client application. +Make sure {zdm-proxy} receives the correct authentication credentials from your client application. You might need to change the credentials set in your client application depending on your origin and target cluster configuration. -With or without {product-proxy}, client applications must provide cluster credentials to authenticate requests. +With or without {zdm-proxy}, client applications must provide cluster credentials to authenticate requests. When a client connects directly to a cluster, it provides the authentication credentials for that cluster. -When a client connects to {product-proxy}, the client still provides only one set of credentials because {product-proxy} is designed to be invisible to the client application. +When a client connects to {zdm-proxy}, the client still provides only one set of credentials because {zdm-proxy} is designed to be invisible to the client application. -{product-proxy} uses the credentials provided by the client to forward requests to one of the clusters. -To forward requests to the other cluster, {product-proxy} uses the credentials defined in `xref:ROOT:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[zdm_proxy_cluster_config.yml]`. +{zdm-proxy} uses the credentials provided by the client to forward requests to one of the clusters. +To forward requests to the other cluster, {zdm-proxy} uses the credentials defined in `xref:ROOT:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[zdm_proxy_cluster_config.yml]`. -.Credential usage by {product-proxy} when authentication is required for both clusters -image::ROOT:zdm-proxy-credential-usage.png[{product-proxy} credentials usage when authentication is required for both clusters, 550] +.Credential usage by {zdm-proxy} when authentication is required for both clusters +image::ROOT:zdm-proxy-credential-usage.png[{zdm-proxy} credentials usage when authentication is required for both clusters, 550] -The credentials that {product-proxy} expects to receive from the client depend on the authentication requirements of the origin and target clusters. +The credentials that {zdm-proxy} expects to receive from the client depend on the authentication requirements of the origin and target clusters. Make sure the client passes the correct credentials for your cluster configuration: * *Authentication required for both clusters*: Your client application must supply credentials for the target cluster. @@ -151,14 +151,14 @@ Then, in your client application, set the `username` to the literal string `toke + [IMPORTANT] ==== -_Don't_ provide a xref:astra-db-serverless:databases:secure-connect-bundle.adoc[{scb}] when connecting your client application to {product-proxy}. +_Don't_ provide a xref:astra-db-serverless:databases:secure-connect-bundle.adoc[{scb}] when connecting your client application to {zdm-proxy}. -If you include the {scb-short}, the driver ignores the {product-proxy} connection points and connects exclusively to your {astra-db} database instead of {product-proxy}. +If you include the {scb-short}, the driver ignores the {zdm-proxy} connection points and connects exclusively to your {astra-db} database instead of {zdm-proxy}. ==== + For connections to early {astra} databases with long-lived tokens created prior to the unified `token` approach, you can set the `username` to the token's `clientId` and the `password` to the token's `secret`. -For more information about cluster credentials for {product-proxy}, see xref:ROOT:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[Cluster and core configuration]. +For more information about cluster credentials for {zdm-proxy}, see xref:ROOT:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[Cluster and core configuration]. == Sample client applications @@ -169,50 +169,50 @@ They are not intended for production use or for production-scale performance tes To test your target cluster's ability to handle production workloads, you can xref:ROOT:enable-async-dual-reads.adoc[enable asynchronous dual reads]. -To assess the performance of {product-proxy}, {company} recommends http://docs.nosqlbench.io/getting-started/[NoSQLBench]. +To assess the performance of {zdm-proxy}, {company} recommends http://docs.nosqlbench.io/getting-started/[NoSQLBench]. ==== -The following sample client applications demonstrate how to use the Java driver with {product-proxy} and the origin and target for that proxy. +The following sample client applications demonstrate how to use the Java driver with {zdm-proxy} and the origin and target for that proxy. See your driver's documentation for code samples that are specific to your chosen driver, including cluster connection examples and statement execution examples. -You can use the provided sample client applications, in addition to your own client applications, to validate that your {product-proxy} deployment orchestrates read and write requests as expected between the origin cluster, target cluster, and client applications. +You can use the provided sample client applications, in addition to your own client applications, to validate that your {zdm-proxy} deployment orchestrates read and write requests as expected between the origin cluster, target cluster, and client applications. -{product-demo}:: -https://github.com/alicel/zdm-demo-client/[{product-demo}] is a minimal Java web application which provides a simple, stripped-down example of an application built to work with {product-proxy}. +{zdm-short} Demo Client:: +https://github.com/alicel/zdm-demo-client/[{zdm-short} Demo Client] is a minimal Java web application which provides a simple, stripped-down example of an application built to work with {zdm-proxy}. After updating connection information you can compile and run the application locally and interact with it using HTTP clients such as `curl` or `wget`. + -You can find the details of building and running {product-demo} in the https://github.com/alicel/zdm-demo-client/blob/master/README.md[README]. +You can find the details of building and running {zdm-short} Demo Client in the https://github.com/alicel/zdm-demo-client/blob/master/README.md[README]. [[_themis_client]] Themis client:: -https://github.com/absurdfarce/themis[Themis] is a Java command-line client application that allows you to write randomly generated data directly to the origin cluster, directly to the target cluster, or indirectly to both clusters through {product-proxy}. -Then, you can use the client application to query the data and confirm that {product-proxy} is reading and writing data from the expected sources. +https://github.com/absurdfarce/themis[Themis] is a Java command-line client application that allows you to write randomly generated data directly to the origin cluster, directly to the target cluster, or indirectly to both clusters through {zdm-proxy}. +Then, you can use the client application to query the data and confirm that {zdm-proxy} is reading and writing data from the expected sources. + -Configuration details for the clusters and {product-proxy} are defined in a YAML file. +Configuration details for the clusters and {zdm-proxy} are defined in a YAML file. For more information, see the https://github.com/absurdfarce/themis/blob/main/README.md[Themis README]. + -In addition to being a validation tool, Themis also provides an example of a larger client application that uses the Java driver to connect to {product-proxy}, or directly to a {cass-short} cluster, and perform operations. +In addition to being a validation tool, Themis also provides an example of a larger client application that uses the Java driver to connect to {zdm-proxy}, or directly to a {cass-short} cluster, and perform operations. The configuration logic, as well as the cluster and session management code, are separated into distinct packages to make them easy to understand. -== Connect the {cql-shell} to {product-proxy} +== Connect the {cql-shell} to {zdm-proxy} The {cql-shell} (`cqlsh`) is a command-line tool that you can use to send CQL and `cqlsh` commands to your {cass-short}-based clusters, including {astra-db}, {dse-short}, {hcd-short}, and {cass} databases. When issuing these commands directly to a cluster, you can use your database's included version of `cqlsh`, or you can download and run a standalone `cqlsh`. -With {product-proxy}, you use a standalone `cqlsh` installation to issue commands to both clusters through {product-proxy}. +With {zdm-proxy}, you use a standalone `cqlsh` installation to issue commands to both clusters through {zdm-proxy}. Your origin and target clusters must have a common `cql_version` between them. -If there is no CQL version that is compatible with both clusters, `cqlsh` won't be able to connect to {product-proxy}. +If there is no CQL version that is compatible with both clusters, `cqlsh` won't be able to connect to {zdm-proxy}. -To connect `cqlsh` to a {product-proxy} instance, do the following: +To connect `cqlsh` to a {zdm-proxy} instance, do the following: -. On a machine that can connect to your {product-proxy} instance, xref:cql:cqlsh:use-cql-shell.adoc[download and install the {cql-shell}]. +. On a machine that can connect to your {zdm-proxy} instance, xref:cql:cqlsh:use-cql-shell.adoc[download and install the {cql-shell}]. + -Any version of `cqlsh` can connect to {product-proxy}, but some clusters require a specific `cqlsh` version. +Any version of `cqlsh` can connect to {zdm-proxy}, but some clusters require a specific `cqlsh` version. -. Start `cqlsh` with your {product-proxy} connection details: +. Start `cqlsh` with your {zdm-proxy} connection details: + [source,shell,subs="+quotes"] ---- @@ -221,8 +221,8 @@ Any version of `cqlsh` can connect to {product-proxy}, but some clusters require + Replace the following: + -* `**ZDM_PROXY_IP**`: The IP address of your {product-proxy} instance. -* `**PORT**`: The port on which the {product-proxy} instance listens for client connections. +* `**ZDM_PROXY_IP**`: The IP address of your {zdm-proxy} instance. +* `**PORT**`: The port on which the {zdm-proxy} instance listens for client connections. If you are using the default port, 9042, you can omit this argument. * `**USERNAME**` and `**PASSWORD**`: Valid cluster authentication credentials, depending on the authentication requirements for your origin and target clusters: + @@ -239,13 +239,13 @@ For {astra-db}, the username is the literal string `token`, and the password is [IMPORTANT] ==== -For {astra-db}, _don't_ use the {scb-short} to connect `cqlsh` to {product-proxy}. +For {astra-db}, _don't_ use the {scb-short} to connect `cqlsh` to {zdm-proxy}. Only use an application token. -If you include the {scb-short}, `cqlsh` ignores all other connection arguments and connects exclusively to your {astra-db} database instead of {product-proxy}. +If you include the {scb-short}, `cqlsh` ignores all other connection arguments and connects exclusively to your {astra-db} database instead of {zdm-proxy}. ==== -- == Next steps -After you connect your client applications to {product-proxy}, you can begin xref:ROOT:migrate-and-validate-data.adoc[Phase 2] of the migration, which is the data migration phase. \ No newline at end of file +After you connect your client applications to {zdm-proxy}, you can begin xref:ROOT:migrate-and-validate-data.adoc[Phase 2] of the migration, which is the data migration phase. \ No newline at end of file diff --git a/modules/ROOT/pages/connect-clients-to-target.adoc b/modules/ROOT/pages/connect-clients-to-target.adoc index d245e21b..fa88d4a7 100644 --- a/modules/ROOT/pages/connect-clients-to-target.adoc +++ b/modules/ROOT/pages/connect-clients-to-target.adoc @@ -4,7 +4,7 @@ Phase 5 is the last phase of the xref:ROOT:introduction.adoc[migration process], after you route all read requests to the target cluster in xref:ROOT:change-read-routing.adoc[Phase 4]. In this final phase, you connect your client applications directly and exclusively to the target cluster. -This removes the dependency on {product-proxy} and the origin cluster, thereby completing the migration process. +This removes the dependency on {zdm-proxy} and the origin cluster, thereby completing the migration process. image::ROOT:migration-phase5ra.svg[In Phase 5, your applications no longer use the proxy and, instead, connect directly to the target cluster] @@ -272,6 +272,6 @@ For more information, see the following: Your migration is now complete, and your target cluster is the source of truth for your client applications and data. -When you are ready, you can decommission your origin cluster and {product-proxy} because these are no longer needed and xref:ROOT:rollback.adoc[seamless rollback] is no longer possible. +When you are ready, you can decommission your origin cluster and {zdm-proxy} because these are no longer needed and xref:ROOT:rollback.adoc[seamless rollback] is no longer possible. If you need to revert to the origin cluster after this point, you must perform a full migration in the opposite direction, with your previous origin cluster as the target, to ensure that all data is rewritten and synchronized back to the origin. \ No newline at end of file diff --git a/modules/ROOT/pages/create-target.adoc b/modules/ROOT/pages/create-target.adoc index a165a355..eb3f09e6 100644 --- a/modules/ROOT/pages/create-target.adoc +++ b/modules/ROOT/pages/create-target.adoc @@ -1,6 +1,6 @@ = Create the target environment -After you review the xref:ROOT:feasibility-checklists.adoc[compatibility requirements] and prepare the xref:ROOT:deployment-infrastructure.adoc[{product-proxy} infrastructure], you must prepare your target cluster for the migration. +After you review the xref:ROOT:feasibility-checklists.adoc[compatibility requirements] and prepare the xref:ROOT:deployment-infrastructure.adoc[{zdm-proxy} infrastructure], you must prepare your target cluster for the migration. This includes the following: @@ -19,7 +19,7 @@ For complex migrations, such as those that involve multi-datacenter clusters, ma . Sign in to the {astra-ui-link} and xref:astra-db-serverless:administration:manage-organizations.adoc#switch-organizations[switch to the organization] where you want to create the new database. + -{product-proxy} can be used with any xref:astra-db-serverless:administration:subscription-plans.adoc[{astra} plan]. +{zdm-proxy} can be used with any xref:astra-db-serverless:administration:subscription-plans.adoc[{astra} plan]. However, paid plans offer premium features that can facilitate your migration, including support for {sstable-sideloader}, more databases, and no automatic database hibernation. . xref:astra-db-serverless:databases:create-database.adoc[Create a database] with your preferred database name, cloud provider, region, and other details. @@ -29,7 +29,7 @@ If the name of this keyspace doesn't match your origin cluster's schema, you can . When your database reaches **Active** status, xref:astra-db-serverless:administration:manage-application-tokens.adoc[create an application token] with a role like {read-write-user-role} or {database-administrator-role}, and then store the credentials (Client ID, Client Secret, and Token) securely. + -These credentials are used by the client application and {product-proxy} to read and write to your target database. +These credentials are used by the client application and {zdm-proxy} to read and write to your target database. Make sure the token's role has sufficient permission to perform the actions required by your client application. . xref:astra-db-serverless:databases:secure-connect-bundle.adoc[Download your database's {scb}]. @@ -79,7 +79,7 @@ For specific infrastructure, installation, and configuration instructions, see t + [TIP] ==== -Because {product-proxy} supports separate connection details for each cluster, you can configure the new cluster as needed, independent of the origin cluster's configuration. +Because {zdm-proxy} supports separate connection details for each cluster, you can configure the new cluster as needed, independent of the origin cluster's configuration. This is a good opportunity to establish your desired configuration state on the new cluster and implement new patterns that might have been unavailable or impractical on the old cluster, such as enabling authentication or configuring TLS encryption. ==== @@ -87,7 +87,7 @@ This is a good opportunity to establish your desired configuration state on the . If you enabled authentication in your cluster, create a user with the required permissions for your client application to use to read and write to the cluster. + -Store the authentication credentials securely for use by your client application and {product-proxy} later in the migration process. +Store the authentication credentials securely for use by your client application and {zdm-proxy} later in the migration process. . Note your cluster's connection details, including the contact points (IP addresses or hostnames) and port number. @@ -105,17 +105,17 @@ For specific changes in each version, see the release notes for your database pl == Migrate to other CQL-compatible data stores -Support for other CQL-compatible data stores isn't guaranteed for {product-proxy}. +Support for other CQL-compatible data stores isn't guaranteed for {zdm-proxy}. -If your origin and target clusters meet the xref:ROOT:feasibility-checklists.adoc[protocol version compatibility requirements], you might be able to use {product-proxy} for your migration. +If your origin and target clusters meet the xref:ROOT:feasibility-checklists.adoc[protocol version compatibility requirements], you might be able to use {zdm-proxy} for your migration. As with any migration, {company} recommends that you test this in isolation before attempting a full-scale production migration. See your data store provider's documentation for information about creating your cluster and schema, generating authentication credentials, and gathering the connection details. == Test the connection to the target cluster -After you create the target cluster, try connecting your client application directly to the target cluster without {product-proxy}. -This ensures that the connection will work when you disconnect {product-proxy} at the end of the migration. +After you create the target cluster, try connecting your client application directly to the target cluster without {zdm-proxy}. +This ensures that the connection will work when you disconnect {zdm-proxy} at the end of the migration. Additionally, {company} recommends running performance tests and measuring benchmarks in a test environment where your client application is connected directly to your target cluster. This helps you understand how your application workloads will perform on the new cluster. diff --git a/modules/ROOT/pages/deploy-proxy-monitoring.adoc b/modules/ROOT/pages/deploy-proxy-monitoring.adoc index 0c4898b2..66dcde97 100644 --- a/modules/ROOT/pages/deploy-proxy-monitoring.adoc +++ b/modules/ROOT/pages/deploy-proxy-monitoring.adoc @@ -1,10 +1,10 @@ -= Deploy {product-proxy} += Deploy {zdm-proxy} :page-aliases: ROOT:tls.adoc -After you xref:setup-ansible-playbooks.adoc[set up the jumphost and {product-automation}], use the Ansible playbooks to deploy your {product-proxy} instances. -Then, xref:ROOT:metrics.adoc[deploy the monitoring stack] to complete your {product-proxy} deployment. +After you xref:setup-ansible-playbooks.adoc[set up the jumphost and {zdm-automation}], use the Ansible playbooks to deploy your {zdm-proxy} instances. +Then, xref:ROOT:metrics.adoc[deploy the monitoring stack] to complete your {zdm-proxy} deployment. -Aside from xref:ROOT:deployment-infrastructure.adoc[preparing the infrastructure], you don't need to install any {product-proxy} dependencies on the {product-proxy} machines. The playbook automatically installs all required software packages. +Aside from xref:ROOT:deployment-infrastructure.adoc[preparing the infrastructure], you don't need to install any {zdm-proxy} dependencies on the {zdm-proxy} machines. The playbook automatically installs all required software packages. == Access the Control Host and locate the configuration files @@ -29,29 +29,29 @@ ubuntu@52772568517c:~$ . List the contents of the `vars` directory, and then find the following YAML configuration files: + * `zdm_proxy_container_config.yml`: Internal configuration for the proxy container itself. -* `zdm_proxy_cluster_config.yml`: Configuration properties to connect {product-proxy} to the origin and target clusters. +* `zdm_proxy_cluster_config.yml`: Configuration properties to connect {zdm-proxy} to the origin and target clusters. This is always required. * `zdm_proxy_core_config.yml`: Important configuration properties that are commonly used and changed during the migration. * `zdm_proxy_advanced_config.yml`: Advanced configuration properties that aren't always required. Leave these to their default values unless you have a specific use case that requires changing them. * `zdm_proxy_custom_tls_config.yml`: Optional TLS encryption properties. -The following sections explain how to configure each of these files before deploying your {product-proxy} instances. +The following sections explain how to configure each of these files before deploying your {zdm-proxy} instances. [[_configure_the_zdm_proxy]] -== Configure {product-proxy} +== Configure {zdm-proxy} -After you locate the configuration files on the Control Host, you must prepare the configuration properties for your {product-proxy} deployment. +After you locate the configuration files on the Control Host, you must prepare the configuration properties for your {zdm-proxy} deployment. === Container configuration . Edit the `zdm_proxy_container_config.yml` file. -. Set your {product-proxy} version. +. Set your {zdm-proxy} version. . Create a strategy to inject configuration parameters. + -In all versions of {product-proxy}, you can use environment variables to set the {product-proxy} configuration. +In all versions of {zdm-proxy}, you can use environment variables to set the {zdm-proxy} configuration. + In versions 2.3.0 and later, you can inject the configuration with a YAML file generated from automation scripts. @@ -61,10 +61,10 @@ In versions 2.3.0 and later, you can inject the configuration with a YAML file g For the cluster and core configuration, you need to provide connection credentials and details for both the origin and target clusters. -.Configuration file change in {product-automation} version 2.2.0 +.Configuration file change in {zdm-automation} version 2.2.0 [IMPORTANT] ==== -Starting in version 2.2.0 of {product-automation}, all origin and target cluster configuration variables are stored in `zdm_proxy_cluster_config.yml`. +Starting in version 2.2.0 of {zdm-automation}, all origin and target cluster configuration variables are stored in `zdm_proxy_cluster_config.yml`. In earlier versions, these variables are in the `zdm_proxy_core_config.yml` file. This change is backward compatible. @@ -89,12 +89,12 @@ For legacy authentication to earlier {astra-db} databases with an older token ge . Edit the `zdm_proxy_cluster_config.yml` file. The `vi` and `nano` text editors are available in the container. -. In the `ORIGIN CONFIGURATION` and `TARGET CONFIGURATION` sections, uncomment and configure all variables that are required for {product-proxy} to connect to the origin and target clusters. +. In the `ORIGIN CONFIGURATION` and `TARGET CONFIGURATION` sections, uncomment and configure all variables that are required for {zdm-proxy} to connect to the origin and target clusters. + [IMPORTANT] ==== You must provide connection details for both `ORIGIN CONFIGURATION` and `TARGET CONFIGURATION`. -If either set is missing, {product-proxy} cannot connect to both clusters. +If either set is missing, {zdm-proxy} cannot connect to both clusters. ==== + The variables are the same in both sections, but they are set separately for each cluster. @@ -130,7 +130,7 @@ The unused option must be unset. For example, if you use `target_astra_db_id` and `target_astra_token`, then `target_astra_secure_connect_bundle_path` must be unset. + -- -** **Both `{asterisk}_astra_db_id` and `{asterisk}_astra_token`**: If you want {product-automation} to automatically download your database's {scb}, use `*_astra_db_id` and `*_astra_token`. +** **Both `{asterisk}_astra_db_id` and `{asterisk}_astra_token`**: If you want {zdm-automation} to automatically download your database's {scb}, use `*_astra_db_id` and `*_astra_token`. Set `*_astra_db_id` to your xref:astra-db-serverless:databases:create-database.adoc#get-db-id[database's ID], and set `*_astra_token` to your application token (`AstraCS:...`). ** **Only `{asterisk}_astra_secure_connect_bundle_path`**: If you want to manually provide your database's {scb-short} to the jumphost, use `*_astra_secure_connect_bundle_path`, and manually upload the {scb-short} to the jumphost: + @@ -193,8 +193,8 @@ Unless you are investigating an issue, leave this set to the default value of `I [IMPORTANT] ==== -The origin credentials, target credentials, and the `primary_cluster` variable are mutable variables that you can change after deploying {product-proxy}. -All other cluster connection configuration variables are immutable; the only way to change these values is by completely recreating the {product-proxy} deployment. +The origin credentials, target credentials, and the `primary_cluster` variable are mutable variables that you can change after deploying {zdm-proxy}. +All other cluster connection configuration variables are immutable; the only way to change these values is by completely recreating the {zdm-proxy} deployment. For more information, see xref:ROOT:manage-proxy-instances.adoc[]. ==== @@ -207,13 +207,13 @@ Only modify the variables in `zdm_proxy_advanced_config.yml` if you have a speci [IMPORTANT] ==== The following advanced configuration variables are immutable. -If you need to change these variables, {company} recommends that you do so _before_ deploying {product-proxy}. -Future changes require you to re-create your entire {product-proxy} deployment. +If you need to change these variables, {company} recommends that you do so _before_ deploying {zdm-proxy}. +Future changes require you to re-create your entire {zdm-proxy} deployment. For more information, see xref:ROOT:manage-proxy-instances.adoc#change-immutable-configuration-variables[Change immutable configuration variables]. ==== Multi-datacenter clusters:: -For xref:ROOT:deployment-infrastructure.adoc#multiple-datacenter-clusters[multi-datacenter origin clusters], specify the name of the datacenter that {product-proxy} should consider local. +For xref:ROOT:deployment-infrastructure.adoc#multiple-datacenter-clusters[multi-datacenter origin clusters], specify the name of the datacenter that {zdm-proxy} should consider local. To do this, set the `origin_local_datacenter` property to the local datacenter name. Similarly, for multi-datacenter target clusters, set the `target_local_datacenter` property to the local datacenter name. These two variables are stored in `zdm_proxy_advanced_config.yml`. @@ -223,17 +223,17 @@ For information about downloading a region-specific {scb-short}, see xref:astra- [#ports] Ports:: -Each {product-proxy} instance listens on port 9042 by default, like a default {cass-short} cluster. +Each {zdm-proxy} instance listens on port 9042 by default, like a default {cass-short} cluster. You can override this by setting `zdm_proxy_listen_port` to your preferred port. -This can be useful if the origin nodes listen on a port that is not 9042 and you want to configure {product-proxy} to listen on that same port to avoid changing the port in your client application configuration. +This can be useful if the origin nodes listen on a port that is not 9042 and you want to configure {zdm-proxy} to listen on that same port to avoid changing the port in your client application configuration. + -{product-proxy} exposes metrics on port 14001 by default. +{zdm-proxy} exposes metrics on port 14001 by default. This port is used by Prometheus to scrape the application-level proxy metrics. This can be changed by setting `metrics_port` to a different value if desired. All other advanced configuration variables in `zdm_proxy_advanced_config.yml` are mutable. -You can seamlessly change them after deploying {product-proxy} with a rolling restart. -Immutable variables require you to re-create the entire deployment and result in downtime for your {product-proxy} deployment. +You can seamlessly change them after deploying {zdm-proxy} with a rolling restart. +Immutable variables require you to re-create the entire deployment and result in downtime for your {zdm-proxy} deployment. For more information, see xref:manage-proxy-instances.adoc[]. [#enable-tls-encryption] @@ -241,26 +241,26 @@ For more information, see xref:manage-proxy-instances.adoc[]. Transportation Layer Security (TLS) encryption is optional and disabled by default. -{product-proxy} supports TLS encryption between {product-proxy} and either or both clusters, and between {product-proxy} and your client application. +{zdm-proxy} supports TLS encryption between {zdm-proxy} and either or both clusters, and between {zdm-proxy} and your client application. To enable TLS encryption, you must provide the necessary files and configure TLS settings in the `zdm_proxy_custom_tls_config.yml` file before running the deployment playbook. -When you deploy the {product-proxy} instances with {product-automation}, the deployment playbook automatically distributes the TLS files and applies the TLS configuration to all {product-proxy} instances. +When you deploy the {zdm-proxy} instances with {zdm-automation}, the deployment playbook automatically distributes the TLS files and applies the TLS configuration to all {zdm-proxy} instances. If you want to enable TLS after the initial deployment, you must rerun the deployment playbook to redeploy the instances with the new TLS configuration. ==== Configure proxy-to-cluster TLS -Use these steps to enable TLS encryption between {product-proxy} and one or both clusters if required. +Use these steps to enable TLS encryption between {zdm-proxy} and one or both clusters if required. Each cluster has its own TLS configuration. One-way TLS and Mutual TLS (mTLS) are both supported, and they can be enabled as needed for each cluster's requirements. -In this case, {product-proxy} acts as the TLS client and the cluster acts as the TLS server. +In this case, {zdm-proxy} acts as the TLS client and the cluster acts as the TLS server. [TIP] ==== This is required for self-managed clusters only. -For {astra-db}, {product-proxy} uses mTLS automatically with the xref:astra-db-serverless:databases:secure-connect-bundle.adoc[{scb-short}]. +For {astra-db}, {zdm-proxy} uses mTLS automatically with the xref:astra-db-serverless:databases:secure-connect-bundle.adoc[{scb-short}]. ==== . Find the required files for each cluster where you want to enable TLS encryption. @@ -273,9 +273,9 @@ All files must be in plain-text, non-binary format. + If your client application and origin cluster already use TLS encryption, then the required files should already be used in the client application's configuration (TLS client files) and the origin cluster's configuration (TLS Server files). -. For each self-managed cluster (origin or target) where you want to enable TLS encrypted {product-proxy} connections, do the following: +. For each self-managed cluster (origin or target) where you want to enable TLS encrypted {zdm-proxy} connections, do the following: + -.. If your TLS files are in a JKS keystore, you must extract them as plain text because {product-proxy} cannot accept a JKS keystore. +.. If your TLS files are in a JKS keystore, you must extract them as plain text because {zdm-proxy} cannot accept a JKS keystore. You must provide the raw files. + -- @@ -341,7 +341,7 @@ docker exec -it zdm-ansible-container bash . Uncomment and populate the TLS configuration variables for the clusters where you want to enable TLS encryption. For example, if you want to enable TLS encryption for both clusters, configure both sets of variables (`origin_tls_{asterisk}` and `target_tls_{asterisk}`). + -In the proxy-to-cluster configuration, the word `server` in the variable names refers to the cluster, which acts as the TLS server, and the word `client` refers to {product-proxy}, which acts as the TLS client. +In the proxy-to-cluster configuration, the word `server` in the variable names refers to the cluster, which acts as the TLS server, and the word `client` refers to {zdm-proxy}, which acts as the TLS client. + Origin cluster TLS encryption variables:: + @@ -363,9 +363,9 @@ Must be unset for one-way TLS. ==== Configure client application-to-proxy TLS -Use these steps to enable TLS encryption between your client application and {product-proxy} if required. +Use these steps to enable TLS encryption between your client application and {zdm-proxy} if required. -In this case, your client application is the TLS client, and {product-proxy} is the TLS server. +In this case, your client application is the TLS client, and {zdm-proxy} is the TLS server. One-way TLS and Mutual TLS (mTLS) are both supported. @@ -376,7 +376,7 @@ If your client application and origin cluster already use TLS encryption, then t . If your TLS files are in a JKS keystore, extract them as plain text. + -{product-proxy} cannot accept a JKS keystore. +{zdm-proxy} cannot accept a JKS keystore. You must provide the raw files. + .. Get the files contained in your JKS keystore and their aliases: @@ -427,7 +427,7 @@ docker exec -it zdm-ansible-container bash . From this shell, edit the `zdm_proxy_tls_config.yml` file at `zdm-proxy-automation/ansible/vars/zdm_proxy_custom_tls_config.yml`. . Uncomment and populate the following TLS configuration variables, which are prefixed with `zdm_proxy_*`. -The word `server` in the variable names refers to {product-proxy}, which acts as the TLS server in this configuration. +The word `server` in the variable names refers to {zdm-proxy}, which acts as the TLS server in this configuration. + * `zdm_proxy_tls_user_dir_path_name`: Use the default value of `/home/ubuntu/zdm_proxy_tls_files`. * `zdm_proxy_tls_server_ca_filename`: Required. Provide the filename (without the path) of the server CA that the proxy must use. @@ -439,7 +439,7 @@ Set to `true` to enable mTLS between the application and the proxy. [#run-the-deployment-playbook] == Run the deployment playbook -After modifying all necessary configuration variables, you are ready to deploy your {product-proxy} instances. +After modifying all necessary configuration variables, you are ready to deploy your {zdm-proxy} instances. . From your shell connected to the Control Host, make sure you are in the `ansible` directory at `/home/ubuntu/zdm-proxy-automation/ansible`. + @@ -452,19 +452,19 @@ If you are in the `vars` directory, then you must go up one level to the `ansibl ansible-playbook deploy_zdm_proxy.yml -i zdm_ansible_inventory ---- -. Wait while {product-proxy} containers are deployed to each of your {product-proxy} machines. +. Wait while {zdm-proxy} containers are deployed to each of your {zdm-proxy} machines. + -The playbook creates one {product-proxy} instance for each proxy host listed in the inventory file. +The playbook creates one {zdm-proxy} instance for each proxy host listed in the inventory file. While the playbook runs, activity is printed to the shell along with any errors. If the entire operation is successful, the final output is a confirmation message. -. Confirm that the {product-proxy} instances are running by checking the Docker logs. +. Confirm that the {zdm-proxy} instances are running by checking the Docker logs. + -Alternatively, after you xref:ROOT:metrics.adoc[deploy the monitoring stack], you can xref:ROOT:metrics.adoc#call-the-liveliness-and-readiness-endpoints[call the `liveliness` and `readiness` endpoints] to confirm that each {product-proxy} instance is running. +Alternatively, after you xref:ROOT:metrics.adoc[deploy the monitoring stack], you can xref:ROOT:metrics.adoc#call-the-liveliness-and-readiness-endpoints[call the `liveliness` and `readiness` endpoints] to confirm that each {zdm-proxy} instance is running. + To check the Docker logs, do the following: + -.. SSH into one of the servers where a deployed {product-proxy} instance is running. +.. SSH into one of the servers where a deployed {zdm-proxy} instance is running. You can do this from within the Ansible Control Host Docker container, or directly from the jumphost machine: + [source,bash,subs="+quotes"] @@ -472,7 +472,7 @@ You can do this from within the Ansible Control Host Docker container, or direct ssh **USER**@**ZDM_PROXY_IP_ADDRESS** ---- -.. Use the `docker logs` command to view the logs for the {product-proxy} instance: +.. Use the `docker logs` command to view the logs for the {zdm-proxy} instance: + [source,bash] ---- @@ -515,10 +515,10 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS == Troubleshoot deployment issues -If the {product-proxy} instances fail to start due to errors in the configuration, edit the configuration files and then rerun the deployment playbook. +If the {zdm-proxy} instances fail to start due to errors in the configuration, edit the configuration files and then rerun the deployment playbook. For specific troubleshooting scenarios, see xref:ROOT:troubleshooting-tips.adoc[]. == Next steps -To continue Phase 1 of the migration, xref:ROOT:metrics.adoc[deploy the {product-proxy} monitoring stack]. \ No newline at end of file +To continue Phase 1 of the migration, xref:ROOT:metrics.adoc[deploy the {zdm-proxy} monitoring stack]. \ No newline at end of file diff --git a/modules/ROOT/pages/deployment-infrastructure.adoc b/modules/ROOT/pages/deployment-infrastructure.adoc index 02d53796..c0986049 100644 --- a/modules/ROOT/pages/deployment-infrastructure.adoc +++ b/modules/ROOT/pages/deployment-infrastructure.adoc @@ -1,60 +1,60 @@ -= Prepare the {product-proxy} deployment infrastructure -:navtitle: Prepare {product-proxy} infrastructure += Prepare the {zdm-proxy} deployment infrastructure +:navtitle: Prepare {zdm-proxy} infrastructure -After confirming that your clusters, data model, and application logic meet the xref:ROOT:feasibility-checklists.adoc[compatibility requirements for {product-proxy}], you must prepare the infrastructure to deploy your {product-proxy} instances. +After confirming that your clusters, data model, and application logic meet the xref:ROOT:feasibility-checklists.adoc[compatibility requirements for {zdm-proxy}], you must prepare the infrastructure to deploy your {zdm-proxy} instances. -{company} recommends using {product-automation} and {product-utility} to manage your {product-proxy} deployment. -If you choose to use these tools, you must also prepare infrastructure for {product-automation}. +{company} recommends using {zdm-automation} and {zdm-utility} to manage your {zdm-proxy} deployment. +If you choose to use these tools, you must also prepare infrastructure for {zdm-automation}. == Choose where to deploy the proxy -A typical {product-proxy} deployment is made up of multiple proxy instances. +A typical {zdm-proxy} deployment is made up of multiple proxy instances. {company} recommends a minimum of three proxy instances for any deployment other than local test or demo environments. -All {product-proxy} instances must be reachable by your client application, and they must be able to connect to your origin and target clusters. -{company} recommends that you deploy {product-proxy} as close to your client application instances as possible. +All {zdm-proxy} instances must be reachable by your client application, and they must be able to connect to your origin and target clusters. +{company} recommends that you deploy {zdm-proxy} as close to your client application instances as possible. For more information, see <>. -You can deploy {product-proxy} on any cloud provider or on-premises, depending on your existing infrastructure. +You can deploy {zdm-proxy} on any cloud provider or on-premises, depending on your existing infrastructure. -The following diagram shows a typical deployment with connectivity between client applications, {product-proxy} instances, and clusters: +The following diagram shows a typical deployment with connectivity between client applications, {zdm-proxy} instances, and clusters: image::ROOT:zdm-during-migration3.png[Connectivity between client applications, proxy instances, and clusters] -The {product-proxy} process is lightweight, requiring only a small amount of resources and no storage to persist its state aside from logs. +The {zdm-proxy} process is lightweight, requiring only a small amount of resources and no storage to persist its state aside from logs. [#multiple-datacenter-clusters] === Multiple datacenter clusters -If you have a multi-datacenter cluster with multiple set of client application instances deployed to geographically distributed datacenters, you must plan a separate {product-proxy} deployment for each datacenter. +If you have a multi-datacenter cluster with multiple set of client application instances deployed to geographically distributed datacenters, you must plan a separate {zdm-proxy} deployment for each datacenter. -In the configuration for each {product-proxy} deployment, specify only the contact points that belong to that datacenter, and set the `xref:ROOT:deploy-proxy-monitoring.adoc#_advanced_configuration_optional[origin_local_datacenter]` and `xref:ROOT:deploy-proxy-monitoring.adoc#_advanced_configuration_optional[target_local_datacenter]` properties as needed. +In the configuration for each {zdm-proxy} deployment, specify only the contact points that belong to that datacenter, and set the `xref:ROOT:deploy-proxy-monitoring.adoc#_advanced_configuration_optional[origin_local_datacenter]` and `xref:ROOT:deploy-proxy-monitoring.adoc#_advanced_configuration_optional[target_local_datacenter]` properties as needed. -If your origin and target clusters are both multi-datacenter clusters, this configuration will be more complicated to correctly orchestrate traffic routing through {product-proxy}. +If your origin and target clusters are both multi-datacenter clusters, this configuration will be more complicated to correctly orchestrate traffic routing through {zdm-proxy}. {company} recommends contacting {support-url}[IBM Support] for assistance with complex multi-region and multi-datacenter migrations. -=== Don't deploy {product-proxy} as a sidecar +=== Don't deploy {zdm-proxy} as a sidecar -Don't deploy {product-proxy} as a sidecar because it was designed to mimic communication with a {cass-short}-based cluster. -For this reason, {company} recommends deploying multiple {product-proxy} instances, each running on a dedicated machine, instance, or VM. +Don't deploy {zdm-proxy} as a sidecar because it was designed to mimic communication with a {cass-short}-based cluster. +For this reason, {company} recommends deploying multiple {zdm-proxy} instances, each running on a dedicated machine, instance, or VM. -For best performance, deploy your {product-proxy} instances as close as possible to your client applications, ideally on the same local network, but don't co-deploy them on the same machines as the client applications. -This way, each client application instance can connect to all {product-proxy} instances, just as it would connect to all nodes in a {cass-short}-based cluster or datacenter. +For best performance, deploy your {zdm-proxy} instances as close as possible to your client applications, ideally on the same local network, but don't co-deploy them on the same machines as the client applications. +This way, each client application instance can connect to all {zdm-proxy} instances, just as it would connect to all nodes in a {cass-short}-based cluster or datacenter. This deployment model provides maximum resilience and failure tolerance guarantees, and it allows the client application driver to continue using the same load balancing and retry mechanisms that it would normally use. -Conversely, deploying a single {product-proxy} instance undermines this resilience mechanism and creates a single point of failure, which can affect client applications if one or more nodes of the underlying origin or target clusters go offline. -In a sidecar deployment, each client application instance would be connecting to a single {product-proxy} instance, and would, therefore, be exposed to this risk. +Conversely, deploying a single {zdm-proxy} instance undermines this resilience mechanism and creates a single point of failure, which can affect client applications if one or more nodes of the underlying origin or target clusters go offline. +In a sidecar deployment, each client application instance would be connecting to a single {zdm-proxy} instance, and would, therefore, be exposed to this risk. [[_machines]] == Hardware requirements -You need a number of machines to run your {product-proxy} instances, plus additional machines for the centralized jumphost, and for running {dsbulk} or {cass-migrator} (which are recommended data migration and validation tools). +You need a number of machines to run your {zdm-proxy} instances, plus additional machines for the centralized jumphost, and for running {dsbulk} or {cass-migrator} (which are recommended data migration and validation tools). This section uses the term _machine_ broadly to refer to a cloud instance (on any cloud provider), a VM, or a physical server. -{product-proxy} instances:: -You need one machine for each {product-proxy} instance. +{zdm-proxy} instances:: +You need one machine for each {zdm-proxy} instance. {company} recommends a minimum of three proxy instances for all migrations that aren't for local testing purposes. + Each machine must meet the following specifications: @@ -69,8 +69,8 @@ Each machine must meet the following specifications: These machines can be on any cloud provider or on-premises, depending on your existing infrastructure, and they should be as close as possible to your client application instances for best performance. Jumphost and monitoring:: -You need at least one machine for the jumphost, which is the centralized manager for the {product-proxy} instances and runs the monitoring stack (Prometheus and Grafana). -With {product-automation}, the jumphost also runs the Ansible Control Host container. +You need at least one machine for the jumphost, which is the centralized manager for the {zdm-proxy} instances and runs the monitoring stack (Prometheus and Grafana). +With {zdm-automation}, the jumphost also runs the Ansible Control Host container. + Typically, a single machine is used for all of these functions, but you can split them across different machines if you prefer. This documentation assumes you are using a single machine. @@ -125,21 +125,21 @@ If you need assistance with your migration, contact {support-url}[IBM Support]. [IMPORTANT] ==== -Don't allow external machines to directly access the {product-proxy} machines. -The jumphost is the only machine that should have direct access to the {product-proxy} machines. +Don't allow external machines to directly access the {zdm-proxy} machines. +The jumphost is the only machine that should have direct access to the {zdm-proxy} machines. ==== Core infrastructure ports:: -The {product-proxy} machines must be reachable by the following: +The {zdm-proxy} machines must be reachable by the following: + * The client application instances on port 9042 * The jumphost on port 22 * The machine running the monitoring stack (the jumphost or a separate machine) on port 14001 Clusters:: -The {product-proxy} machines must be able to connect to the origin and target cluster nodes. +The {zdm-proxy} machines must be able to connect to the origin and target cluster nodes. + -For self-managed clusters like {cass-short} and {dse-short}, {product-proxy} must be able to connect on the {cass-short} Native Protocol port, which is typically 9042. +For self-managed clusters like {cass-short} and {dse-short}, {zdm-proxy} must be able to connect on the {cass-short} Native Protocol port, which is typically 9042. + For {astra}, you must ensure that outbound connectivity is possible over the {astra} endpoint indicated in the xref:astra-db-serverless:databases:secure-connect-bundle.adoc[{scb}]. Connections over xref:astra-db-serverless:administration:manage-private-endpoints.adoc[private endpoints] are supported. @@ -148,8 +148,8 @@ Jumphost and monitoring:: The following connectivity requirements apply to the jumphost and monitoring machine. If you use separate machines for these functions, these requirements apply to both machines. + -* Connect to {product-proxy} instances on port 14001 for metrics collection. -* Connect to {product-proxy} instances on port 22 for {product-automation}, log inspection, and troubleshooting. +* Connect to {zdm-proxy} instances on port 14001 for metrics collection. +* Connect to {zdm-proxy} instances on port 22 for {zdm-automation}, log inspection, and troubleshooting. * Allow incoming, external SSH connections. + [IMPORTANT] @@ -160,14 +160,14 @@ For example, limit access to the IP ranges of your corporate networks or trusted * Expose the Grafana UI on port 3000. + -External connections are required so that {product-automation} can download the necessary software packages (Docker, Prometheus, Grafana) and the {product-proxy} image from DockerHub. +External connections are required so that {zdm-automation} can download the necessary software packages (Docker, Prometheus, Grafana) and the {zdm-proxy} image from DockerHub. -=== Connect to {product-proxy} infrastructure from an external machine +=== Connect to {zdm-proxy} infrastructure from an external machine To connect to the jumphost from an external machine, ensure that its IP address belongs to a permitted IP range. If you are connecting through a VPN that only intercepts connections to selected destinations, you might need to add a route from your VPN IP gateway to the public IP of the jumphost. -A custom SSH config file can simplify the process of connecting to the jumphost and, by extension, the {product-proxy} instances. +A custom SSH config file can simplify the process of connecting to the jumphost and, by extension, the {zdm-proxy} instances. To use the following template, replace all placeholders with the appropriate values for your deployment. Add more entries if you have more than three proxy instances. @@ -208,7 +208,7 @@ Then, use the SSH config file to connect to your jumphost: ssh -F zdm_ssh_config jumphost ---- -Connect to a {product-proxy} instance by specifying the corresponding instance number in your config file: +Connect to a {zdm-proxy} instance by specifying the corresponding instance number in your config file: [source,bash] ---- diff --git a/modules/ROOT/pages/dse-migration-paths.adoc b/modules/ROOT/pages/dse-migration-paths.adoc index fe93df33..e6379042 100644 --- a/modules/ROOT/pages/dse-migration-paths.adoc +++ b/modules/ROOT/pages/dse-migration-paths.adoc @@ -3,17 +3,17 @@ The {dse} migration toolkit includes the xref:ROOT:components.adoc[{company} migration tools] that you can use to migrate your data across {dse-short} and another {cass-reg}-based database, such as {astra-db} or {hcd-short}. -This documentation doesn't describe _all_ possible migration paths; it focuses on migrations using {company} migration tools like {product-proxy}. +This documentation doesn't describe _all_ possible migration paths; it focuses on migrations using {company} migration tools like {zdm-proxy}. [TIP] ==== -Whenever possible, {company} recommends using the {product} ({product-short}) tools when you need to maintain live traffic for your applications while transferring data. +Whenever possible, {company} recommends using the {zdm} tools when you need to maintain live traffic for your applications while transferring data. This is most relevant for full-scale platform migrations where you move your data _and_ switch your applications to connect to your new databases. -The {product-short} tools orchestrate and synchronize read/write requests while you use a data migration tool to copy data from one cluster to the other. +The {zdm-short} tools orchestrate and synchronize read/write requests while you use a data migration tool to copy data from one cluster to the other. Then, you can take as much time as you need to validate the data and simulate production workloads on your new cluster before updating your application code to use the new databases. -For information about clusters that support the {product-short} tools, including supported {dse-short} versions, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. +For information about clusters that support the {zdm-short} tools, including supported {dse-short} versions, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. ==== == Migrate your data @@ -35,7 +35,7 @@ Migrate data from {dse-short}:: When migrating _from_ {dse-short} to another {cass-short}-based database, follow the migration guidance for your target database to determine cluster compatibility, migration options, and recommendations. For example, for {astra-db}, see xref:ROOT:astra-migration-paths.adoc[], and for {hcd-short}, see xref:ROOT:hcd-migration-paths.adoc[]. + -For information about origin and target clusters that are supported by the {product-short} tools, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. +For information about origin and target clusters that are supported by the {zdm-short} tools, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. + If your target database isn't directly compatible with a migration from {dse-short}, you might need to take interim steps to prepare your data for migration, such as upgrading your {dse-short} version, modifying the data in your existing database to be compatible with the target database, or running an extract, transform, load (ETL) pipeline. diff --git a/modules/ROOT/pages/enable-async-dual-reads.adoc b/modules/ROOT/pages/enable-async-dual-reads.adoc index 7be89a87..09606aaa 100644 --- a/modules/ROOT/pages/enable-async-dual-reads.adoc +++ b/modules/ROOT/pages/enable-async-dual-reads.adoc @@ -6,9 +6,9 @@ After migrating and validating your data in xref:ROOT:migrate-and-validate-data. Phase 3 is optional but highly recommended. In this phase, you enable the _asynchronous dual reads_ feature to test the secondary (target) cluster's ability to handle a production workload before you permanently redirect read requests in xref:ROOT:change-read-routing.adoc[Phase 4]. -By default, {product-proxy} sends all reads to the primary (origin) cluster, and then returns the result to the client application. +By default, {zdm-proxy} sends all reads to the primary (origin) cluster, and then returns the result to the client application. -When you enable _asynchronous dual reads_, {product-proxy} sends asynchronous read requests to the secondary cluster in addition to the synchronous read requests that are sent to the primary cluster. +When you enable _asynchronous dual reads_, {zdm-proxy} sends asynchronous read requests to the secondary cluster in addition to the synchronous read requests that are sent to the primary cluster. At this point in the migration process, the secondary cluster is typically the target cluster. Because this feature is intended to test your target cluster's ability to handle a simulated production workload, there is no reason to run it against the origin cluster that is already capable of handling production workloads. @@ -20,13 +20,13 @@ This allows you to assess the target cluster's performance and make any adjustme == Response and error handling with asynchronous dual reads With or without asynchronous dual reads, the client application only receives results from synchronous reads on the primary cluster. -The client never receives results from asynchronous reads on the secondary cluster because these results are used only for {product-proxy}'s asynchronous dual read metrics and testing purposes. +The client never receives results from asynchronous reads on the secondary cluster because these results are used only for {zdm-proxy}'s asynchronous dual read metrics and testing purposes. By design, if an asynchronous read fails or times out, it has no impact on client operations and the client application doesn't receive an error. However, the increased workload from read requests can cause write requests to fail or time out on the secondary cluster. With or without asynchronous dual reads, a failed write on either cluster returns an error to the client application and potentially triggers a retry. -This functionality is intentional so you can simulate production-scale read traffic on the secondary cluster, in addition to the existing write traffic from {product-proxy}'s xref:components.adoc#how-zdm-proxy-handles-reads-and-writes[dual writes], with the least impact to your applications. +This functionality is intentional so you can simulate production-scale read traffic on the secondary cluster, in addition to the existing write traffic from {zdm-proxy}'s xref:components.adoc#how-zdm-proxy-handles-reads-and-writes[dual writes], with the least impact to your applications. To avoid unnecessary failures due to missing unmigrated data, don't enable asynchronous dual reads until you migrate, validate, and reconcile _all_ data from the origin cluster to the target cluster. @@ -34,7 +34,7 @@ To avoid unnecessary failures due to missing unmigrated data, don't enable async == Configure asynchronous dual reads Use the `read_mode` variable to enable or disable asynchronous dual reads. -Then, perform a rolling restart of your {product-proxy} instances to apply the configuration change. +Then, perform a rolling restart of your {zdm-proxy} instances to apply the configuration change. . Edit `vars/zdm_proxy_core_config.yml`, and then set the `read_mode` variable: + @@ -54,7 +54,7 @@ read_mode: PRIMARY_ONLY + The default configuration is disable (`PRIMARY_ONLY`). -. xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[Perform a rolling restart] to apply the configuration change to your {product-proxy} instances. +. xref:ROOT:manage-proxy-instances.adoc#perform-a-rolling-restart-of-the-proxies[Perform a rolling restart] to apply the configuration change to your {zdm-proxy} instances. == Monitor the target cluster's performance @@ -63,7 +63,7 @@ After enabling asynchronous dual reads, observe the target cluster's performance To assess performance, you can monitor the following: * Cluster health metrics like latency, throughput, and error rate -* {product-proxy}'s xref:metrics.adoc#_asynchronous_read_requests_metrics[asynchronous read requests metrics] +* {zdm-proxy}'s xref:metrics.adoc#_asynchronous_read_requests_metrics[asynchronous read requests metrics] If needed, adjust the target cluster's configuration and continue monitoring until the cluster reaches your performance targets. diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index 94c27096..99126fdf 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -1,47 +1,47 @@ -= {product} frequently asked questions -:navtitle: {product-short} FAQs += {zdm} frequently asked questions +:navtitle: {zdm-short} FAQs :page-aliases: ROOT:contributions.adoc, ROOT:glossary.adoc -This page includes common questions about the {company} {product} ({product-short}) tools. +This page includes common questions about the {company} {zdm} tools. == What is a zero-downtime migration? -A zero-downtime migration with the {company} {product-short} tools means you can reliably migrate your client applications and data between {cass-short}-based clusters with no interruption of service. +A zero-downtime migration with the {company} {zdm-short} tools means you can reliably migrate your client applications and data between {cass-short}-based clusters with no interruption of service. -== Which platforms and versions are supported by the {product-short} tools? +== Which platforms and versions are supported by the {zdm-short} tools? See xref:ROOT:zdm-proxy-migration-paths.adoc[]. -== Why should I use the {product-short} tools for my migration? +== Why should I use the {zdm-short} tools for my migration? -There are several benefits to using the {product-short} tools for your migration: +There are several benefits to using the {zdm-short} tools for your migration: Minimal client code changes:: -Depending on cluster compatibility, the {product-short} tools help you migrate to a new or upgraded database platform with minimal changes to your client application code. +Depending on cluster compatibility, the {zdm-short} tools help you migrate to a new or upgraded database platform with minimal changes to your client application code. In some cases, you only need to change the connection string to point to the new cluster at the end of the migration process. Typically, these changes are minimal and non-invasive, especially if your client application uses an externalized property configuration for contact points. Real-time data consistency:: -{product-proxy} orchestrates real-time activity generated by your client applications, ensuring data consistency while you replicate, validate, and test your existing data on the new cluster. -Once you set up {product-proxy}, the dual-writes feature ensures that new writes are sent to both the origin and target clusters, so you can focus on migrating the data that was present before initializing {product-proxy}. +{zdm-proxy} orchestrates real-time activity generated by your client applications, ensuring data consistency while you replicate, validate, and test your existing data on the new cluster. +Once you set up {zdm-proxy}, the dual-writes feature ensures that new writes are sent to both the origin and target clusters, so you can focus on migrating the data that was present before initializing {zdm-proxy}. Safely test the new cluster under full production workloads:: In addition to the dual-writes feature, you can optionally enable asynchronous dual-reads to test the target cluster's ability to handle a production workload before you permanently switch to the target cluster at the end of the migration process. + Client applications aren't interrupted by read errors or latency spikes on the new, target cluster. -Although these errors and metrics are received by {product-proxy} for monitoring and performance benchmarking purposes, they aren't propagated back to the client applications. +Although these errors and metrics are received by {zdm-proxy} for monitoring and performance benchmarking purposes, they aren't propagated back to the client applications. + From the client side, traffic is seamless and uninterrupted during the entire migration process. Seamless rollback without data loss:: If there is a problem during the migration, you can xref:ROOT:rollback.adoc[rollback to the original cluster] without any data loss or interruption of service. -You can allow {product-proxy} to continue orchestrating dual-writes, or redirect your client applications back to the origin cluster and disable {product-proxy}. +You can allow {zdm-proxy} to continue orchestrating dual-writes, or redirect your client applications back to the origin cluster and disable {zdm-proxy}. Endless validation and testing time:: -Because your client applications remain fully operational during the migration, and your clusters are kept in sync by {product-proxy}, you can take as much time as you need to validate and test the target cluster before switching over permanently. +Because your client applications remain fully operational during the migration, and your clusters are kept in sync by {zdm-proxy}, you can take as much time as you need to validate and test the target cluster before switching over permanently. Migrate to a different platform or perform major version upgrades:: -The {product-short} tools support migrations between different {cass-short}-based platforms, such as open-source {cass-reg} to {astra-db}, as well as major version upgrades of the same platform, such as {dse-short} 5.0 to {dse-short} 6.9. +The {zdm-short} tools support migrations between different {cass-short}-based platforms, such as open-source {cass-reg} to {astra-db}, as well as major version upgrades of the same platform, such as {dse-short} 5.0 to {dse-short} 6.9. == What are the requirements for true zero-downtime migrations? @@ -51,99 +51,99 @@ See xref:ROOT:feasibility-checklists.adoc[]. Yes, see xref:ROOT:introduction.adoc[]. -== Is there a {product-short} demo? +== Is there a {zdm-short} demo? -Yes, you can use the xref:ROOT:introduction.adoc#lab[{product-short} interactive lab] to see how the migration process works. +Yes, you can use the xref:ROOT:introduction.adoc#lab[{zdm-short} interactive lab] to see how the migration process works. -== What {product-short} tools are available? +== What {zdm-short} tools are available? -The {product-short} tools are {product-proxy}, {product-utility}, and {product-automation}. +The {zdm-short} tools are {zdm-proxy}, {zdm-utility}, and {zdm-automation}. These tools orchestrate the traffic between your client applications and the origin and target clusters during the migration process. For the actual data migration, there are many tools you can use, such as {sstable-sideloader}, {cass-migrator}, {dsbulk}, and custom data migration scripts. For more information, see xref:ROOT:components.adoc[]. -== What is {product-proxy}? +== What is {zdm-proxy}? Generally speaking, a proxy is a software class functioning as an interface to any other component, connection, or resource, such as a network connection, a server, a large object in memory, or a file. The proxy is a wrapper or agent object that the client calls to access the real object served through that proxy. -In the context of {product-short}, the {product-proxy} is an open-source component designed to seamlessly handle real-time client application activity while a migration is in progress. +In the context of {zdm-short}, the {zdm-proxy} is an open-source component designed to seamlessly handle real-time client application activity while a migration is in progress. For more information, see xref:ROOT:components.adoc[]. -== What are {product-automation} and {product-utility}? +== What are {zdm-automation} and {zdm-utility}? -{product-automation} is an Ansible-based tool that allows you to deploy and manage multiple {product-proxy} instances and the associated monitoring stack. -To simplify the setup, the {product-automation} suite includes {product-utility}, which is an interactive utility that creates a Docker container to act as the Ansible Control Host. +{zdm-automation} is an Ansible-based tool that allows you to deploy and manage multiple {zdm-proxy} instances and the associated monitoring stack. +To simplify the setup, the {zdm-automation} suite includes {zdm-utility}, which is an interactive utility that creates a Docker container to act as the Ansible Control Host. For more information, see xref:ROOT:components.adoc[]. -== Does the {product-short} process migrate clusters? +== Does the {zdm-short} process migrate clusters? -The {product-short} process doesn't directly migrate clusters. -Instead, the {product-short} tools orchestrate live traffic between your existing cluster and a new cluster while you use a data migration tool to replicate and validate data on the new cluster. +The {zdm-short} process doesn't directly migrate clusters. +Instead, the {zdm-short} tools orchestrate live traffic between your existing cluster and a new cluster while you use a data migration tool to replicate and validate data on the new cluster. -{product-proxy} handles real-time requests generated by your client applications during the migration process, and keeps both clusters in sync through dual writes. +{zdm-proxy} handles real-time requests generated by your client applications during the migration process, and keeps both clusters in sync through dual writes. If there is a problem during the migration, you can confidently rollback to the original cluster without data loss or interruption of service. -At the end of the migration process, your client application connects exclusively to your new cluster, and then you decommission {product-proxy} and the old cluster. +At the end of the migration process, your client application connects exclusively to your new cluster, and then you decommission {zdm-proxy} and the old cluster. -== What challenges does {product-short} solve? +== What challenges does {zdm-short} solve? -Before the {product-short} tools were available, migrating client applications between clusters involved granular and intrusive client application code changes, extensive migration preparation, and a window of downtime for the client application's end users. +Before the {zdm-short} tools were available, migrating client applications between clusters involved granular and intrusive client application code changes, extensive migration preparation, and a window of downtime for the client application's end users. -With the {product-short} tools, you can migrate your client applications and data between {cass-short}-based clusters with minimal code changes and no interruption of service. +With the {zdm-short} tools, you can migrate your client applications and data between {cass-short}-based clusters with minimal code changes and no interruption of service. You can have the confidence that you are using tools designed specifically to handle the complexities of live traffic during large enterprise migrations. == What is the pricing model? -{product-proxy}, {product-utility}, {product-automation}, {cass-migrator-short}, and {dsbulk-short} are free and open-sourced. +{zdm-proxy}, {zdm-utility}, {zdm-automation}, {cass-migrator-short}, and {dsbulk-short} are free and open-sourced. {sstable-sideloader} is part of an {astra} *Enterprise* plan, and it incurs costs based on usage. == Where can I get help with my migration? -Technical assistance with the {product-short} process is available from {support-url}[IBM Support] for {dse-short} users, https://www.ibm.com/docs/en/esfac[IBM Elite Support for {cass}] subscribers, and {astra} organizations on an **Enterprise** plan. +Technical assistance with the {zdm-short} process is available from {support-url}[IBM Support] for {dse-short} users, https://www.ibm.com/docs/en/esfac[IBM Elite Support for {cass}] subscribers, and {astra} organizations on an **Enterprise** plan. -For any observed problems with {product-proxy} or the other open-source {product-short} and data migration tools, you can report an issue in their respective GitHub repositories: +For any observed problems with {zdm-proxy} or the other open-source {zdm-short} and data migration tools, you can report an issue in their respective GitHub repositories: -* {product-proxy-repo}[{product-proxy} repository] -* {product-automation-repo}[{product-automation} repository] (includes {product-automation} and {product-utility}) -* {cass-migrator-repo}[{cass-migrator-short} repository] -* {dsbulk-repo}[{dsbulk-short} repository] +* {zdm-proxy-repo-url}[{zdm-proxy} repository] +* {zdm-automation-repo-url}[{zdm-automation} repository] (includes {zdm-automation} and {zdm-utility}) +* {cass-migrator-repo-url}[{cass-migrator-short} repository] +* {dsbulk-repo-url}[{dsbulk-short} repository] -== Can I contribute to {product-proxy}? +== Can I contribute to {zdm-proxy}? -Yes, see `{product-proxy-repo}/blob/main/CONTRIBUTING.md[CONTRIBUTING.md]`. +Yes, see `{zdm-proxy-repo-url}/blob/main/CONTRIBUTING.md[CONTRIBUTING.md]`. -== Does {product-proxy} support Transport Layer Security (TLS)? +== Does {zdm-proxy} support Transport Layer Security (TLS)? Yes, see xref:ROOT:deploy-proxy-monitoring.adoc#enable-tls-encryption[Enable TLS encryption]. -== How does {product-proxy} handle Lightweight Transactions (LWTs)? +== How does {zdm-proxy} handle Lightweight Transactions (LWTs)? See xref:feasibility-checklists.adoc#_lightweight_transactions_and_the_applied_flag[Lightweight Transactions and the applied flag]. -== Can {product-proxy} be deployed as a sidecar? +== Can {zdm-proxy} be deployed as a sidecar? -Don't deploy {product-proxy} as a sidecar. +Don't deploy {zdm-proxy} as a sidecar. For more information, see xref:deployment-infrastructure.adoc#_choosing_where_to_deploy_the_proxy[Choosing where to deploy the proxy]. [#what-are-origin-target-primary-and-secondary-clusters] == What are origin, target, primary, and secondary clusters? -These terms refer to where your data is located during the migration process, and where read and write requests are sent by {product-proxy}: +These terms refer to where your data is located during the migration process, and where read and write requests are sent by {zdm-proxy}: Origin and target:: The _origin cluster_ is your existing database that you are migrating away from. The _target cluster_ is your new database that you are migrating to. + -Origin and target cluster credentials are provided to {product-proxy} so it can establish connections and send requests to both clusters. +Origin and target cluster credentials are provided to {zdm-proxy} so it can establish connections and send requests to both clusters. Primary and secondary:: The _primary cluster_ is the database that is designated as the source of truth for read requests. It receives all read requests by default, and the responses from these read requests are returned to the client application. -The primary cluster is set by {product-automation} through the `primary_cluster` variable, or you can set it directly through the {product-proxy} `ZDM_PRIMARY_CLUSTER` environment variable. +The primary cluster is set by {zdm-automation} through the `primary_cluster` variable, or you can set it directly through the {zdm-proxy} `ZDM_PRIMARY_CLUSTER` environment variable. + The other database is the _secondary cluster_. It doesn't receive read requests unless you enable asynchronous dual-reads. @@ -151,17 +151,17 @@ It doesn't receive read requests unless you enable asynchronous dual-reads. For the majority of the migration process, the origin cluster is the primary cluster, and the target cluster is the secondary cluster. Near the end of the migration, when you have validated that all pre-existing data has been replicated to the target cluster, you set the target cluster as the primary cluster. + -Throughout the entire migration, until you decommission {product-proxy}, both clusters receive all write requests through the dual writes feature. -For more information, see xref:components.adoc#how-zdm-proxy-handles-reads-and-writes[How {product-proxy} handles reads and writes]. +Throughout the entire migration, until you decommission {zdm-proxy}, both clusters receive all write requests through the dual writes feature. +For more information, see xref:components.adoc#how-zdm-proxy-handles-reads-and-writes[How {zdm-proxy} handles reads and writes]. == Is there a difference between cluster and database? -In the context of the {product-short} process, the terms _cluster_ and _database_ are used interchangeably to refer to the source and destination for the data that you are moving during your migration. +In the context of the {zdm-short} process, the terms _cluster_ and _database_ are used interchangeably to refer to the source and destination for the data that you are moving during your migration. -== What happened to {dsbulk-migrator}? +== What happened to {dsbulk-short} Migrator? include::ROOT:partial$dsbulk-migrator-deprecation.adoc[] == See also -* {product-proxy-repo}/blob/main/faq.md[{product-proxy} FAQ on GitHub] \ No newline at end of file +* {zdm-proxy-repo-url}/blob/main/faq.md[{zdm-proxy} FAQ on GitHub] \ No newline at end of file diff --git a/modules/ROOT/pages/feasibility-checklists.adoc b/modules/ROOT/pages/feasibility-checklists.adoc index 85c8eb0e..52392f28 100644 --- a/modules/ROOT/pages/feasibility-checklists.adoc +++ b/modules/ROOT/pages/feasibility-checklists.adoc @@ -1,8 +1,8 @@ -= Compatibility requirements for {product-proxy} += Compatibility requirements for {zdm-proxy} :navtitle: Compatibility requirements :page-aliases: ROOT:preliminary-steps.adoc -True zero downtime migration with {product-proxy} is possible only if your clusters, data model, and application logic meet the compatibility requirements described on this page. +True zero downtime migration with {zdm-proxy} is possible only if your clusters, data model, and application logic meet the compatibility requirements described on this page. You might need to adjust your data model or application logic to ensure compatibility between the origin and target clusters during the migration and ongoing compatibility with the target cluster after the migration. @@ -13,16 +13,16 @@ If you cannot meet these requirements, particularly the cluster and schema compa include::ROOT:partial$cassandra-protocol-versions.adoc[] -When a specific protocol version is requested, {product-proxy} handles protocol negotiation to ensure the requested version is supported by both clusters. -For example, to use protocol `V5` with {product-proxy}, both the origin and target clusters must support `V5`, such as {hcd-short} or open source {cass-reg} 4.0 or later. +When a specific protocol version is requested, {zdm-proxy} handles protocol negotiation to ensure the requested version is supported by both clusters. +For example, to use protocol `V5` with {zdm-proxy}, both the origin and target clusters must support `V5`, such as {hcd-short} or open source {cass-reg} 4.0 or later. Otherwise, a lower protocol version must be used. -If the requested version isn't mutually supported, then {product-proxy} can force the client application to downgrade to a mutually supported protocol version. +If the requested version isn't mutually supported, then {zdm-proxy} can force the client application to downgrade to a mutually supported protocol version. If automatic forced downgrade isn't possible, then the connection fails, and you must modify your client application to request a different protocol version. === Determine your client application's supported and negotiated protocol versions -Outside of a migration scenario (without {product-proxy}), the supported protocol versions depend on your origin cluster's version and client application's driver version. +Outside of a migration scenario (without {zdm-proxy}), the supported protocol versions depend on your origin cluster's version and client application's driver version. Generally, when connecting to a cluster, the driver requests the highest protocol version that it supports. If the cluster supports that version, then the connection uses that version. @@ -32,12 +32,12 @@ For example, if the cluster and driver both support `V5`, then your client appli If you upgrade your cluster, driver, or both to a version with a higher mutually supported protocol version, then the driver automatically starts using the higher version unless you explicitly disable it in your driver configuration. -When you introduce {product-proxy}, the target cluster is integrated into the protocol negotiation process to ensure that the negotiated protocol version is supported by the origin cluster, target cluster, and driver. +When you introduce {zdm-proxy}, the target cluster is integrated into the protocol negotiation process to ensure that the negotiated protocol version is supported by the origin cluster, target cluster, and driver. === Considerations and requirements for `V5` -Required {product-proxy} version:: -Official support for `V5` requires {product-proxy} version 2.4.0 or later. +Required {zdm-proxy} version:: +Official support for `V5` requires {zdm-proxy} version 2.4.0 or later. Use cases requiring `V5`:: You are required to use `V5` only if your client application uses `V5`-specific functionality. @@ -47,7 +47,7 @@ Protocol `V5` has improved integrity checks compared to earlier versions. This can cause slight performance degradation when your client application begins to use `V5` after using an earlier version. + {company} performance tests showed potential throughput reductions ranging from 0 to 15 percent. -This performance impact can occur with and without {product-proxy}. +This performance impact can occur with and without {zdm-proxy}. + [TIP] ==== @@ -57,16 +57,16 @@ If your client application already uses `V5`, it is likely that you already adju If you plan to upgrade to a `V5`-compatible driver before or during your migration, then the potential performance impact depends on which clusters support `V5`: + -- -* **Neither cluster supports `V5`**: You won't notice any protocol-related performance impact before or during the migration because the driver and {product-proxy} cannot negotiate `V5` in this scenario. +* **Neither cluster supports `V5`**: You won't notice any protocol-related performance impact before or during the migration because the driver and {zdm-proxy} cannot negotiate `V5` in this scenario. -* **Only the target cluster supports `V5`**: You won't notice any protocol-related performance impact during the migration because {product-proxy} must negotiate a protocol version that is supported by both clusters. -If the origin cluster doesn't support `V5`, then {product-proxy} cannot negotiate `V5` during the migration, and the driver cannot negotiate `V5` before the migration. +* **Only the target cluster supports `V5`**: You won't notice any protocol-related performance impact during the migration because {zdm-proxy} must negotiate a protocol version that is supported by both clusters. +If the origin cluster doesn't support `V5`, then {zdm-proxy} cannot negotiate `V5` during the migration, and the driver cannot negotiate `V5` before the migration. + However, you might experience a protocol-related performance impact at the end of the migration when you connect your client application directly to the target cluster. -This phase removes {product-proxy} and the origin cluster from the protocol negotiation, allowing the driver to negotiate directly with the target cluster. +This phase removes {zdm-proxy} and the origin cluster from the protocol negotiation, allowing the driver to negotiate directly with the target cluster. If the target cluster supports `V5`, the driver can use `V5` automatically. -* **Both clusters support `V5`**: Unless you <>, you might experience performance impacts because the driver and {product-proxy} can use `V5` automatically in this scenario. +* **Both clusters support `V5`**: Unless you <>, you might experience performance impacts because the driver and {zdm-proxy} can use `V5` automatically in this scenario. Consider upgrading the driver before or after the migration so you can isolate the impact of that change without the added complexity of the migration. As a best practice for any significant version upgrade, run performance tests in lower environments to evaluate the potential impact before making the change in production. -- @@ -74,7 +74,7 @@ As a best practice for any significant version upgrade, run performance tests in [#disallow-or-explicitly-downgrade-the-protocol-version] === Disallow or explicitly downgrade the protocol version -You can restrict protocol versions in the driver and {product-proxy} configuration: +You can restrict protocol versions in the driver and {zdm-proxy} configuration: Driver configuration:: You can explicitly downgrade the protocol version in your client application's driver configuration. @@ -85,21 +85,21 @@ For example: + * Both clusters and the driver support `V5` but you don't want to use `V5`: Configure the protocol version in the driver before the migration if you haven't done so already. * The origin cluster _doesn't_ support `V5` and you want to ensure `V5` isn't used automatically after the migration: Configure the protocol version in the driver at any point before the end of the migration when you connect your client application directly to the target cluster. -* You observe unacceptable performance degradation when using `V5` before the migration (without {product-proxy}): +* You observe unacceptable performance degradation when using `V5` before the migration (without {zdm-proxy}): Either mitigate the performance issues before the migration, or configure the protocol version in the driver before the migration. -{product-proxy} configuration:: +{zdm-proxy} configuration:: You can use the `xref:ROOT:manage-proxy-instances.adoc#blocked-protocol-versions[blocked_protocol_versions]` configuration variable to block specific protocol versions at the proxy level. Make sure at least one mutually supported protocol version isn't blocked. + -This option applies _only_ while {product-proxy} is in use. +This option applies _only_ while {zdm-proxy} is in use. It _doesn't_ persist after the migration. + -Use this option if you observe unacceptable performance degradation when {product-proxy} is active _and_ it negotiates `V5`. -If unacceptable performance degradation occurs _without_ {product-proxy}, then configure the protocol version in the driver instead. -However, be aware that {product-proxy} itself can have a performance impact, regardless of the protocol version. +Use this option if you observe unacceptable performance degradation when {zdm-proxy} is active _and_ it negotiates `V5`. +If unacceptable performance degradation occurs _without_ {zdm-proxy}, then configure the protocol version in the driver instead. +However, be aware that {zdm-proxy} itself can have a performance impact, regardless of the protocol version. -=== Thrift isn't supported by {product-proxy} +=== Thrift isn't supported by {zdm-proxy} If you are using an earlier driver or cluster version that only supports Thrift, you must change your client application to use {cql} statements, and, if needed, upgrade your cluster before starting the migration process. @@ -111,17 +111,17 @@ include::ROOT:partial$migration-scenarios.adoc[] [IMPORTANT] ==== -To successfully migrate with {product-proxy}, the origin and target clusters must have matching schemas, including keyspace names, table names, column names, and data types. -A CQL statement produced by your client application must be able to succeed _without modification_ on both clusters because {product-proxy} routes the exact same CQL statements to both clusters. +To successfully migrate with {zdm-proxy}, the origin and target clusters must have matching schemas, including keyspace names, table names, column names, and data types. +A CQL statement produced by your client application must be able to succeed _without modification_ on both clusters because {zdm-proxy} routes the exact same CQL statements to both clusters. -Even without {product-proxy}, matching schemas are required for most CQL-based data migration tools, such as {sstable-sideloader} and {cass-migrator}. -If it is impossible to match the schemas, your migration cannot use {product-proxy}. +Even without {zdm-proxy}, matching schemas are required for most CQL-based data migration tools, such as {sstable-sideloader} and {cass-migrator}. +If it is impossible to match the schemas, your migration cannot use {zdm-proxy}. See xref:ROOT:components.adoc[] for alternative migration tools and strategies, such as extract, transform, and load (ETL) processes. ==== -{product-proxy} _doesn't_ modify or transform CQL statements with the exception of <>. +{zdm-proxy} _doesn't_ modify or transform CQL statements with the exception of <>. -A CQL statement that your client application sends to {product-proxy} must be able to succeed on both clusters. +A CQL statement that your client application sends to {zdm-proxy} must be able to succeed on both clusters. This means that any keyspace that your client application uses must exist on both the origin and target clusters with the same keyspace name, table names, column names, and compatible data types. However, the keyspaces can have different replication strategies and durable write settings. @@ -133,9 +133,9 @@ Before you decide to omit a column from the target cluster, make sure that it is In some cases, you can change the primary key on the target cluster. For example, if your compound primary key is `PRIMARY KEY (A, B)`, and you always provide parameters for the `A` and `B` columns in your CQL statements, then you could change the key to `PRIMARY KEY (B, A)` when creating the schema on the target because your CQL statements will still run successfully. -== Request and error handling with {product-proxy} +== Request and error handling with {zdm-proxy} -See xref:ROOT:components.adoc#how-zdm-proxy-handles-reads-and-writes[How {product-proxy} handles reads and writes]. +See xref:ROOT:components.adoc#how-zdm-proxy-handles-reads-and-writes[How {zdm-proxy} handles reads and writes]. [[non-idempotent-operations]] == Lightweight Transactions and other non-idempotent operations @@ -149,11 +149,11 @@ Examples of non-idempotent operations include the following: * Collection updates with `+=` and `-=` operators * Non-deterministic functions like `now()` and `uuid()` (see <>) -By design, a {product-short} migration involves two separate clusters, and it is possible for the clusters to have differences due to non-idempotent operations. +By design, a {zdm-short} migration involves two separate clusters, and it is possible for the clusters to have differences due to non-idempotent operations. If you use non-idempotent operations, {company} recommends adding a reconciliation phase to your migration before and after xref:ROOT:change-read-routing.adoc[Phase 4]. This allows you an additional opportunity to resolve any data inconsistencies that are produced by non-idempotent operations. -The xref:ROOT:cassandra-data-migrator.adoc[{cass-migrator-short}] is ideal for detecting and reconciling these types of inconsistencies. +The {cass-migrator-repo-url}[{cass-migrator-short}] is ideal for detecting and reconciling these types of inconsistencies. However, if your application workloads can tolerate inconsistencies produced by LWTs and non-idempotent operations, you might not need to perform any additional validation or reconciliation steps. This depends entirely on your application business logic and requirements. @@ -162,20 +162,20 @@ It is your responsibility to determine whether your workloads can tolerate these [[_lightweight_transactions_and_the_applied_flag]] === Lightweight Transactions and the applied flag -{product-proxy} handles LWTs in the same way as other xref:ROOT:components.adoc#how-zdm-proxy-handles-reads-and-writes[write requests]: It sends the request to both clusters concurrently, and then it waits for both to respond. -{product-proxy} returns a `success` status to the client _only_ if both clusters send successful acknowledgements; otherwise, it returns a `failure` status. +{zdm-proxy} handles LWTs in the same way as other xref:ROOT:components.adoc#how-zdm-proxy-handles-reads-and-writes[write requests]: It sends the request to both clusters concurrently, and then it waits for both to respond. +{zdm-proxy} returns a `success` status to the client _only_ if both clusters send successful acknowledgements; otherwise, it returns a `failure` status. However, clusters can consider an LWT to be successful with or without data modification, because LWTs modify data only if a specific condition is met, and they don't fail if the condition isn't met. -With {product-proxy}, this means both clusters can return successful acknowledgements with _different outcomes on each cluster_. +With {zdm-proxy}, this means both clusters can return successful acknowledgements with _different outcomes on each cluster_. This is because the evaluation of an LWT's condition is based on the state of the data on the cluster at the time of the request. For example, if the condition is met on the origin cluster but not on the target cluster, then the data is modified on the origin cluster only. -If your application logic includes LWTs, you must reconcile any inconsistences caused by LWTs, and understand how the `applied` flag is handled by {product-proxy}: +If your application logic includes LWTs, you must reconcile any inconsistences caused by LWTs, and understand how the `applied` flag is handled by {zdm-proxy}: Repeatedly validate and reconcile data to catch all inconsistencies:: -During the {product-short} process, the clusters aren't fully synchronized until the data has been completely replicated and thoroughly validated at the end of xref:ROOT:migrate-and-validate-data.adoc[Phase 2]. +During the {zdm-short} process, the clusters aren't fully synchronized until the data has been completely replicated and thoroughly validated at the end of xref:ROOT:migrate-and-validate-data.adoc[Phase 2]. Up to that point, LWTs can produce inconsistent data that you must reconcile at the end of Phase 2. -Furthermore, because {product-proxy} is designed to continuously route writes to both clusters, you might need to validate and reconcile the data multiple times to catch additional inconsistencies that occurred while you were reconciling previous inconsistencies. +Furthermore, because {zdm-proxy} is designed to continuously route writes to both clusters, you might need to validate and reconcile the data multiple times to catch additional inconsistencies that occurred while you were reconciling previous inconsistencies. Don't rely on the `applied` flag:: A cluster's response to an LWT includes an `applied` flag. @@ -183,10 +183,10 @@ If `applied` is `True`, then the LWT's condition was met and the data was actual If `applied` is `False`, then the condition wasn't met and the data wasn't modified. + Outside of a migration scenario, this flag is passed to the client application from the one cluster that the client is connected to. -However, with {product-proxy}, the responses from both clusters are passed to {product-proxy}, which then passes only _one_ response back to the client application. -This is intentional because {product-proxy} is designed to be transparent to the client application: The client application believes it is interacting with a single cluster. +However, with {zdm-proxy}, the responses from both clusters are passed to {zdm-proxy}, which then passes only _one_ response back to the client application. +This is intentional because {zdm-proxy} is designed to be transparent to the client application: The client application believes it is interacting with a single cluster. + -Specifically, {product-proxy} returns the response, including the `applied` flag, from the xref:ROOT:faqs.adoc#what-are-origin-target-primary-and-secondary-clusters[primary cluster]. +Specifically, {zdm-proxy} returns the response, including the `applied` flag, from the xref:ROOT:faqs.adoc#what-are-origin-target-primary-and-secondary-clusters[primary cluster]. For the majority of the migration process, the origin cluster is the primary cluster. Near the end of the migration (xref:ROOT:change-read-routing.adoc[Phase 4]), the target cluster becomes the primary cluster. + @@ -204,7 +204,7 @@ If these functions are used for regular non-primary key columns, you must determ However, if these functions are used in any primary key column, then your data migration phase will fail because of data inconsistencies between the two clusters. Effectively, the clusters will never be truly in sync from a programmatic perspective. -{product-proxy} has an option to replace `now()` with a timeUUID calculated at the proxy level to ensure that these records write the same value to both clusters. +{zdm-proxy} has an option to replace `now()` with a timeUUID calculated at the proxy level to ensure that these records write the same value to both clusters. To enable this feature, set `replace_cql_functions` to `true`. For more information, see xref:manage-proxy-instances.adoc#change-mutable-config-variable[Change a mutable configuration variable]. @@ -226,14 +226,14 @@ For more information, see your driver's documentation and xref:6.9@dse:drivers:t [IMPORTANT] ==== -The {product-short} process requires you to perform rolling restarts of your client applications during the migration. +The {zdm-short} process requires you to perform rolling restarts of your client applications during the migration. This is standard practice for client applications that are deployed over multiple instances, and it is a widely used approach to roll out releases and configuration changes. ==== -Throughout the migration process, you must restart {product-proxy} instances to apply configuration changes. +Throughout the migration process, you must restart {zdm-proxy} instances to apply configuration changes. Client applications handle this in the same way as typical rolling restarts of {dse-short} or {cass-short} clusters. -If your applications already tolerate rolling restarts of your origin cluster, then you shouldn't expect any issues during rolling restarts of {product-proxy} instances. +If your applications already tolerate rolling restarts of your origin cluster, then you shouldn't expect any issues during rolling restarts of {zdm-proxy} instances. To ensure that your client application retries requests that fail due to connection errors caused by the rolling restart process, check your driver's retry policies and whether your requests are marked as idempotent. Most drivers treat all statements as non-idempotent by default, and they don't automatically retry them. @@ -245,51 +245,51 @@ For more information, see xref:6.9@dse:drivers:idempotence.adoc[] and xref:6.9@d [IMPORTANT] ==== -LZ4 and Snappy compression algorithms require {product-proxy} version 2.4.0 or later. +LZ4 and Snappy compression algorithms require {zdm-proxy} version 2.4.0 or later. ==== The binary protocol used by {astra}, {dse-short}, {hcd-short}, and open-source {cass-short} supports optional compression of transport-level requests and responses that reduces network traffic at the cost of CPU overhead. -When establishing connections from client applications, {product-proxy} responds with a list of compression algorithms supported by both clusters. +When establishing connections from client applications, {zdm-proxy} responds with a list of compression algorithms supported by both clusters. The compression algorithm configured in your {company}-compatible driver must match any item from the common list, or CQL request compression must be disabled completely. -{product-proxy} cannot decompress and recompress CQL requests using different compression algorithms. +{zdm-proxy} cannot decompress and recompress CQL requests using different compression algorithms. This isn't related to storage compression, which you can configure on specific tables with the `compression` table property. -Storage/table compression doesn't affect the client application or {product-proxy} in any way. +Storage/table compression doesn't affect the client application or {zdm-proxy} in any way. [#zdm-proxy-ignores-token-aware-routing] -== {product-proxy} ignores token-aware routing +== {zdm-proxy} ignores token-aware routing -Token-aware routing isn't enforced when connecting through {product-proxy} because these instances don't hold actual token ranges in the same way as database nodes. -Instead, each {product-proxy} instance has a unique, non-overlapping set of synthetic tokens that simulate token ownership and enable balanced load distribution across the instances. +Token-aware routing isn't enforced when connecting through {zdm-proxy} because these instances don't hold actual token ranges in the same way as database nodes. +Instead, each {zdm-proxy} instance has a unique, non-overlapping set of synthetic tokens that simulate token ownership and enable balanced load distribution across the instances. -Upon receiving a request, a {product-proxy} instance routes the request to appropriate origin and target database nodes, independent of token ownership. +Upon receiving a request, a {zdm-proxy} instance routes the request to appropriate origin and target database nodes, independent of token ownership. -If your clients have token-aware routing enabled, you don't need to disable this behavior while using {product-proxy}. +If your clients have token-aware routing enabled, you don't need to disable this behavior while using {zdm-proxy}. Clients can continue to operate with token-aware routing enabled without negative impacts to functionality or performance. == Authenticator and authorizer configuration -A cluster's _authorizer_ doesn't affect client applications or {product-proxy}, which means that you can use any kind of authorizer configuration on your clusters, and they can use different authorizers. +A cluster's _authorizer_ doesn't affect client applications or {zdm-proxy}, which means that you can use any kind of authorizer configuration on your clusters, and they can use different authorizers. -In contrast, a cluster's _authenticator_ must be compatible with {product-proxy}. -{product-proxy} supports the following cluster authenticator configurations: +In contrast, a cluster's _authenticator_ must be compatible with {zdm-proxy}. +{zdm-proxy} supports the following cluster authenticator configurations: * No authenticator * `PasswordAuthenticator` * `DseAuthenticator` with `internal` or `ldap` scheme -{product-proxy} _doesn't_ support `DseAuthenticator` with `kerberos` scheme. +{zdm-proxy} _doesn't_ support `DseAuthenticator` with `kerberos` scheme. -The origin and target clusters can have different authentication configurations because {product-proxy} treats them independently. +The origin and target clusters can have different authentication configurations because {zdm-proxy} treats them independently. == {dse-short} advanced workloads -This section describes how {product-proxy} handles certain {dse-short} advanced workloads. +This section describes how {zdm-proxy} handles certain {dse-short} advanced workloads. For more {dse-short}-specific migration considerations, see xref:6.9@dse:managing:operations/migrate-data.adoc[]. {dse-short} Graph:: -{product-proxy} handles all {dse-short} Graph requests as write requests even if the traversals are read-only. +{zdm-proxy} handles all {dse-short} Graph requests as write requests even if the traversals are read-only. There is no special handling for these requests, so you must consider the traversals that your client application sends and determine whether the traversals are idempotent. If the traversals are non-idempotent, then your must thoroughly validate and reconcile the data and the end of xref:ROOT:migrate-and-validate-data.adoc[Phase 2] to ensure that the target cluster is truly consistent with the origin cluster. For more information, see <>. @@ -298,9 +298,9 @@ For more information, see <>. They _cannot_ be used with the earlier Graph engine in {dse-short} versions prior to 6.8. {dse-short} Search:: -Read-only {dse-short} Search workloads can be moved directly from the origin to the target without {product-proxy} being involved. -If your client application uses Search and also issues writes, or if you need the read routing capabilities from {product-proxy}, then you can connect your Search workloads to {product-proxy} as long as you are using xref:6.9@dse:drivers:cassandra-drivers-overview.adoc[{dse-short}-compatible drivers] to submit these queries. -This approach means the queries are regular CQL `SELECT` statements, so {product-proxy} handles them as regular read requests. +Read-only {dse-short} Search workloads can be moved directly from the origin to the target without {zdm-proxy} being involved. +If your client application uses Search and also issues writes, or if you need the read routing capabilities from {zdm-proxy}, then you can connect your Search workloads to {zdm-proxy} as long as you are using xref:6.9@dse:drivers:cassandra-drivers-overview.adoc[{dse-short}-compatible drivers] to submit these queries. +This approach means the queries are regular CQL `SELECT` statements, so {zdm-proxy} handles them as regular read requests. + If you use the HTTP API then you can either modify your applications to use the CQL API instead, or you must move those applications directly from the origin to the target when the migration is complete, if that is acceptable for your use case. @@ -308,7 +308,7 @@ If you use the HTTP API then you can either modify your applications to use the This section describes specific considerations for {astra} migrations. If you need to make any changes to your data model or application logic for compatibility with {astra}, do so before starting the migration. -As mentioned previously, this is because {product-proxy} requires that the same CQL statements can be executed successfully on both clusters, and the data migration tools require matching schemas. +As mentioned previously, this is because {zdm-proxy} requires that the same CQL statements can be executed successfully on both clusters, and the data migration tools require matching schemas. {astra} guardrails, limits, and CQL compatibility:: As a managed database-as-a-service (DBaaS) offering, {astra} implements xref:astra-db-serverless:databases:database-limits.adoc[guardrails and limits] on its databases, and xref:astra-db-serverless:cql:develop-with-cql.adoc[{astra} doesn't support all CQL functionality]. @@ -318,8 +318,8 @@ In self-managed clusters, such as {dse-short} and {cass-short}, you can configur However, the xref:astra-db-serverless:databases:database-limits.adoc#cassandra-configuration-properties[{astra} `cassandra.yml`] cannot be changed unless explicitly approved and implemented by {company}. {astra} doesn't support consistency level ONE:: -`CL.ONE` isn't supported by {astra}, and read and write requests sent through {product-proxy} with `CL.ONE` to {astra-db} databases always fail. -{product-proxy} doesn't mute these failures because you need to be aware of them. +`CL.ONE` isn't supported by {astra}, and read and write requests sent through {zdm-proxy} with `CL.ONE` to {astra-db} databases always fail. +{zdm-proxy} doesn't mute these failures because you need to be aware of them. You must adapt your client application to use a consistency level that is supported by both clusters to ensure that the migration is seamless and error-free. {astra} doesn't support the Stargate APIs:: @@ -329,25 +329,25 @@ Before you migrate, you must change your applications to use other programmatic For more information, see xref:astra-db-serverless:api-reference:compare-dataapi-to-stargate.adoc[]. [[_read_only_applications]]Read-only applications:: -In versions 2.1.0 and later, {product-proxy} sends periodic heartbeats to keep idle cluster connections alive. -The default interval is 30,000 milliseconds, and it can be configured with the `xref:ROOT:manage-proxy-instances.adoc#change-mutable-config-variable[heartbeat_interval_ms]` variable, or by directly setting the `ZDM_HEARTBEAT_INTERVAL_MS` environment variable if you aren't using {product-automation}. +In versions 2.1.0 and later, {zdm-proxy} sends periodic heartbeats to keep idle cluster connections alive. +The default interval is 30,000 milliseconds, and it can be configured with the `xref:ROOT:manage-proxy-instances.adoc#change-mutable-config-variable[heartbeat_interval_ms]` variable, or by directly setting the `ZDM_HEARTBEAT_INTERVAL_MS` environment variable if you aren't using {zdm-automation}. + -In {product-proxy} versions earlier than 2.1.0, read-only applications require special handling to avoid connection termination due to inactivity. +In {zdm-proxy} versions earlier than 2.1.0, read-only applications require special handling to avoid connection termination due to inactivity. + -{company} recommends that you use {product-proxy} version 2.1.0 or later to benefit from the heartbeat feature. +{company} recommends that you use {zdm-proxy} version 2.1.0 or later to benefit from the heartbeat feature. If you cannot use version 2.1.0 or later, see the alternatives described in xref:ROOT:troubleshooting-tips.adoc#client-application-closed-connection-errors-every-10-minutes-when-migrating-to-astra-db[Client application closed connection errors every 10 minutes when migrating to {astra-db}]. [#multi-datacenter-clusters-and-other-complex-migrations] == Multi-datacenter clusters and other complex migrations -Complex migration scenarios, such as multi-datacenter migrations or many-to-one migrations, require additional planning to configure {product-proxy} and migrate the data efficiently. +Complex migration scenarios, such as multi-datacenter migrations or many-to-one migrations, require additional planning to configure {zdm-proxy} and migrate the data efficiently. include::ROOT:partial$multi-region-migrations.adoc[] -To configure {product-proxy} for a multi-datacenter migration, see the xref:ROOT:deployment-infrastructure.adoc[{product-proxy} infrastructure guidelines for multi-datacenter clusters]. +To configure {zdm-proxy} for a multi-datacenter migration, see the xref:ROOT:deployment-infrastructure.adoc[{zdm-proxy} infrastructure guidelines for multi-datacenter clusters]. For more guidance on migrations to {astra}, see the xref:sideloader:prepare-sideloader.adoc#additional-preparation-for-specific-migration-scenarios[{sstable-sideloader} preparations for specific migration scenarios]. == Next steps -Next, xref:ROOT:deployment-infrastructure.adoc[prepare the {product-proxy} infrastructure]. \ No newline at end of file +Next, xref:ROOT:deployment-infrastructure.adoc[prepare the {zdm-proxy} infrastructure]. \ No newline at end of file diff --git a/modules/ROOT/pages/hcd-migration-paths.adoc b/modules/ROOT/pages/hcd-migration-paths.adoc index f793e25c..3993a87d 100644 --- a/modules/ROOT/pages/hcd-migration-paths.adoc +++ b/modules/ROOT/pages/hcd-migration-paths.adoc @@ -3,29 +3,29 @@ The {hcd} migration toolkit includes the xref:ROOT:components.adoc[{company} migration tools] that you can use to migrate your data to {hcd-short} from another {cass-reg}-based database, such as {astra-db}, {cass-short}, or {dse-short}. -Whenever possible, {company} strongly recommends using the {product} ({product-short}) tools to orchestrate ongoing read/write traffic when you migrate to {hcd-short}. +Whenever possible, {company} strongly recommends using the {zdm} tools to orchestrate ongoing read/write traffic when you migrate to {hcd-short}. [#zdm-to-hcd] == Zero-downtime migrations to {hcd-short} -The {product} ({product-short}) tools allow you to standup your new {hcd-short} clusters independently of your existing {dse-short} clusters. -Then, {product-proxy} orchestrates live traffic and synchronizes ongoing writes while you migrate data to your new clusters using any {product-short}-compatible data migration and validation tool. -Finally, you can use {product-proxy} to simulate the live workload on your new clusters before permanently switching your traffic over. +The {zdm} tools allow you to standup your new {hcd-short} clusters independently of your existing {dse-short} clusters. +Then, {zdm-proxy} orchestrates live traffic and synchronizes ongoing writes while you migrate data to your new clusters using any {zdm-short}-compatible data migration and validation tool. +Finally, you can use {zdm-proxy} to simulate the live workload on your new clusters before permanently switching your traffic over. -{product-proxy} and {product-automation} provide the safest upgrade approach with blue-green deployment capabilities that eliminate time pressure and ensure optimal availability and operational safety. +{zdm-proxy} and {zdm-automation} provide the safest upgrade approach with blue-green deployment capabilities that eliminate time pressure and ensure optimal availability and operational safety. You can rollback up to the last stage of the migration if necessary. -By orchestrating independent clusters with the {product-short} tools, you can specify your ideal {hcd-short} configuration settings that you otherwise wouldn't be able to change during an in-place cluster upgrade. +By orchestrating independent clusters with the {zdm-short} tools, you can specify your ideal {hcd-short} configuration settings that you otherwise wouldn't be able to change during an in-place cluster upgrade. Incompatibilities in cluster configuration don't disrupt the migration because your existing cluster remains active and unchanged while you set up the new cluster and migrate your data. -=== Data validation with {product-short} +=== Data validation with {zdm-short} -The {product-short} tools don't migrate your data. -During the {product-short} process, you use a xref:ROOT:migrate-and-validate-data.adoc[data migration tool] to rewrite the data from your existing cluster to your new cluster. +The {zdm-short} tools don't migrate your data. +During the {zdm-short} process, you use a xref:ROOT:migrate-and-validate-data.adoc[data migration tool] to rewrite the data from your existing cluster to your new cluster. {company} recommends that you do the following: -* Choose a data migration tool that also includes strong validation capabilities, such as xref:ROOT:cassandra-data-migrator.adoc[{cass-migrator}]. +* Choose a data migration tool that also includes strong validation capabilities, such as {cass-migrator-repo-url}[{cass-migrator}]. * Be aware of incompatible data types that can fail to migrate from your old cluster. Data validation tools can identify inconsistencies as missing or mismatched data, but you still need to have a plan to resolve them. @@ -33,16 +33,16 @@ For example, you might need to modify your applications to use a different data It is crucial that you fully validate and test your new cluster before switching your traffic over to it. -{product-proxy} is ideal for supporting this transition because it allows both clusters to remain in place until you are completely certain you are ready to switch to the new cluster. +{zdm-proxy} is ideal for supporting this transition because it allows both clusters to remain in place until you are completely certain you are ready to switch to the new cluster. Additionally, your old cluster remains untouched and available for rollback or reversion if necessary. -=== Get started with {product-short} and {hcd-short} +=== Get started with {zdm-short} and {hcd-short} -For information about clusters that are eligible for {product} to {hcd-short}, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. +For information about clusters that are eligible for {zdm} to {hcd-short}, see xref:ROOT:zdm-proxy-migration-paths.adoc[]. -To begin your {product} to {hcd-short}, go to xref:ROOT:introduction.adoc[]. +To begin your {zdm} to {hcd-short}, go to xref:ROOT:introduction.adoc[]. -You must set up your {hcd-short} clusters before you can enable the {product-proxy}. +You must set up your {hcd-short} clusters before you can enable the {zdm-proxy}. For information about installing and configuring {hcd-short}, see the xref:hyper-converged-database:get-started:get-started-hcd.adoc[{hcd-short} documentation]. == Migrate your code diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index 65997e64..6777e3b7 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -16,7 +16,7 @@

Bulk-load terabytes while shifting traffic with little or no downtime, securely and reliably.

- xref:ROOT:introduction.adoc[Get started with {product},role="btn btn-primary btn-solid"] + xref:ROOT:introduction.adoc[Get started with {zdm},role="btn btn-primary btn-solid"] xref:ROOT:components.adoc[Compare tools,role="btn btn-neutral btn-outlined"]
@@ -43,11 +43,11 @@ svg::sideloader:astra-migration-toolkit.svg[role="absolute bottom-1/2 translate- svg:common:ROOT:icons/datastax/cloud-backup-restore.svg[role="mx-auto max-w-xs md:mx-0 lg:max-w-none"] -

{product-proxy}

+

{zdm-proxy}

-

To support live migrations, {product-proxy} orchestrates activity-in-transition on your clusters, allowing your applications to run while you migrate data.

+

To support live migrations, {zdm-proxy} orchestrates activity-in-transition on your clusters, allowing your applications to run while you migrate data.

- xref:ROOT:introduction.adoc[Get started with {product-short}] + xref:ROOT:introduction.adoc[Get started with {zdm-short}]
@@ -73,7 +73,7 @@ svg::sideloader:astra-migration-toolkit.svg[role="absolute bottom-1/2 translate-

{cass-migrator-short} can migrate and validate data between {cass-short}-based clusters, with optional logging and reconciliation support.

- xref:ROOT:cassandra-data-migrator.adoc[Get started with {cass-migrator-short}] + {cass-migrator-repo-url}[Get started with {cass-migrator-short}]
diff --git a/modules/ROOT/pages/introduction.adoc b/modules/ROOT/pages/introduction.adoc index 259ef8f1..5a449e64 100644 --- a/modules/ROOT/pages/introduction.adoc +++ b/modules/ROOT/pages/introduction.adoc @@ -1,12 +1,12 @@ -= Phases of the {product} process -:navtitle: About the {product-short} process += Phases of the {zdm} process +:navtitle: About the {zdm-short} process :description: Before you begin, learn about migration concepts, software components, and the sequence of operations. -With the {product} ({product-short}) process, your applications can continue to run while you migrate data from one {cass-short}-based database to another, resulting in little or no downtime and minimal service interruptions. +With the {zdm} process, your applications can continue to run while you migrate data from one {cass-short}-based database to another, resulting in little or no downtime and minimal service interruptions. -The {product-short} process uses {product-proxy}, {product-utility}, and {product-automation} to orchestrate live reads and writes on your databases while you move and validate data with a data migration tool, such as {sstable-sideloader}, {cass-migrator}, or {dsbulk}. +The {zdm-short} process uses {zdm-proxy}, {zdm-utility}, and {zdm-automation} to orchestrate live reads and writes on your databases while you move and validate data with a data migration tool, such as {sstable-sideloader}, {cass-migrator}, or {dsbulk}. -{product-proxy} keeps your databases in sync at all times through its dual-writes feature, which means you can seamlessly stop or abandon the migration at any point before the last phase of the migration (the final cutover to the new database). +{zdm-proxy} keeps your databases in sync at all times through its dual-writes feature, which means you can seamlessly stop or abandon the migration at any point before the last phase of the migration (the final cutover to the new database). For more information about these tools, see xref:ROOT:components.adoc[]. When the migration is complete, all data is present in the new database, and you can switch your client applications to connect exclusively to the new database. @@ -15,15 +15,15 @@ The old database becomes obsolete and can be shut down. .Requirements for zero downtime [IMPORTANT] ==== -True zero downtime migration with {product-proxy} is only possible if your database meets the minimum requirements, including cluster compatibility, that are described in xref:ROOT:feasibility-checklists.adoc[] -If your database doesn't meet these requirements, you can still complete the migration, but you might not be able to use {product-proxy} and some downtime might be necessary to finish the migration. +True zero downtime migration with {zdm-proxy} is only possible if your database meets the minimum requirements, including cluster compatibility, that are described in xref:ROOT:feasibility-checklists.adoc[] +If your database doesn't meet these requirements, you can still complete the migration, but you might not be able to use {zdm-proxy} and some downtime might be necessary to finish the migration. ==== == Migration phases A migration project includes preparation for the migration and five migration phases. -The following sections describe the major events in each phase and how your client applications connect to your origin cluster, {product-proxy}, and the target cluster during each phase. +The following sections describe the major events in each phase and how your client applications connect to your origin cluster, {zdm-proxy}, and the target cluster during each phase. The _origin cluster_ is is your existing {cass-short}-based environment, such as {dse-short} or open-source {cass}. @@ -32,41 +32,41 @@ The _target cluster_ is your new {cass-short}-based environment where you want t === Plan and prepare for your migration From a functional standpoint, before you begin a migration, your client applications perform read/write operations directly with your existing xref:cql:ROOT:index.adoc[CQL]-compatible database, such as {cass}, {dse-short}, {hcd-short}, or {astra-db}. -Over the course of the migration, the contact points change to incorporate {product-proxy} and eventually switch to the target database. +Over the course of the migration, the contact points change to incorporate {zdm-proxy} and eventually switch to the target database. image::ROOT:pre-migration0ra.svg[Before the migration begins, your applications connect exclusively to your origin cluster] To plan and prepare for the migration, do the following: -. xref:ROOT:feasibility-checklists.adoc[Review compatibility requirements for {product-proxy}]: Make sure your clusters, data model, and application logic are compatible with {product-proxy}, and understand how certain operations are handled by {product-proxy}. +. xref:ROOT:feasibility-checklists.adoc[Review compatibility requirements for {zdm-proxy}]: Make sure your clusters, data model, and application logic are compatible with {zdm-proxy}, and understand how certain operations are handled by {zdm-proxy}. If needed, adjust your data model or application logic to ensure compatibility between the two clusters during the migration and ongoing compatibility with the target cluster after the migration. + [IMPORTANT] ==== -To successfully migrate with {product-proxy}, a read or write request produced by your client application must be able to succeed _without modification_ on both your origin (existing) and target (new) clusters. +To successfully migrate with {zdm-proxy}, a read or write request produced by your client application must be able to succeed _without modification_ on both your origin (existing) and target (new) clusters. This means that the origin and target clusters must have matching schemas, including keyspace names, table names, column names, and data types. ==== -. xref:ROOT:deployment-infrastructure.adoc[]: Set up the infrastructure to host the {product-short} tools. +. xref:ROOT:deployment-infrastructure.adoc[]: Set up the infrastructure to host the {zdm-short} tools. . xref:ROOT:create-target.adoc[]: Set up your target cluster, and create schemas that match those on your origin cluster. . xref:ROOT:rollback.adoc[]: Learn when and how you can stop or rollback an in-progress migration. -=== Phase 1: Deploy {product-proxy} and connect client applications +=== Phase 1: Deploy {zdm-proxy} and connect client applications -In this first phase, deploy the {product-proxy} instances and connect client applications to the proxies. +In this first phase, deploy the {zdm-proxy} instances and connect client applications to the proxies. This phase activates the dual-write logic. Writes are sent to both the origin and target databases, while reads are executed on the origin only. -For more information and instructions, see xref:ROOT:phase1.adoc[Phase 1: Deploy and connect {product-proxy}]. +For more information and instructions, see xref:ROOT:phase1.adoc[Phase 1: Deploy and connect {zdm-proxy}]. -image::ROOT:migration-phase1ra.svg[In Phase 1, you deploy and connect {product-proxy}] +image::ROOT:migration-phase1ra.svg[In Phase 1, you deploy and connect {zdm-proxy}] === Phase 2: Migrate data In this phase, you use a data migration tool to copy your existing data to the target database. -{product-proxy} continues to perform dual writes so that you can focus on moving data that was present before you connected {product-proxy}. +{zdm-proxy} continues to perform dual writes so that you can focus on moving data that was present before you connected {zdm-proxy}. Then, you thoroughly validate the migrated data, resolving missing and mismatched records, before moving on to the next phase. For more information and instructions, see xref:ROOT:migrate-and-validate-data.adoc[]. @@ -79,15 +79,15 @@ This phase is optional but recommended. In this phase, you can enable the _asynchronous dual reads_ feature to test the target database's ability to handle a production workload before you permanently switch your applications to the target database at the end of the migration process. -When enabled, {product-proxy} sends asynchronous read requests to the secondary database in addition to the synchronous read requests that are sent to the primary database by default. +When enabled, {zdm-proxy} sends asynchronous read requests to the secondary database in addition to the synchronous read requests that are sent to the primary database by default. -For more information, see xref:ROOT:enable-async-dual-reads.adoc[] and xref:ROOT:components.adoc#how_zdm_proxy_handles_reads_and_writes[How {product-proxy} handles reads and writes]. +For more information, see xref:ROOT:enable-async-dual-reads.adoc[] and xref:ROOT:components.adoc#how_zdm_proxy_handles_reads_and_writes[How {zdm-proxy} handles reads and writes]. image::ROOT:migration-phase3ra.svg[In Phase 3, you test your target cluster's production readiness] === Phase 4: Route reads to the target database -In this phase, read routing on {product-proxy} is switched to the target database so that all reads are executed on the target. +In this phase, read routing on {zdm-proxy} is switched to the target database so that all reads are executed on the target. Writes are still sent to both databases in case you need to rollback the migration. At this point, the target database becomes the primary database. @@ -98,21 +98,21 @@ image::ROOT:migration-phase4ra.svg[In Phase 4, you route reads to the target clu === Phase 5: Connect directly to the target database -In the final phase of the migration, you move your client applications off {product-proxy} and connect them directly to the target database. +In the final phase of the migration, you move your client applications off {zdm-proxy} and connect them directly to the target database. Once this happens, the migration is complete, and you now exclusively use the target database. Whether you choose to destroy to retain the origin database depends on your organization's policies and whether you might need to revert to it in the future. -However, be aware that the origin database is no longer synchronized with the target database, and the origin database won't contain writes that happen after you disconnect {product-proxy}. +However, be aware that the origin database is no longer synchronized with the target database, and the origin database won't contain writes that happen after you disconnect {zdm-proxy}. For more information, see xref:ROOT:connect-clients-to-target.adoc[]. image::ROOT:migration-phase5ra.svg[In Phase 5, you connect your client applications directly and exclusively to the target cluster] [#lab] -== {product} interactive lab +== {zdm} interactive lab -As a companion to the {product-short} documentation, you can use the https://app.gitpod.io/login?navigateTo=%2F%23https%3A%2F%2Fgithub.com%2FDataStax-Academy%2Fzdm-scenario-katapod.git[{product} interactive lab] to try the entire migration process in a demo environment. +As a companion to the {zdm-short} documentation, you can use the https://app.gitpod.io/login?navigateTo=%2F%23https%3A%2F%2Fgithub.com%2FDataStax-Academy%2Fzdm-scenario-katapod.git[{zdm} interactive lab] to try the entire migration process in a demo environment. The lab only requires a GitHub account and a supported browser. All browsers are supported except Safari. diff --git a/modules/ROOT/pages/manage-proxy-instances.adoc b/modules/ROOT/pages/manage-proxy-instances.adoc index a26b1b1c..6e9539f3 100644 --- a/modules/ROOT/pages/manage-proxy-instances.adoc +++ b/modules/ROOT/pages/manage-proxy-instances.adoc @@ -1,13 +1,13 @@ -= Manage your {product-proxy} instances += Manage your {zdm-proxy} instances -After you deploy {product-proxy} instances, you might need to perform various management operations, such as rolling restarts, configuration changes, log inspection, version upgrades, and infrastructure changes. +After you deploy {zdm-proxy} instances, you might need to perform various management operations, such as rolling restarts, configuration changes, log inspection, version upgrades, and infrastructure changes. -If you are using {product-automation}, you can use Ansible playbooks for all of these operations. +If you are using {zdm-automation}, you can use Ansible playbooks for all of these operations. [#perform-a-rolling-restart-of-the-proxies] == Perform a rolling restart of the proxies -Rolling restarts of the {product-proxy} instances are useful to apply configuration changes or to upgrade the {product-proxy} version without impacting the availability of the deployment. +Rolling restarts of the {zdm-proxy} instances are useful to apply configuration changes or to upgrade the {zdm-proxy} version without impacting the availability of the deployment. [IMPORTANT] ==== @@ -15,9 +15,9 @@ A rolling restart is a destructive action because it stops the previous containe xref:ROOT:zdm-logs.adoc[Collect the logs] before you apply the configuration change if you want to keep them. ==== -=== Rolling restart with {product-automation} +=== Rolling restart with {zdm-automation} -If you use {product-automation} to manage your {product-proxy} deployment, you can use a dedicated playbook to perform rolling restarts of all {product-proxy} instances in a deployment: +If you use {zdm-automation} to manage your {zdm-proxy} deployment, you can use a dedicated playbook to perform rolling restarts of all {zdm-proxy} instances in a deployment: . Connect to your Ansible Control Host container. + @@ -48,29 +48,29 @@ ubuntu@52772568517c:~$ ansible-playbook rolling_update_zdm_proxy.yml -i zdm_ansible_inventory ---- -The rolling restart playbook re-creates each {product-proxy} container, one by one. -The {product-proxy} deployment remains available at all times, and you can safely use it throughout this operation. +The rolling restart playbook re-creates each {zdm-proxy} container, one by one. +The {zdm-proxy} deployment remains available at all times, and you can safely use it throughout this operation. If you modified mutable configuration variables, the new containers use the updated configuration files. The playbook performs the following actions automatically: -. {product-automation} stops one container gracefully, and then waits for it to shut down. -. {product-automation} re-creates the container, and then starts it. -. {product-automation} calls the xref:ROOT:metrics.adoc#call-the-liveliness-and-readiness-endpoints[readiness endpoint] to check the container's status: +. {zdm-automation} stops one container gracefully, and then waits for it to shut down. +. {zdm-automation} re-creates the container, and then starts it. +. {zdm-automation} calls the xref:ROOT:metrics.adoc#call-the-liveliness-and-readiness-endpoints[readiness endpoint] to check the container's status: + -* If the status check fails, {product-automation} repeats the check up to six times at 5-second intervals. -If all six attempts fail, {product-automation} interrupts the entire rolling restart process. -* If the check succeeds, {product-automation} waits a fixed amount of time, and then moves on to the next container. +* If the status check fails, {zdm-automation} repeats the check up to six times at 5-second intervals. +If all six attempts fail, {zdm-automation} interrupts the entire rolling restart process. +* If the check succeeds, {zdm-automation} waits a fixed amount of time, and then moves on to the next container. The default pause between containers is 10 seconds. You can change the pause duration in `zdm-proxy-automation/ansible/vars/zdm_playbook_internal_config.yml`. -=== Rolling restart without {product-automation} +=== Rolling restart without {zdm-automation} -If you don't use {product-automation}, you must manually restart each instance. +If you don't use {zdm-automation}, you must manually restart each instance. To avoid downtime, wait for each instance to fully restart and begin receiving traffic before restarting the next instance. -== Inspect {product-proxy} logs +== Inspect {zdm-proxy} logs include::ROOT:partial$zdm-logs-intro.adoc[] For more information, see xref:ROOT:zdm-logs.adoc[]. @@ -78,11 +78,11 @@ For more information, see xref:ROOT:zdm-logs.adoc[]. [[change-mutable-config-variable]] == Change mutable configuration variables -Some, but not all, configuration variables can be changed after you deploy a {product-proxy} instance. +Some, but not all, configuration variables can be changed after you deploy a {zdm-proxy} instance. -This section lists the _mutable_ configuration variables that you can change on an existing {product-proxy} deployment. +This section lists the _mutable_ configuration variables that you can change on an existing {zdm-proxy} deployment. -After you edit mutable variables in their corresponding configuration files (`vars/zdm_proxy_core_config.yml`, `vars/zdm_proxy_cluster_config.yml`, or `vars/zdm_proxy_advanced_config.yml`), you must <> to apply the configuration changes to your {product-proxy} instances. +After you edit mutable variables in their corresponding configuration files (`vars/zdm_proxy_core_config.yml`, `vars/zdm_proxy_cluster_config.yml`, or `vars/zdm_proxy_advanced_config.yml`), you must <> to apply the configuration changes to your {zdm-proxy} instances. === Mutable variables in `vars/zdm_proxy_core_config.yml` @@ -93,7 +93,7 @@ At the start of the migration, the primary cluster is the origin cluster because After all the existing data has been transferred and validated/reconciled on the new cluster, you can switch the primary cluster to the target cluster. `read_mode`:: -Determines how reads are handled by {product-proxy}: +Determines how reads are handled by {zdm-proxy}: + * **`PRIMARY_ONLY` (default)**: Reads are sent synchronously to the primary cluster only. * **`DUAL_ASYNC_ON_SECONDARY`**: Reads are sent synchronously to the primary cluster, and also asynchronously to the secondary cluster. @@ -104,7 +104,7 @@ This is because asynchronous dual reads are primarily intended to help test prod When you are ready to switch `primary_cluster` to `TARGET`, revert `read_mode` to `PRIMARY_ONLY` because there is no need to send writes to both clusters at that point in the migration. `log_level`:: -Set the {product-proxy} log level as `INFO` (default) or `DEBUG`. +Set the {zdm-proxy} log level as `INFO` (default) or `DEBUG`. + Only use `DEBUG` while temporarily troubleshooting an issue. Revert to `INFO` as soon as possible because the extra logging can impact performance slightly. @@ -118,7 +118,7 @@ In `vars/zdm_proxy_cluster_config.yml`, you can change the connection credential === Mutable variables in `vars/zdm_proxy_advanced_config.yml` `zdm_proxy_max_clients_connections`:: -The maximum number of client connections that {product-proxy} can accept. +The maximum number of client connections that {zdm-proxy} can accept. Each client connection results in additional cluster connections and causes the allocation of several in-memory structures. A high number of client connections per proxy instance can cause performance degradation, especially at high throughput. Adjust this variable to limit the total number of connections on each instance. @@ -126,10 +126,10 @@ Adjust this variable to limit the total number of connections on each instance. Default: `1000` `replace_cql_functions`:: -Whether {product-proxy} replaces standard `now()` CQL function calls in write requests with an explicit timeUUID value computed at proxy level. +Whether {zdm-proxy} replaces standard `now()` CQL function calls in write requests with an explicit timeUUID value computed at proxy level. + * **`false` (default)**: Replacement of `now()` is disabled. -* **`true`**: {product-proxy} replaces instances of `now()` in write requests with an explicit timeUUID value before sending the write to each cluster. +* **`true`**: {zdm-proxy} replaces instances of `now()` in write requests with an explicit timeUUID value before sending the write to each cluster. + [IMPORTANT] ==== @@ -149,14 +149,14 @@ For more information, see xref:ROOT:feasibility-checklists.adoc#cql-function-rep [[zdm_proxy_request_timeout_ms]] `zdm_proxy_request_timeout_ms`:: Global timeout in milliseconds of a request at proxy level. -Determines how long {product-proxy} waits for one cluster (for reads) or both clusters (for writes) to reply to a request. -Upon reaching the timeout limit, {product-proxy} abandons the request and no longer considers it pending, which frees up internal resources to processes other requests. +Determines how long {zdm-proxy} waits for one cluster (for reads) or both clusters (for writes) to reply to a request. +Upon reaching the timeout limit, {zdm-proxy} abandons the request and no longer considers it pending, which frees up internal resources to processes other requests. + -When a request is abandoned due to a timeout, {product-proxy} doesn't return any result or error. +When a request is abandoned due to a timeout, {zdm-proxy} doesn't return any result or error. A timeout warning or error is only returned when the client application's own timeout is reached and the request is expired on the driver side. + Make sure `zdm_proxy_request_timeout_ms` is always greater than your client application's client-side timeout. -If the client has an especially high timeout because it routinely generates long-running requests, you must increase the `zdm_proxy_request_timeout_ms` timeout accordingly so that the {product-proxy} doesn't abandon requests prematurely. +If the client has an especially high timeout because it routinely generates long-running requests, you must increase the `zdm_proxy_request_timeout_ms` timeout accordingly so that the {zdm-proxy} doesn't abandon requests prematurely. + Default: `10000` @@ -169,7 +169,7 @@ Default: `30000` Timeout in milliseconds for the initialization (handshake) of the connection that is used solely for asynchronous dual reads between the proxy and the secondary cluster. + Upon reaching the timeout limit, the asynchronous reads aren't sent because the connection failed to be established. -This has no impact on the handling of synchronous requests: {product-proxy} continues to handle all synchronous reads and writes as normal against the primary cluster. +This has no impact on the handling of synchronous requests: {zdm-proxy} continues to handle all synchronous reads and writes as normal against the primary cluster. + Default: `4000` @@ -182,30 +182,30 @@ Default: `30000` `metrics_enabled`:: Whether to enable metrics collection. + -* **`true` (default)**: {product-proxy} collects and exposes metrics. -* **`false`**: {product-proxy} metrics collection is completely disabled. +* **`true` (default)**: {zdm-proxy} collects and exposes metrics. +* **`false`**: {zdm-proxy} metrics collection is completely disabled. This isn't recommended. [[zdm_proxy_max_stream_ids]] `zdm_proxy_max_stream_ids`:: -Set the maximum pool size of available stream IDs managed by {product-proxy} per client connection. +Set the maximum pool size of available stream IDs managed by {zdm-proxy} per client connection. Use the same value as your driver's maximum stream IDs configuration. + In the CQL protocol, every request has a unique stream ID. However, if there are a lot of requests in a given amount of time, errors can occur due to xref:6.9@dse:drivers:speculative-execution.adoc#stream-id-exhaustion[stream ID exhaustion]. + In the client application, the stream IDs are managed internally by the driver. -For most drivers, the maximum is 2048, which is the same default value used by {product-proxy}. +For most drivers, the maximum is 2048, which is the same default value used by {zdm-proxy}. If you have a custom driver configuration with a higher value, make sure `zdm_proxy_max_stream_ids` matches your driver's maximum stream IDs. + Default: `2048` [#blocked-protocol-versions] `blocked_protocol_versions`:: -This variable requires {product-proxy} and {product-automation} version 2.4.0 or later. +This variable requires {zdm-proxy} and {zdm-automation} version 2.4.0 or later. + -Use `blocked_protocol_versions` to block specific {cass-short} Native Protocol versions that are supported by {product-proxy}. -Blocking unsupported versions isn't necessary because {product-proxy} automatically rejects those versions. +Use `blocked_protocol_versions` to block specific {cass-short} Native Protocol versions that are supported by {zdm-proxy}. +Blocking unsupported versions isn't necessary because {zdm-proxy} automatically rejects those versions. + Allowed values include the following: + @@ -237,7 +237,7 @@ For more information, see xref:ROOT:feasibility-checklists.adoc#supported-cassan === Deprecated mutable variables -Deprecated variables will be removed in a future {product-proxy} release. +Deprecated variables will be removed in a future {zdm-proxy} release. Replace them with their recommended alternatives as soon as possible. `forward_client_credentials_to_origin`:: @@ -249,7 +249,7 @@ Whether to use the credentials provided by the client application to connect to + This deprecated variable is no longer functional. Instead, the expected credentials are based on the authentication requirements of the origin and target clusters. -For more information, see xref:ROOT:connect-clients-to-proxy.adoc#connect-applications-to-zdm-proxy[Connect applications to {product-proxy}]. +For more information, see xref:ROOT:connect-clients-to-proxy.adoc#connect-applications-to-zdm-proxy[Connect applications to {zdm-proxy}]. [#change-immutable-configuration-variables] == Change immutable configuration variables @@ -262,15 +262,15 @@ ansible-playbook deploy_zdm_proxy.yml -i zdm_ansible_inventory ---- You can re-run the deployment playbook as many times as necessary. -However, this playbook decommissions and re-creates _all_ {product-proxy} instances simultaneously. -This results in a brief period of time where the entire {product-proxy} deployment is offline because no instances are available. +However, this playbook decommissions and re-creates _all_ {zdm-proxy} instances simultaneously. +This results in a brief period of time where the entire {zdm-proxy} deployment is offline because no instances are available. -For more information, see xref:ROOT:troubleshooting-tips.adoc#configuration-changes-arent-applied-by-zdm-automation[Configuration changes aren't applied by {product-automation}]. +For more information, see xref:ROOT:troubleshooting-tips.adoc#configuration-changes-arent-applied-by-zdm-automation[Configuration changes aren't applied by {zdm-automation}]. [[_upgrade_the_proxy_version]] == Upgrade the proxy version -The same playbook that you use for configuration changes can also be used to upgrade the {product-proxy} version in a rolling fashion. +The same playbook that you use for configuration changes can also be used to upgrade the {zdm-proxy} version in a rolling fashion. All containers are re-created with the given image version. [IMPORTANT] @@ -279,10 +279,10 @@ A version change is a destructive action because the rolling restart playbook re xref:ROOT:zdm-logs.adoc[Collect the logs] before you run the playbook if you want to keep them. ==== -To check your current {product-proxy} version, see xref:ROOT:troubleshooting-tips.adoc#check-version[Check your {product-proxy} version]. +To check your current {zdm-proxy} version, see xref:ROOT:troubleshooting-tips.adoc#check-version[Check your {zdm-proxy} version]. . In `vars/zdm_proxy_container.yml`, set `zdm_proxy_image` to the desired tag. -For available tags, see the https://hub.docker.com/r/datastax/zdm-proxy/tags[{product-proxy} Docker Hub repository]. +For available tags, see the https://hub.docker.com/r/datastax/zdm-proxy/tags[{zdm-proxy} Docker Hub repository]. + [source,yaml,subs="+quotes"] ---- @@ -296,36 +296,36 @@ For example: zdm_proxy_image: datastax/zdm-proxy:2.3.4 ---- -. <> to update all {product-proxy} instances to the new version. +. <> to update all {zdm-proxy} instances to the new version. -== Scale {product-proxy} instances +== Scale {zdm-proxy} instances -The process for scaling your {product-proxy} instance depends on whether you use {product-automation} to manage your deployment. +The process for scaling your {zdm-proxy} instance depends on whether you use {zdm-automation} to manage your deployment. -=== Scale with {product-automation} +=== Scale with {zdm-automation} -{product-automation} doesn't provide a way to scale operations up or down in a rolling fashion. -If you are using {product-automation} and you need a larger {product-proxy} deployment, you can create a new deployment, or you can add instances to an existing deployment. +{zdm-automation} doesn't provide a way to scale operations up or down in a rolling fashion. +If you are using {zdm-automation} and you need a larger {zdm-proxy} deployment, you can create a new deployment, or you can add instances to an existing deployment. Create a new deployment (recommended):: -This option is the recommended way to scale your {product-proxy} deployment because it requires no downtime. +This option is the recommended way to scale your {zdm-proxy} deployment because it requires no downtime. + -Create a new {product-proxy} deployment, and then reconfigure your client application to use the new instance: +Create a new {zdm-proxy} deployment, and then reconfigure your client application to use the new instance: + -. xref:ROOT:setup-ansible-playbooks.adoc[Create a new {product-proxy} deployment] with the desired topology on a new set of machines. -. Change the contact points in the application configuration so that the application instances point to the new {product-proxy} deployment. +. xref:ROOT:setup-ansible-playbooks.adoc[Create a new {zdm-proxy} deployment] with the desired topology on a new set of machines. +. Change the contact points in the application configuration so that the application instances point to the new {zdm-proxy} deployment. . Perform a rolling restart of the application instances to apply the new contact point configuration. + The rolling restart ensures there is no interruption of service. The application instances switch seamlessly from the old deployment to the new one, and they are able to serve requests immediately. -. After restarting all application instances, you can safely remove the old {product-proxy} deployment. +. After restarting all application instances, you can safely remove the old {zdm-proxy} deployment. Add instances to an existing deployment:: This option requires manual configuration and a small amount of downtime. + -Change the topology of your existing {product-proxy} deployment, and then restart the entire deployment to apply the change: +Change the topology of your existing {zdm-proxy} deployment, and then restart the entire deployment to apply the change: + -. Amend the inventory file so that it contains one line for each machine where you want to deploy a {product-proxy} instance. +. Amend the inventory file so that it contains one line for each machine where you want to deploy a {zdm-proxy} instance. + For example, if you want to add three nodes to a deployment with six nodes, then the amended inventory file must contain nine total IPs, including the six existing IPs and the three new IPs. + @@ -337,51 +337,51 @@ ansible-playbook deploy_zdm_proxy.yml -i zdm_ansible_inventory ---- + Rerunning the playbook stops the existing instances, destroys them, and then creates and starts a new deployment with new instances based on the amended inventory. -This results in a brief interruption of service for your entire {product-proxy} deployment. +This results in a brief interruption of service for your entire {zdm-proxy} deployment. -=== Scale without {product-automation} +=== Scale without {zdm-automation} -If you aren't using {product-automation}, use these steps to add, change, or remove {product-proxy} instances. +If you aren't using {zdm-automation}, use these steps to add, change, or remove {zdm-proxy} instances. Add an instance:: + -. Prepare and configure the new {product-proxy} instances appropriately based on your other instances. -Make sure the new instance's configuration references all planned {product-proxy} cluster nodes. -. On all {product-proxy} instances, add the new instance's address to the `ZDM_PROXY_TOPOLOGY_ADDRESSES` environment variable. +. Prepare and configure the new {zdm-proxy} instances appropriately based on your other instances. +Make sure the new instance's configuration references all planned {zdm-proxy} cluster nodes. +. On all {zdm-proxy} instances, add the new instance's address to the `ZDM_PROXY_TOPOLOGY_ADDRESSES` environment variable. Make sure to include all new nodes. -. On the new {product-proxy} instance, set the `ZDM_PROXY_TOPOLOGY_INDEX` to the next sequential integer after the greatest one in your existing deployment. -. Perform a rolling restart of all {product-proxy} instances, one at a time. +. On the new {zdm-proxy} instance, set the `ZDM_PROXY_TOPOLOGY_INDEX` to the next sequential integer after the greatest one in your existing deployment. +. Perform a rolling restart of all {zdm-proxy} instances, one at a time. Vertically scale existing instances:: -Use these steps to increase or decrease resources for existing {product-proxy} instances, such as CPU or memory. +Use these steps to increase or decrease resources for existing {zdm-proxy} instances, such as CPU or memory. To avoid downtime, perform the following steps on one instance at a time: + -. Stop the first {product-proxy} instance that you want to modify. +. Stop the first {zdm-proxy} instance that you want to modify. . Modify the instance's resources as required. Make sure the instance's IP address remains the same. If the IP address changes, you must treat it as a new instance; follow the steps to **Add an instance** instead. -. Restart the modified {product-proxy} instance. +. Restart the modified {zdm-proxy} instance. . Wait until the instance starts, and then confirm that it is receiving traffic. . Repeat these steps to modify each additional instance, one at a time. Remove an instance:: + -. On all {product-proxy} instances, remove the unused instance's address from the `ZDM_PROXY_TOPOLOGY_ADDRESSES` environment variable. -. Perform a rolling restart of all remaining {product-proxy} instances. +. On all {zdm-proxy} instances, remove the unused instance's address from the `ZDM_PROXY_TOPOLOGY_ADDRESSES` environment variable. +. Perform a rolling restart of all remaining {zdm-proxy} instances. . Clean up resources used by the removed instance, such as the container or VM. === Proxy topology addresses enable failover and high availability -When you configure a {product-proxy} deployment, either through {product-automation} or manually-managed {product-proxy} instances, you specify the addresses of your instances. +When you configure a {zdm-proxy} deployment, either through {zdm-automation} or manually-managed {zdm-proxy} instances, you specify the addresses of your instances. These are populated in the `ZDM_PROXY_TOPOLOGY_ADDRESSES` variable, either manually or automatically depending on how you manage your instances. {cass-short} drivers look up nodes on a cluster by querying the `system.peers` table. -{product-proxy} uses the topology addresses to effectively respond to the driver's request for connection nodes. -If there are no topology addresses specified, {product-proxy} defaults to a single-instance configuration. -This means that driver connections use only that one {product-proxy} instance rather than all instances in your {product-proxy} deployment. +{zdm-proxy} uses the topology addresses to effectively respond to the driver's request for connection nodes. +If there are no topology addresses specified, {zdm-proxy} defaults to a single-instance configuration. +This means that driver connections use only that one {zdm-proxy} instance rather than all instances in your {zdm-proxy} deployment. -If that one instance goes down, {product-proxy} won't know that there are other instances available, and your application can experience an outage. -Additionally, if you need to restart {product-proxy} instances, and there is only one instance specified in the topology addresses, your migration will have downtime while that one instance restarts. +If that one instance goes down, {zdm-proxy} won't know that there are other instances available, and your application can experience an outage. +Additionally, if you need to restart {zdm-proxy} instances, and there is only one instance specified in the topology addresses, your migration will have downtime while that one instance restarts. == See also diff --git a/modules/ROOT/pages/metrics.adoc b/modules/ROOT/pages/metrics.adoc index 8966753a..2e95d4e9 100644 --- a/modules/ROOT/pages/metrics.adoc +++ b/modules/ROOT/pages/metrics.adoc @@ -1,28 +1,28 @@ -= Monitor {product-proxy} += Monitor {zdm-proxy} -{product-proxy} can gather a large number of metrics that provide you with insight into its health and operations, communication with client applications and clusters, and request handling. +{zdm-proxy} can gather a large number of metrics that provide you with insight into its health and operations, communication with client applications and clusters, and request handling. This visibility is important to your migration. It builds confidence in the health of your deployment, and it helps you investigate errors or performance degradation. -{company} strongly recommends that you use {product-automation} to deploy the {product-proxy} monitoring stack, or import the pre-built Grafana dashboards into your own monitoring infrastructure. +{company} strongly recommends that you use {zdm-automation} to deploy the {zdm-proxy} monitoring stack, or import the pre-built Grafana dashboards into your own monitoring infrastructure. -Do this after you xref:ROOT:setup-ansible-playbooks.adoc[set up {product-automation}] and xref:ROOT:deploy-proxy-monitoring.adoc[deploy the {product-proxy} instances]. +Do this after you xref:ROOT:setup-ansible-playbooks.adoc[set up {zdm-automation}] and xref:ROOT:deploy-proxy-monitoring.adoc[deploy the {zdm-proxy} instances]. == Deploy the monitoring stack -{product-automation} enables you to easily set up a self-contained monitoring stack that is preconfigured to collect metrics from your {product-proxy} instances and display them in ready-to-use Grafana dashboards. +{zdm-automation} enables you to easily set up a self-contained monitoring stack that is preconfigured to collect metrics from your {zdm-proxy} instances and display them in ready-to-use Grafana dashboards. The monitoring stack is deployed entirely on Docker. It includes the following components, all deployed as Docker containers: -* Prometheus node exporter, which runs on each {product-proxy} host and makes OS- and host-level metrics available to Prometheus. -* Prometheus server, to collect metrics from {product-proxy}, its Golang runtime, and the Prometheus node exporter. +* Prometheus node exporter, which runs on each {zdm-proxy} host and makes OS- and host-level metrics available to Prometheus. +* Prometheus server, to collect metrics from {zdm-proxy}, its Golang runtime, and the Prometheus node exporter. * Grafana, to visualize the metrics in preconfigured dashboards. -After running the monitoring playbook, you will have a fully configured monitoring stack connected to your {product-proxy} deployment. +After running the monitoring playbook, you will have a fully configured monitoring stack connected to your {zdm-proxy} deployment. -Aside from xref:ROOT:deployment-infrastructure.adoc[preparing the infrastructure], you don't need to install any {product-proxy} dependencies on the monitoring host machine. The playbook automatically installs all required software packages. +Aside from xref:ROOT:deployment-infrastructure.adoc[preparing the infrastructure], you don't need to install any {zdm-proxy} dependencies on the monitoring host machine. The playbook automatically installs all required software packages. . Connect to the Ansible Control Host Docker container. You can do this from the jumphost machine by running the following command: @@ -76,9 +76,9 @@ If you deployed your monitoring stack on another machine, replace `**MONITORING_ [#call-the-liveliness-and-readiness-endpoints] == Call the `liveness` and `readiness` HTTP endpoints -{product-short} metrics provide `/health/liveness` and `/health/readiness` HTTP endpoints, that you can call to determine the state of the {product-proxy} instances. +{zdm-short} metrics provide `/health/liveness` and `/health/readiness` HTTP endpoints, that you can call to determine the state of the {zdm-proxy} instances. -For example, after deploying the {product-proxy} monitoring stack, you can use the `liveness` and `readiness` HTTP endpoints to confirm that your {product-proxy} instances are running. +For example, after deploying the {zdm-proxy} monitoring stack, you can use the `liveness` and `readiness` HTTP endpoints to confirm that your {zdm-proxy} instances are running. === Liveliness endpoint @@ -92,7 +92,7 @@ Replace the following: * `**METRICS_PORT**`: The `metrics_port` you set in the `zdm_monitoring_config.yml` file. The default port is 14001. -* `**ZDM_PROXY_PRIVATE_IP**`: The private IP address, hostname, or other valid identifier for the {product-proxy} instance you want to check. +* `**ZDM_PROXY_PRIVATE_IP**`: The private IP address, hostname, or other valid identifier for the {zdm-proxy} instance you want to check. Example request with variables: @@ -120,7 +120,7 @@ Replace the following: * `**METRICS_PORT**`: The `metrics_port` you set in the `zdm_monitoring_config.yml` file. The default port is 14001. -* `**ZDM_PROXY_PRIVATE_IP**`: The private IP address, hostname, or other valid identifier for the {product-proxy} instance you want to check. +* `**ZDM_PROXY_PRIVATE_IP**`: The private IP address, hostname, or other valid identifier for the {zdm-proxy} instance you want to check. Example request with variables: @@ -157,23 +157,23 @@ Example result: } ---- -== Inspect {product-proxy} metrics +== Inspect {zdm-proxy} metrics -{product-proxy} exposes an HTTP endpoint that returns metrics in the Prometheus format. +{zdm-proxy} exposes an HTTP endpoint that returns metrics in the Prometheus format. -After you deploy the monitoring stack, the {product-proxy} Grafana dashboards automatically start displaying these metrics as they are scraped from the instances. +After you deploy the monitoring stack, the {zdm-proxy} Grafana dashboards automatically start displaying these metrics as they are scraped from the instances. -If you already have a Grafana deployment, then you can import the dashboards from the {product-automation-repo}/tree/main/grafana-dashboards[{product-short} dashboard files] in the {product-short} GitHub repository. +If you already have a Grafana deployment, then you can import the dashboards from the {zdm-automation-repo-url}/tree/main/grafana-dashboards[{zdm-short} dashboard files] in the {zdm-short} GitHub repository. -=== Grafana dashboard for {product-proxy} metrics +=== Grafana dashboard for {zdm-proxy} metrics -The {product-proxy} metrics dashboard includes proxy level metrics, node level metrics, and asynchronous read requests metrics. +The {zdm-proxy} metrics dashboard includes proxy level metrics, node level metrics, and asynchronous read requests metrics. -image::ROOT:zdm-grafana-proxy-dashboard1.png[Grafana dashboard shows three categories of {product-short} metrics for the proxy.] +image::ROOT:zdm-grafana-proxy-dashboard1.png[Grafana dashboard shows three categories of {zdm-short} metrics for the proxy.] This dashboard can help you identify issues such as the following: -* If the number of client connections is near 1000 per {product-proxy} instance, be aware that {product-proxy} will start rejecting client connections after accepting 1000 connections. +* If the number of client connections is near 1000 per {zdm-proxy} instance, be aware that {zdm-proxy} will start rejecting client connections after accepting 1000 connections. * If Prepared Statement cache metrics are always increasing, check the **entries** and **misses** metrics. * If you have error metrics reporting for specific error types, evaluate these issues on a case-by-case basis. @@ -182,10 +182,10 @@ This dashboard can help you identify issues such as the following: * **Latency**: + -** **Read Latency**: Total latency measured by {product-proxy} per read request, including post-processing, such as response aggregation. +** **Read Latency**: Total latency measured by {zdm-proxy} per read request, including post-processing, such as response aggregation. This metric has two labels: `reads_origin` and `reads_target`. The label that has data depends on which cluster is receiving the reads, which is the current xref:ROOT:faqs.adoc#what-are-origin-target-primary-and-secondary-clusters[primary cluster]. -** **Write Latency**: Total latency measured by {product-proxy} per write request, including post-processing, such as response aggregation. +** **Write Latency**: Total latency measured by {zdm-proxy} per write request, including post-processing, such as response aggregation. This metric is measured as the total latency across both clusters for a single xref:ROOT:components.adoc#how-zdm-proxy-handles-reads-and-writes[bifurcated write request]. * **Throughput**: Follows the same structure as the latency metrics but measures **Read Throughput** and **Write Throughput**. @@ -196,7 +196,7 @@ This metric is measured as the total latency across both clusters for a single x * **Prepared Statement cache**: + -** **Cache Misses**: If a prepared statement is sent to {product-proxy} but the statement's `preparedID` isn't present in the node's cache, then {product-proxy} sends an `UNPREPARED` response to the client to reprepare the statement. +** **Cache Misses**: If a prepared statement is sent to {zdm-proxy} but the statement's `preparedID` isn't present in the node's cache, then {zdm-proxy} sends an `UNPREPARED` response to the client to reprepare the statement. This metric tracks the number of times this happens. ** **Number of cached prepared statements** @@ -213,7 +213,7 @@ The label that contains data depends on which cluster is currently considered th *** `failed_on=both`: The write request failed on both the origin and target clusters. * **Request Failure Counters**: Number of total request failures. -Resets if the {product-proxy} instance restarts. +Resets if the {zdm-proxy} instance restarts. + ** **Connect Failure Counters**: Has the same labels as the connect failure rate. ** **Read Failure Counters**: Has the same labels as the read failure rate. @@ -227,8 +227,8 @@ For error metrics by error type, see the <<_node_level_metrics,node-level error * **Latency**: Node-level latency metrics report combined read and write latency per cluster, not per request. For latency by request type, see <>. + -** **Origin**: Latency, as measured by {product-proxy}, up to the point that it received a response from the origin connection. -** **Target**: Latency, as measured by {product-proxy}, up to the point it received a response from the target connection. +** **Origin**: Latency, as measured by {zdm-proxy}, up to the point that it received a response from the origin connection. +** **Target**: Latency, as measured by {zdm-proxy}, up to the point it received a response from the target connection. * **Throughput**: Same as the node-level latency metrics. Reads and writes are combined. @@ -263,9 +263,9 @@ These metrics track the following information for asynchronous read requests: === Go runtime metrics dashboard -The Go runtime metrics dashboard is used less often than the {product-proxy} metrics dashboard. +The Go runtime metrics dashboard is used less often than the {zdm-proxy} metrics dashboard. -This dashboard can be helpful for troubleshooting {product-proxy} performance issues. +This dashboard can be helpful for troubleshooting {zdm-proxy} performance issues. It provides metrics for memory usage, Garbage Collection (GC) duration, open FDs (file descriptors), and the number of goroutines. image::ROOT:zdm-golang-dashboard.png[Golang metrics dashboard example is shown.] @@ -279,10 +279,10 @@ Watch for the following problem areas on the Go runtime metrics dashboard: === System-level metrics dashboard -The {product-short} monitoring stack's system-level dashboard metrics are collected through the Prometheus Node Exporter. +The {zdm-short} monitoring stack's system-level dashboard metrics are collected through the Prometheus Node Exporter. This dashboard contains hardware and OS-level metrics for the host on which the proxy runs. This can be useful to check the available resources and identify low-level bottlenecks or issues. == Next steps -To continue Phase 1 of the migration, xref:ROOT:connect-clients-to-proxy.adoc[connect your client applications to {product-proxy}]. \ No newline at end of file +To continue Phase 1 of the migration, xref:ROOT:connect-clients-to-proxy.adoc[connect your client applications to {zdm-proxy}]. \ No newline at end of file diff --git a/modules/ROOT/pages/migrate-and-validate-data.adoc b/modules/ROOT/pages/migrate-and-validate-data.adoc index 91abb545..7a61328d 100644 --- a/modules/ROOT/pages/migrate-and-validate-data.adoc +++ b/modules/ROOT/pages/migrate-and-validate-data.adoc @@ -1,11 +1,11 @@ = Phase 2: Migrate and validate data :page-aliases: ROOT:sideloader-zdm.adoc, ROOT:dsbulk-migrator-overview.adoc, ROOT:dsbulk-migrator.adoc -In xref:ROOT:phase1.adoc[Phase 1], you set up {product-proxy} to orchestrate live traffic to your origin and target clusters. +In xref:ROOT:phase1.adoc[Phase 1], you set up {zdm-proxy} to orchestrate live traffic to your origin and target clusters. -In Phase 2 of {product}, you migrate data from the origin to the target, and then validate the migrated data. +In Phase 2 of {zdm}, you migrate data from the origin to the target, and then validate the migrated data. -image::ROOT:migration-phase2ra.svg[In {product-short} Phase 2, you migrate data from the origin cluster to the target cluster] +image::ROOT:migration-phase2ra.svg[In {zdm-short} Phase 2, you migrate data from the origin cluster to the target cluster] To move and validate data, you can use a dedicated data migration tool, such as {sstable-sideloader}, {cass-migrator}, or {dsbulk}, or your can create your own custom data migration script. @@ -25,11 +25,11 @@ For compatible origin clusters, see xref:ROOT:astra-migration-paths.adoc[]. * *Cloud provider CLI*: Upload snapshots to a dedicated cloud storage bucket for your migration. * *{astra} {devops-api}*: Run the {sstable-sideloader} commands to write the data from cloud storage to your {astra-db} database. -You can use {sstable-sideloader} alone or with {product-proxy}. +You can use {sstable-sideloader} alone or with {zdm-proxy}. For more information and instructions, see xref:sideloader:sideloader-overview.adoc[]. -.Use {sstable-sideloader} with {product-proxy} +.Use {sstable-sideloader} with {zdm-proxy} svg::sideloader:astra-migration-toolkit.svg[] == {cass-migrator} @@ -37,9 +37,9 @@ svg::sideloader:astra-migration-toolkit.svg[] You can use {cass-migrator-short} for data migration and validation between {cass-short}-based databases. It offers extensive functionality and configuration options to support large and complex migrations as well as post-migration data validation. -You can use {cass-migrator-short} alone, with {product-proxy}, or for data validation after using another data migration tool. +You can use {cass-migrator-short} alone, with {zdm-proxy}, or for data validation after using another data migration tool. -For more information, see xref:ROOT:cassandra-data-migrator.adoc[]. +For more information, see the {cass-migrator-repo-url}[{cass-migrator-short} repository]. == {dsbulk} @@ -48,7 +48,7 @@ You can use it to load, unload, and count records. Because {dsbulk-short} doesn't have the same data validation capabilities as {cass-migrator-short}, it is best for migrations that don't require extensive data validation, aside from post-migration row counts. -You can use {dsbulk-short} alone or with {product-proxy}. +You can use {dsbulk-short} alone or with {zdm-proxy}. For more information, see xref:dsbulk:overview:dsbulk-about.adoc[]. @@ -59,16 +59,16 @@ include::ROOT:partial$dsbulk-migrator-deprecation.adoc[] == Other data migration processes -Depending on your origin and target databases, there might be other {product-short}-compatible data migration tools available, or you can write your own custom data migration processes with a tool like {spark-reg}. +Depending on your origin and target databases, there might be other {zdm-short}-compatible data migration tools available, or you can write your own custom data migration processes with a tool like {spark-reg}. -To use a data migration tool with {product-proxy}, it must meet the following requirements: +To use a data migration tool with {zdm-proxy}, it must meet the following requirements: * Built-in data validation functionality or compatibility with another data validation tool, such as {cass-migrator-short}. This is crucial to a successful migration. -* Preserves the data model, including column names and data types, so that {product-proxy} can send the same read/write statements to both databases successfully. +* Preserves the data model, including column names and data types, so that {zdm-proxy} can send the same read/write statements to both databases successfully. + -Migrations that perform significant data transformations might not be compatible with {product-proxy}. +Migrations that perform significant data transformations might not be compatible with {zdm-proxy}. The impact of data transformations depends on your specific data model, database platforms, and the scale of your migration. == Next steps diff --git a/modules/ROOT/pages/phase1.adoc b/modules/ROOT/pages/phase1.adoc index a5916d3b..aab9e190 100644 --- a/modules/ROOT/pages/phase1.adoc +++ b/modules/ROOT/pages/phase1.adoc @@ -1,13 +1,13 @@ -= Deploy and connect {product-proxy} += Deploy and connect {zdm-proxy} -After you plan and prepare for your migration, you can start Phase 1 of the migration process where you deploy and connect {product-proxy}. +After you plan and prepare for your migration, you can start Phase 1 of the migration process where you deploy and connect {zdm-proxy}. -image::ROOT:migration-phase1ra.svg[In migration Phase 1, you deploy {product-proxy} instances, and then connect your client applications to the proxies] +image::ROOT:migration-phase1ra.svg[In migration Phase 1, you deploy {zdm-proxy} instances, and then connect your client applications to the proxies] To complete Phase 1, do the following: . xref:setup-ansible-playbooks.adoc[]. . xref:deploy-proxy-monitoring.adoc[]. -. Deploy the xref:metrics.adoc[{product-proxy} monitoring stack] and learn about {product-proxy} metrics. +. Deploy the xref:metrics.adoc[{zdm-proxy} monitoring stack] and learn about {zdm-proxy} metrics. . xref:connect-clients-to-proxy.adoc[]. -. Learn how to xref:manage-proxy-instances.adoc[manage {product-proxy} instances], including scaling, upgrading, reconfiguring, and restarting them. \ No newline at end of file +. Learn how to xref:manage-proxy-instances.adoc[manage {zdm-proxy} instances], including scaling, upgrading, reconfiguring, and restarting them. \ No newline at end of file diff --git a/modules/ROOT/pages/rollback.adoc b/modules/ROOT/pages/rollback.adoc index dbcfa852..5a1fb6b1 100644 --- a/modules/ROOT/pages/rollback.adoc +++ b/modules/ROOT/pages/rollback.adoc @@ -8,14 +8,14 @@ image::ROOT:migration-all-phases.svg[Migration phases from start to finish.] == Phase 5 is the point of no return -After moving your client applications off the {product-proxy} instances (Phase 5), writes are no longer sent to both the origin and target clusters. +After moving your client applications off the {zdm-proxy} instances (Phase 5), writes are no longer sent to both the origin and target clusters. The data on origin cluster is no longer kept up-to-date, and you lose this seamless rollback option. This is the point at which you commit to using the target cluster permanently. -The {product-proxy} deployment can be destroyed, and the origin cluster is no longer needed by the client applications that have been migrated. +The {zdm-proxy} deployment can be destroyed, and the origin cluster is no longer needed by the client applications that have been migrated. However, should you decide to move back to the origin cluster later, or if you want to move to a new cluster entirely, you can rerun the same migration process. In this case, you use your original target cluster as the new origin cluster, and you set the new target cluster to whatever cluster you want to migrate to (which could even be the original ancestor origin cluster). == Next steps -After preparing the infrastructure for {product-proxy} and your target cluster, begin xref:ROOT:phase1.adoc[Phase 1] of the migration. \ No newline at end of file +After preparing the infrastructure for {zdm-proxy} and your target cluster, begin xref:ROOT:phase1.adoc[Phase 1] of the migration. \ No newline at end of file diff --git a/modules/ROOT/pages/setup-ansible-playbooks.adoc b/modules/ROOT/pages/setup-ansible-playbooks.adoc index 0248a0b3..6c42835c 100644 --- a/modules/ROOT/pages/setup-ansible-playbooks.adoc +++ b/modules/ROOT/pages/setup-ansible-playbooks.adoc @@ -1,37 +1,37 @@ -= Set up {product-automation} with {product-utility} += Set up {zdm-automation} with {zdm-utility} -{product-automation} uses Ansible to deploy and configure the {product-proxy} instances and the associated monitoring stack. +{zdm-automation} uses Ansible to deploy and configure the {zdm-proxy} instances and the associated monitoring stack. Specifically, these configuration processes are defined in Ansible playbooks that you execute from the Ansible Control Host container. -First, you must use {product-utility} to set up the Ansible Control Host container, and then you can xref:deploy-proxy-monitoring.adoc[use {product-automation} to deploy your {product-proxy} instances] and the xref:ROOT:metrics.adoc[monitoring stack]. -After you complete both of these procedures, you will have an active and fully monitored {product-proxy} deployment. +First, you must use {zdm-utility} to set up the Ansible Control Host container, and then you can xref:deploy-proxy-monitoring.adoc[use {zdm-automation} to deploy your {zdm-proxy} instances] and the xref:ROOT:metrics.adoc[monitoring stack]. +After you complete both of these procedures, you will have an active and fully monitored {zdm-proxy} deployment. -{product-utility} is a Golang (Go) executable program that runs anywhere. +{zdm-utility} is a Golang (Go) executable program that runs anywhere. This utility prompts you for a few configuration values, with helpful embedded explanations and error handling, and then it creates and prepares the Ansible Control Host container automatically. -From this container, you will configure and run the {product-automation} Ansible playbooks. +From this container, you will configure and run the {zdm-automation} Ansible playbooks. -image::ROOT:docker-container-and-zdm-utility.png[{product-proxy} connections from Docker container created by {product-utility}] +image::ROOT:docker-container-and-zdm-utility.png[{zdm-proxy} connections from Docker container created by {zdm-utility}] -For more information about {product-utility} and {product-automation}, see xref:ROOT:components.adoc[]. +For more information about {zdm-utility} and {zdm-automation}, see xref:ROOT:components.adoc[]. == Prerequisites -Before running {product-utility} to create the Ansible Control Host container, you must complete the following prerequisites: +Before running {zdm-utility} to create the Ansible Control Host container, you must complete the following prerequisites: -* xref:ROOT:deployment-infrastructure.adoc[Provision all {product-proxy} infrastructure]: This means your server machines are ready, and you know their IP addresses. +* xref:ROOT:deployment-infrastructure.adoc[Provision all {zdm-proxy} infrastructure]: This means your server machines are ready, and you know their IP addresses. + This infrastructure can be on-premise or in any cloud provider of your choice. + -If your {product-proxy} machines use private IPs, which are recommended for production deployments, configure these before running {product-utility}. -If you enable private IPs later, you must reconfigure and redeploy your {product-proxy} instances. -This is a disruptive operation that requires a small amount of downtime because the deployment playbook decommissions and re-creates all {product-proxy} containers simultaneously. +If your {zdm-proxy} machines use private IPs, which are recommended for production deployments, configure these before running {zdm-utility}. +If you enable private IPs later, you must reconfigure and redeploy your {zdm-proxy} instances. +This is a disruptive operation that requires a small amount of downtime because the deployment playbook decommissions and re-creates all {zdm-proxy} containers simultaneously. * https://docs.docker.com/engine/install/#server[Install Docker] on the machine that will run the Ansible Control Host container, and ensure that the `docker` command https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user[doesn't require superuser privileges]. + [TIP] ==== You don't need to install Docker on any other machines. -{product-automation} will install Docker on the other {product-proxy} machines as part of the deployment process. +{zdm-automation} will install Docker on the other {zdm-proxy} machines as part of the deployment process. ==== == Alternative Docker configurations @@ -45,7 +45,7 @@ For instructions, see the Docker documentation on https://docs.docker.com/docker Airgapped local registry:: Local registries that aren't connected to the internet require administrators to manually add containers to their registry. -For {product-utility}, you need the following five containers to install and configure the jumphost, {product-proxy}, and monitoring: +For {zdm-utility}, you need the following five containers to install and configure the jumphost, {zdm-proxy}, and monitoring: + [source,plaintext] ---- @@ -61,24 +61,24 @@ datastax/zdm-proxy:2.x This guide uses a jumphost to run the Ansible Control Host container. A jumphost is a server on a network that is used to access and manage devices in a separate security zone, providing a controlled means of access between them. -The jumphost can be, for example, a Linux server machine that is able to access the server machines that you wish to use for your {product-proxy} deployment. +The jumphost can be, for example, a Linux server machine that is able to access the server machines that you wish to use for your {zdm-proxy} deployment. -The jumphost is used to access the {product-proxy} machines, and run the Ansible Control Host container, from which you run {product-automation}. -In this example, the jumphost also runs the {product-proxy} monitoring stack, which uses Prometheus and Grafana to expose the metrics of all the {product-proxy} instances to preconfigured dashboards. +The jumphost is used to access the {zdm-proxy} machines, and run the Ansible Control Host container, from which you run {zdm-automation}. +In this example, the jumphost also runs the {zdm-proxy} monitoring stack, which uses Prometheus and Grafana to expose the metrics of all the {zdm-proxy} instances to preconfigured dashboards. However, you can run the monitoring stack on a different machine. . Add SSH keys to the jumphost. + -To run {product-automation}, the Ansible Control Host must be able to connect to all other instances of the {product-proxy} deployment. +To run {zdm-automation}, the Ansible Control Host must be able to connect to all other instances of the {zdm-proxy} deployment. For this reason, it needs to have the SSH key required by those instances. + [TIP] ==== -To simplify accessing the jumphost and {product-proxy} instances from your machine, you can create a xref:ROOT:deployment-infrastructure.adoc#connect-to-zdm-proxy-infrastructure-from-an-external-machine[custom SSH configuration file]. +To simplify accessing the jumphost and {zdm-proxy} instances from your machine, you can create a xref:ROOT:deployment-infrastructure.adoc#connect-to-zdm-proxy-infrastructure-from-an-external-machine[custom SSH configuration file]. These steps assume that this file exists. ==== + -From your local machine, transfer (`scp`) the SSH private key for the {product-proxy} deployment to the jumphost. +From your local machine, transfer (`scp`) the SSH private key for the {zdm-proxy} deployment to the jumphost. For example: + [source,bash] @@ -93,15 +93,15 @@ scp -F path/to/zdm_ssh_config zdm_key_name jumphost: ssh -F path/to/zdm_ssh_config jumphost ---- -. From the jumphost, download the latest {product-utility} executable from the {product-automation-repo}/releases[{product-automation} GitHub repository] {product-automation-shield}. +. From the jumphost, download the latest {zdm-utility} executable from the {zdm-automation-repo-url}/releases[{zdm-automation} GitHub repository]. + The package filename format is `zdm-util-**PLATFORM**-**VERSION**.tgz`. -The following example downloads {product-utility} version 2.3.1 for Linux amd64. +The following example downloads {zdm-utility} version 2.3.1 for Linux amd64. To download a different package, change the version and package filename accordingly. + [source,bash,subs="+attributes"] ---- -wget {product-automation-repo}/releases/download/v2.3.1/zdm-util-linux-amd64-v2.3.1.tgz +wget {zdm-automation-repo-url}/releases/download/v2.3.1/zdm-util-linux-amd64-v2.3.1.tgz ---- . Extract the archive: @@ -111,7 +111,7 @@ wget {product-automation-repo}/releases/download/v2.3.1/zdm-util-linux-amd64-v2. tar -xvf zdm-util-linux-amd64-v2.3.1.tgz ---- -. Run {product-utility}: +. Run {zdm-utility}: + [source,bash] ---- @@ -119,20 +119,20 @@ tar -xvf zdm-util-linux-amd64-v2.3.1.tgz ---- . Enter configuration values as you are prompted for them. -{product-utility} creates and initializes the Ansible Control Host container after it has all required configuration values. +{zdm-utility} creates and initializes the Ansible Control Host container after it has all required configuration values. + -{product-utility} validates each value that you enter. +{zdm-utility} validates each value that you enter. In case of invalid values, it prompts you for the variable again and prints a message to help you fix potential problems. You have five attempts to enter valid values. -If all five attempts are invalid, you must rerun {product-utility}. +If all five attempts are invalid, you must rerun {zdm-utility}. + [TIP] ==== -{product-utility} stores your provided configuration values in a file named `ansible_container_init_config` in the current directory. +{zdm-utility} stores your provided configuration values in a file named `ansible_container_init_config` in the current directory. If you run the utility again, it detects this file, and then asks you if you want to use the existing configuration or discard it. If the file contains any invalid values, you are prompted for the missing or invalid parameters only. -You can also pass a custom configuration file to {product-utility} with the optional command-line parameter `-utilConfigFile`. +You can also pass a custom configuration file to {zdm-utility} with the optional command-line parameter `-utilConfigFile`. For example: [source,bash] @@ -159,20 +159,20 @@ For example: + * If you have an existing Ansible inventory file that is present on the jumphost, specify that file. * If you don't have an Ansible inventory file, type kbd:[N]. -{product-utility} prompts you for the required values, and then creates a `zdm_ansible_inventory` file in the working directory. +{zdm-utility} prompts you for the required values, and then creates a `zdm_ansible_inventory` file in the working directory. -. Indicate whether this deployment is for local testing and evaluation (such as when you're creating a demo or experimenting with {product-proxy}) or not. +. Indicate whether this deployment is for local testing and evaluation (such as when you're creating a demo or experimenting with {zdm-proxy}) or not. For production deployments, type kbd:[N]. + [TIP] ==== -Depending on your environment and {product-utility} configuration choices, your prompts might differ from the examples shown here. +Depending on your environment and {zdm-utility} configuration choices, your prompts might differ from the examples shown here. You might receive additional prompts that aren't given here, or you might bypass some prompts. In any case, the utility guides you through the process with helpful hints and troubleshooting guidance. -Regardless of the individual prompts, a successful outcome always results in a configured Ansible Control Host container that is ready to run {product-automation}. +Regardless of the individual prompts, a successful outcome always results in a configured Ansible Control Host container that is ready to run {zdm-automation}. ==== -. For production deployments, enter at least three proxy private IP addresses for the machines that will run the {product-proxy} instances. +. For production deployments, enter at least three proxy private IP addresses for the machines that will run the {zdm-proxy} instances. For local testing and evaluation, only one proxy private IP address is required. + Enter each IP address on a separate line, for example: @@ -186,44 +186,44 @@ Enter each IP address on a separate line, for example: + When you are done entering IP addresses, press kbd:[Return] at the blank prompt. -. Optional: When prompted, enter the private IP address of the machine where you want to deploy the {product-proxy} monitoring stack. +. Optional: When prompted, enter the private IP address of the machine where you want to deploy the {zdm-proxy} monitoring stack. + -You can use the jumphost or a different machine, as long as it can connect to the {product-proxy} instances over TCP on ports 9100 (to collect host-level metrics) and the port on which {product-proxy} exposes its own metrics, typically 14001. +You can use the jumphost or a different machine, as long as it can connect to the {zdm-proxy} instances over TCP on ports 9100 (to collect host-level metrics) and the port on which {zdm-proxy} exposes its own metrics, typically 14001. + You can skip this step if you haven't decided which machine to use for monitoring, or you want to use your own monitoring stack. + [IMPORTANT] ==== -{company} strongly recommends that you configure a monitoring instance to expose xref:ROOT:metrics.adoc[{product-proxy} metrics and preconfigured dashboards] with {product-automation}. -This is the most simplified approach to {product-proxy} montioring, and it is preferred unless you intend (or are required) to use a monitoring stack that you already have. +{company} strongly recommends that you configure a monitoring instance to expose xref:ROOT:metrics.adoc[{zdm-proxy} metrics and preconfigured dashboards] with {zdm-automation}. +This is the most simplified approach to {zdm-proxy} montioring, and it is preferred unless you intend (or are required) to use a monitoring stack that you already have. -Metrics are essential for understanding the performance and health of your {product-proxy} instances, especially for migrations that run over multiple days or weeks. +Metrics are essential for understanding the performance and health of your {zdm-proxy} instances, especially for migrations that run over multiple days or weeks. You cannot rely solely on information in the logs. -Logs report connection or protocol errors, but they don't provide enough information about how {product-proxy} is functioning and how each cluster is responding. +Logs report connection or protocol errors, but they don't provide enough information about how {zdm-proxy} is functioning and how each cluster is responding. In contrast, metrics provide insightful data and graphs illustrating key values and changes over time. ==== + -The following example uses the same IP address as the Ansible Control Host machine, which is the jumphost machine where {product-utility} is running: +The following example uses the same IP address as the Ansible Control Host machine, which is the jumphost machine where {zdm-utility} is running: + [source,bash] ---- 172.18.100.128 ---- -. Review the configuration summary printed to the terminal by {product-utility}. -At this point, {product-utility} has created the Ansible inventory file name `zdm_ansible_inventory` (unless you provided your own custom file), and it has written the {product-utility} configuration to `ansible_container_init_config`. +. Review the configuration summary printed to the terminal by {zdm-utility}. +At this point, {zdm-utility} has created the Ansible inventory file name `zdm_ansible_inventory` (unless you provided your own custom file), and it has written the {zdm-utility} configuration to `ansible_container_init_config`. + image::ROOT:zdm-go-utility-results3.png[A summary of the configuration provided is displayed in the terminal] . Type kbd:[Y] to continue and trigger the following processes: + -.. {product-utility} creates and downloads the image of the Ansible Docker container. -.. {product-utility} creates, configures, and starts the Ansible Control Host container. -.. {product-utility} prints a message when the Ansible container is fully initialized and ready to use. +.. {zdm-utility} creates and downloads the image of the Ansible Docker container. +.. {zdm-utility} creates, configures, and starts the Ansible Control Host container. +.. {zdm-utility} prints a message when the Ansible container is fully initialized and ready to use. + image::ROOT:zdm-go-utility-success3.png[Ansible Docker container success messages] == Next steps -After you use {product-utility} to set up the Ansible Control Host container, xref:deploy-proxy-monitoring.adoc[use {product-automation} to deploy your {product-proxy} instances]. \ No newline at end of file +After you use {zdm-utility} to set up the Ansible Control Host container, xref:deploy-proxy-monitoring.adoc[use {zdm-automation} to deploy your {zdm-proxy} instances]. \ No newline at end of file diff --git a/modules/ROOT/pages/troubleshooting-tips.adoc b/modules/ROOT/pages/troubleshooting-tips.adoc index 72cf5d23..2bf2d8c1 100644 --- a/modules/ROOT/pages/troubleshooting-tips.adoc +++ b/modules/ROOT/pages/troubleshooting-tips.adoc @@ -1,22 +1,22 @@ -= Troubleshoot {product} -:navtitle: Troubleshoot {product-short} += Troubleshoot {zdm} +:navtitle: Troubleshoot {zdm-short} :page-aliases: ROOT:troubleshooting.adoc, ROOT:troubleshooting-scenarios.adoc -:description: Get help with {product}. +:description: Get help with {zdm}. -This page provides general troubleshooting advice and describes some common issues you might encounter with {product} ({product-short}). +This page provides general troubleshooting advice and describes some common issues you might encounter with {zdm}. For additional assistance, you can <>, contact your {company} account representative, or contact {support-url}[IBM Support]. [#proxy-logs] -== Check {product-proxy} logs +== Check {zdm-proxy} logs include::ROOT:partial$zdm-logs-intro.adoc[] For more information, see xref:ROOT:zdm-logs.adoc[]. [#check-version] -== Check your {product-proxy} version +== Check your {zdm-proxy} version -The {product-proxy} version is printed at startup, and it is the first message in the logs, immediately before the long `Parsed configuration` string: +The {zdm-proxy} version is printed at startup, and it is the first message in the logs, immediately before the long `Parsed configuration` string: [source,console] ---- @@ -24,7 +24,7 @@ time="2023-01-13T13:37:28+01:00" level=info msg="Starting ZDM proxy version 2.1. time="2023-01-13T13:37:28+01:00" level=info msg="Parsed configuration: ..." ---- -You can also pass the `-version` flag to {product-proxy} to print the version. +You can also pass the `-version` flag to {zdm-proxy} to print the version. For example, you can use the following Docker command, replacing `**TAG**` with the `zdm_proxy_image` tag set in `vars/zdm_proxy_container.yml`: [source,bash,subs="+quotes"] @@ -32,7 +32,7 @@ For example, you can use the following Docker command, replacing `**TAG**` with docker run --rm datastax/zdm-proxy:**TAG** -version ---- -The output shows the binary version of {product-proxy} that is currently running: +The output shows the binary version of {zdm-proxy} that is currently running: .Result [source,console] @@ -42,15 +42,15 @@ ZDM proxy version 2.1.0 [IMPORTANT] ==== -Don't use `--rm` when you launch the {product-proxy} container. -This flag will prevent you from accessing the logs when {product-proxy} stops or crashes. +Don't use `--rm` when you launch the {zdm-proxy} container. +This flag will prevent you from accessing the logs when {zdm-proxy} stops or crashes. ==== == Query system.peers and system.local to check for configuration issues -Querying `system.peers` and `system.local` can help you investigate {product-proxy} configuration issues: +Querying `system.peers` and `system.local` can help you investigate {zdm-proxy} configuration issues: -. xref:ROOT:connect-clients-to-proxy.adoc#connect-cqlsh-to-zdm-proxy[Connect the {cql-shell} to a {product-proxy} instance]. +. xref:ROOT:connect-clients-to-proxy.adoc#connect-cqlsh-to-zdm-proxy[Connect the {cql-shell} to a {zdm-proxy} instance]. . Query `system.peers`: + @@ -66,9 +66,9 @@ SELECT * FROM system.peers SELECT * FROM system.local ---- -. Repeat for each of your {product-proxy} instances. +. Repeat for each of your {zdm-proxy} instances. + -Because `system.peers` and `system.local` reflect the local {product-proxy} instance's configuration, you must query all instances to get all information and identify potential misconfigurations. +Because `system.peers` and `system.local` reflect the local {zdm-proxy} instance's configuration, you must query all instances to get all information and identify potential misconfigurations. . Compare the results from each instance by searching for values related to an error that you are troubleshooting, such as IP addresses or tokens. + @@ -76,14 +76,14 @@ For example, you might compare `cluster_name` to ensure that all instances are c == Troubleshooting scenarios -The following sections provide troubleshooting advice for specific issues or error messages related to {product}. +The following sections provide troubleshooting advice for specific issues or error messages related to {zdm}. [#configuration-changes-arent-applied-by-zdm-automation] -=== Configuration changes aren't applied by {product-automation} +=== Configuration changes aren't applied by {zdm-automation} -If some configuration changes aren't applied to your {product-proxy} instances after a rolling restart, this typically means that you modified an immutable configuration variable. +If some configuration changes aren't applied to your {zdm-proxy} instances after a rolling restart, this typically means that you modified an immutable configuration variable. -Not all {product-proxy} configuration variables can be changed after deployment, with or without a rolling restart. +Not all {zdm-proxy} configuration variables can be changed after deployment, with or without a rolling restart. For a list of variables that you can change on a live deployment, see xref:manage-proxy-instances.adoc#change-mutable-config-variable[Change mutable configuration variables]. Any configuration variables excluded from the mutable variables list are considered _immutable_, and you must fully redeploy your instances to change them. @@ -92,9 +92,9 @@ Allowing these values to change from a rolling restart could propagate a misconf If you change the value of an immutable configuration variable, you must run the `deploy_zdm_proxy.yml` playbook again. You can run this playbook as many times as needed. -Each time, {product-automation} re-creates your entire {product-proxy} deployment with the new configuration. -However, this doesn't happen in a rolling fashion: The existing {product-proxy} instances are torn down simultaneously, and then they are re-created. -This results in a brief period of downtime where the entire {product-proxy} deployment is unavailable. +Each time, {zdm-automation} re-creates your entire {zdm-proxy} deployment with the new configuration. +However, this doesn't happen in a rolling fashion: The existing {zdm-proxy} instances are torn down simultaneously, and then they are re-created. +This results in a brief period of downtime where the entire {zdm-proxy} deployment is unavailable. === Client application throws unsupported protocol version error @@ -132,9 +132,9 @@ For more information, see the documentation for your version of the Java driver: === Logs report protocol errors but clients connect successfully -`PROTOCOL ERROR` messages in {product-proxy} logs are a normal part of the handshake process while the protocol version is being negotiated. +`PROTOCOL ERROR` messages in {zdm-proxy} logs are a normal part of the handshake process while the protocol version is being negotiated. -These messages indicate that a protocol version downgrades happened because {product-proxy} or one of the clusters doesn't support the version requested by the client. +These messages indicate that a protocol version downgrades happened because {zdm-proxy} or one of the clusters doesn't support the version requested by the client. Protocol version downgrades result from a request by a cluster that doesn't support the version that the client requested. include::ROOT:partial$cassandra-protocol-versions.adoc[] @@ -150,14 +150,14 @@ msg=Invalid or unsupported protocol version (5)).\"\n","stream":"stderr","time": `PROTOCOL ERROR` messages recorded at a higher log level, especially `level=error`, might indicate a bug because this means that the error occurred outside of the handshake process. This is a fatal unexpected error that terminates the connection. -If you observe this behavior in your logs, <> so the issue can be investigated by the {product-short} team. +If you observe this behavior in your logs, <> so the issue can be investigated by the {zdm-short} team. === Proxy fails to start due to invalid or unsupported protocol version -If the {product-proxy} logs contain `debug` messages with `Invalid or unsupported protocol version: 3`, this means that one of the origin clusters doesn't support protocol `V3` or later. +If the {zdm-proxy} logs contain `debug` messages with `Invalid or unsupported protocol version: 3`, this means that one of the origin clusters doesn't support protocol `V3` or later. Specifically, this happens with {cass-short} 2.0 and {dse-short} 4.6. -{product-short} cannot be used for these migrations because the {product-proxy} control connections don't perform protocol version negotiation; they only attempt to use `V3`. +{zdm-short} cannot be used for these migrations because the {zdm-proxy} control connections don't perform protocol version negotiation; they only attempt to use `V3`. .Example: Invalid or unsupported protocol version logs [source,log] @@ -192,33 +192,33 @@ time="2022-10-01T19:58:15+01:00" level=error msg="Couldn't start proxy, retrying Authentication errors indicate that credentials are incorrect or have insufficient permissions. -There are three sets of credentials used with {product-proxy}: +There are three sets of credentials used with {zdm-proxy}: -* Target cluster: Credentials that you set in the {product-proxy} configuration through the `ZDM_TARGET_USERNAME` and `ZDM_TARGET_PASSWORD` settings. +* Target cluster: Credentials that you set in the {zdm-proxy} configuration through the `ZDM_TARGET_USERNAME` and `ZDM_TARGET_PASSWORD` settings. -* Origin cluster: Credentials that you set in the {product-proxy} configuration through the `ZDM_ORIGIN_USERNAME` and `ZDM_ORIGIN_PASSWORD` settings. +* Origin cluster: Credentials that you set in the {zdm-proxy} configuration through the `ZDM_ORIGIN_USERNAME` and `ZDM_ORIGIN_PASSWORD` settings. * Client application: Credentials that the client application sends to the proxy during the connection handshake. These are set in the application configuration, not the proxy configuration. Authentication errors mean that at least one of these three sets of credentials is incorrect or has insufficient permissions. -If the authentication error prevents {product-proxy} from starting, then the issue is in the origin or target cluster credentials. +If the authentication error prevents {zdm-proxy} from starting, then the issue is in the origin or target cluster credentials. The log message shows whether the origin or target handshake failed. -If {product-proxy} starts but the logs contain a message like `Proxy started. Waiting for SIGINT/SIGTERM to shutdown`, then the authentication error occurs when a client application tries to open a connection to the proxy. +If {zdm-proxy} starts but the logs contain a message like `Proxy started. Waiting for SIGINT/SIGTERM to shutdown`, then the authentication error occurs when a client application tries to open a connection to the proxy. This means that the client application itself has invalid credentials, such as an incorrect username/password, expired token, or insufficient permissions. [TIP] ==== Proxy startup messages are reported at `level=info`. -If your configured log level is `warning` or `error`, you won't see these messages in the logs, and you must use another method to determine if {product-proxy} started correctly. -For example, you can check if the {product-proxy} process or Docker container is running, or check for log messages like `Error launching proxy`. +If your configured log level is `warning` or `error`, you won't see these messages in the logs, and you must use another method to determine if {zdm-proxy} started correctly. +For example, you can check if the {zdm-proxy} process or Docker container is running, or check for log messages like `Error launching proxy`. ==== -=== {product-proxy} listens on a custom port, and all applications connect to one proxy instance only +=== {zdm-proxy} listens on a custom port, and all applications connect to one proxy instance only -If {product-proxy} is listening on a custom port (not 9042), you might see either of the following issues: +If {zdm-proxy} is listening on a custom port (not 9042), you might see either of the following issues: * The Grafana dashboard shows only one proxy instance receiving all the connections from the application. * Only one proxy instance has log messages like `level=info msg="Accepted connection from 10.4.77.210:39458"`. @@ -226,7 +226,7 @@ If {product-proxy} is listening on a custom port (not 9042), you might see eithe This happens because the application specifies the custom port as part of the contact points using the format `**PROXY_IP_ADDRESS**:**CUSTOM_PORT**`. -For example, if the {product-proxy} instances were listening on port 14035, the contact points for the {cass-short} Java driver might be specified as `.addContactPoints("172.18.10.36:14035", "172.18.11.48:14035", "172.18.12.61:14035")`. +For example, if the {zdm-proxy} instances were listening on port 14035, the contact points for the {cass-short} Java driver might be specified as `.addContactPoints("172.18.10.36:14035", "172.18.11.48:14035", "172.18.12.61:14035")`. The contact point is the first point of contact to the cluster, but the driver discovers the rest of the nodes through CQL queries. However, this discovery process finds the addresses only, not the ports. @@ -245,7 +245,7 @@ For example, for the Java driver, use `.withPort(**CUSTOM_PORT**)`: === Proxy logs contain SyntaxError no viable alternative at input 'CALL' -{product-proxy} log messages such as the following indicate that the server doesn't recognize the word "CALL" in the query string, which typically means that it is a remote procedure call (RPC): +{zdm-proxy} log messages such as the following indicate that the server doesn't recognize the word "CALL" in the query string, which typically means that it is a remote procedure call (RPC): [source,log] ---- @@ -264,26 +264,26 @@ For example, in the Java driver, you can set `{java-driver-url}/blob/4.x/core/sr === Default Grafana credentials don't work -When you deploy the {product-automation} metrics component, a Grafana instance is deployed that _doesn't_ use Grafana's default `admin`/`admin` credentials. +When you deploy the {zdm-automation} metrics component, a Grafana instance is deployed that _doesn't_ use Grafana's default `admin`/`admin` credentials. -Instead, {product-automation} specifies a custom set of credentials. +Instead, {zdm-automation} specifies a custom set of credentials. -You can find the credentials for your {product-automation} Grafana instance in the `vars/zdm_monitoring_config.yml` file in the {product-automation} directory. +You can find the credentials for your {zdm-automation} Grafana instance in the `vars/zdm_monitoring_config.yml` file in the {zdm-automation} directory. You can also modify these credentials before deploying the metrics stack. === Proxy starts but client cannot connect (connection timed out or connection closed) -If the {product-proxy} logs contain messages like `Couldn't connect to`, `context timed out or cancelled while opening connection`, and `context deadline exceeded`, it can indicate that the {product-proxy} couldn't establish a connection with a particular node. +If the {zdm-proxy} logs contain messages like `Couldn't connect to`, `context timed out or cancelled while opening connection`, and `context deadline exceeded`, it can indicate that the {zdm-proxy} couldn't establish a connection with a particular node. -This can happen because {product-proxy} has connectivity to a specific subset of the nodes. -The control connection, which is established during {product-proxy} startup, cycles through the nodes until it finds one that it can connect to successfully. +This can happen because {zdm-proxy} has connectivity to a specific subset of the nodes. +The control connection, which is established during {zdm-proxy} startup, cycles through the nodes until it finds one that it can connect to successfully. For client connections, each proxy instance cycles through its assigned nodes only. Each proxy instance has a different group of _assigned nodes_, which are a subset of the cluster nodes. Generally, these are unique for each proxy instances to avoid interference with load balancing that is already in place at client-side driver level. The assigned nodes aren't necessarily contact points: Even discovered nodes undergo assignment to proxy instances. -In the following example, {product-proxy} doesn't have connectivity to `10.0.63.20`, which was chosen as the origin node for the incoming client connection, but it connected to `10.0.63.163` during startup: +In the following example, {zdm-proxy} doesn't have connectivity to `10.0.63.20`, which was chosen as the origin node for the incoming client connection, but it connected to `10.0.63.163` during startup: [source,log] ---- @@ -313,21 +313,21 @@ ERRO[0066] [openTCPConnectionWithBackoff] Couldn't connect to 10.0.63.20:9042, r ERRO[0076] Client Handler could not be created: ORIGIN-CONNECTOR context timed out or cancelled while opening connection to ORIGIN: context deadline exceeded ---- -To avoid this issue, ensure that a stable network connection exists between the {product-proxy} instances and all nodes of your origin and target clusters in the client application's local datacenter. +To avoid this issue, ensure that a stable network connection exists between the {zdm-proxy} instances and all nodes of your origin and target clusters in the client application's local datacenter. === Client application driver takes too long to reconnect to a proxy instance -When a {product-proxy} instance comes back online after being unavailable for some time, then your client application might take too long to reconnect. +When a {zdm-proxy} instance comes back online after being unavailable for some time, then your client application might take too long to reconnect. -If {product-proxy} doesn't send topology events to the client application, the driver's reconnection policy determines the time required for the driver to reconnect to the {product-proxy} instance. +If {zdm-proxy} doesn't send topology events to the client application, the driver's reconnection policy determines the time required for the driver to reconnect to the {zdm-proxy} instance. You can restart the client application to force an immediate reconnection attempt. -If you expect {product-proxy} instances to go down frequently, change the driver's reconnection policy to shorten the interval between reconnection attempts. +If you expect {zdm-proxy} instances to go down frequently, change the driver's reconnection policy to shorten the interval between reconnection attempts. -=== {astra} DevOps API errors when using {product-automation} +=== {astra} DevOps API errors when using {zdm-automation} -{product-automation} logs might report errors that contain your {astra} DevOps API endpoint: +{zdm-automation} logs might report errors that contain your {astra} DevOps API endpoint: [source,log] ---- @@ -337,11 +337,11 @@ Connection failure: Remote end closed connection without response", "redirected" ---- This can indicate that the {astra} DevOps API is temporarily unavailable. -You can either wait to retry the operation, or you can xref:astra-db-serverless:databases:secure-connect-bundle.adoc[download your database's {scb} from the {astra-ui}], and then provide the path the {scb-short} zip file in the xref:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[{product-automation} configuration]. +You can either wait to retry the operation, or you can xref:astra-db-serverless:databases:secure-connect-bundle.adoc[download your database's {scb} from the {astra-ui}], and then provide the path the {scb-short} zip file in the xref:deploy-proxy-monitoring.adoc#cluster-and-core-configuration[{zdm-automation} configuration]. === Metadata service returned not successful status code (4xx or 5xx) -If {product-proxy} doesn't start, the logs might contain messages with `not successful status code`: +If {zdm-proxy} doesn't start, the logs might contain messages with `not successful status code`: [source,log] ---- @@ -351,7 +351,7 @@ metadata service (Astra) returned not successful status code There are two possible causes for this: -* The credentials that {product-proxy} is using for {astra-db} don't have sufficient permissions. +* The credentials that {zdm-proxy} is using for {astra-db} don't have sufficient permissions. * The {astra-db} database is hibernated or otherwise unavailable. To resolve this issue, sign in to the {astra-ui}, and then check the xref:astra-db-serverless:databases:database-statuses.adoc[database status]. @@ -364,7 +364,7 @@ Try using an xref:astra-db-serverless:administration:manage-application-tokens.a [[_async_read_timeouts_stream_id_map_exhausted]] === Async read timeouts / stream id map exhausted -When dual reads are enabled, you might find the following messages in the {product-proxy} logs: +When dual reads are enabled, you might find the following messages in the {zdm-proxy} logs: [source,log] ---- @@ -388,63 +388,63 @@ In this case the async connection might not recover. Starting in version 2.1.0, you can tune the maximum number of stream IDs available per connection. The default is 2048, and you can increase it to match your driver configuration with the `xref:manage-proxy-instances.adoc#zdm_proxy_max_stream_ids[zdm_proxy_max_stream_ids]` property. -If you are running a version prior to 2.1.0, upgrade {product-proxy}. +If you are running a version prior to 2.1.0, upgrade {zdm-proxy}. -If these errors are constantly written to the log files over a period of minutes or hours, then you likely need to restart the client application _or_ {product-proxy} to fix the issue. -If you find an error like this, <> so the {product-short} team can investigate it. +If these errors are constantly written to the log files over a period of minutes or hours, then you likely need to restart the client application _or_ {zdm-proxy} to fix the issue. +If you find an error like this, <> so the {zdm-short} team can investigate it. [#client-application-closed-connection-errors-every-10-minutes-when-migrating-to-astra-db] === Client application closed connection errors every 10 minutes when migrating to {astra-db} -This issue is fixed in {product-proxy} 2.1.0. +This issue is fixed in {zdm-proxy} 2.1.0. -In {product-proxy} versions earlier than 2.1.0, the logs can report that the {astra-db} `TARGET-CONNECTOR` is disconnected every 10 minutes. +In {zdm-proxy} versions earlier than 2.1.0, the logs can report that the {astra-db} `TARGET-CONNECTOR` is disconnected every 10 minutes. This happens because {astra-db} terminates idle connections after 10 minutes of inactivity. -In the absence of asynchronous dual reads, the target cluster won't get any traffic when the client application produces only read requests because {product-short} forwards all reads to the origin cluster only. +In the absence of asynchronous dual reads, the target cluster won't get any traffic when the client application produces only read requests because {zdm-short} forwards all reads to the origin cluster only. -To resolve this issue, {company} recommends that you upgrade your {product-proxy} instances to 2.1.0 or later to take advantage of the heartbeats feature, which keeps the connection alive during periods of inactivity. -You can tune the heartbeat interval with the `xref:ROOT:manage-proxy-instances.adoc#change-mutable-config-variable[heartbeat_interval_ms]` variable, or by directly setting the `ZDM_HEARTBEAT_INTERVAL_MS` environment variable if you aren't using {product-automation}. +To resolve this issue, {company} recommends that you upgrade your {zdm-proxy} instances to 2.1.0 or later to take advantage of the heartbeats feature, which keeps the connection alive during periods of inactivity. +You can tune the heartbeat interval with the `xref:ROOT:manage-proxy-instances.adoc#change-mutable-config-variable[heartbeat_interval_ms]` variable, or by directly setting the `ZDM_HEARTBEAT_INTERVAL_MS` environment variable if you aren't using {zdm-automation}. If upgrading is impossible, you can try the following alternatives: -* Don't connect the read-only client applications to {product-proxy}, and then manually ensure that these client applications switch reads to the target at any point after all the data has been migrated, validated, and reconciled on the target cluster. +* Don't connect the read-only client applications to {zdm-proxy}, and then manually ensure that these client applications switch reads to the target at any point after all the data has been migrated, validated, and reconciled on the target cluster. * Implement a mechanism in your client application that creates a new `Session` periodically to avoid the {astra-db} inactivity timeout. * Implement a mechanism in your client application to issue a periodic meaningless write request to prevent the {astra-db} connection from becoming idle. -=== Performance degradation with {product-proxy} +=== Performance degradation with {zdm-proxy} -If you run separate benchmarks against {astra-db} directly, the origin cluster directly, and {product-proxy} with both {astra-db} and the origin cluster, then the results of these tests might show that latency or throughput is worse with {product-proxy} than when connecting to {astra-db} or origin cluster directly. +If you run separate benchmarks against {astra-db} directly, the origin cluster directly, and {zdm-proxy} with both {astra-db} and the origin cluster, then the results of these tests might show that latency or throughput is worse with {zdm-proxy} than when connecting to {astra-db} or origin cluster directly. -This is observed because {product-short} always increases latency and, depending on the nature of the test, reduces throughput. -Whether this performance hit is expected or not depends on the difference between the {product-short} test results and the test results with the cluster that performed the worst. +This is observed because {zdm-short} always increases latency and, depending on the nature of the test, reduces throughput. +Whether this performance hit is expected or not depends on the difference between the {zdm-short} test results and the test results with the cluster that performed the worst. -Writes through {product-short} require successful acknowledgement from both clusters, whereas reads require only the result from the primary cluster, which is typically the origin cluster. -This means that if the origin cluster has better performance than the target cluster, then {product-short} will inevitably have worse write performance than the target cluster alone. +Writes through {zdm-short} require successful acknowledgement from both clusters, whereas reads require only the result from the primary cluster, which is typically the origin cluster. +This means that if the origin cluster has better performance than the target cluster, then {zdm-short} will inevitably have worse write performance than the target cluster alone. -Although it is typical for latency to increase with {product-proxy}, you can minimize performance degradation with {product-proxy}: +Although it is typical for latency to increase with {zdm-proxy}, you can minimize performance degradation with {zdm-proxy}: -* Make sure your {product-proxy} infrastructure or configuration doesn't unnecessarily increase latency. +* Make sure your {zdm-proxy} infrastructure or configuration doesn't unnecessarily increase latency. + -For example, make sure your {product-proxy} instances are in the same availability zone (AZ) as your origin cluster or application instances. +For example, make sure your {zdm-proxy} instances are in the same availability zone (AZ) as your origin cluster or application instances. * Understand the impact of simple and batch statements on latency, as compared to typical prepared statements. + -Avoid simple statements with {product-proxy} because they require significant time for {product-proxy} to parse the queries. +Avoid simple statements with {zdm-proxy} because they require significant time for {zdm-proxy} to parse the queries. + As an alternative, use prepared statements, which are parsed once, and then reused on subsequent requests if repreparation isn't required. -However, inefficient use of prepared statements can degrade performance further, but this would be observed even without {product-proxy}. +However, inefficient use of prepared statements can degrade performance further, but this would be observed even without {zdm-proxy}. * Increase the number of proxies only if the VMs resources (CPU, RAM or network IO) are near capacity. -{product-proxy} doesn't use a lot of RAM, but it uses a lot of CPU and network IO. -Deploying the proxy instances on VMs with faster CPUs and faster network IO might help, but there is no standardized approach to scaling these resources for {product-proxy}. +{zdm-proxy} doesn't use a lot of RAM, but it uses a lot of CPU and network IO. +Deploying the proxy instances on VMs with faster CPUs and faster network IO might help, but there is no standardized approach to scaling these resources for {zdm-proxy}. This is because the ideal balance of resources depends on the workload type and your environment, such as network/VPC configurations and hardware. If you choose to adjust the infrastructure, you must repeat your tests to determine if there was any benefit === Permission errors related to InsightsRpc -If the {product-proxy} logs contain messages such as the following, it's likely that you have an origin {dse-short} cluster where {metrics-collector} is enabled, and the user named in the logs doesn't have sufficient permissions to report Insights data: +If the {zdm-proxy} logs contain messages such as the following, it's likely that you have an origin {dse-short} cluster where {metrics-collector} is enabled, and the user named in the logs doesn't have sufficient permissions to report Insights data: [source,log] ---- @@ -452,7 +452,7 @@ time="2023-05-05T19:14:31Z" level=debug msg="Recording ORIGIN-CONNECTOR other er time="2023-05-05T19:14:31Z" level=debug msg="Recording TARGET-CONNECTOR other error: ERROR SERVER ERROR (code=ErrorCode ServerError [0x00000000], msg=Unexpected persistence error: Unable to authorize statement com.datastax.bdp.cassandra.cql3.RpcCallStatement)" ---- -These are reported as `level=debug`, so {product-proxy} isn't affected by them. +These are reported as `level=debug`, so {zdm-proxy} isn't affected by them. There are two ways to resolve this issue: @@ -487,12 +487,12 @@ Replace **USER** with the actual username given in the logs. [#report-an-issue] == Report an issue -To report an issue or get additional support, submit an issue in the {product-short} component GitHub repositories: +To report an issue or get additional support, submit an issue in the {zdm-short} component GitHub repositories: -* {product-proxy-repo}/issues[{product-proxy} repository] -* {product-automation-repo}/issues[{product-automation} repository] (includes {product-automation} and {product-utility}) -* {cass-migrator-repo}/issues[{cass-migrator-short} repository] -* {dsbulk-repo}/issues[{dsbulk-short} repository] +* {zdm-proxy-repo-url}/issues[{zdm-proxy} repository] +* {zdm-automation-repo-url}/issues[{zdm-automation} repository] (includes {zdm-automation} and {zdm-utility}) +* {cass-migrator-repo-url}/issues[{cass-migrator-short} repository] +* {dsbulk-repo-url}/issues[{dsbulk-short} repository] [IMPORTANT] ==== @@ -503,14 +503,14 @@ Don't include any proprietary or private information in issues, pull requests, o In the issue description, include as much of the following information as possible, and make sure to remove all proprietary and private information before submitting the issue: -* Your <>. +* Your <>. -* <>, ideally at `DEBUG` level, if you can easily reproduce the issue and tolerate restarting the proxy instances to apply the log level configuration change. +* <>, ideally at `DEBUG` level, if you can easily reproduce the issue and tolerate restarting the proxy instances to apply the log level configuration change. * Database deployment type ({dse-short}, {hcd-short}, {cass-short}, or {astra-db}) and version for the origin and target clusters. The version isn't required for {astra-db}. -* Screenshots of the xref:ROOT:metrics.adoc[{product-proxy} metrics] dashboards from Grafana or your chosen visualization tool. +* Screenshots of the xref:ROOT:metrics.adoc[{zdm-proxy} metrics] dashboards from Grafana or your chosen visualization tool. + Direct read access to your metrics dashboard is preferred, if permitted by your security policy. This is particularly helpful for performance-related issues. @@ -534,8 +534,8 @@ For performance-related issues, provide the following additional information: This feature is disabled by default. To determine if this feature is enabled, check the following variables: + -** If you use {product-automation}, check the Ansible advanced configuration variable `replace_cql_functions`. -** If you don't use {product-automation}, check the environment variable `ZDM_REPLACE_CQL_FUNCTIONS`. +** If you use {zdm-automation}, check the Ansible advanced configuration variable `replace_cql_functions`. +** If you don't use {zdm-automation}, check the environment variable `ZDM_REPLACE_CQL_FUNCTIONS`. == See also diff --git a/modules/ROOT/pages/zdm-logs.adoc b/modules/ROOT/pages/zdm-logs.adoc index e0eab094..70947927 100644 --- a/modules/ROOT/pages/zdm-logs.adoc +++ b/modules/ROOT/pages/zdm-logs.adoc @@ -1,35 +1,35 @@ -= Configure and read {product-proxy} logs -:navtitle: Get {product-short} logs -:description: Configure and retrieve {product-proxy} logs. += Configure and read {zdm-proxy} logs +:navtitle: Get {zdm-short} logs +:description: Configure and retrieve {zdm-proxy} logs. include::ROOT:partial$zdm-logs-intro.adoc[] [#set-the-zdm-proxy-log-level] -== Set the {product-proxy} log level +== Set the {zdm-proxy} log level -Set the {product-proxy} log level to print the messages that you need. +Set the {zdm-proxy} log level to print the messages that you need. The default log level is `INFO`, which is adequate for most logging. If you need more detail for temporary troubleshooting, you can set the log level to `DEBUG`. However, this can slightly degrade performance, and {company} recommends that you revert to `INFO` logging as soon as possible. -How you set the log level depends on how you deployed {product-proxy}: +How you set the log level depends on how you deployed {zdm-proxy}: -* If you used {product-automation} to deploy {product-proxy}, set `log_level` in `vars/zdm_proxy_core_config.yml`, and then run the `rolling_update_zdm_proxy.yml` playbook. +* If you used {zdm-automation} to deploy {zdm-proxy}, set `log_level` in `vars/zdm_proxy_core_config.yml`, and then run the `rolling_update_zdm_proxy.yml` playbook. For more information, see xref:manage-proxy-instances.adoc#change-mutable-config-variable[Change a mutable configuration variable]. -* If you didn't use {product-automation} to deploy {product-proxy}, set the `ZDM_LOG_LEVEL` environment variable on each proxy instance, and then restart each instance. +* If you didn't use {zdm-automation} to deploy {zdm-proxy}, set the `ZDM_LOG_LEVEL` environment variable on each proxy instance, and then restart each instance. -== Get {product-proxy} log files +== Get {zdm-proxy} log files -If you used {product-automation} to deploy {product-proxy}, then you can get logs for a single proxy instance, and you can use a playbook to retrieve logs for all instances. +If you used {zdm-automation} to deploy {zdm-proxy}, then you can get logs for a single proxy instance, and you can use a playbook to retrieve logs for all instances. === View or tail logs for one instance -{product-proxy} runs as a Docker container on each proxy host. +{zdm-proxy} runs as a Docker container on each proxy host. -To view the logs for a single {product-proxy} instance, connect to a proxy host, and then run the following command: +To view the logs for a single {zdm-proxy} instance, connect to a proxy host, and then run the following command: [source,bash] ---- @@ -47,7 +47,7 @@ Keep in mind that Docker logs are deleted if the container is re-created. === Collect logs for multiple instances -{product-automation} has a dedicated playbook, `collect_zdm_proxy_logs.yml`, that you can use to collect logs for all {product-proxy} instances in a deployment. +{zdm-automation} has a dedicated playbook, `collect_zdm_proxy_logs.yml`, that you can use to collect logs for all {zdm-proxy} instances in a deployment. You can view the playbook's configuration in `vars/zdm_proxy_log_collection_config.yml`, but no changes are required to run it. @@ -87,9 +87,9 @@ Replace the following: * `**TIMESTAMP**`: The timestamp from the name of your log file archive * `**DESTINATION_DIRECTORY_ON_JUMPHOST**`: The path to the directory where you want to copy the archive -=== Get logs for deployments that don't use {product-automation} +=== Get logs for deployments that don't use {zdm-automation} -If you didn't use {product-automation} to deploy {product-proxy}, you must access the logs another way, depending on your deployment configuration and infrastructure. +If you didn't use {zdm-automation} to deploy {zdm-proxy}, you must access the logs another way, depending on your deployment configuration and infrastructure. For example, if you used Docker, you can use the following command to export a container's logs to a `log.txt` file: @@ -114,17 +114,17 @@ However, they can help you find the source of a problem by providing information * `level=warn`: Reports an event that wasn't fatal to the overall process but might indicate an issue with an individual request or connection. -* `level=error`: Indicates an issue with {product-proxy}, the client application, or the clusters. +* `level=error`: Indicates an issue with {zdm-proxy}, the client application, or the clusters. These messages require further examination. If the meaning of a `warn` or `error` message isn't clear, you can <>. == Common log messages -Here are some of the most common messages in the {product-proxy} logs. +Here are some of the most common messages in the {zdm-proxy} logs. -{product-proxy} startup message:: -If the log level doesn't filter out `info` entries, you can look for a `Proxy started` log message to verify that {product-proxy} started correctly. +{zdm-proxy} startup message:: +If the log level doesn't filter out `info` entries, you can look for a `Proxy started` log message to verify that {zdm-proxy} started correctly. For example: + [source,json] @@ -134,8 +134,8 @@ msg=\"Proxy started. Waiting for SIGINT/SIGTERM to shutdown. \"\n","stream":"stderr","time":"2023-01-13T11:50:48.522097083Z"} ---- -{product-proxy} configuration message:: -If the log level doesn't filter out `info` entries, the first few lines of a {product-proxy} log file contain all configuration variables and values in a long JSON string. +{zdm-proxy} configuration message:: +If the log level doesn't filter out `info` entries, the first few lines of a {zdm-proxy} log file contain all configuration variables and values in a long JSON string. + The following example log message is truncated for readability: + diff --git a/modules/ROOT/pages/zdm-proxy-migration-paths.adoc b/modules/ROOT/pages/zdm-proxy-migration-paths.adoc index 6c13c5c0..734c75c5 100644 --- a/modules/ROOT/pages/zdm-proxy-migration-paths.adoc +++ b/modules/ROOT/pages/zdm-proxy-migration-paths.adoc @@ -1,5 +1,5 @@ -= Cluster compatibility for {product} -:description: Learn which sources and targets are eligible for {product}. += Cluster compatibility for {zdm} +:description: Learn which sources and targets are eligible for {zdm}. True zero downtime migration is only possible if your database meets the minimum requirements described in xref:ROOT:feasibility-checklists.adoc[], including compatibility of the origin (source) and target (destination) clusters. @@ -10,9 +10,9 @@ include::ROOT:partial$migration-scenarios.adoc[] [#incompatible-clusters-and-migrations-with-some-downtime] == Incompatible clusters and migrations with some downtime -If you don't want to use {product-proxy} or your databases don't meet the zero-downtime requirements, you can still complete the migration, but some downtime might be necessary to finish the migration. +If you don't want to use {zdm-proxy} or your databases don't meet the zero-downtime requirements, you can still complete the migration, but some downtime might be necessary to finish the migration. -If your origin cluster is incompatible with {product-proxy}, {product-utility}, and {product-automation}, you might be able to use standalone xref:ROOT:components.adoc#data-migration-tools[data migration tools] such as {dsbulk-short} or a custom data migration script. +If your origin cluster is incompatible with {zdm-proxy}, {zdm-utility}, and {zdm-automation}, you might be able to use standalone xref:ROOT:components.adoc#data-migration-tools[data migration tools] such as {dsbulk-short} or a custom data migration script. Make sure you transform or prepare the data to comply with the target cluster's schema. For more complex migrations, such as RDBMS-to-NoSQL migrations, it is likely that your migration will require downtime for additional processing, such as extract, transform, and load (ETL) operations. diff --git a/modules/ROOT/partials/cassandra-protocol-versions.adoc b/modules/ROOT/partials/cassandra-protocol-versions.adoc index c52b697c..842476da 100644 --- a/modules/ROOT/partials/cassandra-protocol-versions.adoc +++ b/modules/ROOT/partials/cassandra-protocol-versions.adoc @@ -1 +1 @@ -{product-proxy} supports protocol `V2`, `V3`, `V4`, `V5`, `DSE_V1`, and `DSE_V2`. \ No newline at end of file +{zdm-proxy} supports protocol `V2`, `V3`, `V4`, `V5`, `DSE_V1`, and `DSE_V2`. \ No newline at end of file diff --git a/modules/ROOT/partials/dsbulk-migrator-deprecation.adoc b/modules/ROOT/partials/dsbulk-migrator-deprecation.adoc index 36aaedeb..6f4b60db 100644 --- a/modules/ROOT/partials/dsbulk-migrator-deprecation.adoc +++ b/modules/ROOT/partials/dsbulk-migrator-deprecation.adoc @@ -1,3 +1,3 @@ -The {dsbulk-migrator} tool, which was an extension of {dsbulk}, is deprecated. +The {dsbulk-short} Migrator tool, which was an extension of {dsbulk}, is deprecated. This tool is no longer recommended. Instead, use the unload, load, and count commands included with {dsbulk-short}, or use another data migration tool, such as {cass-migrator-short}. \ No newline at end of file diff --git a/modules/ROOT/partials/migration-scenarios.adoc b/modules/ROOT/partials/migration-scenarios.adoc index fa943ddb..36c88071 100644 --- a/modules/ROOT/partials/migration-scenarios.adoc +++ b/modules/ROOT/partials/migration-scenarios.adoc @@ -1,7 +1,7 @@ -You can use {product-proxy} to support migrations between {astra}, {dse}, {hcd}, open-source {cass-reg}, and and other {cass-short}-based databases. +You can use {zdm-proxy} to support migrations between {astra}, {dse}, {hcd}, open-source {cass-reg}, and and other {cass-short}-based databases. Supported migration paths include cross-platform migrations and same-platform upgrades. -{company} tests {product-proxy} compatibility with {astra}, {dse-short}, {hcd-short}, and open-source {cass-short}. +{company} tests {zdm-proxy} compatibility with {astra}, {dse-short}, {hcd-short}, and open-source {cass-short}. Support for other {cass-short}-based databases is possible if the origin and target clusters share a common xref:ROOT:feasibility-checklists.adoc[protocol version]. However, {company} doesn't test all data store providers, and {company} doesn't guarantee full support for any specific data store aside from the platforms and versions listed here. @@ -13,7 +13,7 @@ Migrate from one of the following: * {dse} version 4.7.1 and later. * {cass-reg} version 2.1.6 and later. + -{product-proxy} requires that the origin and target clusters share a common xref:ROOT:feasibility-checklists.adoc[protocol version]. +{zdm-proxy} requires that the origin and target clusters share a common xref:ROOT:feasibility-checklists.adoc[protocol version]. Therefore, {cass-short} 2.0 migrations are only possible when migrating to 2.1 or 2.2 because {cass-short} 2.0 supports only `v2`. * Other {cass-short}-based databases that are based on a compatible {cass-short} version, such as ScyllaDB and Yugabyte. @@ -31,6 +31,6 @@ Check the xref:6.9@dse:planning:compatibility.adoc[built-in {cass-short} version [TIP] ==== -You can use {product-short} for major version upgrades to your current database platform, such as upgrades from {dse-short} 5.0 to {dse-short} 6.9. -Using {product-short} reduces the risk of data loss or corruption due to breaking changes between versions, provides a seamless rollback option, and streamlines the upgrade process, eliminating the need for interim upgrades and progressive manual reconfiguration. +You can use {zdm-short} for major version upgrades to your current database platform, such as upgrades from {dse-short} 5.0 to {dse-short} 6.9. +Using {zdm-short} reduces the risk of data loss or corruption due to breaking changes between versions, provides a seamless rollback option, and streamlines the upgrade process, eliminating the need for interim upgrades and progressive manual reconfiguration. ==== \ No newline at end of file diff --git a/modules/ROOT/partials/zdm-logs-intro.adoc b/modules/ROOT/partials/zdm-logs-intro.adoc index 04bd663d..9486978a 100644 --- a/modules/ROOT/partials/zdm-logs-intro.adoc +++ b/modules/ROOT/partials/zdm-logs-intro.adoc @@ -1 +1 @@ -{product-proxy} logs can help you verify that your {product-proxy} instances are operating normally, investigate how processes are executed, and troubleshoot issues. \ No newline at end of file +{zdm-proxy} logs can help you verify that your {zdm-proxy} instances are operating normally, investigate how processes are executed, and troubleshoot issues. \ No newline at end of file diff --git a/modules/sideloader/pages/prepare-sideloader.adoc b/modules/sideloader/pages/prepare-sideloader.adoc index 1f3ce2d9..f06a37fb 100644 --- a/modules/sideloader/pages/prepare-sideloader.adoc +++ b/modules/sideloader/pages/prepare-sideloader.adoc @@ -161,7 +161,7 @@ If you choose the alternative option, you must modify the commands accordingly f * *{sstable-sideloader} doesn't support encrypted data*: If your origin cluster uses xref:6.9@dse:securing:transparent-data-encryption.adoc[{dse-short} Transparent Data Encryption], be aware that {sstable-sideloader} cannot migrate these SSTables. + If you have a mix of encrypted and unencrypted data, you can use {sstable-sideloader} to migrate the unencrypted data. -After the initial migration, you can use another strategy to move the encrypted data, such as {cass-migrator-repo}[{cass-migrator}] or a manual export and reupload. +After the initial migration, you can use another strategy to move the encrypted data, such as {cass-migrator-repo-url}[{cass-migrator}] or a manual export and reupload. * *{sstable-sideloader} doesn't support secondary indexes*: If you don't remove or replace these in your origin cluster, {sstable-sideloader} ignores these directories when importing the data to your {astra-db} database. @@ -173,7 +173,7 @@ Your administration server must have SSH access to each node in your origin clus {company} recommends that you install the following additional software on your administration server: -* {cass-migrator-repo}[{cass-migrator-short}] to validate imported data and, with {product-proxy}, reconcile it with the origin cluster. +* {cass-migrator-repo-url}[{cass-migrator-short}] to validate imported data and, with {zdm-proxy}, reconcile it with the origin cluster. * https://jqlang.github.io/jq/[jq] to format JSON responses from the {astra} {devops-api}. The {devops-api} commands in this guide use this tool. @@ -237,7 +237,7 @@ The number of node snapshots that you uploaded to the migration bucket doesn't d The success of the import depends primarily on the validity of the schemas and the data in the snapshots. . After the import, validate the migrated data to ensure that it matches the data in the origin cluster. -For example, you can xref:ROOT:cassandra-data-migrator.adoc#cdm-validation-steps[run {cass-migrator-short} in validation mode]. +For example, you can {cass-migrator-repo-url}?tab=readme-ov-file#steps-for-data-validation[run {cass-migrator-short} in validation mode]. ==== Migrate multiple nodes to multiple databases @@ -278,7 +278,7 @@ The number of node snapshots that you uploaded to the migration bucket doesn't d The success of the import depends primarily on the validity of the schemas and the data in the snapshots.\ . After the import, validate the migrated data to ensure that it matches the data in the origin cluster. -For example, you can xref:ROOT:cassandra-data-migrator.adoc#cdm-validation-steps[run {cass-migrator-short} in validation mode]. +For example, you can {cass-migrator-repo-url}?tab=readme-ov-file#steps-for-data-validation[run {cass-migrator-short} in validation mode]. === Multiple migrations to the same database @@ -297,7 +297,7 @@ For example, if you have 10 migration IDs for the same database, you must run 10 Each import must completely finish before starting the next import. After all of the imports are complete, validate the migrated data in your target database to ensure that it matches the data in the origin cluster. -For example, you can xref:ROOT:cassandra-data-migrator.adoc#cdm-validation-steps[run {cass-migrator-short} in validation mode]. +For example, you can {cass-migrator-repo-url}?tab=readme-ov-file#steps-for-data-validation[run {cass-migrator-short} in validation mode]. == Next steps diff --git a/modules/sideloader/pages/sideloader-overview.adoc b/modules/sideloader/pages/sideloader-overview.adoc index c6a9c73a..491b5ea4 100644 --- a/modules/sideloader/pages/sideloader-overview.adoc +++ b/modules/sideloader/pages/sideloader-overview.adoc @@ -4,7 +4,7 @@ {sstable-sideloader} is a service running in {astra-db} that directly imports data from snapshot backups that you've uploaded to {astra-db} from an existing {cass-reg}, {dse}, or {hcd} cluster. -Because it imports data directly, {sstable-sideloader} can offer several advantages over CQL-based tools like xref:dsbulk:overview:dsbulk-about.adoc[{dsbulk}] and xref:ROOT:cassandra-data-migrator.adoc[{cass-migrator}], including faster, more cost-effective data loading, and minimal performance impacts on your origin cluster and target database. +Because it imports data directly, {sstable-sideloader} can offer several advantages over CQL-based tools like xref:dsbulk:overview:dsbulk-about.adoc[{dsbulk}] and {cass-migrator-repo-url}[{cass-migrator}], including faster, more cost-effective data loading, and minimal performance impacts on your origin cluster and target database. == {sstable-sideloader} concepts @@ -44,7 +44,7 @@ Each snapshot for each node in the origin cluster must include all the keyspaces These snapshots are ideal for database migrations because creating snapshots has a negligible performance impact on the origin cluster, and the snapshots preserve metadata like `writetime` and `ttl` values. -When using {sstable-sideloader} with {product-proxy}, {cass-short}'s last-write-wins semantics ensure that new, real-time writes accurately take precedence over historical writes. +When using {sstable-sideloader} with {zdm-proxy}, {cass-short}'s last-write-wins semantics ensure that new, real-time writes accurately take precedence over historical writes. Last-write-wins compares the `writetime` of conflicting records, and then retains the most recent write. For example, if a new write occurs in your target database with a `writetime` of `2023-10-01T12:05:00Z`, and then {sstable-sideloader} migrates a record against the same row with a `writetime` of `2023-10-01T12:00:00Z`, the target database retains the data from the new write because it has the most recent `writetime`. @@ -157,7 +157,7 @@ Once {sstable-sideloader} begins to import SSTable metadata (the next step), you + If the dataset contains tombstones, any read operations on the target database can return inconsistent results during this step. Since compaction is disabled, there is no risk of permanent inconsistencies. -However, in the context of xref:ROOT:introduction.adoc[{product}], it's important that the {product-short} proxy continues to read from the origin cluster. +However, in the context of xref:ROOT:introduction.adoc[{zdm}], it's important that the {zdm-short} proxy continues to read from the origin cluster. . Re-enables compactions on the {astra-db} Serverless database. @@ -172,11 +172,11 @@ For instructions and more information, see xref:sideloader:migrate-sideloader.ad include::sideloader:partial$validate.adoc[] -== Use {sstable-sideloader} with {product-proxy} +== Use {sstable-sideloader} with {zdm-proxy} -If you need to migrate a live database, you can use {sstable-sideloader} instead of {dsbulk-short} or {cass-migrator-short} during of xref:ROOT:migrate-and-validate-data.adoc[Phase 2 of {product}]. +If you need to migrate a live database, you can use {sstable-sideloader} instead of {dsbulk-short} or {cass-migrator-short} during of xref:ROOT:migrate-and-validate-data.adoc[Phase 2 of {zdm}]. -.Use {sstable-sideloader} with {product-proxy} +.Use {sstable-sideloader} with {zdm-proxy} svg::sideloader:astra-migration-toolkit.svg[] == Next steps diff --git a/modules/sideloader/partials/validate.adoc b/modules/sideloader/partials/validate.adoc index bac5da4d..df391ab1 100644 --- a/modules/sideloader/partials/validate.adoc +++ b/modules/sideloader/partials/validate.adoc @@ -1,4 +1,4 @@ After the migration is complete, you can query the migrated data using the xref:astra-db-serverless:cql:develop-with-cql.adoc#connect-to-the-cql-shell[{cql-shell}] (`cqlsh`) or xref:astra-db-serverless:api-reference:row-methods/find-many.adoc[{data-api}]. -You can xref:ROOT:cassandra-data-migrator.adoc#cdm-validation-steps[run {cass-migrator-short} in validation mode] for more thorough validation. +You can {cass-migrator-repo-url}?tab=readme-ov-file#steps-for-data-validation[run {cass-migrator-short} in validation mode] for more thorough validation. {cass-migrator-short} also offers an AutoCorrect mode to reconcile any differences that it detects. \ No newline at end of file From a714614b2859497a50cc197d8627c95ad001c2de Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Mon, 1 Jun 2026 09:32:46 -0700 Subject: [PATCH 2/2] fix attributes because I am a dumdum today --- antora.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/antora.yml b/antora.yml index f5686ba8..052c8915 100644 --- a/antora.yml +++ b/antora.yml @@ -42,13 +42,13 @@ asciidoc: devops-api: 'DevOps API' devops-api-ref-url: 'xref:astra-api-docs:ROOT:attachment$devops-api/index.html' data-api: 'Data API' - product: 'Zero Downtime Migration (ZDM)' - product-short: 'ZDM' - product-proxy: 'ZDM Proxy' - product-proxy-repo: 'https://github.com/datastax/zdm-proxy' - product-utility: 'ZDM Utility' - product-automation: 'ZDM Proxy Automation' - product-automation-repo: 'https://github.com/datastax/zdm-proxy-automation' + zdm: 'Zero Downtime Migration (ZDM)' + zdm-short: 'ZDM' + zdm-proxy: 'ZDM Proxy' + zdm-proxy-repo-url: 'https://github.com/datastax/zdm-proxy' + zdm-utility: 'ZDM Utility' + zdm-automation: 'ZDM Proxy Automation' + zdm-automation-repo-url: 'https://github.com/datastax/zdm-proxy-automation' dsbulk: 'DataStax Bulk Loader (DSBulk)' dsbulk-short: 'DSBulk' dsbulk-repo-url: 'https://github.com/datastax/dsbulk'