From ff822d80bd8ad534ee6964b33d4e1ff1fb394734 Mon Sep 17 00:00:00 2001 From: unset Date: Wed, 22 Apr 2026 19:25:13 +0530 Subject: [PATCH 1/3] Add custom database/namespace/schema support to tableflow catalog integration resources Co-Authored-By: Claude Opus 4.6 (1M context) --- .golangci.yml | 3 + .idea/.gitignore | 10 +++ .idea/cli.iml | 9 +++ .idea/go.imports.xml | 10 +++ .idea/golinter.xml | 7 ++ .idea/modules.xml | 8 +++ .idea/vcs.xml | 6 ++ go.mod | 2 + go.sum | 4 +- .../control-center-local.properties | 2 + .../tableflow/command_catalog_integration.go | 6 ++ .../command_catalog_integration_create.go | 65 +++++++++++++------ .../command_catalog_integration_list.go | 3 + .../command_catalog_integration_update.go | 47 +++++++++++++- logs.txt | 2 + .../create-aws-glue-custom-database.golden | 11 ++++ .../create-snowflake-custom-namespace.golden | 13 ++++ .../create-unity-custom-schema.golden | 13 ++++ .../describe-aws-glue-json.golden | 1 + .../describe-aws-glue.golden | 1 + .../describe-snowflake.golden | 25 +++---- .../catalog-integration/describe-unity.golden | 1 + .../catalog-integration/list-json.golden | 3 + .../tableflow/catalog-integration/list.golden | 10 +-- .../update-aws-glue-custom-database.golden | 11 ++++ .../update-fail-no-flags.golden | 25 +++---- .../catalog-integration/update-name.golden | 1 + .../update-snowflake-custom-namespace.golden | 10 +++ .../update-snowflake-with-name.golden | 25 +++---- .../update-snowflake.golden | 25 +++---- .../update-unity-custom-schema.golden | 13 ++++ test/tableflow_test.go | 6 ++ test/test-server/tableflow_handlers.go | 41 ++++++++---- 33 files changed, 333 insertions(+), 86 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/cli.iml create mode 100644 .idea/go.imports.xml create mode 100644 .idea/golinter.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 internal/local/dir/etc/confluent-control-center/control-center-local.properties create mode 100644 logs.txt create mode 100644 test/fixtures/output/tableflow/catalog-integration/create-aws-glue-custom-database.golden create mode 100644 test/fixtures/output/tableflow/catalog-integration/create-snowflake-custom-namespace.golden create mode 100644 test/fixtures/output/tableflow/catalog-integration/create-unity-custom-schema.golden create mode 100644 test/fixtures/output/tableflow/catalog-integration/update-aws-glue-custom-database.golden create mode 100644 test/fixtures/output/tableflow/catalog-integration/update-snowflake-custom-namespace.golden create mode 100644 test/fixtures/output/tableflow/catalog-integration/update-unity-custom-schema.golden diff --git a/.golangci.yml b/.golangci.yml index 0a5c1915f4..06ef6c5337 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,3 +32,6 @@ linters-settings: custom-order: true nakedret: max-func-lines: 0 + gomoddirectives: + replace-allow-list: + - "github.com/confluentinc/ccloud-sdk-go-v2/tableflow" diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000000..30cf57ed7c --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/cli.iml b/.idea/cli.iml new file mode 100644 index 0000000000..5e764c4f0b --- /dev/null +++ b/.idea/cli.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml new file mode 100644 index 0000000000..644cdf0bd2 --- /dev/null +++ b/.idea/go.imports.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/golinter.xml b/.idea/golinter.xml new file mode 100644 index 0000000000..1ccf3ec6d9 --- /dev/null +++ b/.idea/golinter.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000000..6fcc0dcf27 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000..35eb1ddfbb --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/go.mod b/go.mod index 4a2b92cd5b..7242b40caa 100644 --- a/go.mod +++ b/go.mod @@ -288,3 +288,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.5.0 => github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0 diff --git a/go.sum b/go.sum index 69fbe81319..650b2ee46e 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,8 @@ github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/confluentinc/ccloud-sdk-go-v1-public v0.0.0-20250521223017-0e8f6f971b52 h1:19qEGhkbZa5fopKCe0VPIV+Sasby4Pv10z9ZaktwWso= github.com/confluentinc/ccloud-sdk-go-v1-public v0.0.0-20250521223017-0e8f6f971b52/go.mod h1:62EMf+5uFEt1BJ2q8WMrUoI9VUSxAbDnmZCGRt/MbA0= +github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0 h1:hGvPz/GeneUPf1Lq1nGiqU34R4QoyqR9cpaekUl4+l4= +github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0/go.mod h1:l2yBTc+QGoxMSrilWaTt3t46BhiaeWhmLdh6q6+gC3E= github.com/confluentinc/ccloud-sdk-go-v2/ai v0.1.0 h1:zSF4OQUJXWH2JeAo9rsq13ibk+JFdzITGR8S7cFMpzw= github.com/confluentinc/ccloud-sdk-go-v2/ai v0.1.0/go.mod h1:DoxqzzF3JzvJr3fWkvCiOHFlE0GoYpozWxFZ1Ud9ntA= github.com/confluentinc/ccloud-sdk-go-v2/apikeys v0.4.0 h1:8fWyLwMuy8ec0MVF5Avd54UvbIxhDFhZzanHBVwgxdw= @@ -258,8 +260,6 @@ github.com/confluentinc/ccloud-sdk-go-v2/srcm v0.7.3 h1:ozdDSJHruQIgtxS5hwz8Rp8p github.com/confluentinc/ccloud-sdk-go-v2/srcm v0.7.3/go.mod h1:cD0AeCMBAWBesmWxWCMgVYNABYgHJ/ahCj7b4HP2R2I= github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 h1:WZJYfgXJrvTIYQpCFps/qHF7T8ekgPlX/SFqx4EY2zQ= github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1/go.mod h1:kB+MXWYYg9ohrTCb27LlfpTbuexAzyYAmum105ow0ho= -github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.5.0 h1:Wh3+AsUCncoxRPfs0zCJwBY6/FiJfyN9Q/XO8e6sMRI= -github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.5.0/go.mod h1:unZupel8OU3/o8MRcL9YiJo+56MalsCtHHCc/ZNi0BI= github.com/confluentinc/ccloud-sdk-go-v2/usm v0.1.0 h1:rF9cKecDCowq+oDWjf8rSpXXZHAnVXowIsT/OXF4MOI= github.com/confluentinc/ccloud-sdk-go-v2/usm v0.1.0/go.mod h1:umhEDvQp/5h0ALKBpYTQOmFwaWrvilnbE8Rkzh6oJ4Q= github.com/confluentinc/cmf-sdk-go v0.0.5 h1:TS6S3ClVsM1kanB00mlcmqXczozDTO2t4Du5blDSYvE= diff --git a/internal/local/dir/etc/confluent-control-center/control-center-local.properties b/internal/local/dir/etc/confluent-control-center/control-center-local.properties new file mode 100644 index 0000000000..8602bca970 --- /dev/null +++ b/internal/local/dir/etc/confluent-control-center/control-center-local.properties @@ -0,0 +1,2 @@ +confluent.controlcenter.alertmanager.config.file=abc +confluent.controlcenter.prometheus.rules.file=def diff --git a/internal/tableflow/command_catalog_integration.go b/internal/tableflow/command_catalog_integration.go index 3a9a33780d..9c09ea791f 100644 --- a/internal/tableflow/command_catalog_integration.go +++ b/internal/tableflow/command_catalog_integration.go @@ -39,6 +39,9 @@ type catalogIntegrationOut struct { WorkspaceEndpoint string `human:"Workspace Endpoint,omitempty" serialized:"workspace_endpoint,omitempty"` CatalogName string `human:"Catalog Name,omitempty" serialized:"catalog_name,omitempty"` ClientId string `human:"Client ID,omitempty" serialized:"client_id,omitempty"` + CustomDatabase string `human:"Custom Database,omitempty" serialized:"custom_database,omitempty"` + CustomNamespace string `human:"Custom Namespace,omitempty" serialized:"custom_namespace,omitempty"` + CustomSchema string `human:"Custom Schema,omitempty" serialized:"custom_schema,omitempty"` Suspended bool `human:"Suspended" serialized:"suspended"` Phase string `human:"Phase" serialized:"phase"` ErrorMessage string `human:"Error Message,omitempty" serialized:"error_message,omitempty"` @@ -140,16 +143,19 @@ func printCatalogIntegrationTable(cmd *cobra.Command, catalogIntegration tablefl if catalogIntegrationType == aws { out.ProviderIntegrationId = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueSpec.GetProviderIntegrationId() + out.CustomDatabase = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueSpec.GetCustomDatabase() } if catalogIntegrationType == snowflake { out.Endpoint = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetEndpoint() out.Warehouse = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetWarehouse() out.AllowedScope = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetAllowedScope() + out.CustomNamespace = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetCustomNamespace() } if catalogIntegrationType == unity { out.WorkspaceEndpoint = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetWorkspaceEndpoint() out.CatalogName = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetCatalogName() out.ClientId = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetClientId() + out.CustomSchema = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetCustomSchema() } table := output.NewTable(cmd) diff --git a/internal/tableflow/command_catalog_integration_create.go b/internal/tableflow/command_catalog_integration_create.go index c2c860ef7a..85e43f4d54 100644 --- a/internal/tableflow/command_catalog_integration_create.go +++ b/internal/tableflow/command_catalog_integration_create.go @@ -37,15 +37,18 @@ func (c *command) newCatalogIntegrationCreateCommand() *cobra.Command { addCatalogIntegrationTypeFlag(cmd) cmd.Flags().String("provider-integration", "", "Specify the provider integration id.") + cmd.Flags().String("custom-database", "", "Specify the custom database name for AWS Glue.") cmd.Flags().String("endpoint", "", "Specify the The catalog integration connection endpoint for Snowflake Open Catalog.") cmd.Flags().String("warehouse", "", "Specify the warehouse name of the Snowflake Open Catalog.") cmd.Flags().String("allowed-scope", "", "Specify the allowed scope of the Snowflake Open Catalog.") cmd.Flags().String("client-id", "", "Specify the client id.") cmd.Flags().String("client-secret", "", "Specify the client secret.") + cmd.Flags().String("custom-namespace", "", "Specify the custom namespace for Snowflake Open Catalog.") cmd.Flags().String("workspace-endpoint", "", "Specify the Databricks workspace URL associated with the Unity Catalog.") cmd.Flags().String("catalog-name", "", "Specify the name of the catalog.") cmd.Flags().String("unity-client-id", "", "Specify the Unity client id.") cmd.Flags().String("unity-client-secret", "", "Specify the Unity client secret.") + cmd.Flags().String("custom-schema", "", "Specify the custom schema name for Unity Catalog.") pcmd.AddClusterFlag(cmd, c.AuthenticatedCLICommand) pcmd.AddEnvironmentFlag(cmd, c.AuthenticatedCLICommand) @@ -94,11 +97,19 @@ func (c *command) createCatalogIntegration(cmd *cobra.Command, args []string) er return err } + awsGlueSpec := &tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpec{ + Kind: awsGlueKind, + ProviderIntegrationId: providerIntegration, + } + if cmd.Flags().Changed("custom-database") { + customDatabase, err := cmd.Flags().GetString("custom-database") + if err != nil { + return err + } + awsGlueSpec.SetCustomDatabase(customDatabase) + } createCatalogIntegration.Spec.Config = &tableflowv1.TableflowV1CatalogIntegrationSpecConfigOneOf{ - TableflowV1CatalogIntegrationAwsGlueSpec: &tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpec{ - Kind: awsGlueKind, - ProviderIntegrationId: providerIntegration, - }, + TableflowV1CatalogIntegrationAwsGlueSpec: awsGlueSpec, } } else if strings.ToLower(catalogIntegrationType) == snowflake { if !cmd.Flags().Changed("endpoint") { // we only need to check for one since this flag set is marked as required together @@ -125,15 +136,23 @@ func (c *command) createCatalogIntegration(cmd *cobra.Command, args []string) er return err } + snowflakeSpec := &tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpec{ + Kind: snowflakeKind, + Endpoint: endpoint, + Warehouse: warehouse, + AllowedScope: allowedScope, + ClientId: clientId, + ClientSecret: clientSecret, + } + if cmd.Flags().Changed("custom-namespace") { + customNamespace, err := cmd.Flags().GetString("custom-namespace") + if err != nil { + return err + } + snowflakeSpec.SetCustomNamespace(customNamespace) + } createCatalogIntegration.Spec.Config = &tableflowv1.TableflowV1CatalogIntegrationSpecConfigOneOf{ - TableflowV1CatalogIntegrationSnowflakeSpec: &tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpec{ - Kind: snowflakeKind, - Endpoint: endpoint, - Warehouse: warehouse, - AllowedScope: allowedScope, - ClientId: clientId, - ClientSecret: clientSecret, - }, + TableflowV1CatalogIntegrationSnowflakeSpec: snowflakeSpec, } } else if strings.ToLower(catalogIntegrationType) == unity { if !cmd.Flags().Changed("workspace-endpoint") { // we only need to check for one since this flag set is marked as required together @@ -156,14 +175,22 @@ func (c *command) createCatalogIntegration(cmd *cobra.Command, args []string) er return err } + unitySpec := &tableflowv1.TableflowV1CatalogIntegrationUnitySpec{ + Kind: unityKind, + WorkspaceEndpoint: workspaceEndpoint, + CatalogName: catalogName, + ClientId: clientId, + ClientSecret: clientSecret, + } + if cmd.Flags().Changed("custom-schema") { + customSchema, err := cmd.Flags().GetString("custom-schema") + if err != nil { + return err + } + unitySpec.SetCustomSchema(customSchema) + } createCatalogIntegration.Spec.Config = &tableflowv1.TableflowV1CatalogIntegrationSpecConfigOneOf{ - TableflowV1CatalogIntegrationUnitySpec: &tableflowv1.TableflowV1CatalogIntegrationUnitySpec{ - Kind: unityKind, - WorkspaceEndpoint: workspaceEndpoint, - CatalogName: catalogName, - ClientId: clientId, - ClientSecret: clientSecret, - }, + TableflowV1CatalogIntegrationUnitySpec: unitySpec, } } diff --git a/internal/tableflow/command_catalog_integration_list.go b/internal/tableflow/command_catalog_integration_list.go index 34fee68217..ef0639ff8a 100644 --- a/internal/tableflow/command_catalog_integration_list.go +++ b/internal/tableflow/command_catalog_integration_list.go @@ -67,16 +67,19 @@ func (c *command) listCatalogIntegration(cmd *cobra.Command, args []string) erro if catalogIntegrationType == aws { out.ProviderIntegrationId = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueSpec.GetProviderIntegrationId() + out.CustomDatabase = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueSpec.GetCustomDatabase() } if catalogIntegrationType == snowflake { out.Endpoint = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetEndpoint() out.Warehouse = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetWarehouse() out.AllowedScope = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetAllowedScope() + out.CustomNamespace = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeSpec.GetCustomNamespace() } if catalogIntegrationType == unity { out.WorkspaceEndpoint = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetWorkspaceEndpoint() out.CatalogName = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetCatalogName() out.ClientId = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetClientId() + out.CustomSchema = catalogIntegration.Spec.GetConfig().TableflowV1CatalogIntegrationUnitySpec.GetCustomSchema() } list.Add(out) diff --git a/internal/tableflow/command_catalog_integration_update.go b/internal/tableflow/command_catalog_integration_update.go index 345521e895..c437867e36 100644 --- a/internal/tableflow/command_catalog_integration_update.go +++ b/internal/tableflow/command_catalog_integration_update.go @@ -35,13 +35,16 @@ func (c *command) newCatalogIntegrationUpdateCommand() *cobra.Command { cmd.Flags().String("allowed-scope", "", "Specify the allowed scope of the Snowflake Open Catalog.") cmd.Flags().String("client-id", "", "Specify the client id.") cmd.Flags().String("client-secret", "", "Specify the client secret.") + cmd.Flags().String("custom-database", "", "Specify the custom database name for AWS Glue.") + cmd.Flags().String("custom-namespace", "", "Specify the custom namespace for Snowflake Open Catalog.") + cmd.Flags().String("custom-schema", "", "Specify the custom schema name for Unity Catalog.") pcmd.AddClusterFlag(cmd, c.AuthenticatedCLICommand) pcmd.AddEnvironmentFlag(cmd, c.AuthenticatedCLICommand) pcmd.AddContextFlag(cmd, c.CLICommand) pcmd.AddOutputFlag(cmd) - cmd.MarkFlagsOneRequired("name", "endpoint", "warehouse", "allowed-scope", "client-id", "client-secret") + cmd.MarkFlagsOneRequired("name", "endpoint", "warehouse", "allowed-scope", "client-id", "client-secret", "custom-database", "custom-namespace", "custom-schema") return cmd } @@ -95,9 +98,29 @@ func (c *command) updateCatalogIntegration(cmd *cobra.Command, args []string) er }, }) } + if catalogIntegrationType == unity { + updateCatalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUpdateSpecConfigOneOf{ + TableflowV1CatalogIntegrationUnityUpdateSpec: &tableflowv1.TableflowV1CatalogIntegrationUnityUpdateSpec{ + Kind: unityKind, + }, + }) + } + } + + if cmd.Flags().Changed("custom-database") { + customDatabase, err := cmd.Flags().GetString("custom-database") + if err != nil { + return err + } + updateCatalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUpdateSpecConfigOneOf{ + TableflowV1CatalogIntegrationAwsGlueUpdateSpec: &tableflowv1.TableflowV1CatalogIntegrationAwsGlueUpdateSpec{ + Kind: awsGlueKind, + }, + }) + updateCatalogIntegration.Spec.Config.TableflowV1CatalogIntegrationAwsGlueUpdateSpec.SetCustomDatabase(customDatabase) } - if cmd.Flags().Changed("endpoint") || cmd.Flags().Changed("warehouse") || cmd.Flags().Changed("allowed-scope") || cmd.Flags().Changed("client-id") || cmd.Flags().Changed("client-secret") { + if cmd.Flags().Changed("endpoint") || cmd.Flags().Changed("warehouse") || cmd.Flags().Changed("allowed-scope") || cmd.Flags().Changed("client-id") || cmd.Flags().Changed("client-secret") || cmd.Flags().Changed("custom-namespace") { updateCatalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUpdateSpecConfigOneOf{ TableflowV1CatalogIntegrationSnowflakeUpdateSpec: &tableflowv1.TableflowV1CatalogIntegrationSnowflakeUpdateSpec{ Kind: snowflakeKind, @@ -138,6 +161,26 @@ func (c *command) updateCatalogIntegration(cmd *cobra.Command, args []string) er } updateCatalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeUpdateSpec.SetClientSecret(clientSecret) } + if cmd.Flags().Changed("custom-namespace") { + customNamespace, err := cmd.Flags().GetString("custom-namespace") + if err != nil { + return err + } + updateCatalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeUpdateSpec.SetCustomNamespace(customNamespace) + } + } + + if cmd.Flags().Changed("custom-schema") { + customSchema, err := cmd.Flags().GetString("custom-schema") + if err != nil { + return err + } + updateCatalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUpdateSpecConfigOneOf{ + TableflowV1CatalogIntegrationUnityUpdateSpec: &tableflowv1.TableflowV1CatalogIntegrationUnityUpdateSpec{ + Kind: unityKind, + }, + }) + updateCatalogIntegration.Spec.Config.TableflowV1CatalogIntegrationUnityUpdateSpec.SetCustomSchema(customSchema) } catalogIntegration, err := c.V2Client.UpdateCatalogIntegration(args[0], updateCatalogIntegration) diff --git a/logs.txt b/logs.txt new file mode 100644 index 0000000000..8c24b7c51d --- /dev/null +++ b/logs.txt @@ -0,0 +1,2 @@ +{"Timestamp":"2025-06-16T05:44:23.761Z","Level":"INFO","TaskId":"task-0","Message":"WorkerSourceTask{id=lcc-123-0} Committing offsets for 130 acknowledged messages","Exception":""} +{"Timestamp":"2025-06-16T05:43:23.757Z","Level":"INFO","TaskId":"task-0","Message":"WorkerSourceTask{id=lcc-123-0} Committing offsets for 128 acknowledged messages","Exception":""} diff --git a/test/fixtures/output/tableflow/catalog-integration/create-aws-glue-custom-database.golden b/test/fixtures/output/tableflow/catalog-integration/create-aws-glue-custom-database.golden new file mode 100644 index 0000000000..52caa9537b --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/create-aws-glue-custom-database.golden @@ -0,0 +1,11 @@ ++-------------------------+----------------+ +| ID | tci-abc123 | +| Name | my-aws-glue-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | aws | +| Provider Integration ID | cspi-stgce89r7 | +| Custom Database | my-custom-db | +| Suspended | false | +| Phase | PENDING | ++-------------------------+----------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/create-snowflake-custom-namespace.golden b/test/fixtures/output/tableflow/catalog-integration/create-snowflake-custom-namespace.golden new file mode 100644 index 0000000000..c3b4c8247c --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/create-snowflake-custom-namespace.golden @@ -0,0 +1,13 @@ ++------------------+------------------------------------------------+ +| ID | tci-def456 | +| Name | my-snowflake-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | snowflake | +| Endpoint | https://vuser1_polaris.snowflakecomputing.com/ | +| Warehouse | warehouse | +| Allowed Scope | allowed-scope | +| Custom Namespace | my-custom-ns | +| Suspended | false | +| Phase | PENDING | ++------------------+------------------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/create-unity-custom-schema.golden b/test/fixtures/output/tableflow/catalog-integration/create-unity-custom-schema.golden new file mode 100644 index 0000000000..b031a078c3 --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/create-unity-custom-schema.golden @@ -0,0 +1,13 @@ ++--------------------+------------------------------------+ +| ID | tci-ghi789 | +| Name | my-catalog-integration | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | unity | +| Workspace Endpoint | https://dbc-1.cloud.databricks.com | +| Catalog Name | tableflow-quickstart-catalog | +| Client ID | $CLIENT_ID | +| Custom Schema | my-custom-schema | +| Suspended | false | +| Phase | PENDING | ++--------------------+------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue-json.golden b/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue-json.golden index 24b4a4ef4b..de084c6de1 100644 --- a/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue-json.golden +++ b/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue-json.golden @@ -5,6 +5,7 @@ "kafka_cluster": "lkc-123456", "type": "aws", "provider_integration_id": "cspi-stgce89r7", + "custom_database": "my-custom-db", "suspended": false, "phase": "CONNECTED" } diff --git a/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue.golden b/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue.golden index 1a586c068d..257d07cf97 100644 --- a/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue.golden +++ b/test/fixtures/output/tableflow/catalog-integration/describe-aws-glue.golden @@ -5,6 +5,7 @@ | Kafka Cluster | lkc-123456 | | Type | aws | | Provider Integration ID | cspi-stgce89r7 | +| Custom Database | my-custom-db | | Suspended | false | | Phase | CONNECTED | +-------------------------+----------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/describe-snowflake.golden b/test/fixtures/output/tableflow/catalog-integration/describe-snowflake.golden index c94f2d9d67..e763a5b7f2 100644 --- a/test/fixtures/output/tableflow/catalog-integration/describe-snowflake.golden +++ b/test/fixtures/output/tableflow/catalog-integration/describe-snowflake.golden @@ -1,12 +1,13 @@ -+---------------+------------------------------------------------+ -| ID | tci-def456 | -| Name | my-snowflake-ci | -| Environment | env-596 | -| Kafka Cluster | lkc-123456 | -| Type | snowflake | -| Endpoint | https://vuser1_polaris.snowflakecomputing.com/ | -| Warehouse | warehouse | -| Allowed Scope | allowed-scope | -| Suspended | false | -| Phase | CONNECTED | -+---------------+------------------------------------------------+ ++------------------+------------------------------------------------+ +| ID | tci-def456 | +| Name | my-snowflake-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | snowflake | +| Endpoint | https://vuser1_polaris.snowflakecomputing.com/ | +| Warehouse | warehouse | +| Allowed Scope | allowed-scope | +| Custom Namespace | my-custom-ns | +| Suspended | false | +| Phase | CONNECTED | ++------------------+------------------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/describe-unity.golden b/test/fixtures/output/tableflow/catalog-integration/describe-unity.golden index 12963618f3..ae5307359a 100644 --- a/test/fixtures/output/tableflow/catalog-integration/describe-unity.golden +++ b/test/fixtures/output/tableflow/catalog-integration/describe-unity.golden @@ -7,6 +7,7 @@ | Workspace Endpoint | https://dbc-0e76d5eb-ff10.cloud.databricks.com | | Catalog Name | catalog-name | | Client ID | client-id | +| Custom Schema | my-custom-schema | | Suspended | false | | Phase | CONNECTED | +--------------------+------------------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/list-json.golden b/test/fixtures/output/tableflow/catalog-integration/list-json.golden index 9aa65c0453..9d73086712 100644 --- a/test/fixtures/output/tableflow/catalog-integration/list-json.golden +++ b/test/fixtures/output/tableflow/catalog-integration/list-json.golden @@ -6,6 +6,7 @@ "kafka_cluster": "lkc-123456", "type": "aws", "provider_integration_id": "cspi-stgce89r7", + "custom_database": "my-custom-db", "suspended": false, "phase": "CONNECTED" }, @@ -18,6 +19,7 @@ "endpoint": "https://vuser1_polaris.snowflakecomputing.com/", "warehouse": "warehouse", "allowed_scope": "allowed-scope", + "custom_namespace": "my-custom-ns", "suspended": false, "phase": "PENDING" }, @@ -30,6 +32,7 @@ "workspace_endpoint": "https://dbc-0e76d5eb-ff10.cloud.databricks.com", "catalog_name": "catalog-name", "client_id": "client-id", + "custom_schema": "my-custom-schema", "suspended": false, "phase": "CONNECTED" } diff --git a/test/fixtures/output/tableflow/catalog-integration/list.golden b/test/fixtures/output/tableflow/catalog-integration/list.golden index a90379080f..3f6c596363 100644 --- a/test/fixtures/output/tableflow/catalog-integration/list.golden +++ b/test/fixtures/output/tableflow/catalog-integration/list.golden @@ -1,5 +1,5 @@ - ID | Name | Environment | Kafka Cluster | Type | Provider Integration ID | Endpoint | Warehouse | Allowed Scope | Workspace Endpoint | Catalog Name | Client ID | Suspended | Phase | Error Message --------------+-----------------+-------------+---------------+-----------+-------------------------+------------------------------------------------+-----------+---------------+------------------------------------------------+--------------+-----------+-----------+-----------+---------------- - tci-abc123 | my-aws-glue-ci | env-596 | lkc-123456 | aws | cspi-stgce89r7 | | | | | | | false | CONNECTED | - tci-def456 | my-snowflake-ci | env-596 | lkc-123456 | snowflake | | https://vuser1_polaris.snowflakecomputing.com/ | warehouse | allowed-scope | | | | false | PENDING | - tci-ghi789 | my-unity-ci | env-596 | lkc-123456 | unity | | | | | https://dbc-0e76d5eb-ff10.cloud.databricks.com | catalog-name | client-id | false | CONNECTED | + ID | Name | Environment | Kafka Cluster | Type | Provider Integration ID | Endpoint | Warehouse | Allowed Scope | Workspace Endpoint | Catalog Name | Client ID | Custom Database | Custom Namespace | Custom Schema | Suspended | Phase | Error Message +-------------+-----------------+-------------+---------------+-----------+-------------------------+------------------------------------------------+-----------+---------------+------------------------------------------------+--------------+-----------+-----------------+------------------+------------------+-----------+-----------+---------------- + tci-abc123 | my-aws-glue-ci | env-596 | lkc-123456 | aws | cspi-stgce89r7 | | | | | | | my-custom-db | | | false | CONNECTED | + tci-def456 | my-snowflake-ci | env-596 | lkc-123456 | snowflake | | https://vuser1_polaris.snowflakecomputing.com/ | warehouse | allowed-scope | | | | | my-custom-ns | | false | PENDING | + tci-ghi789 | my-unity-ci | env-596 | lkc-123456 | unity | | | | | https://dbc-0e76d5eb-ff10.cloud.databricks.com | catalog-name | client-id | | | my-custom-schema | false | CONNECTED | diff --git a/test/fixtures/output/tableflow/catalog-integration/update-aws-glue-custom-database.golden b/test/fixtures/output/tableflow/catalog-integration/update-aws-glue-custom-database.golden new file mode 100644 index 0000000000..f882f1b9b5 --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/update-aws-glue-custom-database.golden @@ -0,0 +1,11 @@ ++-------------------------+----------------+ +| ID | tci-abc123 | +| Name | my-aws-glue-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | aws | +| Provider Integration ID | cspi-stgce89r7 | +| Custom Database | new-custom-db | +| Suspended | false | +| Phase | CONNECTED | ++-------------------------+----------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/update-fail-no-flags.golden b/test/fixtures/output/tableflow/catalog-integration/update-fail-no-flags.golden index eb6a0a00ce..379d9ad79b 100644 --- a/test/fixtures/output/tableflow/catalog-integration/update-fail-no-flags.golden +++ b/test/fixtures/output/tableflow/catalog-integration/update-fail-no-flags.golden @@ -1,4 +1,4 @@ -Error: at least one of the flags in the group [name endpoint warehouse allowed-scope client-id client-secret] is required +Error: at least one of the flags in the group [name endpoint warehouse allowed-scope client-id client-secret custom-database custom-namespace custom-schema] is required Usage: confluent tableflow catalog-integration update [flags] @@ -12,16 +12,19 @@ Create a Snowflake catalog integration. $ confluent tableflow catalog-integration update tc-abc123 --endpoint https://vuser1_polaris.snowflakecomputing.com/ --warehouse catalog-name --allowed-scope session:role:R1 --client-id $CLIENT_ID --client-secret $CLIENT_SECRET Flags: - --name string Name of the catalog integration. - --endpoint string Specify the The catalog integration connection endpoint for Snowflake Open Catalog. - --warehouse string Specify the warehouse name of the Snowflake Open Catalog. - --allowed-scope string Specify the allowed scope of the Snowflake Open Catalog. - --client-id string Specify the client id. - --client-secret string Specify the client secret. - --cluster string Kafka cluster ID. - --environment string Environment ID. - --context string CLI context name. - -o, --output string Specify the output format as "human", "json", or "yaml". (default "human") + --name string Name of the catalog integration. + --endpoint string Specify the The catalog integration connection endpoint for Snowflake Open Catalog. + --warehouse string Specify the warehouse name of the Snowflake Open Catalog. + --allowed-scope string Specify the allowed scope of the Snowflake Open Catalog. + --client-id string Specify the client id. + --client-secret string Specify the client secret. + --custom-database string Specify the custom database name for AWS Glue. + --custom-namespace string Specify the custom namespace for Snowflake Open Catalog. + --custom-schema string Specify the custom schema name for Unity Catalog. + --cluster string Kafka cluster ID. + --environment string Environment ID. + --context string CLI context name. + -o, --output string Specify the output format as "human", "json", or "yaml". (default "human") Global Flags: -h, --help Show help for this command. diff --git a/test/fixtures/output/tableflow/catalog-integration/update-name.golden b/test/fixtures/output/tableflow/catalog-integration/update-name.golden index 12a3dbb809..31465ade36 100644 --- a/test/fixtures/output/tableflow/catalog-integration/update-name.golden +++ b/test/fixtures/output/tableflow/catalog-integration/update-name.golden @@ -5,6 +5,7 @@ | Kafka Cluster | lkc-123456 | | Type | aws | | Provider Integration ID | cspi-stgce89r7 | +| Custom Database | my-custom-db | | Suspended | false | | Phase | CONNECTED | +-------------------------+----------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/update-snowflake-custom-namespace.golden b/test/fixtures/output/tableflow/catalog-integration/update-snowflake-custom-namespace.golden new file mode 100644 index 0000000000..a03213b9d5 --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/update-snowflake-custom-namespace.golden @@ -0,0 +1,10 @@ ++------------------+-----------------+ +| ID | tci-def456 | +| Name | my-snowflake-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | snowflake | +| Custom Namespace | new-custom-ns | +| Suspended | false | +| Phase | CONNECTED | ++------------------+-----------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/update-snowflake-with-name.golden b/test/fixtures/output/tableflow/catalog-integration/update-snowflake-with-name.golden index ea849bb290..78441cd5d7 100644 --- a/test/fixtures/output/tableflow/catalog-integration/update-snowflake-with-name.golden +++ b/test/fixtures/output/tableflow/catalog-integration/update-snowflake-with-name.golden @@ -1,12 +1,13 @@ -+---------------+------------------------------------------------+ -| ID | tci-def456 | -| Name | new-name | -| Environment | env-596 | -| Kafka Cluster | lkc-123456 | -| Type | snowflake | -| Endpoint | https://vuser2_polaris.snowflakecomputing.com/ | -| Warehouse | warehouse-2 | -| Allowed Scope | allowed-scope-2 | -| Suspended | false | -| Phase | CONNECTED | -+---------------+------------------------------------------------+ ++------------------+------------------------------------------------+ +| ID | tci-def456 | +| Name | new-name | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | snowflake | +| Endpoint | https://vuser2_polaris.snowflakecomputing.com/ | +| Warehouse | warehouse-2 | +| Allowed Scope | allowed-scope-2 | +| Custom Namespace | my-custom-ns | +| Suspended | false | +| Phase | CONNECTED | ++------------------+------------------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/update-snowflake.golden b/test/fixtures/output/tableflow/catalog-integration/update-snowflake.golden index 8e94d7c447..7dc699d8e5 100644 --- a/test/fixtures/output/tableflow/catalog-integration/update-snowflake.golden +++ b/test/fixtures/output/tableflow/catalog-integration/update-snowflake.golden @@ -1,12 +1,13 @@ -+---------------+------------------------------------------------+ -| ID | tci-def456 | -| Name | my-snowflake-ci | -| Environment | env-596 | -| Kafka Cluster | lkc-123456 | -| Type | snowflake | -| Endpoint | https://vuser2_polaris.snowflakecomputing.com/ | -| Warehouse | warehouse-2 | -| Allowed Scope | allowed-scope-2 | -| Suspended | false | -| Phase | CONNECTED | -+---------------+------------------------------------------------+ ++------------------+------------------------------------------------+ +| ID | tci-def456 | +| Name | my-snowflake-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | snowflake | +| Endpoint | https://vuser2_polaris.snowflakecomputing.com/ | +| Warehouse | warehouse-2 | +| Allowed Scope | allowed-scope-2 | +| Custom Namespace | my-custom-ns | +| Suspended | false | +| Phase | CONNECTED | ++------------------+------------------------------------------------+ diff --git a/test/fixtures/output/tableflow/catalog-integration/update-unity-custom-schema.golden b/test/fixtures/output/tableflow/catalog-integration/update-unity-custom-schema.golden new file mode 100644 index 0000000000..64ce763748 --- /dev/null +++ b/test/fixtures/output/tableflow/catalog-integration/update-unity-custom-schema.golden @@ -0,0 +1,13 @@ ++--------------------+------------------------------------------------+ +| ID | tci-ghi789 | +| Name | my-unity-ci | +| Environment | env-596 | +| Kafka Cluster | lkc-123456 | +| Type | unity | +| Workspace Endpoint | https://dbc-0e76d5eb-ff10.cloud.databricks.com | +| Catalog Name | catalog-name | +| Client ID | client-id | +| Custom Schema | new-custom-schema | +| Suspended | false | +| Phase | CONNECTED | ++--------------------+------------------------------------------------+ diff --git a/test/tableflow_test.go b/test/tableflow_test.go index 1870ea5497..529303a9a1 100644 --- a/test/tableflow_test.go +++ b/test/tableflow_test.go @@ -3,8 +3,11 @@ package test func (s *CLITestSuite) TestTableflowCatalogIntegration() { tests := []CLITest{ {args: "tableflow catalog-integration create my-aws-glue-ci --cluster lkc-123456 --type aws --provider-integration cspi-stgce89r7", fixture: "tableflow/catalog-integration/create-aws-glue.golden"}, + {args: "tableflow catalog-integration create my-aws-glue-ci --cluster lkc-123456 --type aws --provider-integration cspi-stgce89r7 --custom-database my-custom-db", fixture: "tableflow/catalog-integration/create-aws-glue-custom-database.golden"}, {args: "tableflow catalog-integration create my-snowflake-ci --cluster lkc-123456 --type snowflake --endpoint https://vuser1_polaris.snowflakecomputing.com/ --client-id client-id --client-secret client-secret --warehouse warehouse --allowed-scope allowed-scope", fixture: "tableflow/catalog-integration/create-snowflake.golden"}, + {args: "tableflow catalog-integration create my-snowflake-ci --cluster lkc-123456 --type snowflake --endpoint https://vuser1_polaris.snowflakecomputing.com/ --client-id client-id --client-secret client-secret --warehouse warehouse --allowed-scope allowed-scope --custom-namespace my-custom-ns", fixture: "tableflow/catalog-integration/create-snowflake-custom-namespace.golden"}, {args: "tableflow catalog-integration create my-catalog-integration --cluster lkc-123456 --type unity --workspace-endpoint https://dbc-1.cloud.databricks.com --catalog-name tableflow-quickstart-catalog --unity-client-id $CLIENT_ID --unity-client-secret $CLIENT_SECRET", fixture: "tableflow/catalog-integration/create-unity.golden"}, + {args: "tableflow catalog-integration create my-catalog-integration --cluster lkc-123456 --type unity --workspace-endpoint https://dbc-1.cloud.databricks.com --catalog-name tableflow-quickstart-catalog --unity-client-id $CLIENT_ID --unity-client-secret $CLIENT_SECRET --custom-schema my-custom-schema", fixture: "tableflow/catalog-integration/create-unity-custom-schema.golden"}, {args: "tableflow catalog-integration delete tci-abc123 tci-def456 --cluster lkc-123456", input: "y\n", fixture: "tableflow/catalog-integration/delete-multiple.golden"}, {args: "tableflow catalog-integration delete tci-abc123 tci-def456 tci-invalid --cluster lkc-123456", fixture: "tableflow/catalog-integration/delete-invalid.golden", exitCode: 1}, {args: "tableflow catalog-integration list --cluster lkc-123456", fixture: "tableflow/catalog-integration/list.golden"}, @@ -26,6 +29,9 @@ func (s *CLITestSuite) TestTableflowCatalogIntegrationUpdate() { {args: "tableflow catalog-integration update tci-def456 --cluster lkc-123456 --endpoint https://vuser2_polaris.snowflakecomputing.com/ --client-id client-id-2 --client-secret client-secret-2 --warehouse warehouse-2 --allowed-scope allowed-scope-2", fixture: "tableflow/catalog-integration/update-snowflake.golden"}, {args: "tableflow catalog-integration update tci-abc456 --cluster lkc-123456 --name new-name", fixture: "tableflow/catalog-integration/update-name.golden"}, {args: "tableflow catalog-integration update tci-def456 --cluster lkc-123456 --name new-name --endpoint https://vuser2_polaris.snowflakecomputing.com/ --client-id client-id-2 --client-secret client-secret-2 --warehouse warehouse-2 --allowed-scope allowed-scope-2", fixture: "tableflow/catalog-integration/update-snowflake-with-name.golden"}, + {args: "tableflow catalog-integration update tci-abc123 --cluster lkc-123456 --custom-database new-custom-db", fixture: "tableflow/catalog-integration/update-aws-glue-custom-database.golden"}, + {args: "tableflow catalog-integration update tci-def456 --cluster lkc-123456 --custom-namespace new-custom-ns", fixture: "tableflow/catalog-integration/update-snowflake-custom-namespace.golden"}, + {args: "tableflow catalog-integration update tci-ghi789 --cluster lkc-123456 --custom-schema new-custom-schema", fixture: "tableflow/catalog-integration/update-unity-custom-schema.golden"}, {args: "tableflow catalog-integration update tci-abc123 --cluster lkc-123456", fixture: "tableflow/catalog-integration/update-fail-no-flags.golden", exitCode: 1}, } diff --git a/test/test-server/tableflow_handlers.go b/test/test-server/tableflow_handlers.go index 5f55031c23..4d4b1a2459 100644 --- a/test/test-server/tableflow_handlers.go +++ b/test/test-server/tableflow_handlers.go @@ -378,13 +378,26 @@ func handleCatalogIntegrationUpdate(t *testing.T, id string) http.HandlerFunc { switch id { case "tci-abc123": catalogIntegration = getCatalogIntegration(id, body.GetSpec().Environment.Id, body.GetSpec().KafkaCluster.Id, "my-aws-glue-ci", "AwsGlue") + if body.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueUpdateSpec != nil && body.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueUpdateSpec.GetCustomDatabase() != "" { + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationAwsGlueSpec.SetCustomDatabase(body.Spec.GetConfig().TableflowV1CatalogIntegrationAwsGlueUpdateSpec.GetCustomDatabase()) + } case "tci-def456": catalogIntegration = getCatalogIntegration(id, body.GetSpec().Environment.Id, body.GetSpec().KafkaCluster.Id, "my-snowflake-ci", "Snowflake") - catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetEndpoint(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetEndpoint()) - catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetClientId(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetClientId()) - catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetClientSecret(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetClientSecret()) - catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetWarehouse(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetWarehouse()) - catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetAllowedScope(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetAllowedScope()) + if body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec != nil { + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetEndpoint(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetEndpoint()) + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetClientId(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetClientId()) + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetClientSecret(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetClientSecret()) + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetWarehouse(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetWarehouse()) + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetAllowedScope(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetAllowedScope()) + if body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetCustomNamespace() != "" { + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationSnowflakeSpec.SetCustomNamespace(body.Spec.GetConfig().TableflowV1CatalogIntegrationSnowflakeUpdateSpec.GetCustomNamespace()) + } + } + case "tci-ghi789": + catalogIntegration = getCatalogIntegration(id, body.GetSpec().Environment.Id, body.GetSpec().KafkaCluster.Id, "my-unity-ci", "Unity") + if body.Spec.GetConfig().TableflowV1CatalogIntegrationUnityUpdateSpec != nil && body.Spec.GetConfig().TableflowV1CatalogIntegrationUnityUpdateSpec.GetCustomSchema() != "" { + catalogIntegration.Spec.Config.TableflowV1CatalogIntegrationUnitySpec.SetCustomSchema(body.Spec.GetConfig().TableflowV1CatalogIntegrationUnityUpdateSpec.GetCustomSchema()) + } default: catalogIntegration = getCatalogIntegration(id, body.GetSpec().Environment.Id, body.GetSpec().KafkaCluster.Id, "my-aws-glue-ci", "AwsGlue") } @@ -454,27 +467,33 @@ func getCatalogIntegration(id, environment, cluster, name, specConfigKind string switch specConfigKind { case "AwsGlue": - catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(&tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpec{ + awsGlueSpec := &tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpec{ Kind: specConfigKind, ProviderIntegrationId: "cspi-stgce89r7", - })) + } + awsGlueSpec.SetCustomDatabase("my-custom-db") + catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationAwsGlueSpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(awsGlueSpec)) case "Snowflake": - catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(&tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpec{ + snowflakeSpec := &tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpec{ Kind: specConfigKind, Endpoint: "https://vuser1_polaris.snowflakecomputing.com/", ClientId: "client-id", ClientSecret: "client-secret", Warehouse: "warehouse", AllowedScope: "allowed-scope", - })) + } + snowflakeSpec.SetCustomNamespace("my-custom-ns") + catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationSnowflakeSpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(snowflakeSpec)) case "Unity": - catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUnitySpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(&tableflowv1.TableflowV1CatalogIntegrationUnitySpec{ + unitySpec := &tableflowv1.TableflowV1CatalogIntegrationUnitySpec{ Kind: specConfigKind, WorkspaceEndpoint: "https://dbc-0e76d5eb-ff10.cloud.databricks.com", CatalogName: "catalog-name", ClientId: "client-id", ClientSecret: "client-secret", - })) + } + unitySpec.SetCustomSchema("my-custom-schema") + catalogIntegration.Spec.SetConfig(tableflowv1.TableflowV1CatalogIntegrationUnitySpecAsTableflowV1CatalogIntegrationSpecConfigOneOf(unitySpec)) } return catalogIntegration From 4b9536a3f2b32800d3fd6b163b75f73eb79ed696 Mon Sep 17 00:00:00 2001 From: unset Date: Thu, 23 Apr 2026 17:52:43 +0530 Subject: [PATCH 2/3] Update catalog-integration test golden files for custom database/namespace/schema flags --- .idea/.gitignore | 10 -------- .idea/cli.iml | 9 -------- .idea/go.imports.xml | 10 -------- .idea/golinter.xml | 7 ------ .idea/modules.xml | 8 ------- .idea/vcs.xml | 6 ----- .../control-center-local.properties | 2 -- logs.txt | 2 -- .../catalog-integration/create-help.golden | 3 +++ .../tableflow/catalog-integration/list.golden | 8 +++---- .../catalog-integration/update-help.golden | 23 +++++++++++-------- 11 files changed, 20 insertions(+), 68 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/cli.iml delete mode 100644 .idea/go.imports.xml delete mode 100644 .idea/golinter.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 internal/local/dir/etc/confluent-control-center/control-center-local.properties delete mode 100644 logs.txt diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 30cf57ed7c..0000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Ignored default folder with query files -/queries/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/cli.iml b/.idea/cli.iml deleted file mode 100644 index 5e764c4f0b..0000000000 --- a/.idea/cli.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/go.imports.xml b/.idea/go.imports.xml deleted file mode 100644 index 644cdf0bd2..0000000000 --- a/.idea/go.imports.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/golinter.xml b/.idea/golinter.xml deleted file mode 100644 index 1ccf3ec6d9..0000000000 --- a/.idea/golinter.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 6fcc0dcf27..0000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddfbb..0000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/internal/local/dir/etc/confluent-control-center/control-center-local.properties b/internal/local/dir/etc/confluent-control-center/control-center-local.properties deleted file mode 100644 index 8602bca970..0000000000 --- a/internal/local/dir/etc/confluent-control-center/control-center-local.properties +++ /dev/null @@ -1,2 +0,0 @@ -confluent.controlcenter.alertmanager.config.file=abc -confluent.controlcenter.prometheus.rules.file=def diff --git a/logs.txt b/logs.txt deleted file mode 100644 index 8c24b7c51d..0000000000 --- a/logs.txt +++ /dev/null @@ -1,2 +0,0 @@ -{"Timestamp":"2025-06-16T05:44:23.761Z","Level":"INFO","TaskId":"task-0","Message":"WorkerSourceTask{id=lcc-123-0} Committing offsets for 130 acknowledged messages","Exception":""} -{"Timestamp":"2025-06-16T05:43:23.757Z","Level":"INFO","TaskId":"task-0","Message":"WorkerSourceTask{id=lcc-123-0} Committing offsets for 128 acknowledged messages","Exception":""} diff --git a/test/fixtures/output/tableflow/catalog-integration/create-help.golden b/test/fixtures/output/tableflow/catalog-integration/create-help.golden index 4f4520f76a..42f775f20e 100644 --- a/test/fixtures/output/tableflow/catalog-integration/create-help.golden +++ b/test/fixtures/output/tableflow/catalog-integration/create-help.golden @@ -19,15 +19,18 @@ Create a Unity catalog integration. Flags: --type string REQUIRED: Specify the catalog integration type as "aws", "snowflake", or "unity". --provider-integration string Specify the provider integration id. + --custom-database string Specify the custom database name for AWS Glue. --endpoint string Specify the The catalog integration connection endpoint for Snowflake Open Catalog. --warehouse string Specify the warehouse name of the Snowflake Open Catalog. --allowed-scope string Specify the allowed scope of the Snowflake Open Catalog. --client-id string Specify the client id. --client-secret string Specify the client secret. + --custom-namespace string Specify the custom namespace for Snowflake Open Catalog. --workspace-endpoint string Specify the Databricks workspace URL associated with the Unity Catalog. --catalog-name string Specify the name of the catalog. --unity-client-id string Specify the Unity client id. --unity-client-secret string Specify the Unity client secret. + --custom-schema string Specify the custom schema name for Unity Catalog. --cluster string Kafka cluster ID. --environment string Environment ID. --context string CLI context name. diff --git a/test/fixtures/output/tableflow/catalog-integration/list.golden b/test/fixtures/output/tableflow/catalog-integration/list.golden index 3f6c596363..196c4eeda4 100644 --- a/test/fixtures/output/tableflow/catalog-integration/list.golden +++ b/test/fixtures/output/tableflow/catalog-integration/list.golden @@ -1,5 +1,5 @@ - ID | Name | Environment | Kafka Cluster | Type | Provider Integration ID | Endpoint | Warehouse | Allowed Scope | Workspace Endpoint | Catalog Name | Client ID | Custom Database | Custom Namespace | Custom Schema | Suspended | Phase | Error Message + ID | Name | Environment | Kafka Cluster | Type | Provider Integration ID | Endpoint | Warehouse | Allowed Scope | Workspace Endpoint | Catalog Name | Client ID | Custom Database | Custom Namespace | Custom Schema | Suspended | Phase | Error Message -------------+-----------------+-------------+---------------+-----------+-------------------------+------------------------------------------------+-----------+---------------+------------------------------------------------+--------------+-----------+-----------------+------------------+------------------+-----------+-----------+---------------- - tci-abc123 | my-aws-glue-ci | env-596 | lkc-123456 | aws | cspi-stgce89r7 | | | | | | | my-custom-db | | | false | CONNECTED | - tci-def456 | my-snowflake-ci | env-596 | lkc-123456 | snowflake | | https://vuser1_polaris.snowflakecomputing.com/ | warehouse | allowed-scope | | | | | my-custom-ns | | false | PENDING | - tci-ghi789 | my-unity-ci | env-596 | lkc-123456 | unity | | | | | https://dbc-0e76d5eb-ff10.cloud.databricks.com | catalog-name | client-id | | | my-custom-schema | false | CONNECTED | + tci-abc123 | my-aws-glue-ci | env-596 | lkc-123456 | aws | cspi-stgce89r7 | | | | | | | my-custom-db | | | false | CONNECTED | + tci-def456 | my-snowflake-ci | env-596 | lkc-123456 | snowflake | | https://vuser1_polaris.snowflakecomputing.com/ | warehouse | allowed-scope | | | | | my-custom-ns | | false | PENDING | + tci-ghi789 | my-unity-ci | env-596 | lkc-123456 | unity | | | | | https://dbc-0e76d5eb-ff10.cloud.databricks.com | catalog-name | client-id | | | my-custom-schema | false | CONNECTED | diff --git a/test/fixtures/output/tableflow/catalog-integration/update-help.golden b/test/fixtures/output/tableflow/catalog-integration/update-help.golden index 4e8ad71e87..50f864500e 100644 --- a/test/fixtures/output/tableflow/catalog-integration/update-help.golden +++ b/test/fixtures/output/tableflow/catalog-integration/update-help.golden @@ -13,16 +13,19 @@ Create a Snowflake catalog integration. $ confluent tableflow catalog-integration update tc-abc123 --endpoint https://vuser1_polaris.snowflakecomputing.com/ --warehouse catalog-name --allowed-scope session:role:R1 --client-id $CLIENT_ID --client-secret $CLIENT_SECRET Flags: - --name string Name of the catalog integration. - --endpoint string Specify the The catalog integration connection endpoint for Snowflake Open Catalog. - --warehouse string Specify the warehouse name of the Snowflake Open Catalog. - --allowed-scope string Specify the allowed scope of the Snowflake Open Catalog. - --client-id string Specify the client id. - --client-secret string Specify the client secret. - --cluster string Kafka cluster ID. - --environment string Environment ID. - --context string CLI context name. - -o, --output string Specify the output format as "human", "json", or "yaml". (default "human") + --name string Name of the catalog integration. + --endpoint string Specify the The catalog integration connection endpoint for Snowflake Open Catalog. + --warehouse string Specify the warehouse name of the Snowflake Open Catalog. + --allowed-scope string Specify the allowed scope of the Snowflake Open Catalog. + --client-id string Specify the client id. + --client-secret string Specify the client secret. + --custom-database string Specify the custom database name for AWS Glue. + --custom-namespace string Specify the custom namespace for Snowflake Open Catalog. + --custom-schema string Specify the custom schema name for Unity Catalog. + --cluster string Kafka cluster ID. + --environment string Environment ID. + --context string CLI context name. + -o, --output string Specify the output format as "human", "json", or "yaml". (default "human") Global Flags: -h, --help Show help for this command. From 0cedde6c52e62e0e840e2e010ec87dc0eee40f38 Mon Sep 17 00:00:00 2001 From: unset Date: Tue, 28 Apr 2026 00:56:01 +0530 Subject: [PATCH 3/3] Use the updated tableflow public sdk --- .golangci.yml | 3 --- go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 06ef6c5337..0a5c1915f4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,6 +32,3 @@ linters-settings: custom-order: true nakedret: max-func-lines: 0 - gomoddirectives: - replace-allow-list: - - "github.com/confluentinc/ccloud-sdk-go-v2/tableflow" diff --git a/go.mod b/go.mod index 7242b40caa..12fa7c6ec9 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/confluentinc/ccloud-sdk-go-v2/service-quota v0.2.0 github.com/confluentinc/ccloud-sdk-go-v2/srcm v0.7.3 github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 - github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.5.0 + github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.6.0 github.com/confluentinc/ccloud-sdk-go-v2/usm v0.1.0 github.com/confluentinc/cmf-sdk-go v0.0.5 github.com/confluentinc/confluent-kafka-go/v2 v2.14.1 @@ -288,5 +288,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.5.0 => github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0 diff --git a/go.sum b/go.sum index 650b2ee46e..eeb1e55951 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,6 @@ github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= github.com/confluentinc/ccloud-sdk-go-v1-public v0.0.0-20250521223017-0e8f6f971b52 h1:19qEGhkbZa5fopKCe0VPIV+Sasby4Pv10z9ZaktwWso= github.com/confluentinc/ccloud-sdk-go-v1-public v0.0.0-20250521223017-0e8f6f971b52/go.mod h1:62EMf+5uFEt1BJ2q8WMrUoI9VUSxAbDnmZCGRt/MbA0= -github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0 h1:hGvPz/GeneUPf1Lq1nGiqU34R4QoyqR9cpaekUl4+l4= -github.com/confluentinc/ccloud-sdk-go-v2-internal/tableflow v0.25.0/go.mod h1:l2yBTc+QGoxMSrilWaTt3t46BhiaeWhmLdh6q6+gC3E= github.com/confluentinc/ccloud-sdk-go-v2/ai v0.1.0 h1:zSF4OQUJXWH2JeAo9rsq13ibk+JFdzITGR8S7cFMpzw= github.com/confluentinc/ccloud-sdk-go-v2/ai v0.1.0/go.mod h1:DoxqzzF3JzvJr3fWkvCiOHFlE0GoYpozWxFZ1Ud9ntA= github.com/confluentinc/ccloud-sdk-go-v2/apikeys v0.4.0 h1:8fWyLwMuy8ec0MVF5Avd54UvbIxhDFhZzanHBVwgxdw= @@ -260,6 +258,8 @@ github.com/confluentinc/ccloud-sdk-go-v2/srcm v0.7.3 h1:ozdDSJHruQIgtxS5hwz8Rp8p github.com/confluentinc/ccloud-sdk-go-v2/srcm v0.7.3/go.mod h1:cD0AeCMBAWBesmWxWCMgVYNABYgHJ/ahCj7b4HP2R2I= github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1 h1:WZJYfgXJrvTIYQpCFps/qHF7T8ekgPlX/SFqx4EY2zQ= github.com/confluentinc/ccloud-sdk-go-v2/sso v0.0.1/go.mod h1:kB+MXWYYg9ohrTCb27LlfpTbuexAzyYAmum105ow0ho= +github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.6.0 h1:wrmpI4UJgWZ4rX1EYqUxUQrfsKMRDehQsIWjcMU/bzs= +github.com/confluentinc/ccloud-sdk-go-v2/tableflow v0.6.0/go.mod h1:myRmhUEWzpwGqWvdsNk79QH41pkre1G21vmySyGQiWA= github.com/confluentinc/ccloud-sdk-go-v2/usm v0.1.0 h1:rF9cKecDCowq+oDWjf8rSpXXZHAnVXowIsT/OXF4MOI= github.com/confluentinc/ccloud-sdk-go-v2/usm v0.1.0/go.mod h1:umhEDvQp/5h0ALKBpYTQOmFwaWrvilnbE8Rkzh6oJ4Q= github.com/confluentinc/cmf-sdk-go v0.0.5 h1:TS6S3ClVsM1kanB00mlcmqXczozDTO2t4Du5blDSYvE=