From 74bdc8aec14b430bcc83b88c1aa91e246c0335bf Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 9 Jul 2026 12:55:19 +0000 Subject: [PATCH] Release v0.121.0 --- CHANGELOG.md | 31 ++ README.md | 4 +- databricks/sdk/__init__.py | 2 +- databricks/sdk/service/bundledeployments.py | 9 + databricks/sdk/service/catalog.py | 253 +++++++++++++--- databricks/sdk/service/dashboards.py | 2 +- databricks/sdk/service/database.py | 4 +- databricks/sdk/service/disasterrecovery.py | 10 + databricks/sdk/service/jobs.py | 60 ++++ databricks/sdk/service/ml.py | 260 +++++++++++++++-- databricks/sdk/service/postgres.py | 26 +- databricks/sdk/service/serving.py | 4 + databricks/sdk/service/settings.py | 57 +++- databricks/sdk/service/sql.py | 6 +- databricks/sdk/service/workspace.py | 19 +- databricks/sdk/version.py | 2 +- docs/account/iam/workspace_assignment.rst | 8 +- docs/account/provisioning/storage.rst | 7 +- docs/auth-types-reference.md | 271 ++++++++++++++++++ docs/authentication.md | 2 +- docs/dbdataclasses/catalog.rst | 8 + docs/dbdataclasses/ml.rst | 20 +- docs/dbdataclasses/serving.rst | 12 + docs/dbdataclasses/settings.rst | 2 + docs/index.rst | 1 + docs/workspace/catalog/catalogs.rst | 3 +- docs/workspace/catalog/external_locations.rst | 26 +- docs/workspace/catalog/grants.rst | 62 ++++ docs/workspace/catalog/schemas.rst | 8 +- docs/workspace/catalog/secrets_uc.rst | 24 +- .../workspace/catalog/storage_credentials.rst | 12 +- docs/workspace/dashboards/genie.rst | 2 +- docs/workspace/jobs/jobs.rst | 54 +++- docs/workspace/ml/experiments.rst | 6 +- docs/workspace/ml/model_registry.rst | 2 + docs/workspace/postgres/postgres.rst | 5 +- docs/workspace/sql/dashboard_widgets.rst | 4 +- .../sql/query_visualizations_legacy.rst | 2 +- docs/workspace/workspace/repos.rst | 10 +- docs/workspace/workspace/workspace.rst | 25 +- 40 files changed, 1149 insertions(+), 176 deletions(-) create mode 100644 docs/auth-types-reference.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a0ec747..8618891f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Version changelog +## Release v0.121.0 (2026-07-09) + +### API Changes +* Add `list()` and `list_effective()` methods for [w.grants](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/catalog/grants.html) workspace-level service. +* Add `update_time` field for `databricks.sdk.service.bundledeployments.Resource`. +* Add `stable_workspace_id` field for `databricks.sdk.service.disasterrecovery.StableUrl`. +* Add `deployment_id` and `version_id` fields for `databricks.sdk.service.jobs.BaseRun`. +* Add `parent_path` field for `databricks.sdk.service.jobs.CreateJob`. +* Add `parent_path` field for `databricks.sdk.service.jobs.JobSettings`. +* Add `deployment_id` and `version_id` fields for `databricks.sdk.service.jobs.Run`. +* Add `first_distinct` and `last_distinct` fields for `databricks.sdk.service.ml.AggregationFunction`. +* Add `trace_location` field for `databricks.sdk.service.ml.CreateExperiment`. +* Add `trace_location` field for `databricks.sdk.service.ml.Experiment`. +* Add `avro_schema` and `proto_schema` fields for `databricks.sdk.service.ml.SchemaConfig`. +* Add `freshness_target` field for `databricks.sdk.service.ml.StreamingMode`. +* Add `long_rolling` field for `databricks.sdk.service.ml.TimeWindow`. +* Add `git_credential_id` field for `databricks.sdk.service.workspace.CreateRepoRequest`. +* Add `git_credential_id` field for `databricks.sdk.service.workspace.UpdateRepoRequest`. +* Add `cpu_large` and `cpu_medium` enum values for `databricks.sdk.service.serving.ServedModelInputWorkloadType`. +* Add `cpu_large` and `cpu_medium` enum values for `databricks.sdk.service.serving.ServingModelWorkloadType`. +* [Breaking] Change `role` field for `databricks.sdk.service.postgres.DatabaseDatabaseSpec` to be required. +* Change `role` field for `databricks.sdk.service.postgres.DatabaseDatabaseSpec` to be required. +* [Breaking] Remove `include_browse` field for `databricks.sdk.service.catalog.GetSecretRequest`. +* [Breaking] Remove `include_browse` field for `databricks.sdk.service.catalog.ListSecretsRequest`. +* [Breaking] Remove `browse_only` and `external_secret_id` fields for `databricks.sdk.service.catalog.Secret`. +* [Breaking] Remove `first_distinct_n` and `last_distinct_n` fields for `databricks.sdk.service.ml.AggregationFunction`. +* [Breaking] Remove `group_name` field for `databricks.sdk.service.postgres.GenerateDatabaseCredentialRequest`. + +### Documentation +* Document all 18 valid `auth_type` values in a new [Authentication Types Reference](https://github.com/databricks/databricks-sdk-py/blob/main/docs/auth-types-reference.md) page, linked from the README and authentication docs. + ## Release v0.120.0 (2026-07-01) ### API Changes diff --git a/README.md b/README.md index 83d90d8cf..2a9e63a58 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ w = WorkspaceClient(host=input('Databricks Workspace URL: '), azure_client_secret=input('AAD Client Secret: ')) ``` -Please see more examples in [this document](./docs/azure-ad.md). +For more Azure authentication examples, see the [Authentication Types Reference](./docs/auth-types-reference.md#azure-service-principal). ### Google Cloud Platform native authentication @@ -280,7 +280,7 @@ For all authentication methods, you can override the default behavior in client | Argument | Description | Environment variable | |-------------------------|-------------|------------------------| -| `auth_type` | _(String)_ When multiple auth attributes are available in the environment, use the auth type specified by this argument. This argument also holds the currently selected auth. | `DATABRICKS_AUTH_TYPE` | +| `auth_type` | _(String)_ When multiple auth attributes are available in the environment, use the auth type specified by this argument. This argument also holds the currently selected auth. When set explicitly, the SDK only attempts that authentication method, skipping automatic detection of others. See the [Authentication Types Reference](./docs/auth-types-reference.md) for all valid values, required parameters, and usage examples. | `DATABRICKS_AUTH_TYPE` | | `http_timeout_seconds` | _(Integer)_ Number of seconds for HTTP timeout. Default is _60_. | _(None)_ | | `retry_timeout_seconds` | _(Integer)_ Number of seconds to keep retrying HTTP requests. Default is _300 (5 minutes)_. | _(None)_ | | `debug_truncate_bytes` | _(Integer)_ Truncate JSON fields in debug logs above this limit. Default is 96. | `DATABRICKS_DEBUG_TRUNCATE_BYTES` | diff --git a/databricks/sdk/__init__.py b/databricks/sdk/__init__.py index 8dee259f3..0a5f92212 100755 --- a/databricks/sdk/__init__.py +++ b/databricks/sdk/__init__.py @@ -960,7 +960,7 @@ def query_visualizations(self) -> pkg_sql.QueryVisualizationsAPI: @property def query_visualizations_legacy(self) -> pkg_sql.QueryVisualizationsLegacyAPI: - """This is an evolving API that facilitates the addition and removal of vizualisations from existing queries within the Databricks Workspace.""" + """This is an evolving API that facilitates the addition and removal of visualizations from existing queries within the Databricks Workspace.""" return self._query_visualizations_legacy @property diff --git a/databricks/sdk/service/bundledeployments.py b/databricks/sdk/service/bundledeployments.py index a5ecf409e..e1abeedc7 100755 --- a/databricks/sdk/service/bundledeployments.py +++ b/databricks/sdk/service/bundledeployments.py @@ -572,6 +572,10 @@ class Resource: state: Optional[any] = None """Serialized local config state (what the CLI deployed).""" + update_time: Optional[Timestamp] = None + """When the last operation that updated this resource's recorded state was applied. Pairs with + last_action_type and last_version_id (all three advance together on that write).""" + def as_dict(self) -> dict: """Serializes the Resource into a dictionary suitable for use as a JSON request body.""" body = {} @@ -589,6 +593,8 @@ def as_dict(self) -> dict: body["resource_type"] = self.resource_type.value if self.state: body["state"] = self.state + if self.update_time is not None: + body["update_time"] = self.update_time.ToJsonString() return body def as_shallow_dict(self) -> dict: @@ -608,6 +614,8 @@ def as_shallow_dict(self) -> dict: body["resource_type"] = self.resource_type if self.state: body["state"] = self.state + if self.update_time is not None: + body["update_time"] = self.update_time return body @classmethod @@ -621,6 +629,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Resource: resource_key=d.get("resource_key", None), resource_type=_enum(d, "resource_type", DeploymentResourceType), state=d.get("state", None), + update_time=_timestamp(d, "update_time"), ) diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index c081b30c2..70c87f102 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -5793,6 +5793,44 @@ def from_dict(cls, d: Dict[str, Any]) -> ListCredentialsResponse: ) +@dataclass +class ListEffectivePrivilegeAssignmentsResponse: + effective_privilege_assignments: Optional[List[EffectivePrivilegeAssignment]] = None + """The effective privilege assignments for the securable (and optional principal).""" + + next_page_token: Optional[str] = None + """Opaque token to retrieve the next page of results. Absent if there are no more pages. + **page_token** should be set to this value for the next request (for the next page of results).""" + + def as_dict(self) -> dict: + """Serializes the ListEffectivePrivilegeAssignmentsResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.effective_privilege_assignments: + body["effective_privilege_assignments"] = [v.as_dict() for v in self.effective_privilege_assignments] + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ListEffectivePrivilegeAssignmentsResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.effective_privilege_assignments: + body["effective_privilege_assignments"] = self.effective_privilege_assignments + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ListEffectivePrivilegeAssignmentsResponse: + """Deserializes the ListEffectivePrivilegeAssignmentsResponse from a dictionary.""" + return cls( + effective_privilege_assignments=_repeated_dict( + d, "effective_privilege_assignments", EffectivePrivilegeAssignment + ), + next_page_token=d.get("next_page_token", None), + ) + + @dataclass class ListEntityTagAssignmentsResponse: next_page_token: Optional[str] = None @@ -6068,6 +6106,41 @@ def from_dict(cls, d: Dict[str, Any]) -> ListPoliciesResponse: return cls(next_page_token=d.get("next_page_token", None), policies=_repeated_dict(d, "policies", PolicyInfo)) +@dataclass +class ListPrivilegeAssignmentsResponse: + next_page_token: Optional[str] = None + """Opaque token to retrieve the next page of results. Absent if there are no more pages. + **page_token** should be set to this value for the next request (for the next page of results).""" + + privilege_assignments: Optional[List[PrivilegeAssignment]] = None + + def as_dict(self) -> dict: + """Serializes the ListPrivilegeAssignmentsResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + if self.privilege_assignments: + body["privilege_assignments"] = [v.as_dict() for v in self.privilege_assignments] + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ListPrivilegeAssignmentsResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.next_page_token is not None: + body["next_page_token"] = self.next_page_token + if self.privilege_assignments: + body["privilege_assignments"] = self.privilege_assignments + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ListPrivilegeAssignmentsResponse: + """Deserializes the ListPrivilegeAssignmentsResponse from a dictionary.""" + return cls( + next_page_token=d.get("next_page_token", None), + privilege_assignments=_repeated_dict(d, "privilege_assignments", PrivilegeAssignment), + ) + + @dataclass class ListQuotasResponse: next_page_token: Optional[str] = None @@ -9095,10 +9168,6 @@ class Secret: value. The maximum size is 60 KiB (pre-encryption). Accepted content includes passwords, tokens, keys, and other sensitive credential data.""" - browse_only: Optional[bool] = None - """Indicates whether the principal is limited to retrieving metadata for the associated object - through the **BROWSE** privilege when **include_browse** is enabled in the request.""" - comment: Optional[str] = None """User-provided free-form text description of the secret.""" @@ -9121,8 +9190,6 @@ class Secret: longer be used and may be displayed as a warning in the UI. It is purely informational and does not trigger any automatic actions or affect the secret's lifecycle.""" - external_secret_id: Optional[str] = None - full_name: Optional[str] = None """The three-level (fully qualified) name of the secret, in the form of **catalog_name.schema_name.secret_name**.""" @@ -9143,8 +9210,6 @@ class Secret: def as_dict(self) -> dict: """Serializes the Secret into a dictionary suitable for use as a JSON request body.""" body = {} - if self.browse_only is not None: - body["browse_only"] = self.browse_only if self.catalog_name is not None: body["catalog_name"] = self.catalog_name if self.comment is not None: @@ -9159,8 +9224,6 @@ def as_dict(self) -> dict: body["effective_value"] = self.effective_value if self.expire_time is not None: body["expire_time"] = self.expire_time.ToJsonString() - if self.external_secret_id is not None: - body["external_secret_id"] = self.external_secret_id if self.full_name is not None: body["full_name"] = self.full_name if self.metastore_id is not None: @@ -9182,8 +9245,6 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the Secret into a shallow dictionary of its immediate attributes.""" body = {} - if self.browse_only is not None: - body["browse_only"] = self.browse_only if self.catalog_name is not None: body["catalog_name"] = self.catalog_name if self.comment is not None: @@ -9198,8 +9259,6 @@ def as_shallow_dict(self) -> dict: body["effective_value"] = self.effective_value if self.expire_time is not None: body["expire_time"] = self.expire_time - if self.external_secret_id is not None: - body["external_secret_id"] = self.external_secret_id if self.full_name is not None: body["full_name"] = self.full_name if self.metastore_id is not None: @@ -9222,7 +9281,6 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> Secret: """Deserializes the Secret from a dictionary.""" return cls( - browse_only=d.get("browse_only", None), catalog_name=d.get("catalog_name", None), comment=d.get("comment", None), create_time=_timestamp(d, "create_time"), @@ -9230,7 +9288,6 @@ def from_dict(cls, d: Dict[str, Any]) -> Secret: effective_owner=d.get("effective_owner", None), effective_value=d.get("effective_value", None), expire_time=_timestamp(d, "expire_time"), - external_secret_id=d.get("external_secret_id", None), full_name=d.get("full_name", None), metastore_id=d.get("metastore_id", None), name=d.get("name", None), @@ -13725,6 +13782,140 @@ def get_effective( ) return EffectivePermissionsList.from_dict(res) + def list( + self, + securable_type: str, + full_name: str, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + principal: Optional[str] = None, + ) -> Iterator[PrivilegeAssignment]: + """Lists the privilege assignments for a securable. Does not include inherited privileges. Paginated + version of Get Permissions API. + + :param securable_type: str + Type of securable. + :param full_name: str + Full name of securable. + :param page_size: int (optional) + Specifies the maximum number of privilege assignments to return (page length). Every + PrivilegeAssignment present in a single page response is guaranteed to contain all the privileges + granted on the requested Securable for the respective principal. + + If not set, page length is the server configured value. If set to + + - lesser than 0: invalid parameter error + - 0: page length is set to a server configured value + - lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able + to return at least one complete PrivilegeAssignment in a single page response) + - greater than (or equal to) 150: page length is the minimum of this value and a server configured + value + :param page_token: str (optional) + Opaque pagination token to go to next page based on previous query. + :param principal: str (optional) + If provided, only the permissions for the specified principal (user or group) are returned. + + :returns: Iterator over :class:`PrivilegeAssignment` + """ + + query = {} + if page_size is not None: + query["page_size"] = page_size + if page_token is not None: + query["page_token"] = page_token + if principal is not None: + query["principal"] = principal + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Workspace-Id"] = cfg.workspace_id + + while True: + json = self._api.do( + "GET", + f"/api/2.1/unity-catalog/privilege-assignments/{securable_type}/{full_name}", + query=query, + headers=headers, + ) + if "privilege_assignments" in json: + for v in json["privilege_assignments"]: + yield PrivilegeAssignment.from_dict(v) + if "next_page_token" not in json or not json["next_page_token"]: + return + query["page_token"] = json["next_page_token"] + + def list_effective( + self, + securable_type: str, + full_name: str, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None, + principal: Optional[str] = None, + ) -> Iterator[EffectivePrivilegeAssignment]: + """Lists the effective privilege assignments for a securable. Includes inherited privileges. Paginated + version of Get Effective Permissions API. + + :param securable_type: str + Type of securable. + :param full_name: str + Full name of securable. + :param page_size: int (optional) + Specifies the maximum number of privilege assignments to return (page length). Every + EffectivePrivilegeAssignment present in a single page response is guaranteed to contain all the + effective privileges granted on (or inherited by) the requested Securable for the respective + principal. + + If not set, a server-configured default is used. If set to + + - lesser than 0: invalid parameter error + - 0: page length is set to a server configured value + - lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able + to return at least one complete EffectivePrivilegeAssignment in a single page response) + - greater than (or equal to) 150: page length is the minimum of this value and a server configured + value + :param page_token: str (optional) + Opaque pagination token to go to next page based on previous query. + :param principal: str (optional) + If provided, only the effective permissions for the specified principal (user or group) are + returned. + + :returns: Iterator over :class:`EffectivePrivilegeAssignment` + """ + + query = {} + if page_size is not None: + query["page_size"] = page_size + if page_token is not None: + query["page_token"] = page_token + if principal is not None: + query["principal"] = principal + headers = { + "Accept": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Workspace-Id"] = cfg.workspace_id + + while True: + json = self._api.do( + "GET", + f"/api/2.1/unity-catalog/effective-privilege-assignments/{securable_type}/{full_name}", + query=query, + headers=headers, + ) + if "effective_privilege_assignments" in json: + for v in json["effective_privilege_assignments"]: + yield EffectivePrivilegeAssignment.from_dict(v) + if "next_page_token" not in json or not json["next_page_token"]: + return + query["page_token"] = json["next_page_token"] + def update( self, securable_type: str, @@ -16196,7 +16387,7 @@ def delete_secret(self, full_name: str): self._api.do("DELETE", f"/api/2.1/unity-catalog/secrets/{full_name}", headers=headers) - def get_secret(self, full_name: str, *, include_browse: Optional[bool] = None) -> Secret: + def get_secret(self, full_name: str) -> Secret: """Gets a secret by its three-level (fully qualified) name. You must be a metastore admin, the owner of the secret, or have the **MANAGE** privilege on the @@ -16208,16 +16399,10 @@ def get_secret(self, full_name: str, *, include_browse: Optional[bool] = None) - :param full_name: str The three-level (fully qualified) name of the secret (for example, **catalog_name.schema_name.secret_name**). - :param include_browse: bool (optional) - Whether to include secrets in the response for which you only have the **BROWSE** privilege, which - limits access to metadata. :returns: :class:`Secret` """ - query = {} - if include_browse is not None: - query["include_browse"] = include_browse headers = { "Accept": "application/json", } @@ -16226,14 +16411,13 @@ def get_secret(self, full_name: str, *, include_browse: Optional[bool] = None) - if cfg.workspace_id: headers["X-Databricks-Workspace-Id"] = cfg.workspace_id - res = self._api.do("GET", f"/api/2.1/unity-catalog/secrets/{full_name}", query=query, headers=headers) + res = self._api.do("GET", f"/api/2.1/unity-catalog/secrets/{full_name}", headers=headers) return Secret.from_dict(res) def list_secrets( self, *, catalog_name: Optional[str] = None, - include_browse: Optional[bool] = None, page_size: Optional[int] = None, page_token: Optional[str] = None, schema_name: Optional[str] = None, @@ -16250,15 +16434,12 @@ def list_secrets( :param catalog_name: str (optional) The name of the catalog under which to list secrets. Both **catalog_name** and **schema_name** must be specified together. - :param include_browse: bool (optional) - Whether to include secrets in the response for which you only have the **BROWSE** privilege, which - limits access to metadata. :param page_size: int (optional) Maximum number of secrets to return. - - If not specified, at most 10000 secrets are returned. - - If set to a value greater than 0, the page length is the minimum of this value and 10000. - - If set to 0, the page length is set to 10000. + - If not specified, at most 1000 secrets are returned. + - If set to a value greater than 0, the page length is the minimum of this value and 1000. + - If set to 0, the page length is set to 1000. - If set to a value less than 0, an invalid parameter error is returned. :param page_token: str (optional) Opaque pagination token to go to the next page based on previous query. The maximum page length is @@ -16273,8 +16454,6 @@ def list_secrets( query = {} if catalog_name is not None: query["catalog_name"] = catalog_name - if include_browse is not None: - query["include_browse"] = include_browse if page_size is not None: query["page_size"] = page_size if page_token is not None: @@ -16314,8 +16493,12 @@ def update_secret(self, full_name: str, secret: Secret, update_mask: FieldMask) The secret object containing the fields to update. Only fields specified in **update_mask** will be updated. :param update_mask: FieldMask - The field mask specifying which fields of the secret to update. Supported fields: **value**, - **comment**, **owner**, **expire_time**. + The field mask specifying which fields of the secret to update. + + - If **update_mask** is **"*"**, all fields specified in **secret** are updated. + - If **update_mask** specifies one or more fields, only those fields are updated. Each specified + field must be set in **secret**. Supported fields: **value**, **comment**, **owner**, + **expire_time**. To change the secret name, delete and recreate the secret. :returns: :class:`Secret` """ diff --git a/databricks/sdk/service/dashboards.py b/databricks/sdk/service/dashboards.py index 8b8ebb981..60aecc643 100755 --- a/databricks/sdk/service/dashboards.py +++ b/databricks/sdk/service/dashboards.py @@ -2848,7 +2848,7 @@ def delete_conversation_message(self, space_id: str, conversation_id: str, messa def download_message_attachment_visualization(self, name: str) -> DownloadMessageAttachmentVisualizationResponse: """Download a rendered image of a message visualization attachment. The response body is the raw PNG image, not a JSON payload. This is only available if the attachment is a visualization and the message - status is ``COMPLETED``. + status is ``COMPLETED``. This endpoint is not supported for Private Link workspaces. :param name: str The resource name of the attachment to render, in the format diff --git a/databricks/sdk/service/database.py b/databricks/sdk/service/database.py index cbe22f757..759493ecf 100755 --- a/databricks/sdk/service/database.py +++ b/databricks/sdk/service/database.py @@ -1444,8 +1444,8 @@ class SyncedTableSpecTypeOverride: """PostgreSQL-specific target type to use for the column.""" size: Optional[int] = None - """Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR (specifies - the vector dimension, e.g., 1024).""" + """Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR or + PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).""" def as_dict(self) -> dict: """Serializes the SyncedTableSpecTypeOverride into a dictionary suitable for use as a JSON request body.""" diff --git a/databricks/sdk/service/disasterrecovery.py b/databricks/sdk/service/disasterrecovery.py index 90b439070..e964075fa 100755 --- a/databricks/sdk/service/disasterrecovery.py +++ b/databricks/sdk/service/disasterrecovery.py @@ -320,6 +320,11 @@ class StableUrl: name: Optional[str] = None """Fully qualified resource name. Format: accounts/{account_id}/stable-urls/{stable_url_id}.""" + stable_workspace_id: Optional[str] = None + """The stable workspace ID for this stable URL. Generated on creation and immutable thereafter; + identifies the URL across failovers and is the same value embedded in the ``url`` (as the ``w=`` + query parameter for SPOG URLs, or in the ``conn-`` hostname for Private-Link URLs).""" + url: Optional[str] = None """The stable URL endpoint. Generated on creation and immutable thereafter. For non-Private-Link workspaces this is ``https:///?w=``. For Private-Link workspaces this @@ -334,6 +339,8 @@ def as_dict(self) -> dict: body["initial_workspace_id"] = self.initial_workspace_id if self.name is not None: body["name"] = self.name + if self.stable_workspace_id is not None: + body["stable_workspace_id"] = self.stable_workspace_id if self.url is not None: body["url"] = self.url return body @@ -347,6 +354,8 @@ def as_shallow_dict(self) -> dict: body["initial_workspace_id"] = self.initial_workspace_id if self.name is not None: body["name"] = self.name + if self.stable_workspace_id is not None: + body["stable_workspace_id"] = self.stable_workspace_id if self.url is not None: body["url"] = self.url return body @@ -358,6 +367,7 @@ def from_dict(cls, d: Dict[str, Any]) -> StableUrl: failover_group_name=d.get("failover_group_name", None), initial_workspace_id=d.get("initial_workspace_id", None), name=d.get("name", None), + stable_workspace_id=d.get("stable_workspace_id", None), url=d.get("url", None), ) diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 7d001d46e..d8cbdfb4f 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -411,6 +411,11 @@ class BaseRun: """The creator user name. This field won’t be included in the response if the user has already been deleted.""" + deployment_id: Optional[str] = None + """ID of the deployment that produced the job when this run was created. Used to look up deployment + metadata from the Deployment Metadata service. Only set for job runs of jobs with a ``BUNDLE`` + deployment.""" + description: Optional[str] = None """Description of the run""" @@ -529,6 +534,11 @@ class BaseRun: trigger_info: Optional[TriggerInfo] = None + version_id: Optional[str] = None + """ID of the deployment version that produced the job when this run was created. Identifies a + specific snapshot of the deployment in the Deployment Metadata service. Only set for job runs of + jobs with a ``BUNDLE`` deployment.""" + def as_dict(self) -> dict: """Serializes the BaseRun into a dictionary suitable for use as a JSON request body.""" body = {} @@ -542,6 +552,8 @@ def as_dict(self) -> dict: body["cluster_spec"] = self.cluster_spec.as_dict() if self.creator_user_name is not None: body["creator_user_name"] = self.creator_user_name + if self.deployment_id is not None: + body["deployment_id"] = self.deployment_id if self.description is not None: body["description"] = self.description if self.effective_performance_target is not None: @@ -600,6 +612,8 @@ def as_dict(self) -> dict: body["trigger"] = self.trigger.value if self.trigger_info: body["trigger_info"] = self.trigger_info.as_dict() + if self.version_id is not None: + body["version_id"] = self.version_id return body def as_shallow_dict(self) -> dict: @@ -615,6 +629,8 @@ def as_shallow_dict(self) -> dict: body["cluster_spec"] = self.cluster_spec if self.creator_user_name is not None: body["creator_user_name"] = self.creator_user_name + if self.deployment_id is not None: + body["deployment_id"] = self.deployment_id if self.description is not None: body["description"] = self.description if self.effective_performance_target is not None: @@ -673,6 +689,8 @@ def as_shallow_dict(self) -> dict: body["trigger"] = self.trigger if self.trigger_info: body["trigger_info"] = self.trigger_info + if self.version_id is not None: + body["version_id"] = self.version_id return body @classmethod @@ -684,6 +702,7 @@ def from_dict(cls, d: Dict[str, Any]) -> BaseRun: cluster_instance=_from_dict(d, "cluster_instance", ClusterInstance), cluster_spec=_from_dict(d, "cluster_spec", ClusterSpec), creator_user_name=d.get("creator_user_name", None), + deployment_id=d.get("deployment_id", None), description=d.get("description", None), effective_performance_target=_enum(d, "effective_performance_target", PerformanceTarget), effective_usage_policy_id=d.get("effective_usage_policy_id", None), @@ -713,6 +732,7 @@ def from_dict(cls, d: Dict[str, Any]) -> BaseRun: tasks=_repeated_dict(d, "tasks", RunTask), trigger=_enum(d, "trigger", TriggerType), trigger_info=_from_dict(d, "trigger_info", TriggerInfo), + version_id=d.get("version_id", None), ) @@ -3436,6 +3456,10 @@ class JobSettings: parameters: Optional[List[JobParameterDefinition]] = None """Job-level parameter definitions""" + parent_path: Optional[str] = None + """Path of the job parent folder in workspace file tree. If absent, the job doesn't have a + workspace object.""" + performance_target: Optional[PerformanceTarget] = None """The performance mode on a serverless job. This field determines the level of compute performance or cost-efficiency for the run. The performance target does not apply to tasks that run on @@ -3518,6 +3542,8 @@ def as_dict(self) -> dict: body["notification_settings"] = self.notification_settings.as_dict() if self.parameters: body["parameters"] = [v.as_dict() for v in self.parameters] + if self.parent_path is not None: + body["parent_path"] = self.parent_path if self.performance_target is not None: body["performance_target"] = self.performance_target.value if self.queue: @@ -3573,6 +3599,8 @@ def as_shallow_dict(self) -> dict: body["notification_settings"] = self.notification_settings if self.parameters: body["parameters"] = self.parameters + if self.parent_path is not None: + body["parent_path"] = self.parent_path if self.performance_target is not None: body["performance_target"] = self.performance_target if self.queue: @@ -3614,6 +3642,7 @@ def from_dict(cls, d: Dict[str, Any]) -> JobSettings: name=d.get("name", None), notification_settings=_from_dict(d, "notification_settings", JobNotificationSettings), parameters=_repeated_dict(d, "parameters", JobParameterDefinition), + parent_path=d.get("parent_path", None), performance_target=_enum(d, "performance_target", PerformanceTarget), queue=_from_dict(d, "queue", QueueSettings), run_as=_from_dict(d, "run_as", JobRunAs), @@ -5206,6 +5235,11 @@ class Run: """The creator user name. This field won’t be included in the response if the user has already been deleted.""" + deployment_id: Optional[str] = None + """ID of the deployment that produced the job when this run was created. Used to look up deployment + metadata from the Deployment Metadata service. Only set for job runs of jobs with a ``BUNDLE`` + deployment.""" + description: Optional[str] = None """Description of the run""" @@ -5330,6 +5364,11 @@ class Run: trigger_info: Optional[TriggerInfo] = None + version_id: Optional[str] = None + """ID of the deployment version that produced the job when this run was created. Identifies a + specific snapshot of the deployment in the Deployment Metadata service. Only set for job runs of + jobs with a ``BUNDLE`` deployment.""" + def as_dict(self) -> dict: """Serializes the Run into a dictionary suitable for use as a JSON request body.""" body = {} @@ -5343,6 +5382,8 @@ def as_dict(self) -> dict: body["cluster_spec"] = self.cluster_spec.as_dict() if self.creator_user_name is not None: body["creator_user_name"] = self.creator_user_name + if self.deployment_id is not None: + body["deployment_id"] = self.deployment_id if self.description is not None: body["description"] = self.description if self.effective_performance_target is not None: @@ -5405,6 +5446,8 @@ def as_dict(self) -> dict: body["trigger"] = self.trigger.value if self.trigger_info: body["trigger_info"] = self.trigger_info.as_dict() + if self.version_id is not None: + body["version_id"] = self.version_id return body def as_shallow_dict(self) -> dict: @@ -5420,6 +5463,8 @@ def as_shallow_dict(self) -> dict: body["cluster_spec"] = self.cluster_spec if self.creator_user_name is not None: body["creator_user_name"] = self.creator_user_name + if self.deployment_id is not None: + body["deployment_id"] = self.deployment_id if self.description is not None: body["description"] = self.description if self.effective_performance_target is not None: @@ -5482,6 +5527,8 @@ def as_shallow_dict(self) -> dict: body["trigger"] = self.trigger if self.trigger_info: body["trigger_info"] = self.trigger_info + if self.version_id is not None: + body["version_id"] = self.version_id return body @classmethod @@ -5493,6 +5540,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Run: cluster_instance=_from_dict(d, "cluster_instance", ClusterInstance), cluster_spec=_from_dict(d, "cluster_spec", ClusterSpec), creator_user_name=d.get("creator_user_name", None), + deployment_id=d.get("deployment_id", None), description=d.get("description", None), effective_performance_target=_enum(d, "effective_performance_target", PerformanceTarget), effective_usage_policy_id=d.get("effective_usage_policy_id", None), @@ -5524,6 +5572,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Run: tasks=_repeated_dict(d, "tasks", RunTask), trigger=_enum(d, "trigger", TriggerType), trigger_info=_from_dict(d, "trigger_info", TriggerInfo), + version_id=d.get("version_id", None), ) @@ -9594,6 +9643,7 @@ def create( name: Optional[str] = None, notification_settings: Optional[JobNotificationSettings] = None, parameters: Optional[List[JobParameterDefinition]] = None, + parent_path: Optional[str] = None, performance_target: Optional[PerformanceTarget] = None, queue: Optional[QueueSettings] = None, run_as: Optional[JobRunAs] = None, @@ -9666,6 +9716,9 @@ def create( ``email_notifications`` and ``webhook_notifications`` for this job. :param parameters: List[:class:`JobParameterDefinition`] (optional) Job-level parameter definitions + :param parent_path: str (optional) + Path of the job parent folder in workspace file tree. If absent, the job doesn't have a workspace + object. :param performance_target: :class:`PerformanceTarget` (optional) The performance mode on a serverless job. This field determines the level of compute performance or cost-efficiency for the run. The performance target does not apply to tasks that run on Serverless @@ -9742,6 +9795,8 @@ def create( body["notification_settings"] = notification_settings.as_dict() if parameters is not None: body["parameters"] = [v.as_dict() for v in parameters] + if parent_path is not None: + body["parent_path"] = parent_path if performance_target is not None: body["performance_target"] = performance_target.value if queue is not None: @@ -10460,6 +10515,11 @@ def run_now( :param only: List[str] (optional) A list of task keys to run inside of the job. If this field is not provided, all tasks in the job will be run. + + Prefix a task key with ``+`` to also run its upstream tasks, or suffix it with ``+`` to also run its + downstream tasks. For example, ``+my_task`` runs ``my_task`` and everything upstream of it, + ``my_task+`` runs ``my_task`` and everything downstream of it, and ``+my_task+`` runs both. A task + key with no ``+`` runs only that task. :param performance_target: :class:`PerformanceTarget` (optional) The performance mode on a serverless job. The performance target determines the level of compute performance or cost-efficiency for the run. This field overrides the performance target defined on diff --git a/databricks/sdk/service/ml.py b/databricks/sdk/service/ml.py index beb714c48..a059cd7fb 100755 --- a/databricks/sdk/service/ml.py +++ b/databricks/sdk/service/ml.py @@ -195,13 +195,13 @@ class AggregationFunction: first: Optional[FirstFunction] = None - first_distinct_n: Optional[FirstDistinctNFunction] = None + first_distinct: Optional[FirstDistinctFunction] = None first_n: Optional[FirstNFunction] = None last: Optional[LastFunction] = None - last_distinct_n: Optional[LastDistinctNFunction] = None + last_distinct: Optional[LastDistinctFunction] = None last_n: Optional[LastNFunction] = None @@ -235,14 +235,14 @@ def as_dict(self) -> dict: body["count_function"] = self.count_function.as_dict() if self.first: body["first"] = self.first.as_dict() - if self.first_distinct_n: - body["first_distinct_n"] = self.first_distinct_n.as_dict() + if self.first_distinct: + body["first_distinct"] = self.first_distinct.as_dict() if self.first_n: body["first_n"] = self.first_n.as_dict() if self.last: body["last"] = self.last.as_dict() - if self.last_distinct_n: - body["last_distinct_n"] = self.last_distinct_n.as_dict() + if self.last_distinct: + body["last_distinct"] = self.last_distinct.as_dict() if self.last_n: body["last_n"] = self.last_n.as_dict() if self.max: @@ -276,14 +276,14 @@ def as_shallow_dict(self) -> dict: body["count_function"] = self.count_function if self.first: body["first"] = self.first - if self.first_distinct_n: - body["first_distinct_n"] = self.first_distinct_n + if self.first_distinct: + body["first_distinct"] = self.first_distinct if self.first_n: body["first_n"] = self.first_n if self.last: body["last"] = self.last - if self.last_distinct_n: - body["last_distinct_n"] = self.last_distinct_n + if self.last_distinct: + body["last_distinct"] = self.last_distinct if self.last_n: body["last_n"] = self.last_n if self.max: @@ -313,10 +313,10 @@ def from_dict(cls, d: Dict[str, Any]) -> AggregationFunction: avg=_from_dict(d, "avg", AvgFunction), count_function=_from_dict(d, "count_function", CountFunction), first=_from_dict(d, "first", FirstFunction), - first_distinct_n=_from_dict(d, "first_distinct_n", FirstDistinctNFunction), + first_distinct=_from_dict(d, "first_distinct", FirstDistinctFunction), first_n=_from_dict(d, "first_n", FirstNFunction), last=_from_dict(d, "last", LastFunction), - last_distinct_n=_from_dict(d, "last_distinct_n", LastDistinctNFunction), + last_distinct=_from_dict(d, "last_distinct", LastDistinctFunction), last_n=_from_dict(d, "last_n", LastNFunction), max=_from_dict(d, "max", MaxFunction), min=_from_dict(d, "min", MinFunction), @@ -1661,6 +1661,10 @@ class Experiment: tags: Optional[List[ExperimentTag]] = None """Tags: Additional metadata key-value pairs.""" + trace_location: Optional[ExperimentTraceLocation] = None + """The location where the experiment's traces are stored. Unset when traces are stored in the + default MLflow backend. This field cannot be updated after the experiment is created.""" + def as_dict(self) -> dict: """Serializes the Experiment into a dictionary suitable for use as a JSON request body.""" body = {} @@ -1678,6 +1682,8 @@ def as_dict(self) -> dict: body["name"] = self.name if self.tags: body["tags"] = [v.as_dict() for v in self.tags] + if self.trace_location: + body["trace_location"] = self.trace_location.as_dict() return body def as_shallow_dict(self) -> dict: @@ -1697,6 +1703,8 @@ def as_shallow_dict(self) -> dict: body["name"] = self.name if self.tags: body["tags"] = self.tags + if self.trace_location: + body["trace_location"] = self.trace_location return body @classmethod @@ -1710,6 +1718,7 @@ def from_dict(cls, d: Dict[str, Any]) -> Experiment: lifecycle_stage=d.get("lifecycle_stage", None), name=d.get("name", None), tags=_repeated_dict(d, "tags", ExperimentTag), + trace_location=_from_dict(d, "trace_location", ExperimentTraceLocation), ) @@ -1977,6 +1986,33 @@ def from_dict(cls, d: Dict[str, Any]) -> ExperimentTag: return cls(key=d.get("key", None), value=d.get("value", None)) +@dataclass +class ExperimentTraceLocation: + """The storage location for an experiment's traces.""" + + uc_trace_location: Optional[UcTraceLocation] = None + """A Unity Catalog schema where the experiment's traces are stored as Delta tables.""" + + def as_dict(self) -> dict: + """Serializes the ExperimentTraceLocation into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.uc_trace_location: + body["uc_trace_location"] = self.uc_trace_location.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ExperimentTraceLocation into a shallow dictionary of its immediate attributes.""" + body = {} + if self.uc_trace_location: + body["uc_trace_location"] = self.uc_trace_location + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ExperimentTraceLocation: + """Deserializes the ExperimentTraceLocation from a dictionary.""" + return cls(uc_trace_location=_from_dict(d, "uc_trace_location", UcTraceLocation)) + + @dataclass class Feature: full_name: str @@ -2417,7 +2453,7 @@ def from_dict(cls, d: Dict[str, Any]) -> FinalizeLoggedModelResponse: @dataclass -class FirstDistinctNFunction: +class FirstDistinctFunction: """Returns the first N distinct values, ordered by the feature's timeseries column.""" input: str @@ -2427,7 +2463,7 @@ class FirstDistinctNFunction: """The number of distinct values to return.""" def as_dict(self) -> dict: - """Serializes the FirstDistinctNFunction into a dictionary suitable for use as a JSON request body.""" + """Serializes the FirstDistinctFunction into a dictionary suitable for use as a JSON request body.""" body = {} if self.input is not None: body["input"] = self.input @@ -2436,7 +2472,7 @@ def as_dict(self) -> dict: return body def as_shallow_dict(self) -> dict: - """Serializes the FirstDistinctNFunction into a shallow dictionary of its immediate attributes.""" + """Serializes the FirstDistinctFunction into a shallow dictionary of its immediate attributes.""" body = {} if self.input is not None: body["input"] = self.input @@ -2445,8 +2481,8 @@ def as_shallow_dict(self) -> dict: return body @classmethod - def from_dict(cls, d: Dict[str, Any]) -> FirstDistinctNFunction: - """Deserializes the FirstDistinctNFunction from a dictionary.""" + def from_dict(cls, d: Dict[str, Any]) -> FirstDistinctFunction: + """Deserializes the FirstDistinctFunction from a dictionary.""" return cls(input=d.get("input", None), n=d.get("n", None)) @@ -3579,7 +3615,7 @@ def from_dict(cls, d: Dict[str, Any]) -> KafkaSubscriptionMode: @dataclass -class LastDistinctNFunction: +class LastDistinctFunction: """Returns the last N distinct values, ordered by the feature's timeseries column.""" input: str @@ -3589,7 +3625,7 @@ class LastDistinctNFunction: """The number of distinct values to return.""" def as_dict(self) -> dict: - """Serializes the LastDistinctNFunction into a dictionary suitable for use as a JSON request body.""" + """Serializes the LastDistinctFunction into a dictionary suitable for use as a JSON request body.""" body = {} if self.input is not None: body["input"] = self.input @@ -3598,7 +3634,7 @@ def as_dict(self) -> dict: return body def as_shallow_dict(self) -> dict: - """Serializes the LastDistinctNFunction into a shallow dictionary of its immediate attributes.""" + """Serializes the LastDistinctFunction into a shallow dictionary of its immediate attributes.""" body = {} if self.input is not None: body["input"] = self.input @@ -3607,8 +3643,8 @@ def as_shallow_dict(self) -> dict: return body @classmethod - def from_dict(cls, d: Dict[str, Any]) -> LastDistinctNFunction: - """Deserializes the LastDistinctNFunction from a dictionary.""" + def from_dict(cls, d: Dict[str, Any]) -> LastDistinctFunction: + """Deserializes the LastDistinctFunction from a dictionary.""" return cls(input=d.get("input", None), n=d.get("n", None)) @@ -4529,6 +4565,47 @@ def from_dict(cls, d: Dict[str, Any]) -> LoggedModelTag: return cls(key=d.get("key", None), value=d.get("value", None)) +@dataclass +class LongRollingWindow: + """A long (multi-day) rolling window served via the hybrid batch + streaming path. The batch + pipeline maintains daily partial aggregates for the bulk of the window while the streaming + pipeline maintains the most recent day(s), and serving merges them on read. Distinct from + RollingWindow so the control plane can explicitly identify long rolling window features rather + than inferring hybrid behavior from window_duration.""" + + window_duration: Duration + """The duration of the rolling window. Must be positive and span more than two days, so that both + the batch (N-1 day) and stale-path (N-2 day) partial aggregates are well defined. The duration + need not be a whole number of days (e.g. 3 days 15 minutes is allowed).""" + + delay: Optional[Duration] = None + """The delay applied to the end of the rolling window (must be non-negative). For example, delay=1d + shifts the window end 1 day before the evaluation time.""" + + def as_dict(self) -> dict: + """Serializes the LongRollingWindow into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.delay is not None: + body["delay"] = self.delay.ToJsonString() + if self.window_duration is not None: + body["window_duration"] = self.window_duration.ToJsonString() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the LongRollingWindow into a shallow dictionary of its immediate attributes.""" + body = {} + if self.delay is not None: + body["delay"] = self.delay + if self.window_duration is not None: + body["window_duration"] = self.window_duration + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> LongRollingWindow: + """Deserializes the LongRollingWindow from a dictionary.""" + return cls(delay=_duration(d, "delay"), window_duration=_duration(d, "window_duration")) + + @dataclass class MaterializedFeature: """A materialized feature represents a feature that is continuously computed and stored.""" @@ -5715,6 +5792,46 @@ class PermissionLevel(Enum): CAN_READ = "CAN_READ" +@dataclass +class ProtoSchemaSpec: + """A Protocol Buffer schema paired with the name of the message within it that describes the Kafka + payload. A .proto file may declare multiple messages; message_name disambiguates.""" + + schema_text: str + """The raw .proto file text (proto2 and proto3 syntax supported, see + https://protobuf.dev/programming-guides/proto3/ and + https://protobuf.dev/programming-guides/proto2/).""" + + message_name: str + """The fully-qualified name of the message within schema_text that describes the Kafka payload + (e.g. "Event" or "com.example.Event" if schema_text declares a package). Identifies which + message is used to decode each Kafka record — a .proto file may declare multiple messages but + only one represents the payload. Must not be empty.""" + + def as_dict(self) -> dict: + """Serializes the ProtoSchemaSpec into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.message_name is not None: + body["message_name"] = self.message_name + if self.schema_text is not None: + body["schema_text"] = self.schema_text + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ProtoSchemaSpec into a shallow dictionary of its immediate attributes.""" + body = {} + if self.message_name is not None: + body["message_name"] = self.message_name + if self.schema_text is not None: + body["schema_text"] = self.schema_text + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ProtoSchemaSpec: + """Deserializes the ProtoSchemaSpec from a dictionary.""" + return cls(message_name=d.get("message_name", None), schema_text=d.get("schema_text", None)) + + @dataclass class PublishSpec: online_store: str @@ -6647,27 +6764,45 @@ class ScalarDataType(Enum): @dataclass class SchemaConfig: + avro_schema: Optional[str] = None + """Avro schema in JSON format (https://avro.apache.org/docs/current/specification/).""" + json_schema: Optional[str] = None """Schema of the JSON object in standard IETF JSON schema format (https://json-schema.org/).""" + proto_schema: Optional[ProtoSchemaSpec] = None + """Protocol Buffer schema with its payload message name.""" + def as_dict(self) -> dict: """Serializes the SchemaConfig into a dictionary suitable for use as a JSON request body.""" body = {} + if self.avro_schema is not None: + body["avro_schema"] = self.avro_schema if self.json_schema is not None: body["json_schema"] = self.json_schema + if self.proto_schema: + body["proto_schema"] = self.proto_schema.as_dict() return body def as_shallow_dict(self) -> dict: """Serializes the SchemaConfig into a shallow dictionary of its immediate attributes.""" body = {} + if self.avro_schema is not None: + body["avro_schema"] = self.avro_schema if self.json_schema is not None: body["json_schema"] = self.json_schema + if self.proto_schema: + body["proto_schema"] = self.proto_schema return body @classmethod def from_dict(cls, d: Dict[str, Any]) -> SchemaConfig: """Deserializes the SchemaConfig from a dictionary.""" - return cls(json_schema=d.get("json_schema", None)) + return cls( + avro_schema=d.get("avro_schema", None), + json_schema=d.get("json_schema", None), + proto_schema=_from_dict(d, "proto_schema", ProtoSchemaSpec), + ) @dataclass @@ -7399,12 +7534,18 @@ def from_dict(cls, d: Dict[str, Any]) -> StreamSourceConfig: class StreamingMode: """The streaming mode configuration for a streaming materialization pipeline.""" + freshness_target: Optional[str] = None + """The desired data freshness for feature materialization, expressed as a duration string (e.g. "1 + minute").""" + mode: Optional[StreamingModeStreamingModeType] = None """The type of streaming mode used by the materialization pipeline.""" def as_dict(self) -> dict: """Serializes the StreamingMode into a dictionary suitable for use as a JSON request body.""" body = {} + if self.freshness_target is not None: + body["freshness_target"] = self.freshness_target if self.mode is not None: body["mode"] = self.mode.value return body @@ -7412,6 +7553,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the StreamingMode into a shallow dictionary of its immediate attributes.""" body = {} + if self.freshness_target is not None: + body["freshness_target"] = self.freshness_target if self.mode is not None: body["mode"] = self.mode return body @@ -7419,7 +7562,9 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> StreamingMode: """Deserializes the StreamingMode from a dictionary.""" - return cls(mode=_enum(d, "mode", StreamingModeStreamingModeType)) + return cls( + freshness_target=d.get("freshness_target", None), mode=_enum(d, "mode", StreamingModeStreamingModeType) + ) class StreamingModeStreamingModeType(Enum): @@ -7560,6 +7705,9 @@ def from_dict(cls, d: Dict[str, Any]) -> TestRegistryWebhookResponse: class TimeWindow: continuous: Optional[ContinuousWindow] = None + long_rolling: Optional[LongRollingWindow] = None + """A long (multi-day) rolling window served via the hybrid batch + streaming path.""" + rolling: Optional[RollingWindow] = None sliding: Optional[SlidingWindow] = None @@ -7571,6 +7719,8 @@ def as_dict(self) -> dict: body = {} if self.continuous: body["continuous"] = self.continuous.as_dict() + if self.long_rolling: + body["long_rolling"] = self.long_rolling.as_dict() if self.rolling: body["rolling"] = self.rolling.as_dict() if self.sliding: @@ -7584,6 +7734,8 @@ def as_shallow_dict(self) -> dict: body = {} if self.continuous: body["continuous"] = self.continuous + if self.long_rolling: + body["long_rolling"] = self.long_rolling if self.rolling: body["rolling"] = self.rolling if self.sliding: @@ -7597,6 +7749,7 @@ def from_dict(cls, d: Dict[str, Any]) -> TimeWindow: """Deserializes the TimeWindow from a dictionary.""" return cls( continuous=_from_dict(d, "continuous", ContinuousWindow), + long_rolling=_from_dict(d, "long_rolling", LongRollingWindow), rolling=_from_dict(d, "rolling", RollingWindow), sliding=_from_dict(d, "sliding", SlidingWindow), tumbling=_from_dict(d, "tumbling", TumblingWindow), @@ -7750,6 +7903,52 @@ def from_dict(cls, d: Dict[str, Any]) -> TumblingWindow: return cls(window_duration=d.get("window_duration", None)) +@dataclass +class UcTraceLocation: + """A Unity Catalog trace storage location. Traces are stored as Delta tables in the specified + catalog and schema.""" + + catalog: str + """The name of the Unity Catalog catalog.""" + + schema: str + """The name of the Unity Catalog schema within ``catalog``.""" + + table_prefix: Optional[str] = None + """The prefix for the trace tables, which are named ``{catalog}.{schema}.{table_prefix}_otel_*``. + May only contain letters, digits, and underscores, and may be at most 238 characters. When + unset, a server-generated prefix derived from the experiment ID is used.""" + + def as_dict(self) -> dict: + """Serializes the UcTraceLocation into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.catalog is not None: + body["catalog"] = self.catalog + if self.schema is not None: + body["schema"] = self.schema + if self.table_prefix is not None: + body["table_prefix"] = self.table_prefix + return body + + def as_shallow_dict(self) -> dict: + """Serializes the UcTraceLocation into a shallow dictionary of its immediate attributes.""" + body = {} + if self.catalog is not None: + body["catalog"] = self.catalog + if self.schema is not None: + body["schema"] = self.schema + if self.table_prefix is not None: + body["table_prefix"] = self.table_prefix + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> UcTraceLocation: + """Deserializes the UcTraceLocation from a dictionary.""" + return cls( + catalog=d.get("catalog", None), schema=d.get("schema", None), table_prefix=d.get("table_prefix", None) + ) + + @dataclass class UpdateCommentResponse: comment: Optional[CommentObject] = None @@ -7975,7 +8174,12 @@ def __init__(self, api_client): self._api = api_client def create_experiment( - self, name: str, *, artifact_location: Optional[str] = None, tags: Optional[List[ExperimentTag]] = None + self, + name: str, + *, + artifact_location: Optional[str] = None, + tags: Optional[List[ExperimentTag]] = None, + trace_location: Optional[ExperimentTraceLocation] = None, ) -> CreateExperimentResponse: """Creates an experiment with a name. Returns the ID of the newly created experiment. Validates that another experiment with the same name does not already exist and fails if another experiment with the @@ -7995,6 +8199,10 @@ def create_experiment( depends on the storage backend. All storage backends are guaranteed to support tag keys up to 250 bytes in size and tag values up to 5000 bytes in size. All storage backends are also guaranteed to support up to 20 tags per request. + :param trace_location: :class:`ExperimentTraceLocation` (optional) + The location where the experiment's traces are stored. When set, the underlying storage is + provisioned and the experiment's traces are routed to it. When unset, traces are stored in the + default MLflow backend. This field cannot be updated after the experiment is created. :returns: :class:`CreateExperimentResponse` """ @@ -8006,6 +8214,8 @@ def create_experiment( body["name"] = name if tags is not None: body["tags"] = [v.as_dict() for v in tags] + if trace_location is not None: + body["trace_location"] = trace_location.as_dict() headers = { "Accept": "application/json", "Content-Type": "application/json", diff --git a/databricks/sdk/service/postgres.py b/databricks/sdk/service/postgres.py index d50474293..1c1fe3314 100755 --- a/databricks/sdk/service/postgres.py +++ b/databricks/sdk/service/postgres.py @@ -970,6 +970,14 @@ def from_dict(cls, d: Dict[str, Any]) -> DatabaseCredential: @dataclass class DatabaseDatabaseSpec: + role: str + """The name of the role that owns the database. Format: + projects/{project_id}/branches/{branch_id}/roles/{role_id} + + To change the owner, pass valid existing Role name when updating the Database + + A database always has an owner.""" + postgres_database: Optional[str] = None """The name of the Postgres database. @@ -979,14 +987,6 @@ class DatabaseDatabaseSpec: To rename, pass a valid postgres identifier when updating the Database.""" - role: Optional[str] = None - """The name of the role that owns the database. Format: - projects/{project_id}/branches/{branch_id}/roles/{role_id} - - To change the owner, pass valid existing Role name when updating the Database - - A database always has an owner.""" - def as_dict(self) -> dict: """Serializes the DatabaseDatabaseSpec into a dictionary suitable for use as a JSON request body.""" body = {} @@ -3343,8 +3343,8 @@ class SyncedTableSyncedTableSpecTypeOverride: """PostgreSQL-specific target type to use for the column.""" size: Optional[int] = None - """Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR (specifies - the vector dimension, e.g., 1024).""" + """Size parameter for the target type. Required when pg_type is PG_SPECIFIC_TYPE_VECTOR or + PG_SPECIFIC_TYPE_HALFVEC (specifies the vector dimension, e.g., 1024).""" def as_dict(self) -> dict: """Serializes the SyncedTableSyncedTableSpecTypeOverride into a dictionary suitable for use as a JSON request body.""" @@ -3989,7 +3989,6 @@ def generate_database_credential( *, claims: Optional[List[RequestedClaims]] = None, expire_time: Optional[Timestamp] = None, - group_name: Optional[str] = None, ttl: Optional[Duration] = None, ) -> DatabaseCredential: """Generate OAuth credentials for a Postgres database. @@ -4002,9 +4001,6 @@ def generate_database_credential( :param expire_time: Timestamp (optional) Timestamp in UTC of when this credential should expire. Must be at least 300 seconds (5 minutes) and at most 1 hour from the current time. - :param group_name: str (optional) - Databricks workspace group name. When provided, credentials are generated with permissions scoped to - this group. :param ttl: Duration (optional) The requested time-to-live for the generated credential token. Must be at least 300 seconds (5 minutes) and at most 3600 seconds (1 hour). @@ -4019,8 +4015,6 @@ def generate_database_credential( body["endpoint"] = endpoint if expire_time is not None: body["expire_time"] = expire_time.ToJsonString() - if group_name is not None: - body["group_name"] = group_name if ttl is not None: body["ttl"] = ttl.ToJsonString() headers = { diff --git a/databricks/sdk/service/serving.py b/databricks/sdk/service/serving.py index e725ce43f..d6bce71f8 100755 --- a/databricks/sdk/service/serving.py +++ b/databricks/sdk/service/serving.py @@ -3110,6 +3110,8 @@ class ServedModelInputWorkloadType(Enum): """Please keep this in sync with workload types in InferenceEndpointEntities.scala.""" CPU = "CPU" + CPU_LARGE = "CPU_LARGE" + CPU_MEDIUM = "CPU_MEDIUM" GPU_LARGE = "GPU_LARGE" GPU_MEDIUM = "GPU_MEDIUM" GPU_SMALL = "GPU_SMALL" @@ -3924,6 +3926,8 @@ class ServingModelWorkloadType(Enum): """Please keep this in sync with workload types in InferenceEndpointEntities.scala.""" CPU = "CPU" + CPU_LARGE = "CPU_LARGE" + CPU_MEDIUM = "CPU_MEDIUM" GPU_LARGE = "GPU_LARGE" GPU_MEDIUM = "GPU_MEDIUM" GPU_SMALL = "GPU_SMALL" diff --git a/databricks/sdk/service/settings.py b/databricks/sdk/service/settings.py index 12d72dfbe..3edf8d344 100755 --- a/databricks/sdk/service/settings.py +++ b/databricks/sdk/service/settings.py @@ -1066,14 +1066,13 @@ def from_dict(cls, d: Dict[str, Any]) -> CspEnablementAccountSetting: @dataclass class CustomerFacingIngressNetworkPolicy: - """This proto is under development. The network policies applying for ingress traffic. Any changes - here should also be synced to estore/namespaces/lakehousenetworkmanager/latest.proto.""" + """The network policies applying for ingress traffic.""" cross_workspace_access: Optional[CustomerFacingIngressNetworkPolicyCrossWorkspaceAccess] = None private_access: Optional[CustomerFacingIngressNetworkPolicyPrivateAccess] = None - """The network policy restrictions for private access to the workspace. Configures how registered - private endpoints are allowed or denied access.""" + """The network policy restrictions for private access. Configures how requests arriving over + private connectivity are governed.""" public_access: Optional[CustomerFacingIngressNetworkPolicyPublicAccess] = None """The network policy restrictions for public access to the workspace. Configures how public @@ -1121,6 +1120,7 @@ class CustomerFacingIngressNetworkPolicyAccountApiDestination: """Qualifies the breadth of API access for the listed scopes. See ApiScopeQualifier.""" scopes: Optional[List[str]] = None + """The API scopes to match. Use "all-apis" to match any account-level API.""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyAccountApiDestination into a dictionary suitable for use as a JSON request body.""" @@ -1176,6 +1176,8 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyAccou @dataclass class CustomerFacingIngressNetworkPolicyAccountUiDestination: + """The account console UI destination.""" + all_destinations: Optional[bool] = None """Must be set to true.""" @@ -1445,7 +1447,10 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyCross @dataclass class CustomerFacingIngressNetworkPolicyEndpoints: + """A set of registered endpoints, identified by their endpoint IDs.""" + endpoint_ids: Optional[List[str]] = None + """The IDs of the registered endpoints. Must contain at least one endpoint ID.""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyEndpoints into a dictionary suitable for use as a JSON request body.""" @@ -1519,11 +1524,20 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyLakeb @dataclass class CustomerFacingIngressNetworkPolicyPrivateAccess: + """Configures how requests arriving over private connectivity, such as registered endpoints, are + allowed or denied access.""" + restriction_mode: CustomerFacingIngressNetworkPolicyPrivateAccessRestrictionMode + """The restriction mode for private access.""" allow_rules: Optional[List[CustomerFacingIngressNetworkPolicyPrivateIngressRule]] = None + """Allow rules are evaluated after deny rules. A request matching any allow rule is allowed; a + request matching no rule is denied by default. Only applies when restriction_mode is + RESTRICTED_ACCESS.""" deny_rules: Optional[List[CustomerFacingIngressNetworkPolicyPrivateIngressRule]] = None + """Deny rules are evaluated first. A request matching any deny rule is denied, regardless of allow + rules. Only applies when restriction_mode is RESTRICTED_ACCESS.""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyPrivateAccess into a dictionary suitable for use as a JSON request body.""" @@ -1560,20 +1574,37 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyPriva class CustomerFacingIngressNetworkPolicyPrivateAccessRestrictionMode(Enum): + """The restriction mode for private access. In ALLOW_ALL_REGISTERED_ENDPOINTS mode, requests + arriving through any endpoint registered to the account are allowed, and deny rules and allow + rules cannot be set. In RESTRICTED_ACCESS mode, access is restricted based on deny rules and + allow rules; requests that do not match any allow rule are denied.""" + ALLOW_ALL_REGISTERED_ENDPOINTS = "ALLOW_ALL_REGISTERED_ENDPOINTS" RESTRICTED_ACCESS = "RESTRICTED_ACCESS" @dataclass class CustomerFacingIngressNetworkPolicyPrivateIngressRule: + """An ingress rule is enforced when a request satisfies all specified attributes — including + request origin, destination, and authentication.""" + authentication: Optional[CustomerFacingIngressNetworkPolicyAuthentication] = None + """The authenticated identity the request must match. When unset, the rule matches all users and + service principals. On the account-level network policy, scoping to specific identities is not + currently supported, so this field must be unset (the rule matches all users and service + principals).""" destination: Optional[CustomerFacingIngressNetworkPolicyRequestDestination] = None + """The destination the request must match — the resource being accessed, for example the + workspace UI, workspace APIs, or account-level APIs. See RequestDestination.""" label: Optional[str] = None """The label for this ingress rule.""" origin: Optional[CustomerFacingIngressNetworkPolicyPrivateRequestOrigin] = None + """The origin the request must match — the private connectivity the request arrives through, for + example a specific set of registered endpoints or any endpoint registered to the account. See + PrivateRequestOrigin.""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyPrivateIngressRule into a dictionary suitable for use as a JSON request body.""" @@ -1614,13 +1645,25 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyPriva @dataclass class CustomerFacingIngressNetworkPolicyPrivateRequestOrigin: + """The origin of a private access request, identified by the endpoint through which the request + arrives.""" + all_private_access: Optional[bool] = None + """Matches requests arriving over any private connectivity, including registered endpoints and the + workspace's Azure Private Link (ui-api) endpoints. Can only be used in deny rules of + workspace-level network policies. Must be set to true when specified.""" all_registered_endpoints: Optional[bool] = None + """Matches requests arriving through any endpoint registered to the account. Must be set to true + when specified.""" azure_workspace_private_link: Optional[bool] = None + """Matches requests arriving through the workspace's Azure Private Link (ui-api) endpoints. Can + only be used in deny rules of workspace-level network policies. Must be set to true when + specified.""" endpoints: Optional[CustomerFacingIngressNetworkPolicyEndpoints] = None + """Matches requests arriving through any of the specified registered endpoints.""" def as_dict(self) -> dict: """Serializes the CustomerFacingIngressNetworkPolicyPrivateRequestOrigin into a dictionary suitable for use as a JSON request body.""" @@ -1803,12 +1846,14 @@ def from_dict(cls, d: Dict[str, Any]) -> CustomerFacingIngressNetworkPolicyPubli @dataclass class CustomerFacingIngressNetworkPolicyRequestDestination: account_api: Optional[CustomerFacingIngressNetworkPolicyAccountApiDestination] = None + """Matches requests to account-level APIs. Can only be used in the account-level network policy.""" account_databricks_one: Optional[CustomerFacingIngressNetworkPolicyAccountDatabricksOneDestination] = None - """Account DatabricksOne destination is not supported. DO NOT change the stage of this destination - past PRIVATE_PREVIEW.""" + """Account DatabricksOne destination is not supported.""" account_ui: Optional[CustomerFacingIngressNetworkPolicyAccountUiDestination] = None + """Matches requests to the account console UI. Can only be used in the account-level network + policy.""" all_destinations: Optional[bool] = None """When true, match all destinations, no other destination fields can be set. When not set or diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index 7bb2f17c3..748f8c29a 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -8071,7 +8071,7 @@ def create( If this is a textbox widget, the application displays this text. This field is ignored if the widget contains a visualization in the ``visualization`` field. :param visualization_id: str (optional) - Query Vizualization ID returned by :method:queryvisualizations/create. + Query Visualization ID returned by :method:queryvisualizations/create. :returns: :class:`Widget` """ @@ -8141,7 +8141,7 @@ def update( If this is a textbox widget, the application displays this text. This field is ignored if the widget contains a visualization in the ``visualization`` field. :param visualization_id: str (optional) - Query Vizualization ID returned by :method:queryvisualizations/create. + Query Visualization ID returned by :method:queryvisualizations/create. :returns: :class:`Widget` """ @@ -9138,7 +9138,7 @@ def update( class QueryVisualizationsLegacyAPI: - """This is an evolving API that facilitates the addition and removal of vizualisations from existing queries + """This is an evolving API that facilitates the addition and removal of visualizations from existing queries within the Databricks Workspace. Data structures may change over time. **Warning**: This API is deprecated. Please see the latest version of the Databricks SQL API. `Learn more diff --git a/databricks/sdk/service/workspace.py b/databricks/sdk/service/workspace.py index f9028c259..8290f711a 100755 --- a/databricks/sdk/service/workspace.py +++ b/databricks/sdk/service/workspace.py @@ -1963,7 +1963,13 @@ def __init__(self, api_client): self._api = api_client def create( - self, url: str, provider: str, *, path: Optional[str] = None, sparse_checkout: Optional[SparseCheckout] = None + self, + url: str, + provider: str, + *, + git_credential_id: Optional[int] = None, + path: Optional[str] = None, + sparse_checkout: Optional[SparseCheckout] = None, ) -> CreateRepoResponse: """Creates a repo in the workspace and links it to the remote Git repo specified. Note that repos created programmatically must be linked to a remote Git repo, unlike repos created in the browser. @@ -1976,6 +1982,9 @@ def create( Entra ID authentication), ``gitHubEnterprise``, ``bitbucketServer`` (Bitbucket Data Center), ``gitLabEnterpriseEdition`` (GitLab Self-Managed), and ``awsCodeCommit`` (deprecated by AWS, not accepting new customers). + :param git_credential_id: int (optional) + Git credential ID to use when cloning the repository. The Git credential must be configured for the + current user. :param path: str (optional) Desired path for the repo in the workspace. Almost any path in the workspace can be chosen. If repo is created in ``/Repos``, path must be in the format ``/Repos/{folder}/{repo-name}``. @@ -1987,6 +1996,8 @@ def create( """ body = {} + if git_credential_id is not None: + body["git_credential_id"] = git_credential_id if path is not None: body["path"] = path if provider is not None: @@ -2157,6 +2168,7 @@ def update( *, branch: Optional[str] = None, dangerously_force_discard_all: Optional[bool] = None, + git_credential_id: Optional[int] = None, sparse_checkout: Optional[SparseCheckoutUpdate] = None, tag: Optional[str] = None, ): @@ -2178,6 +2190,9 @@ def update( destroyed without warning. Local commits that have been made but not yet pushed to the remote are preserved. + :param git_credential_id: int (optional) + Git credential ID to use for this update operation. The Git credential must be configured for the + current user. :param sparse_checkout: :class:`SparseCheckoutUpdate` (optional) If specified, update the sparse checkout settings. The update will fail if sparse checkout is not enabled for the repo. @@ -2194,6 +2209,8 @@ def update( body["branch"] = branch if dangerously_force_discard_all is not None: body["dangerously_force_discard_all"] = dangerously_force_discard_all + if git_credential_id is not None: + body["git_credential_id"] = git_credential_id if sparse_checkout is not None: body["sparse_checkout"] = sparse_checkout.as_dict() if tag is not None: diff --git a/databricks/sdk/version.py b/databricks/sdk/version.py index 1532cfa22..1b44a902d 100644 --- a/databricks/sdk/version.py +++ b/databricks/sdk/version.py @@ -1 +1 @@ -__version__ = "0.120.0" +__version__ = "0.121.0" diff --git a/docs/account/iam/workspace_assignment.rst b/docs/account/iam/workspace_assignment.rst index ca78b86df..fa9c2ee3e 100755 --- a/docs/account/iam/workspace_assignment.rst +++ b/docs/account/iam/workspace_assignment.rst @@ -43,9 +43,9 @@ a = AccountClient() - workspace_id = os.environ["TEST_WORKSPACE_ID"] + workspace_id = os.environ["DUMMY_WORKSPACE_ID"] - all = a.workspace_assignment.list(list=workspace_id) + all = a.workspace_assignment.list(workspace_id=workspace_id) Get the permission assignments for the specified Databricks account and Databricks workspace. @@ -74,9 +74,9 @@ spn_id = spn.id - workspace_id = os.environ["TEST_WORKSPACE_ID"] + workspace_id = os.environ["DUMMY_WORKSPACE_ID"] - a.workspace_assignment.update( + _ = a.workspace_assignment.update( workspace_id=workspace_id, principal_id=spn_id, permissions=[iam.WorkspacePermission.USER], diff --git a/docs/account/provisioning/storage.rst b/docs/account/provisioning/storage.rst index 41a04deb3..b9f080e36 100644 --- a/docs/account/provisioning/storage.rst +++ b/docs/account/provisioning/storage.rst @@ -16,6 +16,7 @@ .. code-block:: + import os import time from databricks.sdk import AccountClient @@ -23,13 +24,13 @@ a = AccountClient() - bucket = a.storage.create( + storage = a.storage.create( storage_configuration_name=f"sdk-{time.time_ns()}", - root_bucket_info=provisioning.RootBucketInfo(bucket_name=f"sdk-{time.time_ns()}"), + root_bucket_info=provisioning.RootBucketInfo(bucket_name=os.environ["TEST_ROOT_BUCKET"]), ) # cleanup - a.storage.delete(storage_configuration_id=bucket.storage_configuration_id) + a.storage.delete(storage_configuration_id=storage.storage_configuration_id) Creates a Databricks storage configuration for an account. diff --git a/docs/auth-types-reference.md b/docs/auth-types-reference.md new file mode 100644 index 000000000..47837cc1c --- /dev/null +++ b/docs/auth-types-reference.md @@ -0,0 +1,271 @@ +# Authentication Types Reference + +This document lists all authentication types (`auth_type`) supported by the Databricks SDK for Python. + +## Authentication Types Table + +| Auth Type | Description | Required Parameters | Optional Parameters | Environment Variables | +|-----------|-------------|---------------------|---------------------|----------------------| +| `pat` | Personal Access Token authentication - the most common method for programmatic access | `host`, `token` | - | `DATABRICKS_HOST`, `DATABRICKS_TOKEN` | +| `basic` | Basic HTTP authentication using username and password (primarily for AWS) | `host`, `username`, `password` | `account_id` (for account-level operations) | `DATABRICKS_HOST`, `DATABRICKS_USERNAME`, `DATABRICKS_PASSWORD`, `DATABRICKS_ACCOUNT_ID` | +| `oauth-m2m` | OAuth 2.0 Machine-to-Machine (service principal) authentication | `host`, `client_id`, `client_secret` | `scopes`, `authorization_details` | `DATABRICKS_HOST`, `DATABRICKS_CLIENT_ID`, `DATABRICKS_CLIENT_SECRET` | +| `external-browser` | OAuth 2.0 authentication flow using local browser for user login | `host`, `auth_type='external-browser'` | `client_id`, `client_secret` | `DATABRICKS_HOST`, `DATABRICKS_AUTH_TYPE`, `DATABRICKS_CLIENT_ID` | +| `databricks-cli` | Uses tokens from the Databricks CLI (`databricks auth login`) | `host` | `account_id` (for account-level), `databricks_cli_path` | `DATABRICKS_HOST`, `DATABRICKS_ACCOUNT_ID`, `DATABRICKS_CLI_PATH` | +| `azure-client-secret` | Azure Active Directory (AAD) Service Principal authentication | `azure_client_id`, `azure_client_secret` | `azure_tenant_id` (auto-detected from `host` if not set), `host`, `azure_workspace_resource_id`, `azure_environment` | `ARM_CLIENT_ID`, `ARM_CLIENT_SECRET`, `ARM_TENANT_ID`, `DATABRICKS_HOST`, `DATABRICKS_AZURE_RESOURCE_ID`, `ARM_ENVIRONMENT` | +| `azure-cli` | Uses credentials from Azure CLI (`az login`) | `host` (or `azure_workspace_resource_id`) | `azure_tenant_id` | `DATABRICKS_HOST`, `DATABRICKS_AZURE_RESOURCE_ID`, `ARM_TENANT_ID` | +| `github-oidc` | GitHub Actions OIDC authentication (workload identity federation) | `host`, `client_id` | `token_audience`, `account_id` | `DATABRICKS_HOST`, `DATABRICKS_CLIENT_ID`, `DATABRICKS_TOKEN_AUDIENCE`, `DATABRICKS_ACCOUNT_ID` | +| `github-oidc-azure` | GitHub Actions OIDC for Azure Databricks workspaces | `host`, `azure_client_id` | `azure_tenant_id` | `DATABRICKS_HOST`, `ARM_CLIENT_ID`, `ARM_TENANT_ID` | +| `azure-devops-oidc` | Azure DevOps Pipelines OIDC authentication | `host`, `client_id` | `token_audience`, `account_id` | `DATABRICKS_HOST`, `DATABRICKS_CLIENT_ID`, `SYSTEM_ACCESSTOKEN` | +| `google-credentials` | Google Cloud service account authentication using credentials JSON | `host`, `google_credentials` | - | `DATABRICKS_HOST`, `GOOGLE_CREDENTIALS` | +| `google-id` | Google Cloud authentication using service account impersonation | `host`, `google_service_account` | - | `DATABRICKS_HOST`, `DATABRICKS_GOOGLE_SERVICE_ACCOUNT` | +| `metadata-service` | Authentication using Databricks-hosted metadata service | `host`, `metadata_service_url` | - | `DATABRICKS_HOST`, `DATABRICKS_METADATA_SERVICE_URL` | +| `runtime` | Auto-detected authentication when running in Databricks Runtime (notebooks, jobs) | _(auto-detected)_ | - | `DATABRICKS_RUNTIME_VERSION` (auto-set) | +| `runtime-oauth` | OAuth authentication for Databricks Runtime with fine-grained permissions | `scopes` | `authorization_details` | `DATABRICKS_RUNTIME_VERSION` (auto-set) | +| `model-serving` | Auto-detected authentication when running in Databricks Model Serving environment | _(auto-detected)_ | - | `IS_IN_DB_MODEL_SERVING_ENV` or `IS_IN_DATABRICKS_MODEL_SERVING_ENV` (auto-set) | +| `env-oidc` | OIDC token from environment variable | `host` | `oidc_token_env`, `client_id` | `DATABRICKS_HOST`, `DATABRICKS_OIDC_TOKEN`, `DATABRICKS_OIDC_TOKEN_ENV`, `DATABRICKS_CLIENT_ID` | +| `file-oidc` | OIDC token from file path | `host`, `oidc_token_filepath` | `client_id` | `DATABRICKS_HOST`, `DATABRICKS_OIDC_TOKEN_FILEPATH` (alias: `DATABRICKS_OIDC_TOKEN_FILE`), `DATABRICKS_CLIENT_ID` | + +For configuration options that apply to all authentication types (timeouts, debug settings, rate limits), see [Authentication](./authentication.md#additional-configuration-options). + +## Usage Examples + +When you explicitly set `auth_type`, the SDK only attempts that authentication method, skipping the automatic detection of other methods. This is useful when you have multiple credentials configured but want to use a specific one. + +### Personal Access Token (PAT) +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + token="dapi1234567890abcdef", + auth_type="pat" +) +``` + +### Basic Authentication (Username/Password) +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + username="your-username", + password="your-password", + auth_type="basic" +) +``` + +### OAuth Machine-to-Machine (Service Principal) +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + client_id="your-client-id", + client_secret="your-client-secret", + auth_type="oauth-m2m" +) +``` + +### External Browser (OAuth for Users) +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + auth_type="external-browser" +) +``` + +### Databricks CLI +```python +from databricks.sdk import WorkspaceClient + +# Assumes you've run: databricks auth login --host https://your-workspace.cloud.databricks.com +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + auth_type="databricks-cli" +) +``` + +### Azure Service Principal +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://adb-1234567890.azuredatabricks.net", + azure_client_id="your-azure-client-id", + azure_client_secret="your-azure-client-secret", + azure_tenant_id="your-azure-tenant-id", + auth_type="azure-client-secret" +) +``` + +### Azure CLI +```python +from databricks.sdk import WorkspaceClient + +# Assumes you've run: az login +w = WorkspaceClient( + host="https://adb-1234567890.azuredatabricks.net", + auth_type="azure-cli" +) +``` + +### GitHub Actions OIDC +```python +from databricks.sdk import WorkspaceClient + +# In GitHub Actions with OIDC configured +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + client_id="your-databricks-oauth-client-id", + auth_type="github-oidc" +) +``` + +### GitHub Actions OIDC for Azure +```python +from databricks.sdk import WorkspaceClient + +# In GitHub Actions with Azure OIDC configured +w = WorkspaceClient( + host="https://adb-1234567890.azuredatabricks.net", + azure_client_id="your-azure-client-id", + auth_type="github-oidc-azure" +) +``` + +### Azure DevOps OIDC +```python +from databricks.sdk import WorkspaceClient + +# In Azure DevOps with OIDC configured +# Note: SYSTEM_ACCESSTOKEN must be exposed as an environment variable +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + client_id="your-databricks-oauth-client-id", + auth_type="azure-devops-oidc" +) +``` + +### Google Cloud Credentials +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.gcp.databricks.com", + google_credentials="/path/to/service-account-key.json", + auth_type="google-credentials" +) +``` + +### Google Cloud ID (Service Account Impersonation) +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.gcp.databricks.com", + google_service_account="your-service-account@project.iam.gserviceaccount.com", + auth_type="google-id" +) +``` + +### Metadata Service +```python +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + metadata_service_url="http://localhost:8080/metadata", + auth_type="metadata-service" +) +``` + +### Runtime (in Databricks Notebooks) +```python +from databricks.sdk import WorkspaceClient + +# No credentials needed when running in Databricks Runtime +# The runtime auth type is auto-detected +w = WorkspaceClient(auth_type="runtime") +``` + +### Runtime OAuth (in Databricks Notebooks with scoped access) +```python +from databricks.sdk import WorkspaceClient + +# For fine-grained access control in notebooks +w = WorkspaceClient( + scopes="clusters sql", + auth_type="runtime-oauth" +) +``` + +### Environment Variable OIDC +```python +from databricks.sdk import WorkspaceClient + +# OIDC token from DATABRICKS_OIDC_TOKEN environment variable +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + auth_type="env-oidc" +) +``` + +### File-based OIDC +```python +from databricks.sdk import WorkspaceClient + +# OIDC token from a file +w = WorkspaceClient( + host="https://your-workspace.cloud.databricks.com", + oidc_token_filepath="/path/to/oidc-token", + auth_type="file-oidc" +) +``` + +### Model Serving Environment +```python +from databricks.sdk import WorkspaceClient + +# Auto-detected when running in Databricks Model Serving +w = WorkspaceClient(auth_type="model-serving") +``` + +## Authentication Priority Order + +When no `auth_type` is explicitly specified, the SDK attempts authentication methods in this order: + +1. `pat` - Personal Access Token +2. `basic` - Username/Password +3. `metadata-service` - Metadata Service (if URL provided) +4. `oauth-m2m` - OAuth Service Principal +5. `env-oidc` - Environment OIDC token +6. `file-oidc` - File-based OIDC token +7. `github-oidc` - GitHub OIDC +8. `azure-client-secret` - Azure Service Principal +9. `github-oidc-azure` - GitHub OIDC for Azure +10. `azure-cli` - Azure CLI +11. `azure-devops-oidc` - Azure DevOps OIDC +12. `external-browser` - Browser-based OAuth +13. `databricks-cli` - Databricks CLI +14. `runtime-oauth` - Databricks Runtime OAuth +15. `runtime` - Databricks Runtime native +16. `google-credentials` - Google Cloud credentials +17. `google-id` - Google Cloud ID +18. `model-serving` - Model Serving environment + +You can override this order by explicitly setting the `auth_type` parameter. + +## Notes + +- **Auto-detected auth types** (`runtime`, `runtime-oauth`, `model-serving`): These are automatically detected based on environment variables and don't require explicit configuration. +- **Azure authentication**: When using Azure-specific auth types, if `host` is not provided but `azure_workspace_resource_id` is, the SDK will automatically resolve the workspace URL. +- **OIDC authentication**: OIDC-based methods (`github-oidc`, `azure-devops-oidc`, `env-oidc`, `file-oidc`) use token exchange to obtain Databricks tokens from external identity providers. +- **Scopes**: OAuth-based methods support the `scopes` parameter for fine-grained access control (e.g., `scopes="clusters sql"`). + +## See Also + +- [Authentication Overview](./authentication.md) - Default authentication flow and configuration +- [OAuth Documentation](./oauth.md) - OAuth-based authentication details +- [Databricks Authentication Documentation](https://docs.databricks.com/dev-tools/auth.html) - Official Databricks authentication docs diff --git a/docs/authentication.md b/docs/authentication.md index 4307ac3e6..52d375ca4 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -130,7 +130,7 @@ For all authentication methods, you can override the default behavior in client | Argument | Description | Environment variable | |-------------------------|-------------|------------------------| -| `auth_type` | _(String)_ When multiple auth attributes are available in the environment, use the auth type specified by this argument. This argument also holds the currently selected auth. | `DATABRICKS_AUTH_TYPE` | +| `auth_type` | _(String)_ When multiple auth attributes are available in the environment, use the auth type specified by this argument. This argument also holds the currently selected auth. When set explicitly, the SDK only attempts that authentication method, skipping automatic detection of others. See the [Authentication Types Reference](./auth-types-reference.md) for all valid values, required parameters, and usage examples. | `DATABRICKS_AUTH_TYPE` | | `http_timeout_seconds` | _(Integer)_ Number of seconds for HTTP timeout. Default is _60_. | _(None)_ | | `retry_timeout_seconds` | _(Integer)_ Number of seconds to keep retrying HTTP requests. Default is _300 (5 minutes)_. | _(None)_ | | `debug_truncate_bytes` | _(Integer)_ Truncate JSON fields in debug logs above this limit. Default is 96. | `DATABRICKS_DEBUG_TRUNCATE_BYTES` | diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index 765cf5608..8c851257a 100755 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -938,6 +938,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ListEffectivePrivilegeAssignmentsResponse + :members: + :undoc-members: + .. autoclass:: ListEntityTagAssignmentsResponse :members: :undoc-members: @@ -970,6 +974,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ListPrivilegeAssignmentsResponse + :members: + :undoc-members: + .. autoclass:: ListQuotasResponse :members: :undoc-members: diff --git a/docs/dbdataclasses/ml.rst b/docs/dbdataclasses/ml.rst index 7f13a4b8f..c0f3e6022 100755 --- a/docs/dbdataclasses/ml.rst +++ b/docs/dbdataclasses/ml.rst @@ -297,6 +297,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ExperimentTraceLocation + :members: + :undoc-members: + .. autoclass:: Feature :members: :undoc-members: @@ -337,7 +341,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: FirstDistinctNFunction +.. autoclass:: FirstDistinctFunction :members: :undoc-members: @@ -517,7 +521,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: -.. autoclass:: LastDistinctNFunction +.. autoclass:: LastDistinctFunction :members: :undoc-members: @@ -642,6 +646,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: LongRollingWindow + :members: + :undoc-members: + .. autoclass:: MaterializedFeature :members: :undoc-members: @@ -777,6 +785,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: CAN_READ :value: "CAN_READ" +.. autoclass:: ProtoSchemaSpec + :members: + :undoc-members: + .. autoclass:: PublishSpec :members: :undoc-members: @@ -1162,6 +1174,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: UcTraceLocation + :members: + :undoc-members: + .. autoclass:: UpdateCommentResponse :members: :undoc-members: diff --git a/docs/dbdataclasses/serving.rst b/docs/dbdataclasses/serving.rst index 366488016..d30520c2a 100755 --- a/docs/dbdataclasses/serving.rst +++ b/docs/dbdataclasses/serving.rst @@ -393,6 +393,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: CPU :value: "CPU" + .. py:attribute:: CPU_LARGE + :value: "CPU_LARGE" + + .. py:attribute:: CPU_MEDIUM + :value: "CPU_MEDIUM" + .. py:attribute:: GPU_LARGE :value: "GPU_LARGE" @@ -500,6 +506,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: CPU :value: "CPU" + .. py:attribute:: CPU_LARGE + :value: "CPU_LARGE" + + .. py:attribute:: CPU_MEDIUM + :value: "CPU_MEDIUM" + .. py:attribute:: GPU_LARGE :value: "GPU_LARGE" diff --git a/docs/dbdataclasses/settings.rst b/docs/dbdataclasses/settings.rst index c381f4960..3a69f3108 100755 --- a/docs/dbdataclasses/settings.rst +++ b/docs/dbdataclasses/settings.rst @@ -297,6 +297,8 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: CustomerFacingIngressNetworkPolicyPrivateAccessRestrictionMode + The restriction mode for private access. In ALLOW_ALL_REGISTERED_ENDPOINTS mode, requests arriving through any endpoint registered to the account are allowed, and deny rules and allow rules cannot be set. In RESTRICTED_ACCESS mode, access is restricted based on deny rules and allow rules; requests that do not match any allow rule are denied. + .. py:attribute:: ALLOW_ALL_REGISTERED_ENDPOINTS :value: "ALLOW_ALL_REGISTERED_ENDPOINTS" diff --git a/docs/index.rst b/docs/index.rst index 3d3a5dfc5..ccf5e1629 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ We are keen to hear feedback from you on these SDKs. Please `file GitHub issues getting-started authentication + auth-types-reference oauth wait pagination diff --git a/docs/workspace/catalog/catalogs.rst b/docs/workspace/catalog/catalogs.rst index 79e037443..7e84721d0 100755 --- a/docs/workspace/catalog/catalogs.rst +++ b/docs/workspace/catalog/catalogs.rst @@ -162,13 +162,12 @@ import time from databricks.sdk import WorkspaceClient - from databricks.sdk.service import catalog w = WorkspaceClient() created = w.catalogs.create(name=f"sdk-{time.time_ns()}") - _ = w.catalogs.update(name=created.name, isolation_mode=catalog.CatalogIsolationMode.ISOLATED) + _ = w.catalogs.update(name=created.name, comment="updated") # cleanup w.catalogs.delete(name=created.name, force=True) diff --git a/docs/workspace/catalog/external_locations.rst b/docs/workspace/catalog/external_locations.rst index 569bfd1f3..d7957c8a8 100755 --- a/docs/workspace/catalog/external_locations.rst +++ b/docs/workspace/catalog/external_locations.rst @@ -30,20 +30,22 @@ w = WorkspaceClient() - credential = w.storage_credentials.create( + storage_credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + comment="created via SDK", ) - created = w.external_locations.create( + external_location = w.external_locations.create( name=f"sdk-{time.time_ns()}", - credential_name=credential.name, - url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', + credential_name=storage_credential.name, + comment="created via SDK", + url="s3://" + os.environ["TEST_BUCKET"] + "/" + f"sdk-{time.time_ns()}", ) # cleanup - w.storage_credentials.delete(delete=credential.name) - w.external_locations.delete(delete=created.name) + w.storage_credentials.delete(name=storage_credential.name) + w.external_locations.delete(name=external_location.name) Creates a new external location entry in the metastore. The caller must be a metastore admin or have the **CREATE_EXTERNAL_LOCATION** privilege on both the metastore and the associated storage @@ -113,20 +115,20 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', + url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), ) - _ = w.external_locations.get(get=created.name) + _ = w.external_locations.get(name=created.name) # cleanup - w.storage_credentials.delete(delete=credential.name) - w.external_locations.delete(delete=created.name) + w.storage_credentials.delete(name=credential.name) + w.external_locations.delete(name=created.name) Gets an external location from the metastore. The caller must be either a metastore admin, the owner of the external location, or a user that has some privilege on the external location. diff --git a/docs/workspace/catalog/grants.rst b/docs/workspace/catalog/grants.rst index 001bb9753..d709622ee 100644 --- a/docs/workspace/catalog/grants.rst +++ b/docs/workspace/catalog/grants.rst @@ -176,6 +176,68 @@ :returns: :class:`EffectivePermissionsList` + .. py:method:: list(securable_type: str, full_name: str [, page_size: Optional[int], page_token: Optional[str], principal: Optional[str]]) -> Iterator[PrivilegeAssignment] + + Lists the privilege assignments for a securable. Does not include inherited privileges. Paginated + version of Get Permissions API. + + :param securable_type: str + Type of securable. + :param full_name: str + Full name of securable. + :param page_size: int (optional) + Specifies the maximum number of privilege assignments to return (page length). Every + PrivilegeAssignment present in a single page response is guaranteed to contain all the privileges + granted on the requested Securable for the respective principal. + + If not set, page length is the server configured value. If set to + + - lesser than 0: invalid parameter error + - 0: page length is set to a server configured value + - lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able + to return at least one complete PrivilegeAssignment in a single page response) + - greater than (or equal to) 150: page length is the minimum of this value and a server configured + value + :param page_token: str (optional) + Opaque pagination token to go to next page based on previous query. + :param principal: str (optional) + If provided, only the permissions for the specified principal (user or group) are returned. + + :returns: Iterator over :class:`PrivilegeAssignment` + + + .. py:method:: list_effective(securable_type: str, full_name: str [, page_size: Optional[int], page_token: Optional[str], principal: Optional[str]]) -> Iterator[EffectivePrivilegeAssignment] + + Lists the effective privilege assignments for a securable. Includes inherited privileges. Paginated + version of Get Effective Permissions API. + + :param securable_type: str + Type of securable. + :param full_name: str + Full name of securable. + :param page_size: int (optional) + Specifies the maximum number of privilege assignments to return (page length). Every + EffectivePrivilegeAssignment present in a single page response is guaranteed to contain all the + effective privileges granted on (or inherited by) the requested Securable for the respective + principal. + + If not set, a server-configured default is used. If set to + + - lesser than 0: invalid parameter error + - 0: page length is set to a server configured value + - lesser than 150 but greater than 0: invalid parameter error (this is to ensure that server is able + to return at least one complete EffectivePrivilegeAssignment in a single page response) + - greater than (or equal to) 150: page length is the minimum of this value and a server configured + value + :param page_token: str (optional) + Opaque pagination token to go to next page based on previous query. + :param principal: str (optional) + If provided, only the effective permissions for the specified principal (user or group) are + returned. + + :returns: Iterator over :class:`EffectivePrivilegeAssignment` + + .. py:method:: update(securable_type: str, full_name: str [, changes: Optional[List[PermissionsChange]], omit_permissions_in_response: Optional[bool]]) -> UpdatePermissionsResponse diff --git a/docs/workspace/catalog/schemas.rst b/docs/workspace/catalog/schemas.rst index a70b3c472..90f1f6554 100755 --- a/docs/workspace/catalog/schemas.rst +++ b/docs/workspace/catalog/schemas.rst @@ -22,13 +22,13 @@ w = WorkspaceClient() - new_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") + created_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") - created = w.schemas.create(name=f"sdk-{time.time_ns()}", catalog_name=new_catalog.name) + created_schema = w.schemas.create(name=f"sdk-{time.time_ns()}", catalog_name=created_catalog.name) # cleanup - w.catalogs.delete(name=new_catalog.name, force=True) - w.schemas.delete(full_name=created.full_name) + w.catalogs.delete(name=created_catalog.name, force=True) + w.schemas.delete(full_name=created_schema.full_name) Creates a new schema for catalog in the Metastore. The caller must be a metastore admin, or have the **CREATE_SCHEMA** privilege in the parent catalog. diff --git a/docs/workspace/catalog/secrets_uc.rst b/docs/workspace/catalog/secrets_uc.rst index c8a83f809..8b709c0be 100755 --- a/docs/workspace/catalog/secrets_uc.rst +++ b/docs/workspace/catalog/secrets_uc.rst @@ -40,7 +40,7 @@ - .. py:method:: get_secret(full_name: str [, include_browse: Optional[bool]]) -> Secret + .. py:method:: get_secret(full_name: str) -> Secret Gets a secret by its three-level (fully qualified) name. @@ -53,14 +53,11 @@ :param full_name: str The three-level (fully qualified) name of the secret (for example, **catalog_name.schema_name.secret_name**). - :param include_browse: bool (optional) - Whether to include secrets in the response for which you only have the **BROWSE** privilege, which - limits access to metadata. :returns: :class:`Secret` - .. py:method:: list_secrets( [, catalog_name: Optional[str], include_browse: Optional[bool], page_size: Optional[int], page_token: Optional[str], schema_name: Optional[str]]) -> Iterator[Secret] + .. py:method:: list_secrets( [, catalog_name: Optional[str], page_size: Optional[int], page_token: Optional[str], schema_name: Optional[str]]) -> Iterator[Secret] Lists secrets in Unity Catalog. @@ -74,15 +71,12 @@ :param catalog_name: str (optional) The name of the catalog under which to list secrets. Both **catalog_name** and **schema_name** must be specified together. - :param include_browse: bool (optional) - Whether to include secrets in the response for which you only have the **BROWSE** privilege, which - limits access to metadata. :param page_size: int (optional) Maximum number of secrets to return. - - If not specified, at most 10000 secrets are returned. - - If set to a value greater than 0, the page length is the minimum of this value and 10000. - - If set to 0, the page length is set to 10000. + - If not specified, at most 1000 secrets are returned. + - If set to a value greater than 0, the page length is the minimum of this value and 1000. + - If set to 0, the page length is set to 1000. - If set to a value less than 0, an invalid parameter error is returned. :param page_token: str (optional) Opaque pagination token to go to the next page based on previous query. The maximum page length is @@ -111,8 +105,12 @@ The secret object containing the fields to update. Only fields specified in **update_mask** will be updated. :param update_mask: FieldMask - The field mask specifying which fields of the secret to update. Supported fields: **value**, - **comment**, **owner**, **expire_time**. + The field mask specifying which fields of the secret to update. + + - If **update_mask** is **"*"**, all fields specified in **secret** are updated. + - If **update_mask** specifies one or more fields, only those fields are updated. Each specified + field must be set in **secret**. Supported fields: **value**, **comment**, **owner**, + **expire_time**. To change the secret name, delete and recreate the secret. :returns: :class:`Secret` \ No newline at end of file diff --git a/docs/workspace/catalog/storage_credentials.rst b/docs/workspace/catalog/storage_credentials.rst index 35e3511dd..1bebfb8b3 100755 --- a/docs/workspace/catalog/storage_credentials.rst +++ b/docs/workspace/catalog/storage_credentials.rst @@ -30,13 +30,13 @@ w = WorkspaceClient() - credential = w.storage_credentials.create( + created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) # cleanup - w.storage_credentials.delete(delete=credential.name) + w.storage_credentials.delete(name=created.name) Creates a new storage credential. @@ -174,17 +174,17 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) _ = w.storage_credentials.update( name=created.name, comment=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) # cleanup - w.storage_credentials.delete(delete=created.name) + w.storage_credentials.delete(name=created.name) Updates a storage credential on the metastore. diff --git a/docs/workspace/dashboards/genie.rst b/docs/workspace/dashboards/genie.rst index b8204c4ae..6f4ac2c7a 100755 --- a/docs/workspace/dashboards/genie.rst +++ b/docs/workspace/dashboards/genie.rst @@ -98,7 +98,7 @@ Download a rendered image of a message visualization attachment. The response body is the raw PNG image, not a JSON payload. This is only available if the attachment is a visualization and the message - status is ``COMPLETED``. + status is ``COMPLETED``. This endpoint is not supported for Private Link workspaces. :param name: str The resource name of the attachment to render, in the format diff --git a/docs/workspace/jobs/jobs.rst b/docs/workspace/jobs/jobs.rst index 970181e4f..5dad4c7a1 100755 --- a/docs/workspace/jobs/jobs.rst +++ b/docs/workspace/jobs/jobs.rst @@ -125,7 +125,7 @@ .. py:method:: cancel_run_and_wait(run_id: int, timeout: datetime.timedelta = 0:20:00) -> Run - .. py:method:: create( [, access_control_list: Optional[List[JobAccessControlRequest]], budget_policy_id: Optional[str], continuous: Optional[Continuous], deployment: Optional[JobDeployment], description: Optional[str], edit_mode: Optional[JobEditMode], email_notifications: Optional[JobEmailNotifications], environments: Optional[List[JobEnvironment]], format: Optional[Format], git_source: Optional[GitSource], health: Optional[JobsHealthRules], job_clusters: Optional[List[JobCluster]], max_concurrent_runs: Optional[int], name: Optional[str], notification_settings: Optional[JobNotificationSettings], parameters: Optional[List[JobParameterDefinition]], performance_target: Optional[PerformanceTarget], queue: Optional[QueueSettings], run_as: Optional[JobRunAs], schedule: Optional[CronSchedule], tags: Optional[Dict[str, str]], tasks: Optional[List[Task]], timeout_seconds: Optional[int], trigger: Optional[TriggerSettings], usage_policy_id: Optional[str], webhook_notifications: Optional[WebhookNotifications]]) -> CreateResponse + .. py:method:: create( [, access_control_list: Optional[List[JobAccessControlRequest]], budget_policy_id: Optional[str], continuous: Optional[Continuous], deployment: Optional[JobDeployment], description: Optional[str], edit_mode: Optional[JobEditMode], email_notifications: Optional[JobEmailNotifications], environments: Optional[List[JobEnvironment]], format: Optional[Format], git_source: Optional[GitSource], health: Optional[JobsHealthRules], job_clusters: Optional[List[JobCluster]], max_concurrent_runs: Optional[int], name: Optional[str], notification_settings: Optional[JobNotificationSettings], parameters: Optional[List[JobParameterDefinition]], parent_path: Optional[str], performance_target: Optional[PerformanceTarget], queue: Optional[QueueSettings], run_as: Optional[JobRunAs], schedule: Optional[CronSchedule], tags: Optional[Dict[str, str]], tasks: Optional[List[Task]], timeout_seconds: Optional[int], trigger: Optional[TriggerSettings], usage_policy_id: Optional[str], webhook_notifications: Optional[WebhookNotifications]]) -> CreateResponse Usage: @@ -223,6 +223,9 @@ ``email_notifications`` and ``webhook_notifications`` for this job. :param parameters: List[:class:`JobParameterDefinition`] (optional) Job-level parameter definitions + :param parent_path: str (optional) + Path of the job parent folder in workspace file tree. If absent, the job doesn't have a workspace + object. :param performance_target: :class:`PerformanceTarget` (optional) The performance mode on a serverless job. This field determines the level of compute performance or cost-efficiency for the run. The performance target does not apply to tasks that run on Serverless @@ -358,23 +361,21 @@ w.clusters.ensure_cluster_is_running(os.environ["DATABRICKS_CLUSTER_ID"]) and os.environ["DATABRICKS_CLUSTER_ID"] ) - created_job = w.jobs.create( - name=f"sdk-{time.time_ns()}", + run = w.jobs.submit( + run_name=f"sdk-{time.time_ns()}", tasks=[ - jobs.Task( - description="test", + jobs.SubmitTask( existing_cluster_id=cluster_id, notebook_task=jobs.NotebookTask(notebook_path=notebook_path), - task_key="test", - timeout_seconds=0, + task_key=f"sdk-{time.time_ns()}", ) ], - ) + ).result() - by_id = w.jobs.get(job_id=created_job.job_id) + output = w.jobs.get_run_output(run_id=run.tasks[0].run_id) # cleanup - w.jobs.delete(job_id=created_job.job_id) + w.jobs.delete_run(run_id=run.run_id) Get a single job. @@ -523,11 +524,37 @@ .. code-block:: + import os + import time + from databricks.sdk import WorkspaceClient + from databricks.sdk.service import jobs w = WorkspaceClient() - job_list = w.jobs.list(expand_tasks=False) + notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" + + cluster_id = ( + w.clusters.ensure_cluster_is_running(os.environ["DATABRICKS_CLUSTER_ID"]) and os.environ["DATABRICKS_CLUSTER_ID"] + ) + + created_job = w.jobs.create( + name=f"sdk-{time.time_ns()}", + tasks=[ + jobs.Task( + description="test", + existing_cluster_id=cluster_id, + notebook_task=jobs.NotebookTask(notebook_path=notebook_path), + task_key="test", + timeout_seconds=0, + ) + ], + ) + + run_list = w.jobs.list_runs(job_id=created_job.job_id) + + # cleanup + w.jobs.delete(job_id=created_job.job_id) List jobs. @@ -935,6 +962,11 @@ :param only: List[str] (optional) A list of task keys to run inside of the job. If this field is not provided, all tasks in the job will be run. + + Prefix a task key with ``+`` to also run its upstream tasks, or suffix it with ``+`` to also run its + downstream tasks. For example, ``+my_task`` runs ``my_task`` and everything upstream of it, + ``my_task+`` runs ``my_task`` and everything downstream of it, and ``+my_task+`` runs both. A task + key with no ``+`` runs only that task. :param performance_target: :class:`PerformanceTarget` (optional) The performance mode on a serverless job. The performance target determines the level of compute performance or cost-efficiency for the run. This field overrides the performance target defined on diff --git a/docs/workspace/ml/experiments.rst b/docs/workspace/ml/experiments.rst index cc3814b81..de17ba267 100755 --- a/docs/workspace/ml/experiments.rst +++ b/docs/workspace/ml/experiments.rst @@ -11,7 +11,7 @@ Experiments are located in the workspace file tree. You manage experiments using the same tools you use to manage other workspace objects such as folders, notebooks, and libraries. - .. py:method:: create_experiment(name: str [, artifact_location: Optional[str], tags: Optional[List[ExperimentTag]]]) -> CreateExperimentResponse + .. py:method:: create_experiment(name: str [, artifact_location: Optional[str], tags: Optional[List[ExperimentTag]], trace_location: Optional[ExperimentTraceLocation]]) -> CreateExperimentResponse Usage: @@ -47,6 +47,10 @@ depends on the storage backend. All storage backends are guaranteed to support tag keys up to 250 bytes in size and tag values up to 5000 bytes in size. All storage backends are also guaranteed to support up to 20 tags per request. + :param trace_location: :class:`ExperimentTraceLocation` (optional) + The location where the experiment's traces are stored. When set, the underlying storage is + provisioned and the experiment's traces are routed to it. When unset, traces are stored in the + default MLflow backend. This field cannot be updated after the experiment is created. :returns: :class:`CreateExperimentResponse` diff --git a/docs/workspace/ml/model_registry.rst b/docs/workspace/ml/model_registry.rst index 85d4369a5..8e13ecef7 100755 --- a/docs/workspace/ml/model_registry.rst +++ b/docs/workspace/ml/model_registry.rst @@ -88,6 +88,8 @@ w = WorkspaceClient() model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + + mv = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a new registered model with the name specified in the request body. Throws ``RESOURCE_ALREADY_EXISTS`` if a registered model with the given name exists. diff --git a/docs/workspace/postgres/postgres.rst b/docs/workspace/postgres/postgres.rst index 59b9247b6..ba82de1bb 100755 --- a/docs/workspace/postgres/postgres.rst +++ b/docs/workspace/postgres/postgres.rst @@ -260,7 +260,7 @@ :returns: :class:`Operation` - .. py:method:: generate_database_credential(endpoint: str [, claims: Optional[List[RequestedClaims]], expire_time: Optional[Timestamp], group_name: Optional[str], ttl: Optional[Duration]]) -> DatabaseCredential + .. py:method:: generate_database_credential(endpoint: str [, claims: Optional[List[RequestedClaims]], expire_time: Optional[Timestamp], ttl: Optional[Duration]]) -> DatabaseCredential Generate OAuth credentials for a Postgres database. @@ -272,9 +272,6 @@ :param expire_time: Timestamp (optional) Timestamp in UTC of when this credential should expire. Must be at least 300 seconds (5 minutes) and at most 1 hour from the current time. - :param group_name: str (optional) - Databricks workspace group name. When provided, credentials are generated with permissions scoped to - this group. :param ttl: Duration (optional) The requested time-to-live for the generated credential token. Must be at least 300 seconds (5 minutes) and at most 3600 seconds (1 hour). diff --git a/docs/workspace/sql/dashboard_widgets.rst b/docs/workspace/sql/dashboard_widgets.rst index 1293985e7..1217d3236 100644 --- a/docs/workspace/sql/dashboard_widgets.rst +++ b/docs/workspace/sql/dashboard_widgets.rst @@ -20,7 +20,7 @@ If this is a textbox widget, the application displays this text. This field is ignored if the widget contains a visualization in the ``visualization`` field. :param visualization_id: str (optional) - Query Vizualization ID returned by :method:queryvisualizations/create. + Query Visualization ID returned by :method:queryvisualizations/create. :returns: :class:`Widget` @@ -50,7 +50,7 @@ If this is a textbox widget, the application displays this text. This field is ignored if the widget contains a visualization in the ``visualization`` field. :param visualization_id: str (optional) - Query Vizualization ID returned by :method:queryvisualizations/create. + Query Visualization ID returned by :method:queryvisualizations/create. :returns: :class:`Widget` \ No newline at end of file diff --git a/docs/workspace/sql/query_visualizations_legacy.rst b/docs/workspace/sql/query_visualizations_legacy.rst index 6c46e74f5..e51820fcb 100644 --- a/docs/workspace/sql/query_visualizations_legacy.rst +++ b/docs/workspace/sql/query_visualizations_legacy.rst @@ -4,7 +4,7 @@ .. py:class:: QueryVisualizationsLegacyAPI - This is an evolving API that facilitates the addition and removal of vizualisations from existing queries + This is an evolving API that facilitates the addition and removal of visualizations from existing queries within the Databricks Workspace. Data structures may change over time. **Warning**: This API is deprecated. Please see the latest version of the Databricks SQL API. `Learn more diff --git a/docs/workspace/workspace/repos.rst b/docs/workspace/workspace/repos.rst index ac37c25b5..8aff1d364 100755 --- a/docs/workspace/workspace/repos.rst +++ b/docs/workspace/workspace/repos.rst @@ -14,7 +14,7 @@ Within Repos you can develop code in notebooks or other files and follow data science and engineering code development best practices using Git for version control, collaboration, and CI/CD. - .. py:method:: create(url: str, provider: str [, path: Optional[str], sparse_checkout: Optional[SparseCheckout]]) -> CreateRepoResponse + .. py:method:: create(url: str, provider: str [, git_credential_id: Optional[int], path: Optional[str], sparse_checkout: Optional[SparseCheckout]]) -> CreateRepoResponse Usage: @@ -49,6 +49,9 @@ Entra ID authentication), ``gitHubEnterprise``, ``bitbucketServer`` (Bitbucket Data Center), ``gitLabEnterpriseEdition`` (GitLab Self-Managed), and ``awsCodeCommit`` (deprecated by AWS, not accepting new customers). + :param git_credential_id: int (optional) + Git credential ID to use when cloning the repository. The Git credential must be configured for the + current user. :param path: str (optional) Desired path for the repo in the workspace. Almost any path in the workspace can be chosen. If repo is created in ``/Repos``, path must be in the format ``/Repos/{folder}/{repo-name}``. @@ -163,7 +166,7 @@ :returns: :class:`RepoPermissions` - .. py:method:: update(repo_id: int [, branch: Optional[str], dangerously_force_discard_all: Optional[bool], sparse_checkout: Optional[SparseCheckoutUpdate], tag: Optional[str]]) + .. py:method:: update(repo_id: int [, branch: Optional[str], dangerously_force_discard_all: Optional[bool], git_credential_id: Optional[int], sparse_checkout: Optional[SparseCheckoutUpdate], tag: Optional[str]]) Usage: @@ -207,6 +210,9 @@ destroyed without warning. Local commits that have been made but not yet pushed to the remote are preserved. + :param git_credential_id: int (optional) + Git credential ID to use for this update operation. The Git credential must be configured for the + current user. :param sparse_checkout: :class:`SparseCheckoutUpdate` (optional) If specified, update the sparse checkout settings. The update will fail if sparse checkout is not enabled for the repo. diff --git a/docs/workspace/workspace/workspace.rst b/docs/workspace/workspace/workspace.rst index 672f6be98..fd9a2c879 100755 --- a/docs/workspace/workspace/workspace.rst +++ b/docs/workspace/workspace/workspace.rst @@ -82,7 +82,7 @@ notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" - export_response = w.workspace.export_(format=workspace.ExportFormat.SOURCE, path=notebook) + export_response = w.workspace.export(format=workspace.ExportFormat.SOURCE, path=notebook) Exports an object or the contents of an entire directory. @@ -185,16 +185,11 @@ notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" w.workspace.import_( - path=notebook_path, - overwrite=true_, + content=base64.b64encode(("CREATE LIVE TABLE dlt_sample AS SELECT 1").encode()).decode(), format=workspace.ImportFormat.SOURCE, - language=workspace.Language.PYTHON, - content=base64.b64encode( - ( - """print(1) - """ - ).encode() - ).decode(), + language=workspace.Language.SQL, + overwrite=True, + path=notebook_path, ) Imports a workspace object (for example, a notebook or file) or the contents of an entire directory. @@ -240,14 +235,16 @@ .. code-block:: + import os + import time + from databricks.sdk import WorkspaceClient w = WorkspaceClient() - names = [] - for i in w.workspace.list(f"/Users/{w.current_user.me().user_name}", recursive=True): - names.append(i.path) - assert len(names) > 0 + notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" + + objects = w.workspace.list(path=os.path.dirname(notebook)) List workspace objects