diff --git a/.github/workflows/tagging.yml b/.github/workflows/tagging.yml old mode 100755 new mode 100644 index caa826d93..d240f16ee --- a/.github/workflows/tagging.yml +++ b/.github/workflows/tagging.yml @@ -66,7 +66,7 @@ jobs: git config user.email "DECO-SDK-Tagging[bot]@users.noreply.github.com" - name: Install uv - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Run script env: @@ -82,7 +82,7 @@ jobs: - name: Upload created tags artifact if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: created-tags path: created_tags.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 382a307c2..d4e0a85e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Version changelog +## Release v0.119.0 (2026-06-24) + +### New Features and Improvements +* Added a `meta-harness` user-agent dimension that reports the omnigent meta-harness (detected via the `OMNIGENT` environment variable) independently of agent detection. + +### API Changes +* Add `cancel_pending_cluster_enforcement()` method for [w.policy_compliance_for_clusters](https://databricks-sdk-py.readthedocs.io/en/latest/workspace/compute/policy_compliance_for_clusters.html) workspace-level service. +* Add `bundle_root_path` field for `databricks.sdk.service.bundledeployments.WorkspaceInfo`. +* Add `pending_enforcement` field for `databricks.sdk.service.compute.ClusterCompliance`. +* Add `enforce_mode` field for `databricks.sdk.service.compute.EnforceClusterComplianceRequest`. +* Add `enforce_result` field for `databricks.sdk.service.compute.EnforceClusterComplianceResponse`. +* Add `pending_enforcement` field for `databricks.sdk.service.compute.GetClusterComplianceResponse`. +* Add `ai_runtime_task` field for `databricks.sdk.service.jobs.ResolvedValues`. +* Add `ai_runtime_task_output` field for `databricks.sdk.service.jobs.RunOutput`. +* Add `ai_runtime_task` field for `databricks.sdk.service.jobs.RunTask`. +* Add `ai_runtime_task` field for `databricks.sdk.service.jobs.SubmitTask`. +* Add `ai_runtime_task` field for `databricks.sdk.service.jobs.Task`. +* Add `xlarge` enum value for `databricks.sdk.service.apps.ComputeSize`. +* Add `deferred_policy_enforcement_scheduled` and `deferred_policy_enforcement_failed` enum values for `databricks.sdk.service.compute.EventType`. +* [Breaking] Change `replicate_workspace_assets` field for `databricks.sdk.service.disasterrecovery.WorkspaceSet` to no longer be required. +* Change `replicate_workspace_assets` field for `databricks.sdk.service.disasterrecovery.WorkspaceSet` to no longer be required. + ## Release v0.118.0 (2026-06-18) ### API Changes diff --git a/databricks/sdk/service/apps.py b/databricks/sdk/service/apps.py index 44108700d..5552bbdd3 100755 --- a/databricks/sdk/service/apps.py +++ b/databricks/sdk/service/apps.py @@ -1817,6 +1817,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ApplicationStatus: class ComputeSize(Enum): LARGE = "LARGE" MEDIUM = "MEDIUM" + XLARGE = "XLARGE" class ComputeState(Enum): diff --git a/databricks/sdk/service/bundledeployments.py b/databricks/sdk/service/bundledeployments.py index 88b34b02b..8280c802d 100755 --- a/databricks/sdk/service/bundledeployments.py +++ b/databricks/sdk/service/bundledeployments.py @@ -416,30 +416,37 @@ def from_dict(cls, d: Dict[str, Any]) -> ListVersionsResponse: @dataclass class Operation: - """An operation on a single resource performed during a version. Operations are append-only and - record the result of applying a resource change to the workspace.""" + """An operation on a single resource performed during a version. Operations record the result of + applying a resource change to the workspace. Most fields are immutable once recorded; `state`, + `error_message`, `resource_id`, and `status` may be updated afterwards (via UpdateOperation), + guarded by `sequence_id` for optimistic concurrency control.""" action_type: OperationActionType """The type of operation performed on this resource.""" status: OperationStatus - """Whether the operation succeeded or failed.""" + """Whether the operation succeeded or failed. Mutable: may be updated after creation via + UpdateOperation, e.g. when an operation recorded as failed is retried and eventually succeeds. A + succeeded operation cannot carry an `error_message`.""" create_time: Optional[Timestamp] = None """When the operation was recorded.""" error_message: Optional[str] = None """Error message if the operation failed. Set when status is OPERATION_STATUS_FAILED. Captures the - error encountered while applying the resource to the workspace.""" + error encountered while applying the resource to the workspace. Mutable: may be updated after + creation via UpdateOperation; setting it to an empty string clears it. After an update is + applied, an operation whose status is OPERATION_STATUS_SUCCEEDED cannot carry an error_message.""" name: Optional[str] = None """Resource name of the operation. Format: deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}""" resource_id: Optional[str] = None - """ID of the actual resource in the workspace (e.g. the job ID, pipeline ID). Required for every - operation except CREATE and RECREATE, which produce a new resource whose ID is not yet known - when the operation is recorded.""" + """ID of the actual resource in the workspace (e.g. the job ID, pipeline ID). Optional at creation: + CREATE and RECREATE operations produce a new resource whose ID is not yet known when the + operation is recorded. Mutable: may be filled in (or corrected) later via UpdateOperation once + the ID is known.""" resource_key: Optional[str] = None """Resource identifier within the bundle (e.g. "jobs.foo", "pipelines.bar", "jobs.foo.permissions", @@ -451,7 +458,9 @@ class Operation: prefix (e.g. "jobs" → JOB); the caller does not set this field.""" state: Optional[any] = None - """Serialized local config state after the operation. Should be unset for delete operations.""" + """Serialized local config state after the operation. Should be unset for delete operations. + Mutable: may be updated after creation via UpdateOperation. When updating, the caller must echo + the last-observed `sequence_id` as a concurrency precondition.""" def as_dict(self) -> dict: """Serializes the Operation into a dictionary suitable for use as a JSON request body.""" @@ -662,8 +671,10 @@ class Version: """Target name of the deployment, captured at the time of this version.""" version_id: Optional[str] = None - """Monotonically increasing version identifier within the parent deployment. Assigned by the client - on creation.""" + """Version identifier within the parent deployment, assigned by the client on creation. A numeric + string (base-10, fits in a signed 64-bit integer) that is greater than or equal to 1. Version + IDs are strictly increasing within a deployment but are not required to start at 1 or to be + contiguous.""" workspace_info: Optional[WorkspaceInfo] = None """Workspace location of the deployment, captured at the time of this version.""" @@ -787,6 +798,10 @@ class VersionType(Enum): class WorkspaceInfo: """Workspace location of a bundle deployment, captured at deploy time.""" + bundle_root_path: Optional[str] = None + """Path of the bundle root (the directory containing databricks.yml) relative to git_folder_path. + Empty when the deployment is not from a Databricks Git folder.""" + file_path: Optional[str] = None """Absolute workspace path where the deployed bundle files live. Mirrors the workspace.file_path field in DABs bundle config.""" @@ -806,6 +821,8 @@ class WorkspaceInfo: def as_dict(self) -> dict: """Serializes the WorkspaceInfo into a dictionary suitable for use as a JSON request body.""" body = {} + if self.bundle_root_path is not None: + body["bundle_root_path"] = self.bundle_root_path if self.file_path is not None: body["file_path"] = self.file_path if self.git_folder_path is not None: @@ -819,6 +836,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the WorkspaceInfo into a shallow dictionary of its immediate attributes.""" body = {} + if self.bundle_root_path is not None: + body["bundle_root_path"] = self.bundle_root_path if self.file_path is not None: body["file_path"] = self.file_path if self.git_folder_path is not None: @@ -833,6 +852,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> WorkspaceInfo: """Deserializes the WorkspaceInfo from a dictionary.""" return cls( + bundle_root_path=d.get("bundle_root_path", None), file_path=d.get("file_path", None), git_folder_path=d.get("git_folder_path", None), root_path=d.get("root_path", None), @@ -956,16 +976,21 @@ def create_version(self, parent: str, version: Version, version_id: str) -> Vers """Creates a new version under a deployment. Creating a version acquires an exclusive lock on the deployment, preventing concurrent deploys. The - caller provides a `version_id` which the server validates equals `last_version_id + 1` on the - deployment. + caller provides a `version_id`, a numeric string that must be numerically greater than the + deployment's most recent version, and sets the version's `previous_version_id` to the deployment's + most recent version (leaving it unset for the first version), which the server validates to detect + concurrent deploys. :param parent: str The parent deployment where this version will be created. Format: deployments/{deployment_id} :param version: :class:`Version` The version to create. :param version_id: str - The version ID the caller expects to create. The server validates this equals `last_version_id + 1` - on the deployment. If it doesn't match, the server returns `ABORTED`. + The ID to use for the version, which becomes the final component of the version's resource name. A + numeric string (base-10, fits in a signed 64-bit integer) chosen by the caller; must be greater than + or equal to 1. Must be numerically greater than the deployment's most recent version (see + `version.previous_version_id`); it does not need to start at 1 or increase by exactly 1. If the + value is not numerically greater, the server returns `INVALID_PARAMETER_VALUE`. :returns: :class:`Version` """ @@ -1236,7 +1261,7 @@ def list_resources( def list_versions( self, parent: str, *, page_size: Optional[int] = None, page_token: Optional[str] = None ) -> Iterator[Version]: - """Lists versions under a deployment, ordered by version_id descending (most recent first). + """Lists versions under a deployment, ordered numerically by version_id descending (most recent first). :param parent: str The parent deployment. Format: deployments/{deployment_id} diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index a3c648980..ee92056cf 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -1853,8 +1853,6 @@ def from_dict(cls, d: Dict[str, Any]) -> ConnectionInfo: class ConnectionType(Enum): - """Next Id: 127""" - BIGQUERY = "BIGQUERY" CONFLUENCE = "CONFLUENCE" DATABRICKS = "DATABRICKS" @@ -2669,8 +2667,6 @@ class CredentialPurpose(Enum): class CredentialType(Enum): - """Next Id: 20""" - ANY_STATIC_CREDENTIAL = "ANY_STATIC_CREDENTIAL" BEARER_TOKEN = "BEARER_TOKEN" EDGEGRID_AKAMAI = "EDGEGRID_AKAMAI" @@ -8902,8 +8898,6 @@ def from_dict(cls, d: Dict[str, Any]) -> RowFilterOptions: @dataclass class SchemaInfo: - """Next ID: 45""" - browse_only: Optional[bool] = None """Indicates whether the principal is limited to retrieving metadata for the associated object through the BROWSE privilege when include_browse is enabled in the request.""" @@ -9291,10 +9285,6 @@ def from_dict(cls, d: Dict[str, Any]) -> Securable: class SecurableKind(Enum): - """Latest kind: CONNECTION_MARKETO_OAUTH_M2M = 347; Next id: 348. Reserved numbers: 316, 317, 327, - 330, 341 (former ENDPOINT_LLM_*, MODEL_SERVICE_STANDARD, MODEL_SERVICE_SYSTEM_DELTASHARING, - MCP_SERVICE_STANDARD).""" - TABLE_DB_STORAGE = "TABLE_DB_STORAGE" TABLE_DELTA = "TABLE_DELTA" TABLE_DELTASHARING = "TABLE_DELTASHARING" diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index 20bc1e23b..319cf9720 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -13,12 +13,15 @@ from enum import Enum from typing import Any, Callable, Dict, Iterator, List, Optional +from google.protobuf.timestamp_pb2 import Timestamp + from databricks.sdk.service._internal import ( Wait, _enum, _from_dict, _repeated_dict, _repeated_enum, + _timestamp, ) from ..errors import OperationFailed @@ -354,6 +357,27 @@ class AzureAvailability(Enum): SPOT_WITH_FALLBACK_AZURE = "SPOT_WITH_FALLBACK_AZURE" +@dataclass +class CancelPendingClusterEnforcementResponse: + """Response for canceling the pending enforcement for a cluster. If the cancel request succeeds, an + empty response object is returned. Otherwise, an error response is returned.""" + + def as_dict(self) -> dict: + """Serializes the CancelPendingClusterEnforcementResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + return body + + def as_shallow_dict(self) -> dict: + """Serializes the CancelPendingClusterEnforcementResponse into a shallow dictionary of its immediate attributes.""" + body = {} + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> CancelPendingClusterEnforcementResponse: + """Deserializes the CancelPendingClusterEnforcementResponse from a dictionary.""" + return cls() + + @dataclass class CancelResponse: def as_dict(self) -> dict: @@ -920,6 +944,10 @@ class ClusterCompliance: is_compliant: Optional[bool] = None """Whether this cluster is in compliance with the latest version of its policy.""" + pending_enforcement: Optional[PendingEnforcement] = None + """Information about the pending enforcement for the cluster. Only present if a pending enforcement + is scheduled for the cluster.""" + violations: Optional[Dict[str, str]] = None """An object containing key-value mappings representing the first 200 policy validation errors. The keys indicate the path where the policy validation error is occurring. The values indicate an @@ -932,6 +960,8 @@ def as_dict(self) -> dict: body["cluster_id"] = self.cluster_id if self.is_compliant is not None: body["is_compliant"] = self.is_compliant + if self.pending_enforcement: + body["pending_enforcement"] = self.pending_enforcement.as_dict() if self.violations: body["violations"] = self.violations return body @@ -943,6 +973,8 @@ def as_shallow_dict(self) -> dict: body["cluster_id"] = self.cluster_id if self.is_compliant is not None: body["is_compliant"] = self.is_compliant + if self.pending_enforcement: + body["pending_enforcement"] = self.pending_enforcement if self.violations: body["violations"] = self.violations return body @@ -953,6 +985,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ClusterCompliance: return cls( cluster_id=d.get("cluster_id", None), is_compliant=d.get("is_compliant", None), + pending_enforcement=_from_dict(d, "pending_enforcement", PendingEnforcement), violations=d.get("violations", None), ) @@ -3167,6 +3200,9 @@ class EnforceClusterComplianceResponse: """A list of changes that have been made to the cluster settings for the cluster to become compliant with its policy.""" + enforce_result: Optional[EnforcePolicyComplianceForClusterResponseEnforceResult] = None + """Describes whether changes have been applied to the cluster.""" + has_changes: Optional[bool] = None """Whether any changes have been made to the cluster settings for the cluster to become compliant with its policy.""" @@ -3176,6 +3212,8 @@ def as_dict(self) -> dict: body = {} if self.changes: body["changes"] = [v.as_dict() for v in self.changes] + if self.enforce_result is not None: + body["enforce_result"] = self.enforce_result.value if self.has_changes is not None: body["has_changes"] = self.has_changes return body @@ -3185,6 +3223,8 @@ def as_shallow_dict(self) -> dict: body = {} if self.changes: body["changes"] = self.changes + if self.enforce_result is not None: + body["enforce_result"] = self.enforce_result if self.has_changes is not None: body["has_changes"] = self.has_changes return body @@ -3192,7 +3232,369 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> EnforceClusterComplianceResponse: """Deserializes the EnforceClusterComplianceResponse from a dictionary.""" - return cls(changes=_repeated_dict(d, "changes", ClusterSettingsChange), has_changes=d.get("has_changes", None)) + return cls( + changes=_repeated_dict(d, "changes", ClusterSettingsChange), + enforce_result=_enum(d, "enforce_result", EnforcePolicyComplianceForClusterResponseEnforceResult), + has_changes=d.get("has_changes", None), + ) + + +class EnforcePolicyComplianceForClusterEnforceMode(Enum): + ENFORCE_IMMEDIATELY = "ENFORCE_IMMEDIATELY" + WAIT_FOR_TERMINATION = "WAIT_FOR_TERMINATION" + + +@dataclass +class EnforcePolicyComplianceForClusterResponseClusterSettings: + autoscale: Optional[AutoScale] = None + """Parameters needed in order to automatically scale clusters up and down based on load. Note: + autoscaling works best with DB runtime versions 3.0 or later.""" + + autotermination_minutes: Optional[int] = None + """Automatically terminates the cluster after it is inactive for this time in minutes. If not set, + this cluster will not be automatically terminated. If specified, the threshold must be between + 10 and 10000 minutes. Users can also set this value to 0 to explicitly disable automatic + termination.""" + + aws_attributes: Optional[AwsAttributes] = None + """Attributes related to clusters running on Amazon Web Services. If not specified at cluster + creation, a set of default values will be used.""" + + azure_attributes: Optional[AzureAttributes] = None + """Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, + a set of default values will be used.""" + + cluster_log_conf: Optional[ClusterLogConf] = None + """The configuration for delivering spark logs to a long-term storage destination. Three kinds of + destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be + specified for one cluster. If the conf is given, the logs will be delivered to the destination + every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the + destination of executor logs is `$destination/$clusterId/executor`.""" + + cluster_name: Optional[str] = None + """Cluster name requested by the user. This doesn't have to be unique. If not specified at + creation, the cluster name will be an empty string. For job clusters, the cluster name is + automatically set based on the job and job run IDs.""" + + custom_tags: Optional[Dict[str, str]] = None + """Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS + instances and EBS volumes) with these tags in addition to `default_tags`. Notes: + + - Currently, Databricks allows at most 45 custom tags + + - Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster + tags""" + + data_security_mode: Optional[DataSecurityMode] = None + + docker_image: Optional[DockerImage] = None + """Custom docker image BYOC""" + + driver_instance_pool_id: Optional[str] = None + """The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster + uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.""" + + driver_node_type_flexibility: Optional[NodeTypeFlexibility] = None + """Flexible node type configuration for the driver node.""" + + driver_node_type_id: Optional[str] = None + """The node type of the Spark driver. Note that this field is optional; if unset, the driver node + type will be set as the same value as `node_type_id` defined above. + + This field, along with node_type_id, should not be set if virtual_cluster_size is set. If both + driver_node_type_id, node_type_id, and virtual_cluster_size are specified, driver_node_type_id + and node_type_id take precedence.""" + + enable_elastic_disk: Optional[bool] = None + """Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk + space when its Spark workers are running low on disk space.""" + + enable_local_disk_encryption: Optional[bool] = None + """Whether to enable LUKS on cluster VMs' local disks""" + + gcp_attributes: Optional[GcpAttributes] = None + """Attributes related to clusters running on Google Cloud Platform. If not specified at cluster + creation, a set of default values will be used.""" + + init_scripts: Optional[List[InitScriptInfo]] = None + """The configuration for storing init scripts. Any number of destinations can be specified. The + scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, + init script logs are sent to `//init_scripts`.""" + + instance_pool_id: Optional[str] = None + """The optional ID of the instance pool to which the cluster belongs.""" + + is_single_node: Optional[bool] = None + """This field can only be used when `kind = CLASSIC_PREVIEW`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + + node_type_id: Optional[str] = None + """This field encodes, through a single value, the resources available to each of the Spark nodes + in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or + compute intensive workloads. A list of available node types can be retrieved by using the + :method:clusters/listNodeTypes API call.""" + + num_workers: Optional[int] = None + """Number of worker nodes that this cluster should have. A cluster has one Spark Driver and + `num_workers` Executors for a total of `num_workers` + 1 Spark nodes. + + Note: When reading the properties of a cluster, this field reflects the desired number of + workers rather than the actual current number of workers. For instance, if a cluster is resized + from 5 to 10 workers, this field will immediately be updated to reflect the target size of 10 + workers, whereas the workers listed in `spark_info` will gradually increase from 5 to 10 as the + new nodes are provisioned.""" + + policy_id: Optional[str] = None + """The ID of the cluster policy used to create the cluster if applicable.""" + + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + + runtime_engine: Optional[RuntimeEngine] = None + """Determines the cluster's runtime engine, either standard or Photon. + + This field is not compatible with legacy `spark_version` values that contain `-photon-`. Remove + `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`. + + If left unspecified, the runtime engine defaults to standard unless the spark_version contains + -photon-, in which case Photon will be used.""" + + single_user_name: Optional[str] = None + """Single user name if data_security_mode is `SINGLE_USER`""" + + spark_conf: Optional[Dict[str, str]] = None + """An object containing a set of optional, user-specified Spark configuration key-value pairs. + Users can also pass in a string of extra JVM options to the driver and the executors via + `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.""" + + spark_env_vars: Optional[Dict[str, str]] = None + """An object containing a set of optional, user-specified environment variable key-value pairs. + Please note that key-value pair of the form (X,Y) will be exported as is (i.e., `export X='Y'`) + while launching the driver and workers. + + In order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending them + to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all default + databricks managed environmental variables are included as well. + + Example Spark environment variables: `{"SPARK_WORKER_MEMORY": "28000m", "SPARK_LOCAL_DIRS": + "/local_disk0"}` or `{"SPARK_DAEMON_JAVA_OPTS": "$SPARK_DAEMON_JAVA_OPTS + -Dspark.shuffle.service.enabled=true"}`""" + + spark_version: Optional[str] = None + """The Spark version of the cluster, e.g. `3.3.x-scala2.11`. A list of available Spark versions can + be retrieved by using the :method:clusters/sparkVersions API call.""" + + ssh_public_keys: Optional[List[str]] = None + """SSH public key contents that will be added to each Spark node in this cluster. The corresponding + private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can + be specified.""" + + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + + use_ml_runtime: Optional[bool] = None + """This field can only be used when `kind = CLASSIC_PREVIEW`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + + worker_node_type_flexibility: Optional[NodeTypeFlexibility] = None + """Flexible node type configuration for worker nodes.""" + + workload_type: Optional[WorkloadType] = None + + def as_dict(self) -> dict: + """Serializes the EnforcePolicyComplianceForClusterResponseClusterSettings into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.autoscale: + body["autoscale"] = self.autoscale.as_dict() + if self.autotermination_minutes is not None: + body["autotermination_minutes"] = self.autotermination_minutes + if self.aws_attributes: + body["aws_attributes"] = self.aws_attributes.as_dict() + if self.azure_attributes: + body["azure_attributes"] = self.azure_attributes.as_dict() + if self.cluster_log_conf: + body["cluster_log_conf"] = self.cluster_log_conf.as_dict() + if self.cluster_name is not None: + body["cluster_name"] = self.cluster_name + if self.custom_tags: + body["custom_tags"] = self.custom_tags + if self.data_security_mode is not None: + body["data_security_mode"] = self.data_security_mode.value + if self.docker_image: + body["docker_image"] = self.docker_image.as_dict() + if self.driver_instance_pool_id is not None: + body["driver_instance_pool_id"] = self.driver_instance_pool_id + if self.driver_node_type_flexibility: + body["driver_node_type_flexibility"] = self.driver_node_type_flexibility.as_dict() + if self.driver_node_type_id is not None: + body["driver_node_type_id"] = self.driver_node_type_id + if self.enable_elastic_disk is not None: + body["enable_elastic_disk"] = self.enable_elastic_disk + if self.enable_local_disk_encryption is not None: + body["enable_local_disk_encryption"] = self.enable_local_disk_encryption + if self.gcp_attributes: + body["gcp_attributes"] = self.gcp_attributes.as_dict() + if self.init_scripts: + body["init_scripts"] = [v.as_dict() for v in self.init_scripts] + if self.instance_pool_id is not None: + body["instance_pool_id"] = self.instance_pool_id + if self.is_single_node is not None: + body["is_single_node"] = self.is_single_node + if self.kind is not None: + body["kind"] = self.kind.value + if self.node_type_id is not None: + body["node_type_id"] = self.node_type_id + if self.num_workers is not None: + body["num_workers"] = self.num_workers + if self.policy_id is not None: + body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.runtime_engine is not None: + body["runtime_engine"] = self.runtime_engine.value + if self.single_user_name is not None: + body["single_user_name"] = self.single_user_name + if self.spark_conf: + body["spark_conf"] = self.spark_conf + if self.spark_env_vars: + body["spark_env_vars"] = self.spark_env_vars + if self.spark_version is not None: + body["spark_version"] = self.spark_version + if self.ssh_public_keys: + body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size + if self.use_ml_runtime is not None: + body["use_ml_runtime"] = self.use_ml_runtime + if self.worker_node_type_flexibility: + body["worker_node_type_flexibility"] = self.worker_node_type_flexibility.as_dict() + if self.workload_type: + body["workload_type"] = self.workload_type.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the EnforcePolicyComplianceForClusterResponseClusterSettings into a shallow dictionary of its immediate attributes.""" + body = {} + if self.autoscale: + body["autoscale"] = self.autoscale + if self.autotermination_minutes is not None: + body["autotermination_minutes"] = self.autotermination_minutes + if self.aws_attributes: + body["aws_attributes"] = self.aws_attributes + if self.azure_attributes: + body["azure_attributes"] = self.azure_attributes + if self.cluster_log_conf: + body["cluster_log_conf"] = self.cluster_log_conf + if self.cluster_name is not None: + body["cluster_name"] = self.cluster_name + if self.custom_tags: + body["custom_tags"] = self.custom_tags + if self.data_security_mode is not None: + body["data_security_mode"] = self.data_security_mode + if self.docker_image: + body["docker_image"] = self.docker_image + if self.driver_instance_pool_id is not None: + body["driver_instance_pool_id"] = self.driver_instance_pool_id + if self.driver_node_type_flexibility: + body["driver_node_type_flexibility"] = self.driver_node_type_flexibility + if self.driver_node_type_id is not None: + body["driver_node_type_id"] = self.driver_node_type_id + if self.enable_elastic_disk is not None: + body["enable_elastic_disk"] = self.enable_elastic_disk + if self.enable_local_disk_encryption is not None: + body["enable_local_disk_encryption"] = self.enable_local_disk_encryption + if self.gcp_attributes: + body["gcp_attributes"] = self.gcp_attributes + if self.init_scripts: + body["init_scripts"] = self.init_scripts + if self.instance_pool_id is not None: + body["instance_pool_id"] = self.instance_pool_id + if self.is_single_node is not None: + body["is_single_node"] = self.is_single_node + if self.kind is not None: + body["kind"] = self.kind + if self.node_type_id is not None: + body["node_type_id"] = self.node_type_id + if self.num_workers is not None: + body["num_workers"] = self.num_workers + if self.policy_id is not None: + body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.runtime_engine is not None: + body["runtime_engine"] = self.runtime_engine + if self.single_user_name is not None: + body["single_user_name"] = self.single_user_name + if self.spark_conf: + body["spark_conf"] = self.spark_conf + if self.spark_env_vars: + body["spark_env_vars"] = self.spark_env_vars + if self.spark_version is not None: + body["spark_version"] = self.spark_version + if self.ssh_public_keys: + body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size + if self.use_ml_runtime is not None: + body["use_ml_runtime"] = self.use_ml_runtime + if self.worker_node_type_flexibility: + body["worker_node_type_flexibility"] = self.worker_node_type_flexibility + if self.workload_type: + body["workload_type"] = self.workload_type + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> EnforcePolicyComplianceForClusterResponseClusterSettings: + """Deserializes the EnforcePolicyComplianceForClusterResponseClusterSettings from a dictionary.""" + return cls( + autoscale=_from_dict(d, "autoscale", AutoScale), + autotermination_minutes=d.get("autotermination_minutes", None), + aws_attributes=_from_dict(d, "aws_attributes", AwsAttributes), + azure_attributes=_from_dict(d, "azure_attributes", AzureAttributes), + cluster_log_conf=_from_dict(d, "cluster_log_conf", ClusterLogConf), + cluster_name=d.get("cluster_name", None), + custom_tags=d.get("custom_tags", None), + data_security_mode=_enum(d, "data_security_mode", DataSecurityMode), + docker_image=_from_dict(d, "docker_image", DockerImage), + driver_instance_pool_id=d.get("driver_instance_pool_id", None), + driver_node_type_flexibility=_from_dict(d, "driver_node_type_flexibility", NodeTypeFlexibility), + driver_node_type_id=d.get("driver_node_type_id", None), + enable_elastic_disk=d.get("enable_elastic_disk", None), + enable_local_disk_encryption=d.get("enable_local_disk_encryption", None), + gcp_attributes=_from_dict(d, "gcp_attributes", GcpAttributes), + init_scripts=_repeated_dict(d, "init_scripts", InitScriptInfo), + instance_pool_id=d.get("instance_pool_id", None), + is_single_node=d.get("is_single_node", None), + kind=_enum(d, "kind", Kind), + node_type_id=d.get("node_type_id", None), + num_workers=d.get("num_workers", None), + policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), + runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), + single_user_name=d.get("single_user_name", None), + spark_conf=d.get("spark_conf", None), + spark_env_vars=d.get("spark_env_vars", None), + spark_version=d.get("spark_version", None), + ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), + use_ml_runtime=d.get("use_ml_runtime", None), + worker_node_type_flexibility=_from_dict(d, "worker_node_type_flexibility", NodeTypeFlexibility), + workload_type=_from_dict(d, "workload_type", WorkloadType), + ) + + +class EnforcePolicyComplianceForClusterResponseEnforceResult(Enum): + APPLIED = "APPLIED" + DEFERRED = "DEFERRED" + NO_CHANGES = "NO_CHANGES" @dataclass @@ -3475,6 +3877,8 @@ class EventType(Enum): DBFS_DOWN = "DBFS_DOWN" DECOMMISSION_ENDED = "DECOMMISSION_ENDED" DECOMMISSION_STARTED = "DECOMMISSION_STARTED" + DEFERRED_POLICY_ENFORCEMENT_FAILED = "DEFERRED_POLICY_ENFORCEMENT_FAILED" + DEFERRED_POLICY_ENFORCEMENT_SCHEDULED = "DEFERRED_POLICY_ENFORCEMENT_SCHEDULED" DID_NOT_EXPAND_DISK = "DID_NOT_EXPAND_DISK" DRIVER_HEALTHY = "DRIVER_HEALTHY" DRIVER_NOT_RESPONDING = "DRIVER_NOT_RESPONDING" @@ -3647,6 +4051,10 @@ class GetClusterComplianceResponse: """Whether the cluster is compliant with its policy or not. Clusters could be out of compliance if the policy was updated after the cluster was last edited.""" + pending_enforcement: Optional[PendingEnforcement] = None + """Information about the pending enforcement for the cluster. Only present if a pending enforcement + is scheduled for the cluster.""" + violations: Optional[Dict[str, str]] = None """An object containing key-value mappings representing the first 200 policy validation errors. The keys indicate the path where the policy validation error is occurring. The values indicate an @@ -3657,6 +4065,8 @@ def as_dict(self) -> dict: body = {} if self.is_compliant is not None: body["is_compliant"] = self.is_compliant + if self.pending_enforcement: + body["pending_enforcement"] = self.pending_enforcement.as_dict() if self.violations: body["violations"] = self.violations return body @@ -3666,6 +4076,8 @@ def as_shallow_dict(self) -> dict: body = {} if self.is_compliant is not None: body["is_compliant"] = self.is_compliant + if self.pending_enforcement: + body["pending_enforcement"] = self.pending_enforcement if self.violations: body["violations"] = self.violations return body @@ -3673,7 +4085,11 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, Any]) -> GetClusterComplianceResponse: """Deserializes the GetClusterComplianceResponse from a dictionary.""" - return cls(is_compliant=d.get("is_compliant", None), violations=d.get("violations", None)) + return cls( + is_compliant=d.get("is_compliant", None), + pending_enforcement=_from_dict(d, "pending_enforcement", PendingEnforcement), + violations=d.get("violations", None), + ) @dataclass @@ -6409,6 +6825,76 @@ def from_dict(cls, d: Dict[str, Any]) -> NodeTypeFlexibility: return cls(alternate_node_type_ids=d.get("alternate_node_type_ids", None)) +@dataclass +class PendingEnforcement: + """Represents a pending enforcement on a cluster, which contains the changes to make to the cluster + configuration when the cluster is next terminated or restarted.""" + + enforcement_status: Optional[PendingEnforcementEnforcementStatus] = None + """Whether the pending enforcement will be applied. A pending enforcement begins in `ACTIVE` state. + If the enforcement fails to apply too many times, the state transitions to `INACTIVE`. + Afterwards, the enforcement must be re-scheduled to become `ACTIVE` again.""" + + initiate_time: Optional[Timestamp] = None + """The time the pending enforcement was initiated.""" + + initiator_user: Optional[str] = None + """The user who initiated the pending enforcement.""" + + target_changes: Optional[List[ClusterSettingsChange]] = None + """A list of changes that will be made to the cluster configuration when the pending enforcement is + applied.""" + + target_spec: Optional[EnforcePolicyComplianceForClusterResponseClusterSettings] = None + """The new configuration to apply upon cluster termination or restart.""" + + def as_dict(self) -> dict: + """Serializes the PendingEnforcement into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.enforcement_status is not None: + body["enforcement_status"] = self.enforcement_status.value + if self.initiate_time is not None: + body["initiate_time"] = self.initiate_time.ToJsonString() + if self.initiator_user is not None: + body["initiator_user"] = self.initiator_user + if self.target_changes: + body["target_changes"] = [v.as_dict() for v in self.target_changes] + if self.target_spec: + body["target_spec"] = self.target_spec.as_dict() + return body + + def as_shallow_dict(self) -> dict: + """Serializes the PendingEnforcement into a shallow dictionary of its immediate attributes.""" + body = {} + if self.enforcement_status is not None: + body["enforcement_status"] = self.enforcement_status + if self.initiate_time is not None: + body["initiate_time"] = self.initiate_time + if self.initiator_user is not None: + body["initiator_user"] = self.initiator_user + if self.target_changes: + body["target_changes"] = self.target_changes + if self.target_spec: + body["target_spec"] = self.target_spec + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> PendingEnforcement: + """Deserializes the PendingEnforcement from a dictionary.""" + return cls( + enforcement_status=_enum(d, "enforcement_status", PendingEnforcementEnforcementStatus), + initiate_time=_timestamp(d, "initiate_time"), + initiator_user=d.get("initiator_user", None), + target_changes=_repeated_dict(d, "target_changes", ClusterSettingsChange), + target_spec=_from_dict(d, "target_spec", EnforcePolicyComplianceForClusterResponseClusterSettings), + ) + + +class PendingEnforcementEnforcementStatus(Enum): + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + + @dataclass class PendingInstanceError: """Error message of a failed pending instances""" @@ -10919,23 +11405,69 @@ class PolicyComplianceForClustersAPI: def __init__(self, api_client): self._api = api_client + def cancel_pending_cluster_enforcement( + self, cluster_id: str, *, allow_missing: Optional[bool] = None + ) -> CancelPendingClusterEnforcementResponse: + """Cancels a pending enforcement on a cluster. After canceling the pending enforcement, the cluster will + no longer update on the next termination or restart. Pending enforcements cannot be canceled when a + cluster is in `TERMINATING` state. Only workspace admins can cancel pending enforcements. + + :param cluster_id: str + The ID of the cluster to cancel the pending enforcement for. + :param allow_missing: bool (optional) + If true and no pending enforcement exists, the request will succeed but no action will be taken. + + :returns: :class:`CancelPendingClusterEnforcementResponse` + """ + + body = {} + if allow_missing is not None: + body["allow_missing"] = allow_missing + if cluster_id is not None: + body["cluster_id"] = cluster_id + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + + cfg = self._api._cfg + if cfg.workspace_id: + headers["X-Databricks-Workspace-Id"] = cfg.workspace_id + + res = self._api.do( + "POST", "/api/2.0/policies/clusters:cancelPendingClusterEnforcement", body=body, headers=headers + ) + return CancelPendingClusterEnforcementResponse.from_dict(res) + def enforce_compliance( - self, cluster_id: str, *, validate_only: Optional[bool] = None + self, + cluster_id: str, + *, + enforce_mode: Optional[EnforcePolicyComplianceForClusterEnforceMode] = None, + validate_only: Optional[bool] = None, ) -> EnforceClusterComplianceResponse: - """Updates a cluster to be compliant with the current version of its policy. A cluster can be updated if - it is in a `RUNNING` or `TERMINATED` state. - - If a cluster is updated while in a `RUNNING` state, it will be restarted so that the new attributes - can take effect. + """Updates a cluster to be compliant with the current version of its policy. If a cluster is updated while in a `TERMINATED` state, it will remain `TERMINATED`. The next time the cluster is started, the new attributes will take effect. + For clusters in other states, the behavior depends on the `enforce_mode` used. + Clusters created by the Databricks Jobs, SDP, or Models services cannot be enforced by this API. Instead, use the "Enforce job policy compliance" API to enforce policy compliance on jobs. :param cluster_id: str The ID of the cluster you want to enforce policy compliance on. + :param enforce_mode: :class:`EnforcePolicyComplianceForClusterEnforceMode` (optional) + Determines how changes should be made to clusters that are not in `TERMINATED` state. + + - `ENFORCE_IMMEDIATELY`: If the cluster is in a `RUNNING` state, it will be restarted so that the + new attributes can take effect. For other states aside from `TERMINATED` state, the request will be + rejected. - `WAIT_FOR_TERMINATION`: The cluster is not immediately edited. Instead, a pending + enforcement is scheduled to update the cluster when it terminates or restarts. When this occurs, + `enforce_result` will contain `DEFERRED`. Only workspace admins can use this mode. + + Regardless of the enforce mode, clusters in `TERMINATED` state are immediately edited. :param validate_only: bool (optional) If set, previews the changes that would be made to a cluster to enforce compliance but does not update the cluster. @@ -10946,6 +11478,8 @@ def enforce_compliance( body = {} if cluster_id is not None: body["cluster_id"] = cluster_id + if enforce_mode is not None: + body["enforce_mode"] = enforce_mode.value if validate_only is not None: body["validate_only"] = validate_only headers = { diff --git a/databricks/sdk/service/dashboards.py b/databricks/sdk/service/dashboards.py index b3cbaf296..39b929ebe 100755 --- a/databricks/sdk/service/dashboards.py +++ b/databricks/sdk/service/dashboards.py @@ -3862,7 +3862,7 @@ def list( The flag to include dashboards located in the trash. If unspecified, only active dashboards will be returned. :param view: :class:`DashboardView` (optional) - `DASHBOARD_VIEW_BASIC`only includes summary metadata from the dashboard. + `DASHBOARD_VIEW_BASIC` only includes summary metadata from the dashboard. :returns: Iterator over :class:`Dashboard` """ diff --git a/databricks/sdk/service/dataquality.py b/databricks/sdk/service/dataquality.py index 4b1d26912..1c0986330 100644 --- a/databricks/sdk/service/dataquality.py +++ b/databricks/sdk/service/dataquality.py @@ -686,7 +686,7 @@ class Refresh: """The Refresh object gives information on a refresh of the data quality monitoring pipeline.""" object_type: str - """The type of the monitored object. Can be one of the following: `schema`or `table`.""" + """The type of the monitored object. Can be one of the following: `schema` or `table`.""" object_id: str """The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`. @@ -952,7 +952,7 @@ def create_refresh(self, object_type: str, object_id: str, refresh: Refresh) -> **USE_SCHEMA** on the table's parent schema, and **MANAGE** on the table. :param object_type: str - The type of the monitored object. Can be one of the following: `schema`or `table`. + The type of the monitored object. Can be one of the following: `schema` or `table`. :param object_id: str The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`. diff --git a/databricks/sdk/service/disasterrecovery.py b/databricks/sdk/service/disasterrecovery.py index b688d018e..b24a990a9 100755 --- a/databricks/sdk/service/disasterrecovery.py +++ b/databricks/sdk/service/disasterrecovery.py @@ -323,7 +323,7 @@ class StableUrl: url: Optional[str] = None """The stable URL endpoint. Generated on creation and immutable thereafter. For non-Private-Link - workspaces this is `https:///?c=`. For Private-Link workspaces this is + workspaces this is `https:///?w=`. For Private-Link workspaces this is the per-connection hostname.""" def as_dict(self) -> dict: @@ -447,8 +447,9 @@ class WorkspaceSet: """Workspace IDs in this set. The system derives and validates regions. All workspaces must be in the Mission Critical tier.""" - replicate_workspace_assets: bool - """Whether to enable control plane DR (notebooks, jobs, clusters, etc.) for this set.""" + replicate_workspace_assets: Optional[bool] = None + """Whether to enable control plane DR (notebooks, jobs, clusters, etc.) for this set. Defaults to + false.""" stable_url_names: Optional[List[str]] = None """Resource names of stable URLs associated with this workspace set. Format: diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 92d11f24a..646aa4514 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -29,6 +29,126 @@ # all definitions in this file are in alphabetical order +@dataclass +class AiRuntimeTask: + """AiRuntimeTask: multi-node GPU compute task definition for Databricks AI Runtime workloads. + + Jobs-framework-level concepts (retries, per-task timeout, idempotency token, usage/budget + policy, permissions) live on the surrounding TaskSettings / run-submit request and are + intentionally NOT duplicated here. Users compose `ai_runtime_task` with the standard Jobs/DABs + task wrapper to get those.""" + + experiment: str + """MLflow experiment name for this run. If an experiment with this name already exists under the + calling user, the run is appended to it; otherwise a new experiment is created. To target a + specific MLflow storage location (for example, when running as a service principal), set + `mlflow_experiment_directory`.""" + + deployments: List[DeploymentSpec] + """Deployment specs for this task. Exactly one deployment is currently supported (a single entry + where every node runs the same command); this is a current-Preview constraint. Role-split + workloads (driver + worker, parameter server, separate eval node, etc.) with multiple entries + are the eventual intent but not yet supported.""" + + code_source_path: Optional[str] = None + """Optional workspace or UC volume path of the uploaded code-source archive. The CLI packages the + user's local code directory into an archive and populates this. Customers calling the Jobs API + directly should upload their archive to the workspace or a UC volume first and supply the + resulting path here. + + When set, the training node exposes the value via the `$CODE_SOURCE` environment variable.""" + + mlflow_experiment_directory: Optional[str] = None + """Optional workspace directory under which the MLflow experiment named in `experiment` is created. + Must start with `/Workspace`. Set this when running as a service principal that has no default + user directory; for regular users the experiment defaults to the user's home directory.""" + + mlflow_run: Optional[str] = None + """Optional display name for the MLflow run created under `experiment`. If omitted, MLflow + generates a default name.""" + + def as_dict(self) -> dict: + """Serializes the AiRuntimeTask into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.code_source_path is not None: + body["code_source_path"] = self.code_source_path + if self.deployments: + body["deployments"] = [v.as_dict() for v in self.deployments] + if self.experiment is not None: + body["experiment"] = self.experiment + if self.mlflow_experiment_directory is not None: + body["mlflow_experiment_directory"] = self.mlflow_experiment_directory + if self.mlflow_run is not None: + body["mlflow_run"] = self.mlflow_run + return body + + def as_shallow_dict(self) -> dict: + """Serializes the AiRuntimeTask into a shallow dictionary of its immediate attributes.""" + body = {} + if self.code_source_path is not None: + body["code_source_path"] = self.code_source_path + if self.deployments: + body["deployments"] = self.deployments + if self.experiment is not None: + body["experiment"] = self.experiment + if self.mlflow_experiment_directory is not None: + body["mlflow_experiment_directory"] = self.mlflow_experiment_directory + if self.mlflow_run is not None: + body["mlflow_run"] = self.mlflow_run + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> AiRuntimeTask: + """Deserializes the AiRuntimeTask from a dictionary.""" + return cls( + code_source_path=d.get("code_source_path", None), + deployments=_repeated_dict(d, "deployments", DeploymentSpec), + experiment=d.get("experiment", None), + mlflow_experiment_directory=d.get("mlflow_experiment_directory", None), + mlflow_run=d.get("mlflow_run", None), + ) + + +@dataclass +class AiRuntimeTaskOutput: + """AiRuntimeTaskOutput: output identifiers for an AiRuntimeTask run — the MLflow experiment and + run IDs the task wrote to. + + Run lifecycle and termination status are not on this message; they live on the surrounding + `RunTask.status` field (see `runs.proto:RunTask.status`).""" + + mlflow_experiment_id: Optional[str] = None + """MLflow experiment ID the run was logged to. Use it to look up the experiment in MLflow APIs or + the workspace MLflow UI.""" + + mlflow_run_id: Optional[str] = None + """MLflow run ID for this task execution. Use it to look up the run in MLflow APIs or the workspace + MLflow UI.""" + + def as_dict(self) -> dict: + """Serializes the AiRuntimeTaskOutput into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.mlflow_experiment_id is not None: + body["mlflow_experiment_id"] = self.mlflow_experiment_id + if self.mlflow_run_id is not None: + body["mlflow_run_id"] = self.mlflow_run_id + return body + + def as_shallow_dict(self) -> dict: + """Serializes the AiRuntimeTaskOutput into a shallow dictionary of its immediate attributes.""" + body = {} + if self.mlflow_experiment_id is not None: + body["mlflow_experiment_id"] = self.mlflow_experiment_id + if self.mlflow_run_id is not None: + body["mlflow_run_id"] = self.mlflow_run_id + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> AiRuntimeTaskOutput: + """Deserializes the AiRuntimeTaskOutput from a dictionary.""" + return cls(mlflow_experiment_id=d.get("mlflow_experiment_id", None), mlflow_run_id=d.get("mlflow_run_id", None)) + + class AlertEvaluationState(Enum): """Same alert evaluation state as in redash-v2/api/proto/alertsv2/alerts.proto""" @@ -914,6 +1034,56 @@ def from_dict(cls, d: Dict[str, Any]) -> ComputeConfig: ) +@dataclass +class ComputeSpec: + """ComputeSpec: compute configuration — accelerator type and total accelerator count across all + nodes.""" + + accelerator_type: ComputeSpecAcceleratorType + """Hardware accelerator type (for example, `GPU_1xA10` or `GPU_8xH100`). The number of accelerators + per node is encoded in the enum value — `GPU_8xH100` means 8 H100 GPUs per node.""" + + accelerator_count: int + """Total number of accelerators across all nodes. Must be a positive multiple of the per-node + accelerator count encoded in `accelerator_type`. For example, `GPU_8xH100` with + `accelerator_count: 16` allocates 2 nodes (8 GPUs per node).""" + + def as_dict(self) -> dict: + """Serializes the ComputeSpec into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.accelerator_count is not None: + body["accelerator_count"] = self.accelerator_count + if self.accelerator_type is not None: + body["accelerator_type"] = self.accelerator_type.value + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ComputeSpec into a shallow dictionary of its immediate attributes.""" + body = {} + if self.accelerator_count is not None: + body["accelerator_count"] = self.accelerator_count + if self.accelerator_type is not None: + body["accelerator_type"] = self.accelerator_type + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ComputeSpec: + """Deserializes the ComputeSpec from a dictionary.""" + return cls( + accelerator_count=d.get("accelerator_count", None), + accelerator_type=_enum(d, "accelerator_type", ComputeSpecAcceleratorType), + ) + + +class ComputeSpecAcceleratorType(Enum): + """Customer-facing AcceleratorType: hardware accelerator type for the AiRuntime workload. Per-node + accelerator count is encoded in the value name (e.g. `GPU_8xH100` means 8 H100s per node).""" + + GPU_1X_A10 = "GPU_1xA10" + GPU_1X_H100 = "GPU_1xH100" + GPU_8X_H100 = "GPU_8xH100" + + class Condition(Enum): ALL_UPDATED = "ALL_UPDATED" ANY_UPDATED = "ANY_UPDATED" @@ -1645,6 +1815,59 @@ def from_dict(cls, d: Dict[str, Any]) -> DbtTask: ) +@dataclass +class DeploymentSpec: + """DeploymentSpec: configuration for one deployment within an AiRuntimeTask. Each entry in + `AiRuntimeTask.deployments` describes a group of nodes that share the same command and compute. + Many single-program training algorithms use a single entry where every node runs the same + command; role-split workloads (driver + worker, parameter server, separate eval node, etc.) use + multiple entries.""" + + command_path: str + """Workspace path of the bash script to execute on each node in this deployment. The CLI uploads + the user's script and populates this. Customers calling the Jobs API directly should upload + their script to the workspace first and supply the resulting path here.""" + + compute: ComputeSpec + """Compute resources allocated to each node in this deployment.""" + + name: Optional[str] = None + """Optional human-readable name for this deployment (for example, `driver`, `worker`, + `param_server`). Used for log and UI display. Distinct names are recommended so deployments can + be told apart, but uniqueness is not enforced.""" + + def as_dict(self) -> dict: + """Serializes the DeploymentSpec into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.command_path is not None: + body["command_path"] = self.command_path + if self.compute: + body["compute"] = self.compute.as_dict() + if self.name is not None: + body["name"] = self.name + return body + + def as_shallow_dict(self) -> dict: + """Serializes the DeploymentSpec into a shallow dictionary of its immediate attributes.""" + body = {} + if self.command_path is not None: + body["command_path"] = self.command_path + if self.compute: + body["compute"] = self.compute + if self.name is not None: + body["name"] = self.name + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> DeploymentSpec: + """Deserializes the DeploymentSpec from a dictionary.""" + return cls( + command_path=d.get("command_path", None), + compute=_from_dict(d, "compute", ComputeSpec), + name=d.get("name", None), + ) + + @dataclass class EnforcePolicyComplianceForJobResponseJobClusterSettingsChange: """Represents a change to the job cluster's settings that would be required for the job clusters to @@ -4779,6 +5002,10 @@ def from_dict(cls, d: Dict[str, Any]) -> ResolvedStringParamsValues: @dataclass class ResolvedValues: + ai_runtime_task: Optional[ResolvedValuesAiRuntimeTaskResolvedValues] = None + """Resolved values for an AI Runtime task — env_vars with `{{tasks..values.}}` + references substituted to concrete values before submission to the training service.""" + condition_task: Optional[ResolvedConditionTaskValues] = None dbt_task: Optional[ResolvedDbtTaskValues] = None @@ -4804,6 +5031,8 @@ class ResolvedValues: def as_dict(self) -> dict: """Serializes the ResolvedValues into a dictionary suitable for use as a JSON request body.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task.as_dict() if self.condition_task: body["condition_task"] = self.condition_task.as_dict() if self.dbt_task: @@ -4831,6 +5060,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the ResolvedValues into a shallow dictionary of its immediate attributes.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task if self.condition_task: body["condition_task"] = self.condition_task if self.dbt_task: @@ -4859,6 +5090,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> ResolvedValues: """Deserializes the ResolvedValues from a dictionary.""" return cls( + ai_runtime_task=_from_dict(d, "ai_runtime_task", ResolvedValuesAiRuntimeTaskResolvedValues), condition_task=_from_dict(d, "condition_task", ResolvedConditionTaskValues), dbt_task=_from_dict(d, "dbt_task", ResolvedDbtTaskValues), notebook_task=_from_dict(d, "notebook_task", ResolvedNotebookTaskValues), @@ -4873,6 +5105,28 @@ def from_dict(cls, d: Dict[str, Any]) -> ResolvedValues: ) +@dataclass +class ResolvedValuesAiRuntimeTaskResolvedValues: + """Resolved env_vars for an AiRuntimeTask after dynamic-value substitution. Mirrors the task's + `resolved_parameters_field` (env_vars) so Jobs can expand `{{tasks..values.}}` + references before submission.""" + + def as_dict(self) -> dict: + """Serializes the ResolvedValuesAiRuntimeTaskResolvedValues into a dictionary suitable for use as a JSON request body.""" + body = {} + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ResolvedValuesAiRuntimeTaskResolvedValues into a shallow dictionary of its immediate attributes.""" + body = {} + return body + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> ResolvedValuesAiRuntimeTaskResolvedValues: + """Deserializes the ResolvedValuesAiRuntimeTaskResolvedValues from a dictionary.""" + return cls() + + @dataclass class Run: """Run was retrieved successfully""" @@ -5611,6 +5865,12 @@ def from_dict(cls, d: Dict[str, Any]) -> RunNowResponse: class RunOutput: """Run output was retrieved successfully.""" + ai_runtime_task_output: Optional[AiRuntimeTaskOutput] = None + """The output of an AiRuntimeTask, if available — MLflow identifiers, artifact paths, and + per-replica allocated compute. Run lifecycle / termination status lives on the surrounding + framework `RunTask.status` (`runs.proto:RunTask.status` of type `RunStatus`), not on this + output. See `tasks/genai/ai_runtime_task.proto:AiRuntimeTaskOutput`.""" + alert_output: Optional[AlertTaskOutput] = None """The output of an alert task, if available""" @@ -5669,6 +5929,8 @@ class RunOutput: def as_dict(self) -> dict: """Serializes the RunOutput into a dictionary suitable for use as a JSON request body.""" body = {} + if self.ai_runtime_task_output: + body["ai_runtime_task_output"] = self.ai_runtime_task_output.as_dict() if self.alert_output: body["alert_output"] = self.alert_output.as_dict() if self.clean_rooms_notebook_output: @@ -5704,6 +5966,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the RunOutput into a shallow dictionary of its immediate attributes.""" body = {} + if self.ai_runtime_task_output: + body["ai_runtime_task_output"] = self.ai_runtime_task_output if self.alert_output: body["alert_output"] = self.alert_output if self.clean_rooms_notebook_output: @@ -5740,6 +6004,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> RunOutput: """Deserializes the RunOutput from a dictionary.""" return cls( + ai_runtime_task_output=_from_dict(d, "ai_runtime_task_output", AiRuntimeTaskOutput), alert_output=_from_dict(d, "alert_output", AlertTaskOutput), clean_rooms_notebook_output=_from_dict( d, "clean_rooms_notebook_output", CleanRoomsNotebookTaskCleanRoomsNotebookTaskOutput @@ -6043,6 +6308,10 @@ class RunTask: field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" + ai_runtime_task: Optional[AiRuntimeTask] = None + """The task runs a multi-node GPU compute workload on Databricks AI Runtime. External-facing + surface; mirrors the AIR CLI (fka SGCLI) v2 YAML schema.""" + alert_task: Optional[AlertTask] = None """The task evaluates a Databricks alert and sends notifications to subscribers when the `alert_task` field is present.""" @@ -6261,6 +6530,8 @@ class RunTask: def as_dict(self) -> dict: """Serializes the RunTask into a dictionary suitable for use as a JSON request body.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task.as_dict() if self.alert_task: body["alert_task"] = self.alert_task.as_dict() if self.attempt_number is not None: @@ -6374,6 +6645,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the RunTask into a shallow dictionary of its immediate attributes.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task if self.alert_task: body["alert_task"] = self.alert_task if self.attempt_number is not None: @@ -6488,6 +6761,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> RunTask: """Deserializes the RunTask from a dictionary.""" return cls( + ai_runtime_task=_from_dict(d, "ai_runtime_task", AiRuntimeTask), alert_task=_from_dict(d, "alert_task", AlertTask), attempt_number=d.get("attempt_number", None), clean_rooms_notebook_task=_from_dict(d, "clean_rooms_notebook_task", CleanRoomsNotebookTask), @@ -7384,6 +7658,10 @@ class SubmitTask: field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" + ai_runtime_task: Optional[AiRuntimeTask] = None + """The task runs a multi-node GPU compute workload on Databricks AI Runtime. External-facing + surface; mirrors the AIR CLI (fka SGCLI) v2 YAML schema.""" + alert_task: Optional[AlertTask] = None """The task evaluates a Databricks alert and sends notifications to subscribers when the `alert_task` field is present.""" @@ -7524,6 +7802,8 @@ class SubmitTask: def as_dict(self) -> dict: """Serializes the SubmitTask into a dictionary suitable for use as a JSON request body.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task.as_dict() if self.alert_task: body["alert_task"] = self.alert_task.as_dict() if self.clean_rooms_notebook_task: @@ -7605,6 +7885,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the SubmitTask into a shallow dictionary of its immediate attributes.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task if self.alert_task: body["alert_task"] = self.alert_task if self.clean_rooms_notebook_task: @@ -7687,6 +7969,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> SubmitTask: """Deserializes the SubmitTask from a dictionary.""" return cls( + ai_runtime_task=_from_dict(d, "ai_runtime_task", AiRuntimeTask), alert_task=_from_dict(d, "alert_task", AlertTask), clean_rooms_notebook_task=_from_dict(d, "clean_rooms_notebook_task", CleanRoomsNotebookTask), compute=_from_dict(d, "compute", Compute), @@ -7934,6 +8217,10 @@ class Task: field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" + ai_runtime_task: Optional[AiRuntimeTask] = None + """The task runs a multi-node GPU compute workload on Databricks AI Runtime. External-facing + surface; mirrors the AIR CLI (fka SGCLI) v2 YAML schema.""" + alert_task: Optional[AlertTask] = None """The task evaluates a Databricks alert and sends notifications to subscribers when the `alert_task` field is present.""" @@ -8082,6 +8369,8 @@ class Task: def as_dict(self) -> dict: """Serializes the Task into a dictionary suitable for use as a JSON request body.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task.as_dict() if self.alert_task: body["alert_task"] = self.alert_task.as_dict() if self.clean_rooms_notebook_task: @@ -8165,6 +8454,8 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the Task into a shallow dictionary of its immediate attributes.""" body = {} + if self.ai_runtime_task: + body["ai_runtime_task"] = self.ai_runtime_task if self.alert_task: body["alert_task"] = self.alert_task if self.clean_rooms_notebook_task: @@ -8249,6 +8540,7 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, Any]) -> Task: """Deserializes the Task from a dictionary.""" return cls( + ai_runtime_task=_from_dict(d, "ai_runtime_task", AiRuntimeTask), alert_task=_from_dict(d, "alert_task", AlertTask), clean_rooms_notebook_task=_from_dict(d, "clean_rooms_notebook_task", CleanRoomsNotebookTask), compute=_from_dict(d, "compute", Compute), diff --git a/databricks/sdk/service/oauth2.py b/databricks/sdk/service/oauth2.py index 14f0bd79a..aaa94a780 100755 --- a/databricks/sdk/service/oauth2.py +++ b/databricks/sdk/service/oauth2.py @@ -973,12 +973,24 @@ class AccountFederationPolicyAPI: Databricks automatically fetches the public keys from your issuer’s well known endpoint. Databricks strongly recommends relying on your issuer’s well known endpoint for discovering public keys. - An example federation policy is: ``` issuer: "https://idp.mycompany.com/oidc" audiences: ["databricks"] - subject_claim: "sub" ``` + An example federation policy is: + + .. code-block:: + + issuer: "https://idp.mycompany.com/oidc" + audiences: ["databricks"] + subject_claim: "sub" An example JWT token body that matches this policy and could be used to authenticate to Databricks as user - `username@mycompany.com` is: ``` { "iss": "https://idp.mycompany.com/oidc", "aud": "databricks", "sub": - "username@mycompany.com" } ``` + `username@mycompany.com` is: + + .. code-block:: + + { + "iss": "https://idp.mycompany.com/oidc", + "aud": "databricks", + "sub": "username@mycompany.com" + } You may also need to configure your IdP to generate tokens for your users to exchange with Databricks, if your users do not already have the ability to generate tokens that are compatible with your federation @@ -1526,13 +1538,23 @@ class ServicePrincipalFederationPolicyAPI: fetches the public keys from the issuer’s well known endpoint. Databricks strongly recommends relying on the issuer’s well known endpoint for discovering public keys. - An example service principal federation policy, for a Github Actions workload, is: ``` issuer: - "https://token.actions.githubusercontent.com" audiences: ["https://github.com/my-github-org"] subject: - "repo:my-github-org/my-repo:environment:prod" ``` + An example service principal federation policy, for a Github Actions workload, is: + + .. code-block:: + + issuer: "https://token.actions.githubusercontent.com" + audiences: ["https://github.com/my-github-org"] + subject: "repo:my-github-org/my-repo:environment:prod" + + An example JWT token body that matches this policy and could be used to authenticate to Databricks is: + + .. code-block:: - An example JWT token body that matches this policy and could be used to authenticate to Databricks is: ``` - { "iss": "https://token.actions.githubusercontent.com", "aud": "https://github.com/my-github-org", "sub": - "repo:my-github-org/my-repo:environment:prod" } ``` + { + "iss": "https://token.actions.githubusercontent.com", + "aud": "https://github.com/my-github-org", + "sub": "repo:my-github-org/my-repo:environment:prod" + } You may also need to configure the workload runtime to generate tokens for your workloads. diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index aef0af4ea..9d0a5b834 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -9518,7 +9518,13 @@ def execute_statement( output of `SELECT concat('id-', id) AS strCol, id AS intCol, null AS nullCol FROM range(3)` would look like this: - ``` [ [ "id-1", "1", null ], [ "id-2", "2", null ], [ "id-3", "3", null ], ] ``` + .. code-block:: + + [ + [ "id-1", "1", null ], + [ "id-2", "2", null ], + [ "id-3", "3", null ], + ] When specifying `format=JSON_ARRAY` and `disposition=EXTERNAL_LINKS`, each chunk in the result contains compact JSON with no indentation or extra whitespace. @@ -9532,7 +9538,12 @@ def execute_statement( chunk in the result would contain a header row with column names. For example, the output of `SELECT concat('id-', id) AS strCol, id AS intCol, null as nullCol FROM range(3)` would look like this: - ``` strCol,intCol,nullCol id-1,1,null id-2,2,null id-3,3,null ``` + .. code-block:: + + strCol,intCol,nullCol + id-1,1,null + id-2,2,null + id-3,3,null [Apache Arrow streaming format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format [RFC 4180]: https://www.rfc-editor.org/rfc/rfc4180 @@ -9556,7 +9567,9 @@ def execute_statement( For example, the following statement contains two parameters, `my_name` and `my_date`: - ``` SELECT * FROM my_table WHERE name = :my_name AND date = :my_date ``` + .. code-block:: + + SELECT * FROM my_table WHERE name = :my_name AND date = :my_date The parameters can be passed in the request body as follows: diff --git a/databricks/sdk/useragent.py b/databricks/sdk/useragent.py index af0d054a7..90bdb55f4 100644 --- a/databricks/sdk/useragent.py +++ b/databricks/sdk/useragent.py @@ -12,6 +12,7 @@ RUNTIME_KEY = "runtime" CICD_KEY = "cicd" AUTH_KEY = "auth" +META_HARNESS_KEY = "meta-harness" _product_name = "unknown" _product_version = "0.0.0" @@ -175,6 +176,9 @@ def to_string( agent = agent_provider() if agent: base.append(("agent", agent)) + meta_harness = meta_harness_provider() + if meta_harness: + base.append((META_HARNESS_KEY, meta_harness)) return " ".join(f"{k}/{v}" for k, v in base) @@ -329,3 +333,37 @@ def _agent_env_fallback() -> str: if not v: return "" return _sanitize_agent_value(v)[:_MAX_AGENT_FALLBACK_LEN] + + +@dataclass(frozen=True) +class _MetaHarnessRecord: + env_var: str + product: str + + +# Known agent meta-harnesses, detected independently of agents (a meta-harness +# is not an agent). Keep in sync with databricks-sdk-go and databricks-sdk-java. +_KNOWN_META_HARNESSES: List[_MetaHarnessRecord] = [ + _MetaHarnessRecord("OMNIGENT", "omnigent"), # https://github.com/omnigent-ai/omnigent +] + +# None = not computed, "" = computed but no meta-harness found. +_meta_harness_provider = None + + +def meta_harness_provider() -> str: + """Detect a known agent meta-harness by presence-only env var, else "". + + Returns "multiple" if more than one matched. Cached after the first call. + """ + global _meta_harness_provider + if _meta_harness_provider is not None: + return _meta_harness_provider + matches = [h.product for h in _KNOWN_META_HARNESSES if h.env_var in os.environ] + if len(matches) == 1: + _meta_harness_provider = matches[0] + elif len(matches) > 1: + _meta_harness_provider = "multiple" + else: + _meta_harness_provider = "" + return _meta_harness_provider diff --git a/databricks/sdk/version.py b/databricks/sdk/version.py index cc8b2f720..862c3b518 100644 --- a/databricks/sdk/version.py +++ b/databricks/sdk/version.py @@ -1 +1 @@ -__version__ = "0.118.0" +__version__ = "0.119.0" diff --git a/docs/account/iam/workspace_assignment.rst b/docs/account/iam/workspace_assignment.rst index 133b16f3d..2a8043172 100755 --- a/docs/account/iam/workspace_assignment.rst +++ b/docs/account/iam/workspace_assignment.rst @@ -43,9 +43,9 @@ a = AccountClient() - workspace_id = os.environ["TEST_WORKSPACE_ID"] + workspace_id = os.environ["DUMMY_WORKSPACE_ID"] - all = a.workspace_assignment.list(list=workspace_id) + all = a.workspace_assignment.list(workspace_id=workspace_id) Get the permission assignments for the specified Databricks account and Databricks workspace. @@ -74,9 +74,9 @@ spn_id = spn.id - workspace_id = os.environ["DUMMY_WORKSPACE_ID"] + workspace_id = os.environ["TEST_WORKSPACE_ID"] - _ = a.workspace_assignment.update( + a.workspace_assignment.update( workspace_id=workspace_id, principal_id=spn_id, permissions=[iam.WorkspacePermission.USER], diff --git a/docs/account/oauth2/federation_policy.rst b/docs/account/oauth2/federation_policy.rst index a8957e5f2..594d92756 100644 --- a/docs/account/oauth2/federation_policy.rst +++ b/docs/account/oauth2/federation_policy.rst @@ -30,12 +30,24 @@ Databricks automatically fetches the public keys from your issuer’s well known endpoint. Databricks strongly recommends relying on your issuer’s well known endpoint for discovering public keys. - An example federation policy is: ``` issuer: "https://idp.mycompany.com/oidc" audiences: ["databricks"] - subject_claim: "sub" ``` + An example federation policy is: + + .. code-block:: + + issuer: "https://idp.mycompany.com/oidc" + audiences: ["databricks"] + subject_claim: "sub" An example JWT token body that matches this policy and could be used to authenticate to Databricks as user - `username@mycompany.com` is: ``` { "iss": "https://idp.mycompany.com/oidc", "aud": "databricks", "sub": - "username@mycompany.com" } ``` + `username@mycompany.com` is: + + .. code-block:: + + { + "iss": "https://idp.mycompany.com/oidc", + "aud": "databricks", + "sub": "username@mycompany.com" + } You may also need to configure your IdP to generate tokens for your users to exchange with Databricks, if your users do not already have the ability to generate tokens that are compatible with your federation diff --git a/docs/account/oauth2/service_principal_federation_policy.rst b/docs/account/oauth2/service_principal_federation_policy.rst index 3f7f275ee..cda84e9e3 100644 --- a/docs/account/oauth2/service_principal_federation_policy.rst +++ b/docs/account/oauth2/service_principal_federation_policy.rst @@ -33,13 +33,23 @@ fetches the public keys from the issuer’s well known endpoint. Databricks strongly recommends relying on the issuer’s well known endpoint for discovering public keys. - An example service principal federation policy, for a Github Actions workload, is: ``` issuer: - "https://token.actions.githubusercontent.com" audiences: ["https://github.com/my-github-org"] subject: - "repo:my-github-org/my-repo:environment:prod" ``` + An example service principal federation policy, for a Github Actions workload, is: - An example JWT token body that matches this policy and could be used to authenticate to Databricks is: ``` - { "iss": "https://token.actions.githubusercontent.com", "aud": "https://github.com/my-github-org", "sub": - "repo:my-github-org/my-repo:environment:prod" } ``` + .. code-block:: + + issuer: "https://token.actions.githubusercontent.com" + audiences: ["https://github.com/my-github-org"] + subject: "repo:my-github-org/my-repo:environment:prod" + + An example JWT token body that matches this policy and could be used to authenticate to Databricks is: + + .. code-block:: + + { + "iss": "https://token.actions.githubusercontent.com", + "aud": "https://github.com/my-github-org", + "sub": "repo:my-github-org/my-repo:environment:prod" + } You may also need to configure the workload runtime to generate tokens for your workloads. diff --git a/docs/account/provisioning/storage.rst b/docs/account/provisioning/storage.rst index b9f080e36..25ee5abaa 100644 --- a/docs/account/provisioning/storage.rst +++ b/docs/account/provisioning/storage.rst @@ -16,7 +16,6 @@ .. code-block:: - import os import time from databricks.sdk import AccountClient @@ -26,11 +25,8 @@ storage = a.storage.create( storage_configuration_name=f"sdk-{time.time_ns()}", - root_bucket_info=provisioning.RootBucketInfo(bucket_name=os.environ["TEST_ROOT_BUCKET"]), + root_bucket_info=provisioning.RootBucketInfo(bucket_name=f"sdk-{time.time_ns()}"), ) - - # cleanup - a.storage.delete(storage_configuration_id=storage.storage_configuration_id) Creates a Databricks storage configuration for an account. diff --git a/docs/dbdataclasses/apps.rst b/docs/dbdataclasses/apps.rst index d00fe91ba..5ea085de8 100755 --- a/docs/dbdataclasses/apps.rst +++ b/docs/dbdataclasses/apps.rst @@ -417,6 +417,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: MEDIUM :value: "MEDIUM" + .. py:attribute:: XLARGE + :value: "XLARGE" + .. py:class:: ComputeState .. py:attribute:: ACTIVE diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index e6eece4a5..2c1f1629a 100755 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -283,8 +283,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: ConnectionType - Next Id: 127 - .. py:attribute:: BIGQUERY :value: "BIGQUERY" @@ -448,8 +446,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: CredentialType - Next Id: 20 - .. py:attribute:: ANY_STATIC_CREDENTIAL :value: "ANY_STATIC_CREDENTIAL" @@ -1555,8 +1551,6 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: SecurableKind - Latest kind: CONNECTION_MARKETO_OAUTH_M2M = 347; Next id: 348. Reserved numbers: 316, 317, 327, 330, 341 (former ENDPOINT_LLM_*, MODEL_SERVICE_STANDARD, MODEL_SERVICE_SYSTEM_DELTASHARING, MCP_SERVICE_STANDARD). - .. py:attribute:: TABLE_DB_STORAGE :value: "TABLE_DB_STORAGE" diff --git a/docs/dbdataclasses/compute.rst b/docs/dbdataclasses/compute.rst index 74e5400f8..de2f9a44e 100755 --- a/docs/dbdataclasses/compute.rst +++ b/docs/dbdataclasses/compute.rst @@ -51,6 +51,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: SPOT_WITH_FALLBACK_AZURE :value: "SPOT_WITH_FALLBACK_AZURE" +.. autoclass:: CancelPendingClusterEnforcementResponse + :members: + :undoc-members: + .. autoclass:: CancelResponse :members: :undoc-members: @@ -414,6 +418,29 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. py:class:: EnforcePolicyComplianceForClusterEnforceMode + + .. py:attribute:: ENFORCE_IMMEDIATELY + :value: "ENFORCE_IMMEDIATELY" + + .. py:attribute:: WAIT_FOR_TERMINATION + :value: "WAIT_FOR_TERMINATION" + +.. autoclass:: EnforcePolicyComplianceForClusterResponseClusterSettings + :members: + :undoc-members: + +.. py:class:: EnforcePolicyComplianceForClusterResponseEnforceResult + + .. py:attribute:: APPLIED + :value: "APPLIED" + + .. py:attribute:: DEFERRED + :value: "DEFERRED" + + .. py:attribute:: NO_CHANGES + :value: "NO_CHANGES" + .. autoclass:: Environment :members: :undoc-members: @@ -476,6 +503,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: DECOMMISSION_STARTED :value: "DECOMMISSION_STARTED" + .. py:attribute:: DEFERRED_POLICY_ENFORCEMENT_FAILED + :value: "DEFERRED_POLICY_ENFORCEMENT_FAILED" + + .. py:attribute:: DEFERRED_POLICY_ENFORCEMENT_SCHEDULED + :value: "DEFERRED_POLICY_ENFORCEMENT_SCHEDULED" + .. py:attribute:: DID_NOT_EXPAND_DISK :value: "DID_NOT_EXPAND_DISK" @@ -926,6 +959,18 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: PendingEnforcement + :members: + :undoc-members: + +.. py:class:: PendingEnforcementEnforcementStatus + + .. py:attribute:: ACTIVE + :value: "ACTIVE" + + .. py:attribute:: INACTIVE + :value: "INACTIVE" + .. autoclass:: PendingInstanceError :members: :undoc-members: diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index 5fc8275a9..48c9612a2 100755 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -4,6 +4,14 @@ Jobs These dataclasses are used in the SDK to represent API requests and responses for services in the ``databricks.sdk.service.jobs`` module. .. py:currentmodule:: databricks.sdk.service.jobs +.. autoclass:: AiRuntimeTask + :members: + :undoc-members: + +.. autoclass:: AiRuntimeTaskOutput + :members: + :undoc-members: + .. py:class:: AlertEvaluationState Same alert evaluation state as in redash-v2/api/proto/alertsv2/alerts.proto @@ -153,6 +161,23 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ComputeSpec + :members: + :undoc-members: + +.. py:class:: ComputeSpecAcceleratorType + + Customer-facing AcceleratorType: hardware accelerator type for the AiRuntime workload. Per-node accelerator count is encoded in the value name (e.g. `GPU_8xH100` means 8 H100s per node). + + .. py:attribute:: GPU_1X_A10 + :value: "GPU_1X_A10" + + .. py:attribute:: GPU_1X_H100 + :value: "GPU_1X_H100" + + .. py:attribute:: GPU_8X_H100 + :value: "GPU_8X_H100" + .. py:class:: Condition .. py:attribute:: ALL_UPDATED @@ -266,6 +291,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: DeploymentSpec + :members: + :undoc-members: + .. autoclass:: EnforcePolicyComplianceForJobResponseJobClusterSettingsChange :members: :undoc-members: @@ -686,6 +715,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ResolvedValuesAiRuntimeTaskResolvedValues + :members: + :undoc-members: + .. autoclass:: Run :members: :undoc-members: diff --git a/docs/workspace/bundledeployments/bundle_deployments.rst b/docs/workspace/bundledeployments/bundle_deployments.rst index 90b06a38f..bd86ceff6 100644 --- a/docs/workspace/bundledeployments/bundle_deployments.rst +++ b/docs/workspace/bundledeployments/bundle_deployments.rst @@ -69,16 +69,21 @@ Creates a new version under a deployment. Creating a version acquires an exclusive lock on the deployment, preventing concurrent deploys. The - caller provides a `version_id` which the server validates equals `last_version_id + 1` on the - deployment. + caller provides a `version_id`, a numeric string that must be numerically greater than the + deployment's most recent version, and sets the version's `previous_version_id` to the deployment's + most recent version (leaving it unset for the first version), which the server validates to detect + concurrent deploys. :param parent: str The parent deployment where this version will be created. Format: deployments/{deployment_id} :param version: :class:`Version` The version to create. :param version_id: str - The version ID the caller expects to create. The server validates this equals `last_version_id + 1` - on the deployment. If it doesn't match, the server returns `ABORTED`. + The ID to use for the version, which becomes the final component of the version's resource name. A + numeric string (base-10, fits in a signed 64-bit integer) chosen by the caller; must be greater than + or equal to 1. Must be numerically greater than the deployment's most recent version (see + `version.previous_version_id`); it does not need to start at 1 or increase by exactly 1. If the + value is not numerically greater, the server returns `INVALID_PARAMETER_VALUE`. :returns: :class:`Version` @@ -203,7 +208,7 @@ .. py:method:: list_versions(parent: str [, page_size: Optional[int], page_token: Optional[str]]) -> Iterator[Version] - Lists versions under a deployment, ordered by version_id descending (most recent first). + Lists versions under a deployment, ordered numerically by version_id descending (most recent first). :param parent: str The parent deployment. Format: deployments/{deployment_id} diff --git a/docs/workspace/catalog/catalogs.rst b/docs/workspace/catalog/catalogs.rst index de2330320..e5410c43d 100755 --- a/docs/workspace/catalog/catalogs.rst +++ b/docs/workspace/catalog/catalogs.rst @@ -24,10 +24,10 @@ w = WorkspaceClient() - created = w.catalogs.create(name=f"sdk-{time.time_ns()}") + created_catalog = w.catalogs.create(name=f"sdk-{time.time_ns()}") # cleanup - w.catalogs.delete(name=created.name, force=True) + w.catalogs.delete(name=created_catalog.name, force=True) Creates a new catalog instance in the parent metastore if the caller is a metastore admin or has the **CREATE_CATALOG** privilege. @@ -159,12 +159,13 @@ import time from databricks.sdk import WorkspaceClient + from databricks.sdk.service import catalog w = WorkspaceClient() created = w.catalogs.create(name=f"sdk-{time.time_ns()}") - _ = w.catalogs.update(name=created.name, comment="updated") + _ = w.catalogs.update(name=created.name, isolation_mode=catalog.CatalogIsolationMode.ISOLATED) # cleanup w.catalogs.delete(name=created.name, force=True) diff --git a/docs/workspace/catalog/external_locations.rst b/docs/workspace/catalog/external_locations.rst index 1ddc9ab58..c5dbea161 100755 --- a/docs/workspace/catalog/external_locations.rst +++ b/docs/workspace/catalog/external_locations.rst @@ -30,22 +30,20 @@ w = WorkspaceClient() - storage_credential = w.storage_credentials.create( + credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), - comment="created via SDK", + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) - external_location = w.external_locations.create( + created = w.external_locations.create( name=f"sdk-{time.time_ns()}", - credential_name=storage_credential.name, - comment="created via SDK", - url="s3://" + os.environ["TEST_BUCKET"] + "/" + f"sdk-{time.time_ns()}", + credential_name=credential.name, + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) # cleanup - w.storage_credentials.delete(name=storage_credential.name) - w.external_locations.delete(name=external_location.name) + w.storage_credentials.delete(delete=credential.name) + w.external_locations.delete(delete=created.name) Creates a new external location entry in the metastore. The caller must be a metastore admin or have the **CREATE_EXTERNAL_LOCATION** privilege on both the metastore and the associated storage @@ -150,11 +148,10 @@ .. code-block:: from databricks.sdk import WorkspaceClient - from databricks.sdk.service import catalog w = WorkspaceClient() - all = w.external_locations.list(catalog.ListExternalLocationsRequest()) + all = w.external_locations.list() Gets an array of external locations (__ExternalLocationInfo__ objects) from the metastore. The caller must be a metastore admin, the owner of the external location, or a user that has some privilege on @@ -201,24 +198,24 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) _ = w.external_locations.update( name=created.name, credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) # cleanup - w.storage_credentials.delete(name=credential.name) - w.external_locations.delete(name=created.name) + w.storage_credentials.delete(delete=credential.name) + w.external_locations.delete(delete=created.name) Updates an external location in the metastore. The caller must be the owner of the external location, or be a metastore admin. In the second case, the admin can only update the name of the external diff --git a/docs/workspace/compute/policy_compliance_for_clusters.rst b/docs/workspace/compute/policy_compliance_for_clusters.rst index c008ce7a1..86e24912c 100755 --- a/docs/workspace/compute/policy_compliance_for_clusters.rst +++ b/docs/workspace/compute/policy_compliance_for_clusters.rst @@ -13,22 +13,44 @@ The get and list compliance APIs allow you to view the policy compliance status of a cluster. The enforce compliance API allows you to update a cluster to be compliant with the current version of its policy. - .. py:method:: enforce_compliance(cluster_id: str [, validate_only: Optional[bool]]) -> EnforceClusterComplianceResponse + .. py:method:: cancel_pending_cluster_enforcement(cluster_id: str [, allow_missing: Optional[bool]]) -> CancelPendingClusterEnforcementResponse - Updates a cluster to be compliant with the current version of its policy. A cluster can be updated if - it is in a `RUNNING` or `TERMINATED` state. + Cancels a pending enforcement on a cluster. After canceling the pending enforcement, the cluster will + no longer update on the next termination or restart. Pending enforcements cannot be canceled when a + cluster is in `TERMINATING` state. Only workspace admins can cancel pending enforcements. - If a cluster is updated while in a `RUNNING` state, it will be restarted so that the new attributes - can take effect. + :param cluster_id: str + The ID of the cluster to cancel the pending enforcement for. + :param allow_missing: bool (optional) + If true and no pending enforcement exists, the request will succeed but no action will be taken. + + :returns: :class:`CancelPendingClusterEnforcementResponse` + + + .. py:method:: enforce_compliance(cluster_id: str [, enforce_mode: Optional[EnforcePolicyComplianceForClusterEnforceMode], validate_only: Optional[bool]]) -> EnforceClusterComplianceResponse + + Updates a cluster to be compliant with the current version of its policy. If a cluster is updated while in a `TERMINATED` state, it will remain `TERMINATED`. The next time the cluster is started, the new attributes will take effect. + For clusters in other states, the behavior depends on the `enforce_mode` used. + Clusters created by the Databricks Jobs, SDP, or Models services cannot be enforced by this API. Instead, use the "Enforce job policy compliance" API to enforce policy compliance on jobs. :param cluster_id: str The ID of the cluster you want to enforce policy compliance on. + :param enforce_mode: :class:`EnforcePolicyComplianceForClusterEnforceMode` (optional) + Determines how changes should be made to clusters that are not in `TERMINATED` state. + + - `ENFORCE_IMMEDIATELY`: If the cluster is in a `RUNNING` state, it will be restarted so that the + new attributes can take effect. For other states aside from `TERMINATED` state, the request will be + rejected. - `WAIT_FOR_TERMINATION`: The cluster is not immediately edited. Instead, a pending + enforcement is scheduled to update the cluster when it terminates or restarts. When this occurs, + `enforce_result` will contain `DEFERRED`. Only workspace admins can use this mode. + + Regardless of the enforce mode, clusters in `TERMINATED` state are immediately edited. :param validate_only: bool (optional) If set, previews the changes that would be made to a cluster to enforce compliance but does not update the cluster. diff --git a/docs/workspace/dashboards/lakeview.rst b/docs/workspace/dashboards/lakeview.rst index 21167041b..7caebd3e1 100755 --- a/docs/workspace/dashboards/lakeview.rst +++ b/docs/workspace/dashboards/lakeview.rst @@ -141,7 +141,7 @@ The flag to include dashboards located in the trash. If unspecified, only active dashboards will be returned. :param view: :class:`DashboardView` (optional) - `DASHBOARD_VIEW_BASIC`only includes summary metadata from the dashboard. + `DASHBOARD_VIEW_BASIC` only includes summary metadata from the dashboard. :returns: Iterator over :class:`Dashboard` diff --git a/docs/workspace/dataquality/data_quality.rst b/docs/workspace/dataquality/data_quality.rst index b40ceac0a..79f36643f 100644 --- a/docs/workspace/dataquality/data_quality.rst +++ b/docs/workspace/dataquality/data_quality.rst @@ -71,7 +71,7 @@ **USE_SCHEMA** on the table's parent schema, and **MANAGE** on the table. :param object_type: str - The type of the monitored object. Can be one of the following: `schema`or `table`. + The type of the monitored object. Can be one of the following: `schema` or `table`. :param object_id: str The UUID of the request object. It is `schema_id` for `schema`, and `table_id` for `table`. diff --git a/docs/workspace/iam/current_user.rst b/docs/workspace/iam/current_user.rst index 1429199cc..0918d992e 100755 --- a/docs/workspace/iam/current_user.rst +++ b/docs/workspace/iam/current_user.rst @@ -17,7 +17,7 @@ w = WorkspaceClient() - me2 = w.current_user.me() + me = w.current_user.me() Get details about the current method caller's identity. diff --git a/docs/workspace/iam/permissions.rst b/docs/workspace/iam/permissions.rst index 531208e2f..1f836a4b1 100755 --- a/docs/workspace/iam/permissions.rst +++ b/docs/workspace/iam/permissions.rst @@ -44,7 +44,7 @@ obj = w.workspace.get_status(path=notebook_path) - levels = w.permissions.get_permission_levels(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) + _ = w.permissions.get(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) Gets the permissions of an object. Objects can inherit permissions from their parent objects or root object. diff --git a/docs/workspace/jobs/jobs.rst b/docs/workspace/jobs/jobs.rst index b8dd2e902..ea5f34dc8 100755 --- a/docs/workspace/jobs/jobs.rst +++ b/docs/workspace/jobs/jobs.rst @@ -359,21 +359,23 @@ w.clusters.ensure_cluster_is_running(os.environ["DATABRICKS_CLUSTER_ID"]) and os.environ["DATABRICKS_CLUSTER_ID"] ) - run = w.jobs.submit( - run_name=f"sdk-{time.time_ns()}", + created_job = w.jobs.create( + name=f"sdk-{time.time_ns()}", tasks=[ - jobs.SubmitTask( + jobs.Task( + description="test", existing_cluster_id=cluster_id, notebook_task=jobs.NotebookTask(notebook_path=notebook_path), - task_key=f"sdk-{time.time_ns()}", + task_key="test", + timeout_seconds=0, ) ], - ).result() + ) - output = w.jobs.get_run_output(run_id=run.tasks[0].run_id) + by_id = w.jobs.get(job_id=created_job.job_id) # cleanup - w.jobs.delete_run(run_id=run.run_id) + w.jobs.delete(job_id=created_job.job_id) Get a single job. @@ -522,37 +524,11 @@ .. code-block:: - import os - import time - from databricks.sdk import WorkspaceClient - from databricks.sdk.service import jobs w = WorkspaceClient() - notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" - - cluster_id = ( - w.clusters.ensure_cluster_is_running(os.environ["DATABRICKS_CLUSTER_ID"]) and os.environ["DATABRICKS_CLUSTER_ID"] - ) - - created_job = w.jobs.create( - name=f"sdk-{time.time_ns()}", - tasks=[ - jobs.Task( - description="test", - existing_cluster_id=cluster_id, - notebook_task=jobs.NotebookTask(notebook_path=notebook_path), - task_key="test", - timeout_seconds=0, - ) - ], - ) - - run_list = w.jobs.list_runs(job_id=created_job.job_id) - - # cleanup - w.jobs.delete(job_id=created_job.job_id) + job_list = w.jobs.list(expand_tasks=False) List jobs. diff --git a/docs/workspace/ml/model_registry.rst b/docs/workspace/ml/model_registry.rst index 2d34256e4..601ffd87d 100755 --- a/docs/workspace/ml/model_registry.rst +++ b/docs/workspace/ml/model_registry.rst @@ -91,6 +91,8 @@ w = WorkspaceClient() model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + + created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a new registered model with the name specified in the request body. Throws `RESOURCE_ALREADY_EXISTS` if a registered model with the given name exists. @@ -120,7 +122,7 @@ model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - mv = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") + created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a model version. diff --git a/docs/workspace/sharing/providers.rst b/docs/workspace/sharing/providers.rst index 302039578..8f66df54a 100755 --- a/docs/workspace/sharing/providers.rst +++ b/docs/workspace/sharing/providers.rst @@ -101,12 +101,25 @@ .. code-block:: + import time + from databricks.sdk import WorkspaceClient - from databricks.sdk.service import sharing w = WorkspaceClient() - all = w.providers.list(sharing.ListProvidersRequest()) + public_share_recipient = """{ + "shareCredentialsVersion":1, + "bearerToken":"dapiabcdefghijklmonpqrstuvwxyz", + "endpoint":"https://sharing.delta.io/delta-sharing/" + } + """ + + created = w.providers.create(name=f"sdk-{time.time_ns()}", recipient_profile_str=public_share_recipient) + + shares = w.providers.list_shares(name=created.name) + + # cleanup + w.providers.delete(name=created.name) Gets an array of available authentication providers. The caller must either be a metastore admin, have the **USE_PROVIDER** privilege on the providers, or be the owner of the providers. Providers not owned diff --git a/docs/workspace/sql/queries.rst b/docs/workspace/sql/queries.rst index f0081b3f2..0dfb63fbf 100644 --- a/docs/workspace/sql/queries.rst +++ b/docs/workspace/sql/queries.rst @@ -29,7 +29,7 @@ display_name=f"sdk-{time.time_ns()}", warehouse_id=srcs[0].warehouse_id, description="test query from Go SDK", - query_text="SHOW TABLES", + query_text="SELECT 1", ) ) diff --git a/docs/workspace/sql/statement_execution.rst b/docs/workspace/sql/statement_execution.rst index b8e12e388..64d2317b4 100755 --- a/docs/workspace/sql/statement_execution.rst +++ b/docs/workspace/sql/statement_execution.rst @@ -199,7 +199,13 @@ output of `SELECT concat('id-', id) AS strCol, id AS intCol, null AS nullCol FROM range(3)` would look like this: - ``` [ [ "id-1", "1", null ], [ "id-2", "2", null ], [ "id-3", "3", null ], ] ``` + .. code-block:: + + [ + [ "id-1", "1", null ], + [ "id-2", "2", null ], + [ "id-3", "3", null ], + ] When specifying `format=JSON_ARRAY` and `disposition=EXTERNAL_LINKS`, each chunk in the result contains compact JSON with no indentation or extra whitespace. @@ -213,7 +219,12 @@ chunk in the result would contain a header row with column names. For example, the output of `SELECT concat('id-', id) AS strCol, id AS intCol, null as nullCol FROM range(3)` would look like this: - ``` strCol,intCol,nullCol id-1,1,null id-2,2,null id-3,3,null ``` + .. code-block:: + + strCol,intCol,nullCol + id-1,1,null + id-2,2,null + id-3,3,null [Apache Arrow streaming format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format [RFC 4180]: https://www.rfc-editor.org/rfc/rfc4180 @@ -237,7 +248,9 @@ For example, the following statement contains two parameters, `my_name` and `my_date`: - ``` SELECT * FROM my_table WHERE name = :my_name AND date = :my_date ``` + .. code-block:: + + SELECT * FROM my_table WHERE name = :my_name AND date = :my_date The parameters can be passed in the request body as follows: diff --git a/docs/workspace/workspace/workspace.rst b/docs/workspace/workspace/workspace.rst index 784acdc27..e957855d6 100755 --- a/docs/workspace/workspace/workspace.rst +++ b/docs/workspace/workspace/workspace.rst @@ -181,11 +181,18 @@ notebook_path = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" w.workspace.import_( - content=base64.b64encode(("CREATE LIVE TABLE dlt_sample AS SELECT 1").encode()).decode(), - format=workspace.ImportFormat.SOURCE, - language=workspace.Language.SQL, - overwrite=True, path=notebook_path, + overwrite=true_, + format=workspace.ImportFormat.SOURCE, + language=workspace.Language.PYTHON, + content=base64.b64encode( + ( + """import time + time.sleep(10) + dbutils.notebook.exit('hello') + """ + ).encode() + ).decode(), ) Imports a workspace object (for example, a notebook or file) or the contents of an entire directory. @@ -229,14 +236,16 @@ .. code-block:: + import os + import time + from databricks.sdk import WorkspaceClient w = WorkspaceClient() - names = [] - for i in w.workspace.list(f"/Users/{w.current_user.me().user_name}", recursive=True): - names.append(i.path) - assert len(names) > 0 + notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" + + objects = w.workspace.list(path=os.path.dirname(notebook)) List workspace objects diff --git a/tests/databricks/sdk/service/lrotesting.py b/tests/databricks/sdk/service/lrotesting.py index 6f2014d90..fc5039916 100755 --- a/tests/databricks/sdk/service/lrotesting.py +++ b/tests/databricks/sdk/service/lrotesting.py @@ -310,7 +310,13 @@ def cancel_operation(self, name: str): self._api.do("POST", f"/api/2.0/lro-testing/operations/{name}/cancel", headers=headers) def create_test_resource(self, resource: TestResource) -> CreateTestResourceOperation: - """Simple method to create test resource for LRO testing + """Simple method to create test resource for LRO testing. + + Example: + + .. code-block:: python + + op = w.lro_testing.create_test_resource(resource=resource) :param resource: :class:`TestResource` The resource to create diff --git a/tests/test_config.py b/tests/test_config.py index e13aaa46a..67d14a975 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -82,6 +82,7 @@ def system(self): monkeypatch.setattr(useragent, "_extra", []) monkeypatch.setattr(useragent, "_cicd_provider", None) monkeypatch.setattr(useragent, "_agent_provider", None) + monkeypatch.setattr(useragent, "_meta_harness_provider", None) monkeypatch.setattr(platform, "python_version", lambda: "3.0.0") monkeypatch.setattr(platform, "uname", MockUname) diff --git a/tests/test_core.py b/tests/test_core.py index 364c830a9..4c1b3a699 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -275,6 +275,7 @@ def system(self): monkeypatch.setattr(useragent, "_extra", []) monkeypatch.setattr(useragent, "_cicd_provider", None) monkeypatch.setattr(useragent, "_agent_provider", None) + monkeypatch.setattr(useragent, "_meta_harness_provider", None) monkeypatch.setattr(platform, "python_version", lambda: "3.0.0") monkeypatch.setattr(platform, "uname", MockUname) diff --git a/tests/test_user_agent.py b/tests/test_user_agent.py index 3c542e752..2fb78ae44 100644 --- a/tests/test_user_agent.py +++ b/tests/test_user_agent.py @@ -68,11 +68,12 @@ def clean_useragent_env(): original_env = os.environ.copy() os.environ.clear() - # Clear cached CICD and agent providers. + # Clear cached CICD, agent, and meta-harness providers. from databricks.sdk import useragent useragent._cicd_provider = None useragent._agent_provider = None + useragent._meta_harness_provider = None yield @@ -81,6 +82,7 @@ def clean_useragent_env(): os.environ.update(original_env) useragent._cicd_provider = None useragent._agent_provider = None + useragent._meta_harness_provider = None def test_user_agent_cicd_no_provider(clean_useragent_env): @@ -493,3 +495,64 @@ def test_agent_provider_cached(clean_useragent_env): os.environ["CLAUDECODE"] = "1" assert useragent.agent_provider() == "cursor" + + +def test_meta_harness_provider_no_meta_harness(clean_useragent_env): + from databricks.sdk import useragent + + assert useragent.meta_harness_provider() == "" + + +def test_meta_harness_provider_omnigent(clean_useragent_env): + os.environ["OMNIGENT"] = "1" + from databricks.sdk import useragent + + assert useragent.meta_harness_provider() == "omnigent" + + +def test_meta_harness_provider_omnigent_empty_value_still_counts_as_set(clean_useragent_env): + # Presence-only matcher: an empty value still fires. + os.environ["OMNIGENT"] = "" + from databricks.sdk import useragent + + assert useragent.meta_harness_provider() == "omnigent" + + +def test_meta_harness_provider_cached(clean_useragent_env): + os.environ["OMNIGENT"] = "1" + from databricks.sdk import useragent + + assert useragent.meta_harness_provider() == "omnigent" + + # Change the environment: the cached result should persist. + del os.environ["OMNIGENT"] + + assert useragent.meta_harness_provider() == "omnigent" + + +def test_user_agent_string_includes_meta_harness(clean_useragent_env): + os.environ["OMNIGENT"] = "1" + from databricks.sdk import useragent + + ua = useragent.to_string() + assert "meta-harness/omnigent" in ua + + +def test_user_agent_string_no_meta_harness(clean_useragent_env): + from databricks.sdk import useragent + + ua = useragent.to_string() + assert "meta-harness/" not in ua + + +def test_meta_harness_independent_of_agent(clean_useragent_env): + # omnigent spawns the real agent CLI, so both env vars are set: the UA must + # carry both dimensions and omnigent must not trip the agent "multiple" signal. + os.environ["CLAUDECODE"] = "1" + os.environ["OMNIGENT"] = "1" + from databricks.sdk import useragent + + ua = useragent.to_string() + assert "agent/claude-code" in ua + assert "meta-harness/omnigent" in ua + assert useragent.agent_provider() == "claude-code"