diff --git a/README.md b/README.md index 4fa974e..2550348 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,16 @@ dsctl project list dsctl use project etl-prod dsctl workflow list dsctl workflow get daily-etl +dsctl template datasource --type MYSQL dsctl workflow run daily-etl --project etl-prod dsctl workflow-instance digest dsctl task-instance list --workflow-instance +dsctl task-instance list --project etl-prod --state FAILURE dsctl task-instance log +dsctl --columns id,name,state workflow-instance list --project etl-prod +dsctl --output-format table workflow-instance list --project etl-prod +dsctl --output-format tsv --columns id,name,state task-instance list --workflow-instance +dsctl --output-format tsv --columns '*' task-instance list --workflow-instance ``` ## Command Surface @@ -80,10 +86,11 @@ Stable user-facing commands today: - `dsctl doctor` - `dsctl schema` - `dsctl capabilities` +- `dsctl enum names` - `dsctl enum list ENUM` - `dsctl lint workflow FILE` - `dsctl task-type list` -- `dsctl env list|get|create|update|delete` +- `dsctl environment list|get|create|update|delete` - `dsctl cluster list|get|create|update|delete` - `dsctl datasource list|get|create|update|delete|test` - `dsctl namespace list|get|available|create|delete` @@ -93,6 +100,7 @@ Stable user-facing commands today: - `dsctl task-group list|get|create|update|close|start` - `dsctl task-group queue list|force-start|set-priority` - `dsctl alert-plugin list|get|schema|create|update|delete|test` +- `dsctl alert-plugin definition list` - `dsctl alert-group list|get|create|update|delete` - `dsctl tenant list|get|create|update|delete` - `dsctl user list|get|create|update|delete` @@ -107,7 +115,7 @@ Stable user-facing commands today: - `dsctl project-preference get|update|enable|disable` - `dsctl project-worker-group list|set|clear` - `dsctl schedule list|get|preview|explain|create|update|delete|online|offline` -- `dsctl template workflow|params|task` +- `dsctl template workflow|params|environment|cluster|datasource|task` - `dsctl workflow list|get|describe|digest|create|edit|online|offline|run|run-task|backfill|delete` - `dsctl workflow lineage list|get|dependent-tasks` - `dsctl workflow-instance list|get|parent|digest|update|watch|stop|rerun|recover-failed|execute-task` diff --git a/docs/development/architecture.md b/docs/development/architecture.md index 0b68e95..724b431 100644 --- a/docs/development/architecture.md +++ b/docs/development/architecture.md @@ -29,10 +29,11 @@ The current stable CLI surface is: - `dsctl doctor` - `dsctl schema` - `dsctl capabilities` +- `dsctl enum names` - `dsctl enum list ENUM` - `dsctl lint workflow FILE` - `dsctl task-type list` -- `dsctl env list|get|create|update|delete` +- `dsctl environment list|get|create|update|delete` - `dsctl cluster list|get|create|update|delete` - `dsctl datasource list|get|create|update|delete|test` - `dsctl namespace list|get|available|create|delete` @@ -42,6 +43,7 @@ The current stable CLI surface is: - `dsctl task-group list|get|create|update|close|start` - `dsctl task-group queue list|force-start|set-priority` - `dsctl alert-plugin list|get|schema|create|update|delete|test` +- `dsctl alert-plugin definition list` - `dsctl alert-group list|get|create|update|delete` - `dsctl tenant list|get|create|update|delete` - `dsctl user list|get|create|update|delete` @@ -56,7 +58,7 @@ The current stable CLI surface is: - `dsctl project-preference get|update|enable|disable` - `dsctl project-worker-group list|set|clear` - `dsctl schedule list|get|preview|explain|create|update|delete|online|offline` -- `dsctl template workflow|params|task` +- `dsctl template workflow|params|environment|cluster|datasource|task` - `dsctl workflow list|get|describe|digest|create|edit|online|offline|run|run-task|backfill|delete` - `dsctl workflow lineage list|get|dependent-tasks` - `dsctl workflow-instance list|get|parent|digest|update|watch|stop|rerun|recover-failed|execute-task` @@ -72,7 +74,7 @@ The most useful split is: | Plane | Core objects | | --- | --- | -| Governance | user, access-token, tenant, queue, worker group, env, cluster, datasource, namespace, resource, alert plugin, alert group | +| Governance | user, access-token, tenant, queue, worker group, environment, cluster, datasource, namespace, resource, alert plugin, alert group | | Project | project, project parameter, project preference, project worker-group, task group | | Design | workflow, task, relation, workflow lineage, schedule | | Runtime | command, workflow-instance, task-instance, audit log, logs, health | @@ -279,7 +281,7 @@ Hard rules: Current naming rules: -- governance and definition resources such as env, datasource, namespace, +- governance and definition resources such as environment, datasource, namespace, queue, worker-group, tenant, user, project, workflow, and task are name-first - `resource` is path-first and uses the DS `fullName` path selector directly diff --git a/docs/development/live-testing.md b/docs/development/live-testing.md index 73fbbbd..9c02807 100644 --- a/docs/development/live-testing.md +++ b/docs/development/live-testing.md @@ -291,10 +291,10 @@ future. | --- | --- | --- | --- | | meta | `version`, `context` | No | local metadata and merged local context | | schema | `schema`, `capabilities` | No | static CLI self-description | -| enum | `enum list` | No | generated enum metadata | +| enum | `enum names`, `enum list` | No | generated enum metadata | | use | `use project|workflow|--clear` | No | local context persistence | | lint | `lint workflow FILE` | No | local validation only | -| template | `template workflow|task` | No | local template rendering | +| template | `template workflow|params|environment|cluster|datasource|task` | No | local template rendering | ### Meta And Diagnostics @@ -309,14 +309,14 @@ future. | Surface | Commands | Default persona | Live required | Minimum live coverage | | --- | --- | --- | --- | --- | -| env | `env list|get|create|update|delete` | delegated governance user | Yes | CRUD round-trip plus not-found or validation case | +| environment | `environment list|get|create|update|delete` | delegated governance user | Yes | CRUD round-trip plus not-found or validation case | | cluster | `cluster list|get|create|update|delete` | delegated governance user | Yes | CRUD round-trip plus delete cleanup | | datasource | `datasource list|get|create|update|delete|test` | delegated governance user | Yes | CRUD round-trip and real connectivity test where the capability exists | | namespace | `namespace list|get|available|create|delete` | delegated governance user | Yes | list and availability plus create/delete round-trip | | resource | `resource list|view|upload|create|mkdir|download|delete` | `etl-developer` | Yes | file and directory lifecycle with content round-trip | | queue | `queue list|get|create|update|delete` | `admin-bootstrap` or delegated governance user | Yes | CRUD round-trip plus permission boundary | | worker-group | `worker-group list|get|create|update|delete` | `admin-bootstrap` or delegated governance user | Yes | CRUD round-trip plus selector correctness | -| alert-plugin | `alert-plugin list|get|schema|create|update|delete|test` | `admin-bootstrap` or delegated governance user | Yes | schema/read paths plus create/update/delete and plugin test where installed | +| alert-plugin | `alert-plugin list|get|definition list|schema|create|update|delete|test` | `admin-bootstrap` or delegated governance user | Yes | definition/schema/read paths plus create/update/delete and plugin test where installed | | alert-group | `alert-group list|get|create|update|delete` | delegated governance user | Yes | CRUD round-trip and referenceable group payload shape | | tenant | `tenant list|get|create|update|delete` | `admin-bootstrap` | Yes | CRUD round-trip plus non-admin denial case | | user | `user list|get|create|update|delete|grant project|datasource|namespace|revoke project|datasource|namespace` | `admin-bootstrap` | Yes | CRUD plus grant/revoke effect and non-admin denial case | @@ -347,7 +347,7 @@ Some resources still require live coverage, but only in a compatible cluster: - datasource connection tests need a reachable backend - alert-plugin tests need installed plugin instances or plugin backends -- namespace, resource, env, and cluster operations may depend on deployment +- namespace, resource, environment, and cluster operations may depend on deployment topology and storage configuration The rule is not “skip forever”. The rule is: @@ -383,7 +383,7 @@ Current verified coverage includes: namespace capability/error paths - project surfaces: `task-type`, `project`, `project-parameter`, `project-preference`, `project-worker-group` -- runtime-adjacent governance: `cluster`, `env`, `resource` +- runtime-adjacent governance: `cluster`, `environment`, `resource` - workflow runtime surfaces: `workflow`, `task`, `workflow-instance`, `task-instance`, parent/sub-workflow relation reads, finished-instance DAG update with and without definition sync, schedule-triggered runtime, @@ -403,11 +403,12 @@ live additions. - `resource view` is not reliable through the DS view endpoint because the upstream controller misuses the `limit` parameter. The adapter now reads the download endpoint and applies the line window client-side. -- `workflow-instance` task inspection in DS 3.4.1 is backed by - `GET /projects/{projectCode}/workflow-instances/{id}/tasks`, and the useful - payload lives under the DS `dataList` envelope as - `{ "taskList": [...], "workflowInstanceState": ... }`. Do not assume a page - envelope on this path. +- `task-instance list` in DS 3.4.1 is backed by the project-scoped + `GET /projects/{projectCode}/task-instances` paging query. The CLI narrows + the common per-run inspection path by sending `workflowInstanceId`, and it + uses the same path for broader project-scoped runtime triage filters. + Workflow-definition filtering is intentionally not exposed here because the + upstream BATCH query does not reliably apply `workflowDefinitionName`. - `workflow describe` returns one root sentinel relation with `preTaskCode=0`. That row is part of the DS DAG encoding and should not be confused with a user-authored dependency edge. @@ -435,9 +436,9 @@ live additions. - schedule create also needs the workflow definition to be `ONLINE`, and the current cluster requires explicit `tenantCode` plus a valid nonzero `environmentCode`. -- `alert-plugin schema Script` exposes only the DS plugin definition metadata, - not a rich parameter form. The live suite should treat that as the upstream - contract unless source confirms otherwise. +- `alert-plugin definition list` discovers supported plugin definitions, while + `alert-plugin schema PLUGIN` fetches the full DS UI parameter form for one + definition when the upstream detail endpoint exposes it. - `alert-plugin test` against the current Script plugin returns DS result code `110014` when no executable script backend is configured. This is a valid cluster capability failure, not a transport error. diff --git a/docs/development/roadmap.md b/docs/development/roadmap.md index 856aaf4..2d7fc1b 100644 --- a/docs/development/roadmap.md +++ b/docs/development/roadmap.md @@ -123,7 +123,8 @@ dsctl use --clear - [x] `--dry-run` support - [x] `tests/models/test_workflow_spec.py` — YAML parsing tests - [x] `tests/services/test_workflow.py` — create workflow tests -- [x] `dsctl template workflow` and `dsctl template task SHELL|SQL|HTTP|...` +- [x] `dsctl template workflow`, `dsctl template environment`, `dsctl template cluster`, and + `dsctl template task SHELL|SQL|HTTP|...` - [x] YAML `schedule:` block support during `workflow create` - [x] extend task-type coverage for DS logical/compound nodes: `SUB_WORKFLOW`, `DEPENDENT`, `SWITCH`, `CONDITIONS` @@ -331,10 +332,10 @@ for production use. - [x] `queue` — CRUD - [x] `worker-group` — CRUD - [x] `task-group` — lifecycle plus task-group queue list/force-start/priority -- [x] `alert-plugin` — list/get/schema/create/update/delete/test +- [x] `alert-plugin` — list/get/definition list/schema/create/update/delete/test - [x] `alert-group` — CRUD - [x] `tenant` — CRUD -- [x] `env` — environment CRUD +- [x] `environment` — environment CRUD - [x] `cluster` — cluster CRUD - [x] `monitor` — health, servers, database stats - [x] `audit` — list, model-types, operation-types @@ -363,7 +364,7 @@ surface. - [x] `dsctl lint` for local workflow design-time checks - [x] `dsctl doctor` for runtime and governance diagnostics - [x] `dsctl schema` — JSON tool definition output for the current stable surface -- [x] `dsctl enum list ` — enum value discovery +- [x] `dsctl enum names`, `dsctl enum list ` — enum value discovery - [x] `dsctl task-type list` — live DS task-type discovery with favourite flags - [x] audit log inspection and audit filter metadata discovery - [x] workflow lineage inspection and dependent-task discovery @@ -450,7 +451,7 @@ live contract test. - [x] live contract cases for: `tenant`, `user`, `access-token`, `queue`, `worker-group` - [x] live contract cases for: - `env`, `cluster`, `alert-group` + `environment`, `cluster`, `alert-group` - [x] permission-boundary tests for admin-only mutations - [x] grant/revoke live tests where cluster policy allows them @@ -462,7 +463,8 @@ have admin-path live coverage plus at least one denial or boundary case. - [x] `datasource` live tests in an environment with real reachable backends - [x] `alert-plugin` live tests in an environment with installed plugin backends -- [x] capability-gated `env`, `cluster`, `namespace`, and `resource` scenarios +- [x] capability-gated `environment`, `cluster`, `namespace`, and `resource` + scenarios in compatible deployments where needed - [x] any skipped suite records the missing capability explicitly diff --git a/docs/reference/cli-contract.md b/docs/reference/cli-contract.md index 6cfc5fa..1927f1f 100644 --- a/docs/reference/cli-contract.md +++ b/docs/reference/cli-contract.md @@ -8,15 +8,18 @@ is not described here, treat it as roadmap work rather than contract. Current stable commands: - global option `--env-file PATH` +- global option `--output-format {json,table,tsv}` +- global option `--columns CSV` - `dsctl version` - `dsctl context` - `dsctl doctor` - `dsctl schema` - `dsctl capabilities` +- `dsctl enum names` - `dsctl enum list ENUM` - `dsctl lint workflow FILE` - `dsctl task-type list` -- `dsctl env list|get|create|update|delete` +- `dsctl environment list|get|create|update|delete` - `dsctl cluster list|get|create|update|delete` - `dsctl datasource list|get|create|update|delete|test` - `dsctl namespace list|get|available|create|delete` @@ -26,6 +29,7 @@ Current stable commands: - `dsctl task-group list|get|create|update|close|start` - `dsctl task-group queue list|force-start|set-priority` - `dsctl alert-plugin list|get|schema|create|update|delete|test` +- `dsctl alert-plugin definition list` - `dsctl alert-group list|get|create|update|delete` - `dsctl tenant list|get|create|update|delete` - `dsctl user list|get|create|update|delete` @@ -42,7 +46,7 @@ Current stable commands: - `dsctl project-preference get|update|enable|disable` - `dsctl project-worker-group list|set|clear` - `dsctl schedule list|get|preview|explain|create|update|delete|online|offline` -- `dsctl template workflow|params|task` +- `dsctl template workflow|params|environment|cluster|datasource|task` - `dsctl workflow list|get|describe|digest|create|edit|online|offline|run|run-task|backfill|delete` - `dsctl workflow lineage list|get|dependent-tasks` - `dsctl workflow-instance list|get|parent|digest|update|watch|stop|rerun|recover-failed|execute-task` @@ -67,7 +71,7 @@ Current examples: ```bash dsctl project get etl-prod dsctl project-parameter get warehouse_db --project etl-prod -dsctl env get prod +dsctl environment get prod dsctl cluster get k8s-prod dsctl datasource get warehouse dsctl lint workflow workflow.yaml @@ -108,9 +112,58 @@ Example: dsctl --env-file cluster.env context ``` +### `--output-format {json,table,tsv}` + +Controls display rendering. The default is `json`. + +Rules: + +- `json` returns the standard JSON envelope and remains the stable machine + contract; when `--columns` is present, only the command data payload at the + canonical row/object path is projected +- `table` renders row/object-oriented data as a plain text table for terminal + scanning +- `tsv` renders the same row model as tab-separated text for shell pipelines +- row-oriented formats use each command's `data_shape` metadata when present + and fall back to runtime shape inference for simple list payloads +- global options are passed before the command group, for example: + +```bash +dsctl --output-format table workflow-instance list --project etl-prod +``` + +### `--columns CSV` + +Selects top-level row/object fields. For `json`, this narrows the standard +envelope data payload at the command's canonical row/object path. For `table` +and `tsv`, it selects rendered display columns. + +Rules: + +- comma-separated values keep the requested order +- only top-level row/object fields are selected +- `--columns '*'` selects all top-level row fields; quote `*` so the shell does + not expand it as a filesystem glob +- unknown columns are a `user_input_error` when rows are available to validate + against +- errors are never projected; failed commands keep the full structured error + payload + +Example: + +```bash +dsctl --columns id,name,state workflow-instance list --project etl-prod +dsctl --output-format tsv --columns id,name,state,host task-instance list --workflow-instance 901 +dsctl --output-format tsv --columns '*' task-instance list --workflow-instance 901 +``` + ## Output Envelope Every stable command returns the standard JSON envelope from `src/dsctl/output.py`. +This statement applies to the default `--output-format json` mode. Explicit +`--columns` projection keeps the envelope and narrows only the command `data` +payload. Row-oriented display formats are an alternate rendering layer over the +same command result. Success shape: @@ -188,6 +241,26 @@ Field rules: - all stable warnings emitted by the CLI include aligned `warning_details` - every dry-run result includes one standard warning detail with code `dry_run_no_request_sent` +- `resolved` records facts selected or adopted by this command invocation, such + as resolved resource identities, normalized selectors, applied filters, or + the active output view +- `resolved.view`, when present, is reserved for commands whose single stable + action can return more than one `data` shape; it is not required for commands + whose `action` already uniquely identifies the output shape +- discovery candidates and allowed values do not belong in `resolved`; expose + them through command `data`, schema `choices`, `discovery_command`, enum + commands, capabilities, or structured error `details` +- command schema entries may include `data_shape` metadata with a stable + low-entropy row/object model for renderers, JSON projection, and AI agents + +Current `data_shape` fields: + +- `kind`: one of `page`, `collection`, `object`, or `summary` +- `row_path`: dot-path from the standard JSON envelope to the canonical row + collection or object, such as `data.totalList` or `data` +- `default_columns`: suggested compact display columns +- `column_discovery`: currently `runtime_row_keys`, meaning full column + discovery comes from the JSON row payload ## `dsctl version` @@ -252,6 +325,46 @@ Current guarantees: ## `dsctl schema` Returns the stable machine-readable command schema for the current CLI surface. +This is the authoritative self-description for command invocation: arguments, +options, choices, selectors, defaults, and supported composite keys. + +Options: + +- `--group GROUP` +- `--command ACTION` +- `--list-groups` +- `--list-commands` + +Selection rules: + +- omit all scope options to return the full schema, including `capabilities` +- `--group` returns one command-group schema by stable group name such as + `task-instance` +- `--group` values come from `dsctl schema --list-groups` +- `--command` returns one command schema by stable action such as + `task-instance.list` or `version` +- `--command` values come from `dsctl schema --list-commands` +- `--list-groups` returns compact rows with `name`, `summary`, + `command_count`, and `schema_command` +- `--list-commands` returns compact rows with `action`, `group`, `name`, + `summary`, and `schema_command` +- `--list-commands` uses `group: null` for root-level commands such as + `version` +- `--group`, `--command`, `--list-groups`, and `--list-commands` are mutually + exclusive +- scoped schema payloads keep the standard schema header and `commands` tree + shape but omit `capabilities`; use `dsctl capabilities` for feature + discovery +- scoped `--group` and `--command` payloads also include `rows` for compact + table/tsv rendering; JSON callers that need the full contract should continue + reading `commands` +- `--group` rows list commands in the group with `kind`, `action`, `name`, + `summary`, and `schema_command` +- `--command` rows flatten the command contract into `command`, `argument`, + `option`, `payload`, and `data_shape` rows so terminal output does not + collapse nested contract data into one large value cell +- scoped schema `resolved.schema.view` is `group`, `command`, `groups`, or + `commands` Current `data` fields: @@ -266,26 +379,60 @@ Current `data` fields: - `confirmation` - `capabilities` - `commands` +- `rows` for scoped `--group` and `--command` views Current guarantees: - describes only the current stable surface +- uses `DS_VERSION` and `--env-file` when rendering embedded capability + metadata, matching `dsctl capabilities` - includes selector semantics for name-first, path-first, and id-first resources - includes the standard success/error envelope contract - includes the stable structured error envelope and `error.source` contract - command arguments and options may include additive metadata such as - `choices`, `examples`, and `supported_keys` when the CLI can expose a - tighter contract for composite inputs + `choices`, `examples`, `supported_keys`, and `discovery_command` when the + CLI can expose a tighter contract for composite inputs +- command entries that accept file payloads may include compact `payload` + metadata; when present, `payload.template_command` is the preferred + progressive-discovery command for a concrete payload template - includes task template type and variant discovery under `capabilities.templates.task` +- `--group`, `--command`, `--list-groups`, and `--list-commands` are additive + scoped or discovery views over the same command tree, not a different schema + mode - `schema_version` changes for breaking schema changes; additive fields may appear within the same version - is tested against the actual registered command tree +- command entries that expose row/object-oriented output include `data_shape`; + this is the authoritative model for `--columns` and + `--output-format table|tsv` +- schema and capabilities output metadata expose `json_column_projection` when + JSON `--columns` projection is supported ## `dsctl capabilities` Returns stable version and surface capability discovery for the current CLI and selected DS version. +This payload is intentionally lighter than `dsctl schema`: it answers what +resource families and feature groups exist, not how to invoke every command. +Agents that need to construct commands should read `dsctl schema`. + +Options: + +- `--summary` +- `--section SECTION` + +Selection rules: + +- omit both options to return full capability discovery +- `--summary` returns lightweight capability discovery with `cli`, `ds`, + `self_description`, `resources`, `planes`, `runtime`, `schedule`, `monitor`, + `enums`, and a summarized `authoring` section +- `--section` returns one top-level section plus the standard `cli`, `ds`, and + `self_description` header +- valid sections are `selection`, `output`, `errors`, `resources`, `planes`, + `authoring`, `schedule`, `monitor`, `enums`, and `runtime` +- `--summary` and `--section` are mutually exclusive Current `data` fields: @@ -322,6 +469,12 @@ Current guarantees: - exposes untemplated upstream task types for authoring gap analysis - keeps live runtime task-type discovery out of the static capability payload; use `dsctl task-type list` for cluster/user-visible DS task types +- does not describe command arguments or options; use `dsctl schema` for that +- exposes `data.self_description.command_invocation_source="schema"` and + `data.self_description.capabilities_scope="feature_discovery"` so tools can + distinguish feature discovery from command invocation metadata +- `--summary` and `--section` are additive scoped views over the same feature + discovery data, not output-format modes - is intended as the lightweight companion to `dsctl schema` ## `dsctl use` @@ -386,14 +539,31 @@ Current `data` fields: - `checks` - `diagnostics` +## `dsctl enum names` + +Returns compact discovery rows for generated enum names supported by the +current selected DS contract. + +Rules: + +- this command is local-only and does not require DS connectivity +- each row includes the stable enum discovery name and the corresponding + `dsctl enum list` command +- schema entries that require an enum discovery name should point to this + command with `discovery_command` + +Successful output returns a list of rows with: + +- `name` +- `list_command` + ## `dsctl enum list ENUM` Returns one generated enum and its members for the current supported DS version. Rules: -- `ENUM` uses the stable enum discovery names exposed by `dsctl schema` and - `dsctl capabilities` +- `ENUM` uses the stable enum discovery names exposed by `dsctl enum names` - class-name aliases such as `ReleaseState` are also accepted - enum member metadata is projected from generated enum attributes and kept under `members[].attributes` @@ -410,7 +580,8 @@ Successful output returns: ## `dsctl task-type list` -Returns the DS task type list for the current user/runtime. +Returns the live DS task-type catalog for the configured cluster and current +user. Rules: @@ -419,6 +590,7 @@ Rules: user's `isCollection` favourite flag - unlike `capabilities`, this payload depends on the configured cluster and authenticated user +- unlike `template task --list`, this is not the local YAML template catalog - `resolved.source` is always `favourite/taskTypes` `data.taskTypes` projects the DS `FavTaskDto` records and keeps DS-native field @@ -461,11 +633,15 @@ When `--all` is used, the CLI materializes all fetched items into one page-shaped response. `resolved.all` indicates the response was client-aggregated. +Use `dsctl project list` to discover project names and numeric codes. + ## `dsctl project get PROJECT` Accepts a project name or a numeric project code, resolves the stable project identity, then fetches the current project payload. +Use `dsctl project list` to discover project names and numeric codes. + `resolved.project` includes: - `code` @@ -482,6 +658,7 @@ Updates one project resolved by name or code. Rules: +- use `dsctl project list` to discover project names and numeric codes - omitting `--name` preserves the current name - omitting both `--description` and `--clear-description` preserves the current description @@ -495,6 +672,7 @@ Deletes one resolved project. Rules: - `PROJECT` may be a project name or numeric code +- use `dsctl project list` to discover project names and numeric codes - `--force` is required Successful output returns: @@ -515,6 +693,9 @@ Options: - `--page-size N` - `--all` +Use `dsctl project list` to discover projects. Use +`dsctl enum list data-type` to discover project-parameter data-type values. + The payload keeps DS paging field names: - `totalList` @@ -548,6 +729,8 @@ Selection rules: - `--project` falls back to context, then config - `PROJECT_PARAMETER` is name-first within that project, with a numeric `code` shortcut +- use `dsctl project-parameter list` inside the selected project to discover + parameter names and codes `resolved.projectParameter` includes: @@ -564,6 +747,7 @@ Rules: - `--name` is required - `--value` is required - `--data-type` defaults to `VARCHAR` +- use `dsctl enum list data-type` to discover data-type values ## `dsctl project-parameter update PROJECT_PARAMETER` @@ -575,6 +759,9 @@ Rules: - requires at least one of `--name`, `--value`, or `--data-type` - omitting `--name`, `--value`, or `--data-type` preserves the current remote value for that field +- use `dsctl project-parameter list` inside the selected project to discover + parameter names and codes +- use `dsctl enum list data-type` to discover data-type values ## `dsctl project-parameter delete PROJECT_PARAMETER --force` @@ -584,6 +771,8 @@ Rules: - `--project` falls back to context, then config - `PROJECT_PARAMETER` may be a project-parameter name or numeric code +- use `dsctl project-parameter list` inside the selected project to discover + parameter names and codes - `--force` is required Successful output returns: @@ -599,6 +788,7 @@ project. Rules: - `--project` follows the standard `flag > context` selection rule +- use `dsctl project list` to discover project names and numeric codes - DS may return `data: null` when the selected project has no stored preference - `state=1` means the stored preference is enabled as a project-level default-value source for CLI and UI surfaces that explicitly support it @@ -630,6 +820,7 @@ Options: Rules: +- use `dsctl project list` to discover project names and numeric codes - the input must decode to one JSON object - the CLI normalizes that object into a compact JSON string before sending it as DS `projectPreferences` @@ -645,6 +836,7 @@ source for one selected project. Rules: - `--project` follows the standard `flag > context` selection rule +- use `dsctl project list` to discover project names and numeric codes - DS uses integer state `1` for enabled - enabling project preference does not mutate existing workflow/task/schedule rows; it only affects clients that choose to consume it as a default source @@ -660,6 +852,7 @@ source for one selected project. Rules: - `--project` follows the standard `flag > context` selection rule +- use `dsctl project list` to discover project names and numeric codes - DS uses integer state `0` for disabled - disabling project preference does not delete the stored JSON payload - the same missing-row warning semantics as `enable` apply @@ -671,6 +864,7 @@ Lists the worker groups currently reported for one selected project. Rules: - selection uses `--project`, then context, then config +- use `dsctl project list` to discover project names and numeric codes - upstream `GET /projects/{projectCode}/worker-group` may return both explicitly assigned worker groups and worker groups still implied by tasks or schedules - output is a JSON array, not a paging wrapper @@ -694,7 +888,9 @@ Replaces the explicit worker-group assignment set for one selected project. Rules: - selection uses `--project`, then context, then config +- use `dsctl project list` to discover project names and numeric codes - repeat `--worker-group NAME` to keep multiple worker groups assigned +- use `dsctl worker-group list` to discover worker-group names - the CLI normalizes duplicates after trimming whitespace - the CLI rejects an empty assignment set; use `clear --force` instead - successful output returns the current upstream-reported worker-group list after @@ -713,6 +909,7 @@ Removes the explicit worker-group assignment set for one selected project. Rules: +- use `dsctl project list` to discover project names and numeric codes - `--force` is required - successful output returns the current upstream-reported worker-group list after the mutation @@ -720,7 +917,7 @@ Rules: schedules; when that happens, the CLI emits a warning aligned with one `warning_details[]` item using code `project_worker_group_still_in_use` -## `dsctl env list` +## `dsctl environment list` Returns a DS-style paging object. @@ -744,7 +941,7 @@ When `--all` is used, the CLI materializes all fetched items into one page-shaped response. `resolved.all` indicates the response was client-aggregated. -## `dsctl env get ENVIRONMENT` +## `dsctl environment get ENVIRONMENT` Accepts an environment name or a numeric environment code, resolves the stable environment identity, then fetches the current environment payload. @@ -755,21 +952,27 @@ environment identity, then fetches the current environment payload. - `name` - `description` -## `dsctl env create` +## `dsctl environment create` Creates one environment. Options: - `--name TEXT` required -- `--config TEXT` required +- exactly one of `--config TEXT` or `--config-file PATH` - `--description TEXT` - `--worker-group NAME` repeatable +Rules: + +- `config` is DS environment shell/export text, not JSON +- prefer `--config-file` for multiline configs +- run `dsctl template environment` for a starter config file + Successful output returns the refreshed environment payload. `resolved.environment` contains the created `code`, `name`, and `description`. -## `dsctl env update ENVIRONMENT` +## `dsctl environment update ENVIRONMENT` Updates one resolved environment while preserving omitted fields. @@ -777,6 +980,7 @@ Options: - `--name TEXT` - `--config TEXT` +- `--config-file PATH` - `--description TEXT` - `--clear-description` - `--worker-group NAME` repeatable @@ -787,11 +991,12 @@ Rules: - `ENVIRONMENT` may be an environment name or numeric code - at least one field change is required - `--description` and `--clear-description` are mutually exclusive +- `--config` and `--config-file` are mutually exclusive - `--worker-group` and `--clear-worker-groups` are mutually exclusive - omitted `name`, `config`, `description`, and worker groups preserve the current remote values -## `dsctl env delete ENVIRONMENT --force` +## `dsctl environment delete ENVIRONMENT --force` Deletes one resolved environment. @@ -855,9 +1060,16 @@ Creates one cluster. Options: - `--name TEXT` required -- `--config TEXT` required +- exactly one of `--config TEXT` or `--config-file PATH` - `--description TEXT` +Rules: + +- `config` is DS cluster config JSON text; in DS 3.4.1 the UI submits + `{"k8s": "...", "yarn": ""}` +- prefer `--config-file` for multiline Kubernetes kubeconfigs +- run `dsctl template cluster` for a starter config file + Successful output returns the refreshed cluster payload. `resolved.cluster` contains the created `code`, `name`, and `description`. @@ -869,6 +1081,7 @@ Options: - `--name TEXT` - `--config TEXT` +- `--config-file PATH` - `--description TEXT` - `--clear-description` @@ -877,6 +1090,7 @@ Rules: - `CLUSTER` may be a cluster name or numeric code - at least one field change is required - `--description` and `--clear-description` are mutually exclusive +- `--config` and `--config-file` are mutually exclusive - omitted `name`, `config`, and `description` preserve the current remote values @@ -921,10 +1135,15 @@ Current datasource list item fields: - `note` - `type` - `userId` -- `userName` +- `userName` — DS datasource owner/creator user, not the datasource + connection username - `createTime` - `updateTime` +`datasource list` keeps DS-native field names. For connection credentials, use +`datasource get DATASOURCE`; in that detail payload, `userName` is the +datasource connection username accepted by datasource create/update payloads. + ## `dsctl datasource get DATASOURCE` Accepts a datasource name or a numeric datasource id, resolves the stable @@ -954,8 +1173,12 @@ Rules: - the file must contain one JSON object - the payload must include string fields `name` and `type` +- `type` is normalized against the generated DS `DbType` enum; discover values + with `dsctl enum list db-type` - the payload must not include `id` - masked password placeholders such as `******` are rejected for create +- run `dsctl template datasource` to choose a supported type, then + `dsctl template datasource --type TYPE` and write `data.json` to the file ## `dsctl datasource update DATASOURCE` @@ -970,9 +1193,13 @@ Rules: - `DATASOURCE` may be a datasource name or numeric id - the file must contain one JSON object - the payload must include string fields `name` and `type` +- `type` is normalized against the generated DS `DbType` enum; discover values + with `dsctl enum list db-type` - if the payload includes `id`, it must match the selected datasource id - if the payload contains `password: "******"` from a prior `datasource get`, the CLI sends an empty password so DS preserves the existing secret +- start from `dsctl datasource get DATASOURCE` or + `dsctl template datasource --type TYPE` when preparing an update file - when that warning is present, the aligned `warning_details[]` item uses code `datasource_update_preserved_existing_password` @@ -1091,6 +1318,7 @@ Selection and behavior: - this command is admin-only because DS 3.4.1 namespace create is admin-only - `--cluster-code` is the DS cluster code stored in the namespace record +- run `dsctl cluster list` to discover cluster codes - the returned `data` payload keeps the DS-native namespace shape - `data.clusterName` may be `null` in the immediate create response because the DS create path does not always project the cluster name @@ -1128,6 +1356,8 @@ Rules: - when `--dir` is omitted, the CLI resolves the upstream resource base directory - selectors are DS `fullName` paths rather than opaque names +- run `dsctl resource list` or `dsctl resource list --dir DIR` to discover + resource paths - the paging payload keeps DS field names Current resource list item fields: @@ -1154,6 +1384,7 @@ Options: Rules: - `RESOURCE` is a DS `fullName` path +- run `dsctl resource list --dir DIR` to discover resource paths - `resolved.resource` returns the normalized path metadata - `data.content` contains the returned text window @@ -1169,6 +1400,7 @@ Options: Rules: +- run `dsctl resource list` to discover destination directory paths - when `--name` is omitted, the local leaf filename is reused remotely - the returned `data` payload is a CLI projection because DS upload does not return an entity body @@ -1188,6 +1420,8 @@ Rules: - `--name` must include a file extension because DS online-create accepts `fileName` and `suffix` separately +- use `dsctl resource upload --file PATH` when the content already lives in a + local file - the returned `data` payload is a CLI projection because DS online-create does not return an entity body @@ -1202,6 +1436,7 @@ Options: Rules: - `NAME` is one leaf directory name, not a path +- run `dsctl resource list` to discover parent directory paths - the returned `data` payload is a CLI projection because DS directory create does not return an entity body @@ -1214,6 +1449,10 @@ Options: - `--output PATH` - `--overwrite` +Rules: + +- run `dsctl resource list --dir DIR` to discover resource paths + Successful output returns: - `data.fullName` @@ -1227,6 +1466,7 @@ Deletes one resource selected by DS `fullName` path. Rules: +- run `dsctl resource list --dir DIR` to discover resource paths - `RESOURCE` is path-first, not name-first - `--force` is required - `data.resource.isDirectory` may be `null` when the selector does not prove the @@ -1265,6 +1505,8 @@ Current queue list item fields: Accepts a queue name or a numeric queue id, resolves the stable queue identity, then returns the current queue payload. +Run `dsctl queue list` to discover queue names and ids. + `resolved.queue` includes: - `id` @@ -1282,8 +1524,8 @@ Options: Rules: -- `queueName` is the human-facing queue name -- `queue` is the underlying DolphinScheduler queue value +- `queueName` is the human-facing DS queue name used as the selector label +- `queue` is the underlying YARN queue value stored in DS ## `dsctl queue update QUEUE` @@ -1297,6 +1539,7 @@ Options: Rules: - `QUEUE` may be a queue name or numeric id +- run `dsctl queue list` to discover queue names and ids - at least one field change is required - omitted `queueName` and `queue` preserve the current remote values @@ -1307,6 +1550,7 @@ Deletes one resolved queue. Rules: - `QUEUE` may be a queue name or numeric id +- run `dsctl queue list` to discover queue names and ids - `--force` is required Successful output returns: @@ -1357,6 +1601,8 @@ Current worker-group list item fields: Accepts a worker-group name or a numeric worker-group id, resolves the stable worker-group identity, then returns the current worker-group payload. +Run `dsctl worker-group list` to discover worker-group names and ids. + `resolved.workerGroup` includes: - `id` @@ -1377,6 +1623,7 @@ Options: Rules: - repeated `--addr` values are joined into the upstream `addrList` +- run `dsctl monitor server worker` to discover worker server addresses - omitting `--addr` creates the worker group with an empty `addrList` ## `dsctl worker-group update WORKER_GROUP` @@ -1394,6 +1641,9 @@ Options: Rules: - `WORKER_GROUP` may be a worker-group name or numeric id +- run `dsctl worker-group list` to discover worker-group names and ids +- run `dsctl monitor server worker` to discover worker server addresses for + `--addr` - at least one field change is required - omitted fields preserve the current remote values - `--addr` and `--clear-addrs` are mutually exclusive @@ -1407,6 +1657,7 @@ Deletes one resolved worker group. Rules: - `WORKER_GROUP` may be a worker-group name or numeric id +- run `dsctl worker-group list` to discover worker-group names and ids - `--force` is required - config-derived worker-group rows cannot be deleted through the CRUD endpoint @@ -1434,6 +1685,7 @@ Rules: task-group paging API - with `--project`, the CLI resolves project selection and uses DS's project-scoped task-group list shape +- run `dsctl project list` to discover project names and codes for `--project` - `--project` cannot be combined with `--search` or `--status` because DolphinScheduler 3.4.1 does not expose that filter shape - `--status` accepts `open`, `closed`, `1`, or `0` @@ -1465,6 +1717,8 @@ Current task-group list item fields: Accepts a task-group name or a numeric task-group id, resolves the stable task-group identity, then returns the current task-group payload. +Run `dsctl task-group list` to discover task-group names and ids. + `resolved.taskGroup` includes: - `id` @@ -1485,6 +1739,7 @@ Options: Rules: - project selection uses `flag > context` +- run `dsctl project list` to discover project names and codes for `--project` - `groupSize` must be greater than or equal to `1` - omitted description is sent as an empty string @@ -1504,6 +1759,7 @@ Options: Rules: - `TASK_GROUP` may be a task-group name or numeric id +- run `dsctl task-group list` to discover task-group names and ids - at least one field change is required - omitted fields preserve the current remote values - `--clear-description` sends an empty description @@ -1518,6 +1774,7 @@ Closes one resolved task group and returns the refreshed task-group payload. Rules: - `TASK_GROUP` may be a task-group name or numeric id +- run `dsctl task-group list` to discover task-group names and ids - closing an already closed task group returns `invalid_state` with a suggestion to run `task-group start` @@ -1528,6 +1785,7 @@ Starts one resolved task group and returns the refreshed task-group payload. Rules: - `TASK_GROUP` may be a task-group name or numeric id +- run `dsctl task-group list` to discover task-group names and ids - starting an already open task group returns `invalid_state` with a suggestion to keep it open or run `task-group close` @@ -1547,6 +1805,7 @@ Options: Rules: - `TASK_GROUP` may be a task-group name or numeric id +- run `dsctl task-group list` to discover task-group names and ids - `--task-instance` filters by task-instance name - `--workflow-instance` filters by workflow-instance name - `--status` accepts `WAIT_QUEUE`, `ACQUIRE_SUCCESS`, `RELEASE`, `-1`, `1`, @@ -1573,6 +1832,8 @@ The payload keeps DS paging field names. Current queue item fields: Force-starts one waiting task-group queue row by numeric queue id. +Run `dsctl task-group queue list TASK_GROUP` to discover queue ids. + Successful output returns: - `data.queueId` @@ -1592,6 +1853,7 @@ Options: Rules: - `QUEUE_ID` is id-first and does not use context +- run `dsctl task-group queue list TASK_GROUP` to discover queue ids - `--priority` must be greater than or equal to `0` Successful output returns: @@ -1640,6 +1902,8 @@ Rules: Accepts an alert-plugin instance name or a numeric alert-plugin id, resolves the stable identity, then returns the current alert-plugin payload. +Run `dsctl alert-plugin list` to discover alert-plugin instance names and ids. + `resolved.alertPlugin` includes: - `id` @@ -1647,6 +1911,33 @@ the stable identity, then returns the current alert-plugin payload. - `pluginDefineId` - `alertPluginName` +## `dsctl alert-plugin definition list` + +Lists the alert-plugin definitions supported by the current DolphinScheduler +runtime. This command returns plugin definitions such as `Feishu`, `Email`, or +`Slack`; it does not return configured alert-plugin instances. + +Current definition list payload fields: + +- `definitions` +- `count` +- `schema_command` + +Current definition row fields: + +- `id` +- `pluginName` +- `pluginType` +- `createTime` +- `updateTime` + +Rules: + +- use this command to discover valid `--plugin` values for + `alert-plugin create` +- use `alert-plugin schema PLUGIN` to fetch the full parameter schema for one + returned definition + ## `dsctl alert-plugin schema PLUGIN` Accepts an alert UI plugin definition name or a numeric plugin-definition id, @@ -1658,14 +1949,22 @@ Current plugin definition fields: - `pluginName` - `pluginType` - `pluginParams` +- `pluginParamFields` - `createTime` - `updateTime` Rules: -- `PLUGIN` must resolve to an `ALERT` UI plugin definition +- `PLUGIN` must resolve to an alert UI plugin definition; plugin-definition + names are matched exactly first, then case-insensitively when unique +- run `dsctl alert-plugin definition list` to discover plugin definitions +- name resolution fetches the plugin-detail endpoint after locating the id + because the upstream list endpoint returns only definition summaries - `pluginParams` is the DS-native UI param-list schema used by create/update and test-send flows +- `pluginParamFields` is a compact derived summary of the same schema for + field discovery; it includes `field`, `type`, `required`, `defaultValue`, + and options when present ## `dsctl alert-plugin create` @@ -1675,15 +1974,21 @@ Options: - `--name TEXT` required - `--plugin TEXT` required +- `--param KEY=VALUE` - `--params-json JSON` - `--file PATH` Rules: - `--plugin` accepts an alert UI plugin definition name or numeric id -- pass exactly one of `--params-json` or `--file` -- the params payload must be a DS-native JSON array of UI param objects, not a - plain key/value JSON object +- run `dsctl alert-plugin definition list` to discover `--plugin` values +- pass exactly one of `--param`, `--params-json`, or `--file` +- `--param` may be repeated; it overlays fields from the upstream plugin + schema and then submits DS-native UI params to DolphinScheduler +- field names from `--param` are matched exactly first, then + case-insensitively when unique +- `--params-json` and `--file` accept a DS-native JSON array of UI param + objects, not a plain key/value JSON object - use `dsctl alert-plugin schema PLUGIN` to fetch the upstream param template, fill each item's `value`, then submit it unchanged @@ -1696,17 +2001,21 @@ Updates one resolved alert-plugin instance while preserving omitted fields. Options: - `--name TEXT` +- `--param KEY=VALUE` - `--params-json JSON` - `--file PATH` Rules: - `ALERT_PLUGIN` may be an alert-plugin instance name or numeric id +- run `dsctl alert-plugin list` to discover alert-plugin instance names and ids - at least one field change is required - omitted params preserve the current upstream `pluginInstanceParams` -- when params are provided, pass exactly one of `--params-json` or `--file` -- the params payload format is the same DS-native JSON array accepted by - `create` +- when params are provided, pass exactly one of `--param`, `--params-json`, or + `--file` +- `--param` overlays the current upstream UI params; omitted fields keep their + current values +- `--params-json` and `--file` replace the full DS-native UI params array ## `dsctl alert-plugin delete ALERT_PLUGIN --force` @@ -1715,6 +2024,7 @@ Deletes one resolved alert-plugin instance. Rules: - `ALERT_PLUGIN` may be an alert-plugin instance name or numeric id +- run `dsctl alert-plugin list` to discover alert-plugin instance names and ids - `--force` is required Successful output returns: @@ -1729,6 +2039,7 @@ Sends one test alert using the resolved alert-plugin instance. Rules: - `ALERT_PLUGIN` may be an alert-plugin instance name or numeric id +- run `dsctl alert-plugin list` to discover alert-plugin instance names and ids - the CLI reuses the current upstream `pluginDefineId` and `pluginInstanceParams` from the resolved instance @@ -1770,12 +2081,15 @@ Current alert-group list item fields: Rules: - `search` is passed through to the upstream alert-group name filter +- use `dsctl alert-group list` to discover alert-group names and ids ## `dsctl alert-group get ALERT_GROUP` Accepts an alert-group name or a numeric alert-group id, resolves the stable alert-group identity, then returns the current alert-group payload. +Use `dsctl alert-group list` to discover alert-group names and ids. + `resolved.alertGroup` includes: - `id` @@ -1794,6 +2108,7 @@ Options: Rules: +- use `dsctl alert-plugin list` to discover alert plugin instance ids - repeated `--instance-id` values are deduplicated before the upstream request - omitting `--instance-id` sends an empty upstream `alertInstanceIds` string @@ -1812,6 +2127,8 @@ Options: Rules: - `ALERT_GROUP` may be an alert-group name or numeric id +- use `dsctl alert-group list` to discover alert-group names and ids +- use `dsctl alert-plugin list` to discover alert plugin instance ids - at least one field change is required - omitted fields preserve the current remote values - `--instance-id` and `--clear-instance-ids` are mutually exclusive @@ -1824,6 +2141,7 @@ Deletes one resolved alert group. Rules: - `ALERT_GROUP` may be an alert-group name or numeric id +- use `dsctl alert-group list` to discover alert-group names and ids - `--force` is required - DS 3.4.1 does not allow deleting the default alert group @@ -1866,6 +2184,7 @@ Current tenant list item fields: Rules: - `search` is passed through to the upstream tenant-code filter +- use `dsctl tenant list` to discover tenant codes and ids - `queueName` is usually present from the paging endpoint - `queue` may be `null` because the DS tenant paging query does not always project the underlying queue value @@ -1886,6 +2205,7 @@ identity, then returns the current tenant payload. Rules: +- use `dsctl tenant list` to discover tenant codes and ids - `queueName` is the more reliable upstream tenant queue label - `queue` may still be `null` on real clusters when the DS tenant detail query does not project the underlying queue value @@ -1903,6 +2223,7 @@ Options: Rules: - `--queue` accepts a queue name or numeric id +- use `dsctl queue list` to discover queue names and ids - the CLI resolves `--queue` to the upstream `queueId` ## `dsctl tenant update TENANT` @@ -1920,6 +2241,8 @@ Rules: - `TENANT` may be a tenant code or numeric id - `--queue` accepts a queue name or numeric id +- use `dsctl tenant list` to discover tenant codes and ids +- use `dsctl queue list` to discover queue names and ids - at least one field change is required - omitted fields preserve the current remote values - `--description` and `--clear-description` are mutually exclusive @@ -1931,6 +2254,7 @@ Deletes one resolved tenant. Rules: - `TENANT` may be a tenant code or numeric id +- use `dsctl tenant list` to discover tenant codes and ids - `--force` is required Successful output returns: @@ -1976,6 +2300,7 @@ Current user list item fields: Rules: - `search` is passed through to the upstream user-name filter +- use `dsctl user list` to discover user names and ids - `queue` is the effective queue surfaced by the upstream paging view - `queueName` is the tenant queue name joined by the upstream paging view @@ -2000,6 +2325,7 @@ Current get-only extra fields: Rules: - `USER` may be a user name or numeric id +- use `dsctl user list` to discover user names and ids - `queue` remains the effective queue shown by the merged upstream user views ## `dsctl user create` @@ -2020,6 +2346,8 @@ Rules: - `--tenant` accepts a tenant code or numeric id - `--queue` is the raw queue-name override stored on the user record +- use `dsctl tenant list` to discover tenant codes and ids +- use `dsctl queue list` to discover queue names - `--state 1` means enabled and `--state 0` means disabled ## `dsctl user update USER` @@ -2042,9 +2370,12 @@ Options: Rules: - `USER` may be a user name or numeric id +- use `dsctl user list` to discover user names and ids - omitted fields preserve the current remote values - `--tenant` accepts a tenant code or numeric id - `--queue` is the raw queue-name override stored on the user record +- use `dsctl tenant list` to discover tenant codes and ids +- use `dsctl queue list` to discover queue names - `--phone` and `--clear-phone` are mutually exclusive - `--queue` and `--clear-queue` are mutually exclusive - at least one field change is required @@ -2056,6 +2387,7 @@ Deletes one resolved user. Rules: - `USER` may be a user name or numeric id +- use `dsctl user list` to discover user names and ids - `--force` is required Successful output returns: @@ -2071,6 +2403,7 @@ Rules: - `USER` may be a user name or numeric id - `PROJECT` may be a project name or numeric code +- use `dsctl user list` and `dsctl project list` to discover selectors - the command uses the DS additive project grant path rather than replacing the full user grant set @@ -2089,6 +2422,7 @@ Rules: - `USER` may be a user name or numeric id - `PROJECT` may be a project name or numeric code +- use `dsctl user list` and `dsctl project list` to discover selectors Successful output returns: @@ -2108,6 +2442,7 @@ Rules: - `USER` may be a user name or numeric id - each `--datasource` accepts a datasource name or numeric id +- use `dsctl user list` and `dsctl datasource list` to discover selectors - the CLI reads the user's currently authorized datasources, merges the requested datasources into that set, then writes the full set back through the DS datasource-grant endpoint @@ -2132,6 +2467,7 @@ Rules: - `USER` may be a user name or numeric id - each `--datasource` accepts a datasource name or numeric id +- use `dsctl user list` and `dsctl datasource list` to discover selectors - the CLI reads the user's currently authorized datasources, subtracts the requested datasources from that set, then writes the remaining full set back through the DS datasource-grant endpoint @@ -2156,6 +2492,7 @@ Rules: - `USER` may be a user name or numeric id - each `--namespace` accepts a namespace name or numeric id +- use `dsctl user list` and `dsctl namespace list` to discover selectors - namespace names may be ambiguous across clusters; when that happens, the CLI returns a resolution error and expects a numeric namespace id - the CLI reads the user's currently authorized namespaces, merges the @@ -2182,6 +2519,7 @@ Rules: - `USER` may be a user name or numeric id - each `--namespace` accepts a namespace name or numeric id +- use `dsctl user list` and `dsctl namespace list` to discover selectors - namespace names may be ambiguous across clusters; when that happens, the CLI returns a resolution error and expects a numeric namespace id - the CLI reads the user's currently authorized namespaces, subtracts the @@ -2233,6 +2571,7 @@ Accepts one numeric access-token id. Selection rules: - `ACCESS_TOKEN` is id-first +- use `dsctl access-token list` to discover token ids `resolved.accessToken` includes: @@ -2247,7 +2586,9 @@ Creates one access token. Rules: - `--user` is required and accepts a user name or numeric id -- `--expire-time` is required +- use `dsctl user list` to discover user names and ids +- `--expire-time` is required and follows the DS format + `YYYY-MM-DD HH:MM:SS` - `--token` is optional; omitting it lets DS generate one ## `dsctl access-token update ACCESS_TOKEN` @@ -2259,6 +2600,8 @@ Rules: - requires at least one of `--user`, `--expire-time`, `--token`, or `--regenerate-token` - `--user` accepts a user name or numeric id +- use `dsctl access-token list` to discover token ids +- use `dsctl user list` to discover user names and ids - omitted `--user` and `--expire-time` preserve the current remote values - omitted `--token` preserves the current token unless `--regenerate-token` is used @@ -2270,6 +2613,7 @@ Deletes one access token by numeric id. Rules: +- use `dsctl access-token list` to discover token ids - `--force` is required Successful output returns: @@ -2284,7 +2628,9 @@ Generates one token string without persisting it. Rules: - `--user` is required and accepts a user name or numeric id -- `--expire-time` is required +- use `dsctl user list` to discover user names and ids +- `--expire-time` is required and follows the DS format + `YYYY-MM-DD HH:MM:SS` Successful output returns: @@ -2376,6 +2722,8 @@ Rules: - `--model-type` is repeatable and mapped to DS `modelTypes` - `--operation-type` is repeatable and mapped to DS `operationTypes` +- use `dsctl audit model-types` to discover model-type filter values +- use `dsctl audit operation-types` to discover operation-type filter values - `--start` and `--end` must use DS datetime format `YYYY-MM-DD HH:MM:SS` - when both `--start` and `--end` are provided, `end` must be greater than or equal to `start` @@ -2444,6 +2792,7 @@ Selection rules: - `--project` wins - then stored context project +- use `dsctl project list` to discover project names and codes The `data` payload is a JSON array of workflow summaries: @@ -2459,6 +2808,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: positional argument, then context workflow +- use `dsctl project list` and `dsctl workflow list` to discover selectors Formats: @@ -2469,6 +2819,8 @@ Formats: Returns one workflow DAG as structured JSON: +Use `dsctl project list` and `dsctl workflow list` to discover selectors. + - `data.workflow` - `data.tasks` - `data.relations` @@ -2477,6 +2829,8 @@ Returns one workflow DAG as structured JSON: Returns one compact workflow graph summary derived from the workflow DAG. +Use `dsctl project list` and `dsctl workflow list` to discover selectors. + Current `data` fields: - `workflow` @@ -2511,6 +2865,8 @@ Options: Rules: +- use `dsctl template workflow` to start a workflow YAML file +- use `dsctl project list` to discover project names and codes for `--project` - project selection precedence is: - explicit `--project` - then `workflow.project` from the YAML file @@ -2613,6 +2969,7 @@ Rules: - explicit positional `WORKFLOW` - then workflow context - project selection precedence remains `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors - the patch is applied against the current live workflow YAML export, then compiled back into one legacy whole-definition update payload - current stable patch operations are: @@ -2710,6 +3067,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors Rules: @@ -2736,6 +3094,7 @@ Returns the project-wide workflow lineage graph for one resolved project. Selection rules: - project selection: `flag > context` +- use `dsctl project list` to discover project names and codes Current `data` fields: @@ -2766,6 +3125,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors Rules: @@ -2781,6 +3141,8 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` - optional task filter is explicit-only through `--task` +- use `dsctl project list`, `dsctl workflow list`, and `dsctl task list` to + discover selectors Options: @@ -2822,6 +3184,9 @@ Options: Rules: - `--workflow` and `--search` are mutually exclusive +- use `dsctl project list` to discover project names and codes for `--project` +- use `dsctl workflow list` inside the selected project to discover workflow + names and codes for `--workflow` - the payload keeps DS paging field names: - `totalList` - `total` @@ -2849,10 +3214,13 @@ Fetches one schedule by numeric id. Selection rules: - `SCHEDULE_ID` is id-first and does not use context +- use `dsctl schedule list` inside the selected project to discover schedule ids ## `dsctl template workflow` Returns the current stable workflow YAML template inside `data.yaml`. +`data.lines[]` provides the same template as row-oriented `line_no` and `line` +values for table and tsv output. Rules: @@ -2920,12 +3288,121 @@ Rules: - SQL tasks can publish result columns whose names match OUT parameter `prop` values +## `dsctl template environment` + +Returns one DS environment shell/export config template. + +Default table and tsv output render `data.lines[]` so multiline config content +does not collapse into one large value cell. + +Current `data` fields: + +- `filename` +- `config` +- `lines` +- `target_commands` +- `source_options` +- `upstream_request_shape` +- `rules` + +Rules: + +- `data.config` is the file content accepted by + `environment create --config-file` and `environment update --config-file` +- `data.lines[]` contains row-oriented `line` and `purpose` values for compact + terminal scanning +- DS stores this value as the raw `EnvironmentController` form field `config` +- environment paths must exist on DolphinScheduler worker hosts + +## `dsctl template cluster` + +Returns one DS cluster config JSON template. + +Default table and tsv output render `data.fields[]` so multiline kubeconfig +content does not collapse into one large value cell. + +Current `data` fields: + +- `filename` +- `config` +- `payload` +- `fields` +- `rows` +- `target_commands` +- `source_options` +- `upstream_request_shape` +- `upstream_ui_shape` +- `rules` + +Rules: + +- `data.config` is the file content accepted by + `cluster create --config-file` and `cluster update --config-file` +- DS stores this value as the raw `ClusterController` form field `config` +- DS 3.4.1 reads the `k8s` JSON field as Kubernetes kubeconfig content +- keep `yarn` as an empty string unless your DS deployment uses it + +## `dsctl template datasource` + +Returns datasource JSON payload-template type discovery when `--type` is +omitted, or one DS-native datasource JSON payload template when `--type TYPE` +is passed. + +Options: + +- `--type TYPE` + +Default index fields: + +- `data.default_type` +- `data.template_command` +- `data.template_command_pattern` +- `data.target_commands` +- `data.type_enum` +- `data.type_discovery_command` +- `data.supported_types` +- `data.rows` + +`resolved.view` is `list` for the default output. The supported type list lives +in `data.supported_types`; `resolved` does not duplicate it. + +Typed template fields: + +- `data.type` +- `data.target_commands` +- `data.source_option` +- `data.payload` +- `data.json` +- `data.fields` +- `data.rows` +- `data.rules` + +`resolved.view` is `template` and `resolved.datasource_type` is the normalized +DS `DbType` value selected by `--type`. + +Rules: + +- `type` matching is case-insensitive and accepts common generated `DbType` + aliases such as `mysql` and `aliyun-serverless-spark` +- `data.payload` is the object shape accepted by `datasource create --file` +- `data.json` is the same payload rendered as pretty JSON +- `data.fields` is grounded in generated `BaseDataSourceParamDTO` plus known + plugin-specific JSON fields for the selected datasource type only +- `data.rows` is the row-oriented table/tsv view; index output lists + datasource types, typed output lists payload fields +- typed template output does not repeat global `type` choices; the selected + value is `data.type` and `resolved.datasource_type`, while full type + discovery lives in the default index and `dsctl enum list db-type` + ## `dsctl template task TASK_TYPE` -Returns one task YAML template inside `data.yaml`. +Returns one task YAML template inside `data.yaml`. `data.rows[]` provides the +row-oriented table/tsv view: line rows for a concrete template, and compact +task-type rows for `--list`. Options: +- `--list` - `--variant VARIANT` Current stable task template coverage includes every DS 3.4.1 upstream default @@ -2949,6 +3426,8 @@ The remaining upstream default task types return generic templates with raw Rules: - task type matching is case-insensitive +- `--list` keeps the stable action `template.task` and returns + `resolved.mode=list` - the normalized type is returned as `resolved.task_type` - `resolved.task_category` reports the upstream DS category - `resolved.template_kind` is `typed` or `generic` @@ -2963,6 +3442,7 @@ Rules: - `data.generic_task_types` - `data.task_types_by_category` - `data.task_templates` + - `data.rows` `data.task_templates.TYPE` exposes: @@ -3013,6 +3493,7 @@ Supported forms: Rules: +- use `dsctl schedule list` inside the selected project to discover schedule ids - preview by id does not accept `--project`, `--cron`, `--start`, `--end`, or `--timezone` - ad hoc preview requires all of `--cron`, `--start`, `--end`, and @@ -3020,6 +3501,8 @@ Rules: - `--cron` must be a DolphinScheduler Quartz cron expression with 6 or 7 fields and seconds first - ad hoc preview resolves project selection with `flag > context` +- use `dsctl project list` to discover project names and codes for ad hoc + `--project` Successful output returns: @@ -3047,6 +3530,15 @@ Rules: - without `SCHEDULE_ID`, explain models `schedule.create` selection and risk rules +- use `dsctl schedule list` inside the selected project to discover schedule ids + for update-form explain +- use `dsctl project list` and `dsctl workflow list` to discover create-form + selectors +- use `dsctl alert-group list`, `dsctl worker-group list`, + `dsctl tenant list`, and `dsctl environment list` to discover optional + create/update selector values +- `--failure-strategy`, `--warning-type`, and `--priority` use generated DS + enum values exposed by `dsctl enum list` - create-form tenant selection: `flag > enabled project preference.tenant > current-user tenantCode > "default"` - create-form omitted `warningType`, `warningGroupId`, @@ -3112,6 +3604,13 @@ Selection rules: - workflow selection: `flag > context` - tenant selection: `flag > enabled project preference.tenant > current-user tenantCode > "default"` +- use `dsctl project list` and `dsctl workflow list` to discover project and + workflow selectors +- use `dsctl alert-group list`, `dsctl worker-group list`, + `dsctl tenant list`, and `dsctl environment list` to discover optional + selector values +- `--failure-strategy`, `--warning-type`, and `--priority` use generated DS + enum values exposed by `dsctl enum list` Required options: @@ -3163,6 +3662,11 @@ Updates one schedule by numeric id. Rules: - `SCHEDULE_ID` is id-first and does not use context +- use `dsctl schedule list` inside the selected project to discover schedule ids +- use `dsctl alert-group list`, `dsctl worker-group list`, and + `dsctl environment list` to discover optional update selector values +- `--failure-strategy`, `--warning-type`, and `--priority` use generated DS + enum values exposed by `dsctl enum list` - omitted fields preserve current remote values - at least one field change is required - `--confirm-risk TOKEN` accepts a token previously returned in a @@ -3183,6 +3687,7 @@ Deletes one schedule by numeric id. Rules: - `SCHEDULE_ID` is id-first and does not use context +- use `dsctl schedule list` inside the selected project to discover schedule ids - `--force` is required - deleting an online schedule returns `invalid_state` @@ -3198,6 +3703,7 @@ Brings one schedule online and returns the refreshed schedule payload. Rules: - `SCHEDULE_ID` is id-first and does not use context +- use `dsctl schedule list` inside the selected project to discover schedule ids - the DS 3.4.1 adapter resolves the bound workflow to recover `projectCode` for the legacy online endpoint - bringing a schedule online requires the bound workflow to already be online @@ -3209,6 +3715,7 @@ Brings one schedule offline and returns the refreshed schedule payload. Rules: - `SCHEDULE_ID` is id-first and does not use context +- use `dsctl schedule list` inside the selected project to discover schedule ids - the DS 3.4.1 adapter resolves the bound workflow to recover `projectCode` for the legacy offline endpoint @@ -3220,6 +3727,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors The `data` payload is a JSON array of task summaries: @@ -3236,6 +3744,8 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` - task is resolved by name or numeric code within the selected workflow +- use `dsctl task list` inside the selected workflow to discover task names and + codes ## `dsctl task update` @@ -3271,6 +3781,10 @@ Current stable `--set` keys: Rules: - selection precedence matches `task get` +- use `dsctl task list` inside the selected workflow to discover task names and + codes +- use `dsctl schema --command task.update` to discover supported `--set` keys, + examples, and machine-readable metadata - the CLI compiles the update into the DS native `updateTaskWithUpstream` form request - `command` updates are supported only for `SHELL`, `PYTHON`, and @@ -3305,6 +3819,10 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors +- use `dsctl worker-group list`, `dsctl tenant list`, + `dsctl alert-group list`, and `dsctl environment list` to discover optional + runtime selectors - worker group selection: `flag > enabled project preference.workerGroup > "default"` - tenant selection: @@ -3364,6 +3882,8 @@ Selection rules: - project selection: `flag > context` - workflow selection: `argument > context` - task selection: `--task` name or code within the workflow definition +- use `dsctl project list`, `dsctl workflow list`, and `dsctl task list` to + discover selectors - worker group selection: `flag > enabled project preference.workerGroup > "default"` - tenant selection: @@ -3420,6 +3940,11 @@ Selection rules: - project selection: `flag > context` - workflow selection: `argument > context` - optional task selection: `--task` name or code within the workflow definition +- use `dsctl project list`, `dsctl workflow list`, and `dsctl task list` to + discover selectors +- use `dsctl worker-group list`, `dsctl tenant list`, + `dsctl alert-group list`, and `dsctl environment list` to discover optional + runtime selectors - worker group, tenant, warning, priority, environment, start params, and execution dry-run rules match `dsctl workflow run` @@ -3466,6 +3991,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors Rules: @@ -3484,6 +4010,7 @@ Selection rules: - project selection: `flag > context` - workflow selection: `flag > context` +- use `dsctl project list` and `dsctl workflow list` to discover selectors Rules: @@ -3504,14 +4031,35 @@ Options: - `--all` - `--project TEXT` - `--workflow TEXT` +- `--search TEXT` +- `--executor TEXT` +- `--host TEXT` +- `--start TEXT` +- `--end TEXT` - `--state TEXT` Selection rules: - workflow-instance resources are id-first and do not consume project/workflow context -- `--project` and `--workflow` are filter strings forwarded to the DS v2 list - API +- discover workflow-instance ids with `dsctl workflow-instance list` +- discover `--project` values with `dsctl project list` +- discover `--workflow` values with `dsctl workflow list` +- discover `--state` values with + `dsctl enum list workflow-execution-status` +- without `--project`, the CLI uses the DS v2 workflow-instance list API and + supports global `--workflow`, `--host`, `--start`, `--end`, and `--state` + filters +- with `--project`, the CLI resolves the project code and uses the + project-scoped DS workflow-instance list API; `--workflow` is then resolved + as a workflow definition name or code inside that project +- `--search` filters workflow-instance names through upstream `searchVal` and + requires `--project` +- `--executor` filters by exact executor user name and requires `--project` +- `--host` filters by upstream host substring +- `--start` and `--end` filter workflow-instance `start_time` using DS datetime + format `YYYY-MM-DD HH:MM:SS`; both are optional, and when both are present + `--end` must be greater than or equal to `--start` - `--state` accepts DS workflow execution status names such as `RUNNING_EXECUTION` and `SUCCESS` - with `--all`, the CLI fetches remaining pages up to the standard safety limit @@ -3556,6 +4104,7 @@ Selection rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` ## `dsctl workflow-instance parent` @@ -3565,6 +4114,7 @@ Selection rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover sub-workflow instance ids with `dsctl workflow-instance list` - the CLI first fetches the sub-workflow instance payload, then recovers its owning `projectCode` for the DS 3.4.1 relation endpoint - the workflow instance must itself be a DS sub-workflow instance @@ -3582,6 +4132,7 @@ Selection rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - the CLI fetches the owning workflow-instance payload, then auto-exhausts the task-instance list for that workflow instance inside the standard page safety limit @@ -3623,6 +4174,7 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - the workflow instance must already be in one DS final state - the CLI requires `dagData` from the workflow-instance payload and rebuilds a live workflow spec snapshot from that instance DAG before applying the patch @@ -3655,6 +4207,7 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - the CLI checks the current DS workflow execution status before sending the stop request - states that are not stoppable return `invalid_state` @@ -3677,6 +4230,7 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - default polling interval is `5` seconds - default timeout is `600` seconds - `--timeout-seconds 0` means wait indefinitely @@ -3692,6 +4246,7 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - the workflow instance must already be in one DS final state - if DS accepts the request but the refreshed state is still final, the command succeeds and adds a warning describing the current state; the aligned @@ -3708,6 +4263,7 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` - the workflow instance must currently be in DS `FAILURE` - if DS accepts the request but the refreshed state is still final, the command succeeds and adds a warning describing the current state; the aligned @@ -3729,6 +4285,9 @@ Rules: - workflow-instance resources are id-first and do not consume project/workflow context +- discover workflow-instance ids with `dsctl workflow-instance list` +- discover `--task` values with + `dsctl task-instance list --workflow-instance WORKFLOW_INSTANCE` - the workflow instance must already be in one DS final state - the CLI resolves `--task` against the owning workflow definition recovered from the workflow instance payload @@ -3740,28 +4299,58 @@ Rules: `warning_details[]` item uses code `workflow_instance_action_state_after_request` with `action="execute-task"` and `expect_non_final=true` - succeeds and adds a warning describing the current state ## `dsctl task-instance list` -Lists task instances inside one workflow instance. +Lists task instances through the project-scoped DS task-instance paging query. Options: - `--workflow-instance ID` +- `--project TEXT` +- `--workflow-instance-name TEXT` - `--page-no N` - `--page-size N` - `--all` - `--search TEXT` +- `--task TEXT` +- `--task-code N` +- `--executor TEXT` - `--state TEXT` +- `--host TEXT` +- `--start TEXT` +- `--end TEXT` +- `--execute-type TEXT` Selection rules: - task-instance resources are id-first -- `--workflow-instance` is required because the DS 3.4.1 task-instance list API - remains project-scoped under the hood +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` +- discover `--project` values with `dsctl project list` +- discover `--task-code` values with `dsctl task list` +- discover `--state` values with `dsctl enum list task-execution-status` +- discover `--execute-type` values with + `dsctl enum list task-execute-type` +- `--workflow-instance` narrows the project-scoped query to one workflow + instance; when it is omitted, pass `--project` or set project context +- when `--workflow-instance` is present, the CLI resolves the owning project + from the workflow instance; an explicit `--project` must match that project +- workflow-definition filtering is not part of the stable `task-instance list` + contract for DS 3.4.1 because the upstream BATCH task-instance paging query + does not reliably apply `workflowDefinitionName`; use + `workflow-instance list --workflow ...` first, then pass the returned + workflow-instance id to `task-instance list --workflow-instance` +- `--workflow-instance-name` filters by the upstream workflow-instance name - `--state` accepts DS task execution status names such as `RUNNING_EXECUTION` and `SUCCESS` +- `--execute-type` accepts DS task execute type names such as `BATCH` and + `STREAM` +- `--search` is the upstream free-text `searchVal` filter; use `--task` for an + exact task instance name filter +- `--start` and `--end` filter task start time using DS datetime format + `YYYY-MM-DD HH:MM:SS`; both are optional, and when both are present `--end` + must be greater than or equal to `--start` - with `--all`, the CLI fetches remaining pages up to the standard safety limit and materializes one DS-style page payload @@ -3802,6 +4391,8 @@ Fetches one task instance by id within one workflow instance. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 direct get endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance @@ -3819,6 +4410,8 @@ Options: Rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 direct get endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance @@ -3837,6 +4430,8 @@ Returns the child workflow instance for one `SUB_WORKFLOW` task instance. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 relation endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance @@ -3855,10 +4450,13 @@ Fetches the tail of one task-instance log. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` - `--workflow-instance` is not required because the DS logger API reads log chunks by task-instance id - `--tail` means “keep the last N lines” and is implemented by chunking the DS logger API until exhaustion +- DS result code `10103` for an empty task log path is translated to stable + error type `task_not_dispatched` The `data` payload is a JSON object: @@ -3872,6 +4470,8 @@ Forces one failed task instance into `FORCED_SUCCESS`. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 mutation endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance @@ -3888,6 +4488,8 @@ Requests one savepoint for a running task instance. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 mutation endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance @@ -3905,6 +4507,8 @@ Requests stop for one task instance. Selection rules: - task-instance resources are id-first +- discover task-instance ids with `dsctl task-instance list` +- discover `--workflow-instance` values with `dsctl workflow-instance list` - `--workflow-instance` is required because the DS 3.4.1 mutation endpoint is still project-scoped and the CLI recovers `projectCode` from the owning workflow instance diff --git a/docs/reference/domain-model.md b/docs/reference/domain-model.md index f8b217b..5f3f6f8 100644 --- a/docs/reference/domain-model.md +++ b/docs/reference/domain-model.md @@ -38,7 +38,7 @@ The most useful mental split is: | Plane | Core objects | Why it matters | | --- | --- | --- | -| Governance | user, tenant, queue, worker group, env, datasource, resource, alert group | execution context, authorization, and cluster-level diagnostics | +| Governance | user, tenant, queue, worker group, environment, datasource, resource, alert group | execution context, authorization, and cluster-level diagnostics | | Project | project | default selection boundary and ownership boundary | | Design | workflow, task, relation, schedule | what should run | | Runtime | command, workflow-instance, task-instance, logs, health | what actually ran | @@ -61,7 +61,7 @@ workflow trigger └── task-instance[*] workflow/task runtime resolution - └── tenant / queue / worker group / env / datasource / resource / alert group + └── tenant / queue / worker group / environment / datasource / resource / alert group ``` Important consequences: diff --git a/docs/user/commands.md b/docs/user/commands.md index 1c54310..565c534 100644 --- a/docs/user/commands.md +++ b/docs/user/commands.md @@ -5,6 +5,27 @@ The stable CLI surface is documented in the machine-readable behavior contract for command names, output envelopes, error shape, warnings, and dry-run behavior. +Use `dsctl schema` for exact command arguments, options, choices, and selector +rules. Use `dsctl capabilities` for lightweight feature discovery; it is not an +argument schema. + +For agent or scripted discovery, prefer scoped self-description calls when the +full payload is unnecessary: + +```bash +dsctl capabilities --summary +dsctl capabilities --section runtime +dsctl schema --list-groups +dsctl schema --list-commands +dsctl schema --group task-instance +dsctl schema --command task-instance.list +dsctl enum names +``` + +`schema --group` values come from `dsctl schema --list-groups`. +`schema --command` values come from `dsctl schema --list-commands`. +`enum list ENUM` values come from `dsctl enum names`. + ## Discovery ```bash @@ -12,7 +33,12 @@ dsctl version dsctl context dsctl doctor dsctl schema +dsctl schema --list-groups +dsctl schema --list-commands +dsctl schema --command task-instance.list dsctl capabilities +dsctl capabilities --summary +dsctl enum names dsctl enum list WorkflowExecutionStatus ``` @@ -20,8 +46,14 @@ dsctl enum list WorkflowExecutionStatus ```bash dsctl project list -dsctl env list +dsctl environment list +dsctl template environment +dsctl environment create --name stock-etl --config-file env.sh +dsctl template cluster +dsctl cluster create --name k8s-prod --config-file cluster-config.json dsctl datasource list +dsctl schema --command datasource.create +dsctl template datasource --type MYSQL dsctl resource list / dsctl worker-group list dsctl alert-group list @@ -45,12 +77,13 @@ dsctl workflow run-task daily-etl --task load --project etl-prod dsctl workflow-instance digest dsctl workflow-instance watch dsctl task-instance list --workflow-instance +dsctl task-instance list --project etl-prod --state FAILURE dsctl task-instance log ``` ## Output Contract -All stable commands return the standard JSON envelope: +All stable commands return the standard JSON envelope by default: ```json { @@ -65,3 +98,25 @@ All stable commands return the standard JSON envelope: Errors use a stable `error.type` and include structured details when the CLI can derive them without guessing. + +For scan-friendly terminal output, pass a global output renderer before the +command group: + +```bash +dsctl --columns id,name,state workflow-instance list --project etl-prod +dsctl --output-format table workflow-instance list --project etl-prod +dsctl --output-format tsv --columns id,name,state task-instance list --workflow-instance +dsctl --output-format tsv --columns '*' task-instance list --workflow-instance +``` + +Use `dsctl schema --command ` and inspect `data_shape` to discover the +canonical row/object path and default display columns for row-oriented +commands. +For quick terminal inspection of one command contract, use table output; scoped +schema views include compact rows for arguments, options, payload hints, and +data-shape metadata: + +```bash +dsctl --output-format table schema --command datasource.create +dsctl --output-format table --columns flag,description,discovery_command schema --command environment.create +``` diff --git a/docs/user/runtime.md b/docs/user/runtime.md index 9169891..83472be 100644 --- a/docs/user/runtime.md +++ b/docs/user/runtime.md @@ -40,17 +40,34 @@ Use `--dry-run` to inspect the request without sending it. Inspect progress: ```bash +dsctl workflow-instance list --project etl-prod --start "2026-04-11 00:00:00" --end "2026-04-11 23:59:59" dsctl workflow-instance digest dsctl workflow-instance watch ``` +`workflow-instance list` is the primary runtime query surface. Use +`--project` for project-scoped filters such as `--search` and `--executor`. +Without `--project`, global filters are limited to `--workflow`, `--host`, +`--start`, `--end`, and `--state`. + Inspect task logs: ```bash dsctl task-instance list --workflow-instance +dsctl task-instance list --project etl-prod --state FAILURE --start "2026-04-11 00:00:00" --end "2026-04-11 23:59:59" dsctl task-instance log ``` +`task-instance list` uses the project-scoped DS task-instance paging query. Use +`--workflow-instance` for the common per-run inspection path, or use `--project` +plus filters such as `--task`, `--executor`, `--host`, `--state`, `--start`, +and `--end` for runtime triage across workflow instances. To inspect task +instances for one workflow definition, first run +`dsctl workflow-instance list --project etl-prod --workflow daily-etl`, then +pass the returned instance id to `task-instance list --workflow-instance`. Use +`--search` only for the upstream free-text `searchVal` filter; use `--task` for +an exact task instance name filter. + Control runtime state: ```bash diff --git a/docs/user/workflow-authoring.md b/docs/user/workflow-authoring.md index dc27c5f..16c2685 100644 --- a/docs/user/workflow-authoring.md +++ b/docs/user/workflow-authoring.md @@ -19,6 +19,10 @@ dsctl workflow create --file workflow.yaml --dry-run ## Discovery Flow +Use `dsctl task-type list` when you need the live DS task-type catalog for the +configured cluster and current user. Use `dsctl template task --list` when you +need the local YAML template catalog and per-task template variants. + `dsctl template workflow` returns a minimal full workflow. It is intentionally small so generated files do not include unrelated optional fields. diff --git a/src/dsctl/app.py b/src/dsctl/app.py index ba505ed..a8b0bd1 100644 --- a/src/dsctl/app.py +++ b/src/dsctl/app.py @@ -1,10 +1,24 @@ +import sys from pathlib import Path from typing import Annotated import typer -from dsctl.cli_runtime import AppState +from dsctl.cli_runtime import AppState, set_app_state from dsctl.commands.registry import register_all_commands +from dsctl.errors import UserInputError +from dsctl.output_formats import ( + OutputFormat, + RenderOptions, + parse_columns, +) + +_ROOT_OPTIONS_WITH_VALUES = frozenset({"--env-file", "--output-format", "--columns"}) +_ROOT_OPTION_EXAMPLES = { + "--env-file": "dsctl --env-file cluster.env ...", + "--output-format": "dsctl --output-format table ...", + "--columns": "dsctl --columns id,name,state ...", +} app = typer.Typer( add_completion=False, @@ -32,14 +46,107 @@ def main_callback( resolve_path=True, ), ] = None, + output_format: Annotated[ + str, + typer.Option( + "--output-format", + help=( + "Render the standard envelope as json, or render row/object " + "views as table/tsv." + ), + ), + ] = "json", + columns: Annotated[ + str | None, + typer.Option( + "--columns", + help=( + "Comma-separated row/object fields to render or project. In json " + "mode this narrows the standard envelope data payload." + ), + ), + ] = None, ) -> None: """Initialize shared command state.""" - ctx.obj = AppState(env_file=env_file) + format_choice = _parse_output_format(output_format) + try: + parsed_columns = parse_columns(columns) + except UserInputError as exc: + raise typer.BadParameter(exc.message) from exc + state = AppState( + env_file=env_file, + render_options=RenderOptions( + output_format=format_choice, + columns=parsed_columns, + ), + ) + ctx.obj = state + set_app_state(state) def main() -> None: """Run the Typer application.""" + misplaced = _misplaced_root_option(sys.argv[1:]) + if misplaced is not None: + _show_misplaced_root_option_error(misplaced) + raise SystemExit(2) app() register_all_commands(app) + + +def _parse_output_format(value: str) -> OutputFormat: + """Parse one Typer string option into the stable output-format literal.""" + normalized = value.lower() + if normalized == "json": + return "json" + if normalized == "table": + return "table" + if normalized == "tsv": + return "tsv" + message = f"Unsupported output format: {value}" + raise typer.BadParameter(message) + + +def _misplaced_root_option(args: list[str]) -> str | None: + """Return a root-only option that appears after the command path.""" + seen_command = False + index = 0 + while index < len(args): + arg = args[index] + if arg == "--": + return None + + option = _root_option_name(arg) + if option is not None: + if seen_command: + return option + index += 1 if "=" in arg else 2 + continue + + if arg.startswith("-"): + index += 1 + continue + + seen_command = True + index += 1 + return None + + +def _root_option_name(token: str) -> str | None: + for option in _ROOT_OPTIONS_WITH_VALUES: + if token == option or token.startswith(f"{option}="): + return option + return None + + +def _show_misplaced_root_option_error(option: str) -> None: + example = _ROOT_OPTION_EXAMPLES[option] + typer.echo( + ( + f"{option} is a global dsctl option. Put it before the command " + f"group, for example: {example}" + ), + err=True, + ) diff --git a/src/dsctl/cli_runtime.py b/src/dsctl/cli_runtime.py index 4734f1e..5961471 100644 --- a/src/dsctl/cli_runtime.py +++ b/src/dsctl/cli_runtime.py @@ -1,11 +1,13 @@ from collections.abc import Callable -from dataclasses import dataclass +from contextvars import ContextVar +from dataclasses import dataclass, field from pathlib import Path import typer from dsctl.errors import DsctlError -from dsctl.output import CommandResult, error_payload, print_json, success_payload +from dsctl.output import CommandResult, error_payload, success_payload +from dsctl.output_formats import RenderOptions, render_payload, validate_render_options @dataclass(frozen=True) @@ -13,6 +15,14 @@ class AppState: """Global CLI state shared across commands.""" env_file: Path | None + render_options: RenderOptions = field(default_factory=RenderOptions) + + +_DEFAULT_APP_STATE = AppState(env_file=None) +_CURRENT_APP_STATE: ContextVar[AppState] = ContextVar( + "dsctl_current_app_state", + default=_DEFAULT_APP_STATE, +) def get_app_state(ctx: typer.Context) -> AppState: @@ -24,12 +34,37 @@ def get_app_state(ctx: typer.Context) -> AppState: raise RuntimeError(message) +def set_app_state(state: AppState) -> None: + """Store the active app state for shared emitters.""" + _CURRENT_APP_STATE.set(state) + + def emit_result(action: str, builder: Callable[[], CommandResult]) -> None: - """Render a command result as the standard JSON envelope.""" + """Render a command result with the active global display settings.""" + render_options = _CURRENT_APP_STATE.get().render_options try: - result = builder() - except DsctlError as exc: - print_json(error_payload(action, exc)) - raise typer.Exit(code=1) from None + try: + validate_render_options(render_options) + result = builder() + payload = success_payload(action, result) + except DsctlError as exc: + payload = error_payload(action, exc) + typer.echo( + render_payload(payload, action=action, options=render_options), + ) + raise typer.Exit(code=1) from None - print_json(success_payload(action, result)) + try: + rendered = render_payload(payload, action=action, options=render_options) + except DsctlError as exc: + typer.echo( + render_payload( + error_payload(action, exc), + action=action, + options=render_options, + ), + ) + raise typer.Exit(code=1) from None + typer.echo(rendered) + finally: + _CURRENT_APP_STATE.set(_DEFAULT_APP_STATE) diff --git a/src/dsctl/cli_surface.py b/src/dsctl/cli_surface.py index 26ced26..5d9199d 100644 --- a/src/dsctl/cli_surface.py +++ b/src/dsctl/cli_surface.py @@ -5,7 +5,7 @@ USE_RESOURCE = "use" ENUM_RESOURCE = "enum" LINT_RESOURCE = "lint" -ENV_RESOURCE = "env" +ENV_RESOURCE = "environment" CLUSTER_RESOURCE = "cluster" DATASOURCE_RESOURCE = "datasource" NAMESPACE_RESOURCE = "namespace" @@ -97,7 +97,7 @@ def _surface_command(name: str, *commands: SurfaceCommand) -> SurfaceCommand: _surface_command(PROJECT_RESOURCE), _surface_command(WORKFLOW_RESOURCE), ), - ENUM_RESOURCE: (_surface_command("list"),), + ENUM_RESOURCE: (_surface_command("names"), _surface_command("list")), LINT_RESOURCE: (_surface_command("workflow"),), ENV_RESOURCE: ( _surface_command("list"), @@ -173,6 +173,10 @@ def _surface_command(name: str, *commands: SurfaceCommand) -> SurfaceCommand: _surface_command("update"), _surface_command("delete"), _surface_command("test"), + _surface_command( + "definition", + _surface_command("list"), + ), ), ALERT_GROUP_RESOURCE: ( _surface_command("list"), @@ -264,6 +268,9 @@ def _surface_command(name: str, *commands: SurfaceCommand) -> SurfaceCommand: TEMPLATE_RESOURCE: ( _surface_command(WORKFLOW_RESOURCE), _surface_command("params"), + _surface_command(ENV_RESOURCE), + _surface_command(CLUSTER_RESOURCE), + _surface_command(DATASOURCE_RESOURCE), _surface_command(TASK_RESOURCE), ), TASK_TYPE_RESOURCE: (_surface_command("list"),), diff --git a/src/dsctl/commands/access_token.py b/src/dsctl/commands/access_token.py index 2f0244f..8aa649a 100644 --- a/src/dsctl/commands/access_token.py +++ b/src/dsctl/commands/access_token.py @@ -17,6 +17,8 @@ no_args_is_help=True, ) +ACCESS_ID_HELP = "Access-token id. Run `dsctl access-token list` to discover values." + def register_access_token_commands(app: typer.Typer) -> None: """Register the `access-token` command group.""" @@ -78,7 +80,7 @@ def get_command( ctx: typer.Context, access_token: Annotated[ int, - typer.Argument(help="Access-token id."), + typer.Argument(help=ACCESS_ID_HELP), ], ) -> None: """Get one access token by numeric id.""" @@ -98,14 +100,14 @@ def create_command( str, typer.Option( "--user", - help="User name or numeric id.", + help="User name or numeric id. Run `dsctl user list` to discover values.", ), ], expire_time: Annotated[ str, typer.Option( "--expire-time", - help="Token expiration time.", + help="Token expiration time, for example '2027-01-01 00:00:00'.", ), ], token: Annotated[ @@ -135,21 +137,24 @@ def update_command( ctx: typer.Context, access_token: Annotated[ int, - typer.Argument(help="Access-token id."), + typer.Argument(help=ACCESS_ID_HELP), ], *, user: Annotated[ str | None, typer.Option( "--user", - help="Updated user name or numeric id.", + help=( + "Updated user name or numeric id. Run `dsctl user list` to " + "discover values." + ), ), ] = None, expire_time: Annotated[ str | None, typer.Option( "--expire-time", - help="Updated token expiration time.", + help=("Updated token expiration time, for example '2027-01-01 00:00:00'."), ), ] = None, token: Annotated[ @@ -188,7 +193,7 @@ def delete_command( ctx: typer.Context, access_token: Annotated[ int, - typer.Argument(help="Access-token id."), + typer.Argument(help=ACCESS_ID_HELP), ], *, force: Annotated[ @@ -220,14 +225,14 @@ def generate_command( str, typer.Option( "--user", - help="User name or numeric id.", + help="User name or numeric id. Run `dsctl user list` to discover values.", ), ], expire_time: Annotated[ str, typer.Option( "--expire-time", - help="Token expiration time.", + help="Token expiration time, for example '2027-01-01 00:00:00'.", ), ], ) -> None: diff --git a/src/dsctl/commands/alert_group.py b/src/dsctl/commands/alert_group.py index dd9b21a..af235dd 100644 --- a/src/dsctl/commands/alert_group.py +++ b/src/dsctl/commands/alert_group.py @@ -21,6 +21,10 @@ no_args_is_help=True, ) +ALERT_GROUP_HELP = ( + "Alert-group name or numeric id. Run `dsctl alert-group list` to discover values." +) + def register_alert_group_commands(app: typer.Typer) -> None: """Register the `alert-group` command group.""" @@ -82,7 +86,7 @@ def get_command( ctx: typer.Context, alert_group: Annotated[ str, - typer.Argument(help="Alert-group name or numeric id."), + typer.Argument(help=ALERT_GROUP_HELP), ], ) -> None: """Get one alert group by name or id.""" @@ -110,7 +114,10 @@ def create_command( typer.Option( "--instance-id", min=1, - help="Alert plugin instance id to bind to this group. Repeat as needed.", + help=( + "Alert plugin instance id to bind to this group. Repeat as " + "needed; run `dsctl alert-plugin list` to discover ids." + ), ), ] = None, description: Annotated[ @@ -140,7 +147,7 @@ def update_command( ctx: typer.Context, alert_group: Annotated[ str, - typer.Argument(help="Alert-group name or numeric id."), + typer.Argument(help=ALERT_GROUP_HELP), ], *, name: Annotated[ @@ -155,7 +162,10 @@ def update_command( typer.Option( "--instance-id", min=1, - help="Alert plugin instance id to bind to this group. Repeat as needed.", + help=( + "Alert plugin instance id to bind to this group. Repeat as " + "needed; run `dsctl alert-plugin list` to discover ids." + ), ), ] = None, clear_instance_ids: Annotated[ @@ -225,7 +235,7 @@ def delete_command( ctx: typer.Context, alert_group: Annotated[ str, - typer.Argument(help="Alert-group name or numeric id."), + typer.Argument(help=ALERT_GROUP_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/alert_plugin.py b/src/dsctl/commands/alert_plugin.py index 31be16e..3ca87da 100644 --- a/src/dsctl/commands/alert_plugin.py +++ b/src/dsctl/commands/alert_plugin.py @@ -9,6 +9,7 @@ delete_alert_plugin_result, get_alert_plugin_result, get_alert_plugin_schema_result, + list_alert_plugin_definitions_result, list_alert_plugins_result, send_test_alert_plugin_result, update_alert_plugin_result, @@ -18,10 +19,20 @@ help="Manage DolphinScheduler alert plugin instances.", no_args_is_help=True, ) +alert_plugin_definition_app = typer.Typer( + help="Discover supported DolphinScheduler alert plugin definitions.", + no_args_is_help=True, +) + +ALERT_PLUGIN_HELP = ( + "Alert-plugin instance name or numeric id. Run `dsctl alert-plugin list` " + "to discover values." +) def register_alert_plugin_commands(app: typer.Typer) -> None: """Register the `alert-plugin` command group.""" + alert_plugin_app.add_typer(alert_plugin_definition_app, name="definition") app.add_typer(alert_plugin_app, name="alert-plugin") @@ -80,7 +91,7 @@ def get_command( ctx: typer.Context, alert_plugin: Annotated[ str, - typer.Argument(help="Alert-plugin instance name or numeric id."), + typer.Argument(help=ALERT_PLUGIN_HELP), ], ) -> None: """Get one alert-plugin instance by name or id.""" @@ -97,7 +108,12 @@ def schema_command( ctx: typer.Context, plugin: Annotated[ str, - typer.Argument(help="Alert UI plugin definition name or numeric id."), + typer.Argument( + help=( + "Alert UI plugin definition name or numeric id. Run " + "`dsctl alert-plugin definition list` to discover values." + ), + ), ], ) -> None: """Get one alert-plugin definition schema by name or id.""" @@ -109,6 +125,17 @@ def schema_command( ) +@alert_plugin_definition_app.command("list") +def list_definition_command(ctx: typer.Context) -> None: + """List supported alert-plugin definitions, not configured instances.""" + state = get_app_state(ctx) + env_file = None if state.env_file is None else str(state.env_file) + emit_result( + "alert-plugin.definition.list", + lambda: list_alert_plugin_definitions_result(env_file=env_file), + ) + + @alert_plugin_app.command("create") def create_command( ctx: typer.Context, @@ -124,14 +151,20 @@ def create_command( str, typer.Option( "--plugin", - help="Alert UI plugin definition name or numeric id.", + help=( + "Alert UI plugin definition name or numeric id. Run " + "`dsctl alert-plugin definition list` to discover values." + ), ), ], params_json: Annotated[ str | None, typer.Option( "--params-json", - help="DS-native alert-plugin UI params JSON array.", + help=( + "DS-native alert-plugin UI params JSON array. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect fields." + ), ), ] = None, file: Annotated[ @@ -141,11 +174,25 @@ def create_command( dir_okay=False, exists=True, file_okay=True, - help="Path to one DS-native alert-plugin UI params JSON file.", + help=( + "Path to one DS-native alert-plugin UI params JSON file. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect fields." + ), readable=True, resolve_path=True, ), ] = None, + params: Annotated[ + list[str] | None, + typer.Option( + "--param", + help=( + "Alert-plugin UI param in KEY=VALUE form. Repeat for multiple " + "fields; run `dsctl alert-plugin schema PLUGIN` to inspect " + "keys." + ), + ), + ] = None, ) -> None: """Create one alert-plugin instance.""" state = get_app_state(ctx) @@ -157,6 +204,7 @@ def create_command( plugin=plugin, params_json=params_json, file=file, + params=params, env_file=env_file, ), ) @@ -167,7 +215,7 @@ def update_command( ctx: typer.Context, alert_plugin: Annotated[ str, - typer.Argument(help="Alert-plugin instance name or numeric id."), + typer.Argument(help=ALERT_PLUGIN_HELP), ], *, name: Annotated[ @@ -181,7 +229,10 @@ def update_command( str | None, typer.Option( "--params-json", - help="Replacement DS-native alert-plugin UI params JSON array.", + help=( + "Replacement DS-native alert-plugin UI params JSON array. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect fields." + ), ), ] = None, file: Annotated[ @@ -191,11 +242,25 @@ def update_command( dir_okay=False, exists=True, file_okay=True, - help="Path to one replacement DS-native alert-plugin UI params JSON file.", + help=( + "Path to one replacement DS-native alert-plugin UI params JSON " + "file. Run `dsctl alert-plugin schema PLUGIN` to inspect fields." + ), readable=True, resolve_path=True, ), ] = None, + params: Annotated[ + list[str] | None, + typer.Option( + "--param", + help=( + "Replacement alert-plugin UI param in KEY=VALUE form. Repeat " + "for multiple fields; omitted fields keep current values. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect keys." + ), + ), + ] = None, ) -> None: """Update one alert-plugin instance.""" state = get_app_state(ctx) @@ -207,6 +272,7 @@ def update_command( name=name, params_json=params_json, file=file, + params=params, env_file=env_file, ), ) @@ -217,7 +283,7 @@ def delete_command( ctx: typer.Context, alert_plugin: Annotated[ str, - typer.Argument(help="Alert-plugin instance name or numeric id."), + typer.Argument(help=ALERT_PLUGIN_HELP), ], *, force: Annotated[ @@ -246,7 +312,7 @@ def test_command( ctx: typer.Context, alert_plugin: Annotated[ str, - typer.Argument(help="Alert-plugin instance name or numeric id."), + typer.Argument(help=ALERT_PLUGIN_HELP), ], ) -> None: """Send one test alert using one existing alert-plugin instance.""" diff --git a/src/dsctl/commands/audit.py b/src/dsctl/commands/audit.py index ab00753..b1a0d08 100644 --- a/src/dsctl/commands/audit.py +++ b/src/dsctl/commands/audit.py @@ -28,14 +28,20 @@ def list_command( list[str] | None, typer.Option( "--model-type", - help="Audit model type filter. Repeat as needed.", + help=( + "Audit model type filter. Repeat as needed; run " + "`dsctl audit model-types` to discover values." + ), ), ] = None, operation_types: Annotated[ list[str] | None, typer.Option( "--operation-type", - help="Audit operation type filter. Repeat as needed.", + help=( + "Audit operation type filter. Repeat as needed; run " + "`dsctl audit operation-types` to discover values." + ), ), ] = None, start: Annotated[ diff --git a/src/dsctl/commands/capabilities.py b/src/dsctl/commands/capabilities.py index 4a6e12e..5dbae51 100644 --- a/src/dsctl/commands/capabilities.py +++ b/src/dsctl/commands/capabilities.py @@ -1,7 +1,18 @@ +from typing import Annotated + import typer from dsctl.cli_runtime import emit_result, get_app_state -from dsctl.services.capabilities import get_capabilities_result +from dsctl.services.capabilities import ( + CAPABILITIES_SECTION_CHOICES, + get_capabilities_result, +) + +CAPABILITIES_SECTION_HELP = ( + "Return one top-level capability section. Supported: " + f"{', '.join(CAPABILITIES_SECTION_CHOICES)}. Discover values with " + "`dsctl schema --command capabilities`." +) def register_capabilities_commands(app: typer.Typer) -> None: @@ -9,8 +20,32 @@ def register_capabilities_commands(app: typer.Typer) -> None: app.command("capabilities")(capabilities_command) -def capabilities_command(ctx: typer.Context) -> None: +def capabilities_command( + ctx: typer.Context, + *, + summary: Annotated[ + bool, + typer.Option( + "--summary", + help="Return lightweight capability discovery.", + ), + ] = False, + section: Annotated[ + str | None, + typer.Option( + "--section", + help=CAPABILITIES_SECTION_HELP, + ), + ] = None, +) -> None: """Print stable version and surface capability discovery.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) - emit_result("capabilities", lambda: get_capabilities_result(env_file=env_file)) + emit_result( + "capabilities", + lambda: get_capabilities_result( + env_file=env_file, + summary=summary, + section=section, + ), + ) diff --git a/src/dsctl/commands/cluster.py b/src/dsctl/commands/cluster.py index f671007..b089037 100644 --- a/src/dsctl/commands/cluster.py +++ b/src/dsctl/commands/cluster.py @@ -1,4 +1,5 @@ -from typing import Annotated +from pathlib import Path +from typing import Annotated, cast import typer @@ -20,6 +21,10 @@ no_args_is_help=True, ) +CLUSTER_HELP = ( + "Cluster name or numeric code. Run `dsctl cluster list` to discover values." +) + def register_cluster_commands(app: typer.Typer) -> None: """Register the `cluster` command group.""" @@ -81,7 +86,7 @@ def get_command( ctx: typer.Context, cluster: Annotated[ str, - typer.Argument(help="Cluster name or numeric code."), + typer.Argument(help=CLUSTER_HELP), ], ) -> None: """Get one cluster by name or code.""" @@ -105,12 +110,31 @@ def create_command( ), ], config: Annotated[ - str, + str | None, typer.Option( "--config", - help="Cluster config payload.", + help=( + "Inline DS cluster config JSON. Prefer --config-file for " + "multiline Kubernetes configs; run `dsctl template cluster` " + "for an example." + ), ), - ], + ] = None, + config_file: Annotated[ + Path | None, + typer.Option( + "--config-file", + dir_okay=False, + exists=True, + file_okay=True, + help=( + "Path to one DS cluster config JSON file. Run " + "`dsctl template cluster` for an example." + ), + readable=True, + resolve_path=True, + ), + ] = None, description: Annotated[ str | None, typer.Option( @@ -119,14 +143,21 @@ def create_command( ), ] = None, ) -> None: - """Create one cluster.""" + """Create one cluster; pass --config or --config-file.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( "cluster.create", lambda: create_cluster_result( name=name, - config=config, + config=cast( + "str", + _cluster_config_from_options( + config=config, + config_file=config_file, + required=True, + ), + ), description=description, env_file=env_file, ), @@ -138,7 +169,7 @@ def update_command( ctx: typer.Context, cluster: Annotated[ str, - typer.Argument(help="Cluster name or numeric code."), + typer.Argument(help=CLUSTER_HELP), ], *, name: Annotated[ @@ -152,7 +183,25 @@ def update_command( str | None, typer.Option( "--config", - help="Updated cluster config. Omit to keep the current config.", + help=( + "Updated inline DS cluster config JSON. Omit to keep the current " + "config; prefer --config-file for multiline Kubernetes configs." + ), + ), + ] = None, + config_file: Annotated[ + Path | None, + typer.Option( + "--config-file", + dir_okay=False, + exists=True, + file_okay=True, + help=( + "Path to an updated DS cluster config JSON file. Omit both " + "config options to keep the current config." + ), + readable=True, + resolve_path=True, ), ] = None, description: Annotated[ @@ -170,7 +219,7 @@ def update_command( ), ] = False, ) -> None: - """Update one cluster.""" + """Update one cluster; config may come from --config-file.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) @@ -188,7 +237,11 @@ def build_result() -> CommandResult: return update_cluster_result( cluster, name=name, - config=config, + config=_cluster_config_from_options( + config=config, + config_file=config_file, + required=False, + ), description=description_update, env_file=env_file, ) @@ -201,7 +254,7 @@ def delete_command( ctx: typer.Context, cluster: Annotated[ str, - typer.Argument(help="Cluster name or numeric code."), + typer.Argument(help=CLUSTER_HELP), ], *, force: Annotated[ @@ -223,3 +276,33 @@ def delete_command( env_file=env_file, ), ) + + +def _cluster_config_from_options( + *, + config: str | None, + config_file: Path | None, + required: bool, +) -> str | None: + if config is not None and config_file is not None: + message = "--config and --config-file are mutually exclusive" + raise UserInputError( + message, + suggestion=( + "Pass inline config with --config or read it from --config-file." + ), + ) + if config_file is not None: + return config_file.read_text(encoding="utf-8") + if config is not None: + return config + if required: + message = "Cluster config is required" + raise UserInputError( + message, + suggestion=( + "Pass --config TEXT or --config-file PATH. Run " + "`dsctl template cluster` for an example JSON config." + ), + ) + return None diff --git a/src/dsctl/commands/datasource.py b/src/dsctl/commands/datasource.py index 2a7a1be..fddf3d7 100644 --- a/src/dsctl/commands/datasource.py +++ b/src/dsctl/commands/datasource.py @@ -14,10 +14,17 @@ ) datasource_app = typer.Typer( - help="Manage DolphinScheduler datasources.", + help=( + "Manage DolphinScheduler datasources. Create/update use DS-native JSON " + "payload files." + ), no_args_is_help=True, ) +DATASOURCE_HELP = ( + "Datasource name or numeric id. Run `dsctl datasource list` to discover values." +) + def register_datasource_commands(app: typer.Typer) -> None: """Register the `datasource` command group.""" @@ -59,7 +66,7 @@ def list_command( ), ] = False, ) -> None: - """List datasources with optional filtering and pagination controls.""" + """List datasource identities and summary fields.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( @@ -79,7 +86,7 @@ def get_command( ctx: typer.Context, datasource: Annotated[ str, - typer.Argument(help="Datasource name or numeric id."), + typer.Argument(help=DATASOURCE_HELP), ], ) -> None: """Get one datasource by name or id.""" @@ -102,7 +109,12 @@ def create_command( dir_okay=False, exists=True, file_okay=True, - help="Path to one DS-native datasource JSON payload file.", + help=( + "Path to one DS-native datasource JSON payload file. Start " + "with `dsctl template datasource`, then " + "`dsctl template datasource --type TYPE` and pass the saved " + "data.json path here." + ), readable=True, resolve_path=True, ), @@ -122,7 +134,7 @@ def update_command( ctx: typer.Context, datasource: Annotated[ str, - typer.Argument(help="Datasource name or numeric id."), + typer.Argument(help=DATASOURCE_HELP), ], *, file: Annotated[ @@ -132,7 +144,13 @@ def update_command( dir_okay=False, exists=True, file_okay=True, - help="Path to one DS-native datasource JSON payload file.", + help=( + "Path to one DS-native datasource JSON payload file. Start from " + "`dsctl datasource get DATASOURCE` or " + "`dsctl template datasource --type TYPE`, then pass the saved " + "JSON path here. Masked password ****** preserves the existing " + "password." + ), readable=True, resolve_path=True, ), @@ -156,7 +174,7 @@ def delete_command( ctx: typer.Context, datasource: Annotated[ str, - typer.Argument(help="Datasource name or numeric id."), + typer.Argument(help=DATASOURCE_HELP), ], *, force: Annotated[ @@ -167,7 +185,7 @@ def delete_command( ), ] = False, ) -> None: - """Delete one datasource.""" + """Delete one datasource by name or id.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( @@ -185,10 +203,10 @@ def test_command( ctx: typer.Context, datasource: Annotated[ str, - typer.Argument(help="Datasource name or numeric id."), + typer.Argument(help=DATASOURCE_HELP), ], ) -> None: - """Run one datasource connection test.""" + """Run one datasource connection test after create or update.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( diff --git a/src/dsctl/commands/enums.py b/src/dsctl/commands/enums.py index af32456..0978d7e 100644 --- a/src/dsctl/commands/enums.py +++ b/src/dsctl/commands/enums.py @@ -5,10 +5,10 @@ import typer from dsctl.cli_runtime import emit_result, get_app_state -from dsctl.services.enums import list_enum_result +from dsctl.services.enums import list_enum_names_result, list_enum_result enum_app = typer.Typer( - help="Discover generated DolphinScheduler enums.", + help="Discover generated DolphinScheduler enums. Start with `dsctl enum names`.", no_args_is_help=True, ) @@ -18,13 +18,24 @@ def register_enum_commands(app: typer.Typer) -> None: app.add_typer(enum_app, name="enum") +@enum_app.command("names") +def names_command(ctx: typer.Context) -> None: + """List supported generated enum discovery names.""" + state = get_app_state(ctx) + env_file = None if state.env_file is None else str(state.env_file) + emit_result("enum.names", lambda: list_enum_names_result(env_file=env_file)) + + @enum_app.command("list") def list_command( ctx: typer.Context, enum_name: Annotated[ str, typer.Argument( - help="Stable enum discovery name.", + help=( + "Stable enum discovery name. Run `dsctl enum names` to list " + "supported values." + ), show_choices=False, ), ], diff --git a/src/dsctl/commands/env.py b/src/dsctl/commands/env.py index 047a0dd..6eb4a89 100644 --- a/src/dsctl/commands/env.py +++ b/src/dsctl/commands/env.py @@ -1,4 +1,5 @@ -from typing import Annotated +from pathlib import Path +from typing import Annotated, cast import typer @@ -21,10 +22,18 @@ no_args_is_help=True, ) +ENVIRONMENT_HELP = ( + "Environment name or numeric code. Run `dsctl environment list` to discover values." +) +WORKER_GROUP_HELP = ( + "Worker group to bind to this environment. Repeat as needed; run " + "`dsctl worker-group list` to discover values." +) + def register_env_commands(app: typer.Typer) -> None: - """Register the `env` command group.""" - app.add_typer(env_app, name="env") + """Register the `environment` command group.""" + app.add_typer(env_app, name="environment") @env_app.command("list") @@ -66,7 +75,7 @@ def list_command( state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( - "env.list", + "environment.list", lambda: list_environments_result( env_file=env_file, search=search, @@ -82,14 +91,14 @@ def get_command( ctx: typer.Context, environment: Annotated[ str, - typer.Argument(help="Environment name or numeric code."), + typer.Argument(help=ENVIRONMENT_HELP), ], ) -> None: """Get one environment by name or code.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( - "env.get", + "environment.get", lambda: get_environment_result(environment, env_file=env_file), ) @@ -106,12 +115,32 @@ def create_command( ), ], config: Annotated[ - str, + str | None, typer.Option( "--config", - help="Environment config payload.", + help=( + "Inline DS environment shell/export config. Prefer " + "--config-file for multiline configs; run " + "`dsctl template environment` " + "for an example." + ), ), - ], + ] = None, + config_file: Annotated[ + Path | None, + typer.Option( + "--config-file", + dir_okay=False, + exists=True, + file_okay=True, + help=( + "Path to a DS environment shell/export config file. Run " + "`dsctl template environment` for an example." + ), + readable=True, + resolve_path=True, + ), + ] = None, description: Annotated[ str | None, typer.Option( @@ -123,18 +152,25 @@ def create_command( list[str] | None, typer.Option( "--worker-group", - help="Worker group to bind to this environment. Repeat as needed.", + help=WORKER_GROUP_HELP, ), ] = None, ) -> None: - """Create one environment.""" + """Create one environment; pass --config or --config-file.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( - "env.create", + "environment.create", lambda: create_environment_result( name=name, - config=config, + config=cast( + "str", + _environment_config_from_options( + config=config, + config_file=config_file, + required=True, + ), + ), description=description, worker_groups=worker_groups, env_file=env_file, @@ -147,7 +183,7 @@ def update_command( ctx: typer.Context, environment: Annotated[ str, - typer.Argument(help="Environment name or numeric code."), + typer.Argument(help=ENVIRONMENT_HELP), ], *, name: Annotated[ @@ -161,7 +197,26 @@ def update_command( str | None, typer.Option( "--config", - help="Updated environment config. Omit to keep the current config.", + help=( + "Updated inline DS environment shell/export config. Omit to " + "keep the current config; prefer --config-file for multiline " + "configs." + ), + ), + ] = None, + config_file: Annotated[ + Path | None, + typer.Option( + "--config-file", + dir_okay=False, + exists=True, + file_okay=True, + help=( + "Path to an updated DS environment shell/export config file. " + "Omit both config options to keep the current config." + ), + readable=True, + resolve_path=True, ), ] = None, description: Annotated[ @@ -182,7 +237,7 @@ def update_command( list[str] | None, typer.Option( "--worker-group", - help="Worker group to bind to this environment. Repeat as needed.", + help=WORKER_GROUP_HELP, ), ] = None, clear_worker_groups: Annotated[ @@ -193,7 +248,7 @@ def update_command( ), ] = False, ) -> None: - """Update one environment.""" + """Update one environment; config may come from --config-file.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) @@ -224,13 +279,17 @@ def build_result() -> CommandResult: return update_environment_result( environment, name=name, - config=config, + config=_environment_config_from_options( + config=config, + config_file=config_file, + required=False, + ), description=description_update, worker_groups=worker_groups_update, env_file=env_file, ) - emit_result("env.update", build_result) + emit_result("environment.update", build_result) @env_app.command("delete") @@ -238,7 +297,7 @@ def delete_command( ctx: typer.Context, environment: Annotated[ str, - typer.Argument(help="Environment name or numeric code."), + typer.Argument(help=ENVIRONMENT_HELP), ], *, force: Annotated[ @@ -253,10 +312,40 @@ def delete_command( state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( - "env.delete", + "environment.delete", lambda: delete_environment_result( environment, force=force, env_file=env_file, ), ) + + +def _environment_config_from_options( + *, + config: str | None, + config_file: Path | None, + required: bool, +) -> str | None: + if config is not None and config_file is not None: + message = "--config and --config-file are mutually exclusive" + raise UserInputError( + message, + suggestion=( + "Pass inline config with --config or read it from --config-file." + ), + ) + if config_file is not None: + return config_file.read_text(encoding="utf-8") + if config is not None: + return config + if required: + message = "Environment config is required" + raise UserInputError( + message, + suggestion=( + "Pass --config TEXT or --config-file PATH. Run " + "`dsctl template environment` for an example shell/export config." + ), + ) + return None diff --git a/src/dsctl/commands/namespace.py b/src/dsctl/commands/namespace.py index 68ab3f7..30d595e 100644 --- a/src/dsctl/commands/namespace.py +++ b/src/dsctl/commands/namespace.py @@ -16,6 +16,10 @@ no_args_is_help=True, ) +NAMESPACE_HELP = ( + "Namespace name or numeric id. Run `dsctl namespace list` to discover values." +) + def register_namespace_commands(app: typer.Typer) -> None: """Register the `namespace` command group.""" @@ -77,7 +81,7 @@ def get_command( ctx: typer.Context, namespace: Annotated[ str, - typer.Argument(help="Namespace name or numeric id."), + typer.Argument(help=NAMESPACE_HELP), ], ) -> None: """Get one namespace by name or id.""" @@ -116,7 +120,7 @@ def create_command( typer.Option( "--cluster-code", min=1, - help="Owning cluster code.", + help="Owning cluster code. Run `dsctl cluster list` to discover codes.", ), ], ) -> None: @@ -138,7 +142,7 @@ def delete_command( ctx: typer.Context, namespace: Annotated[ str, - typer.Argument(help="Namespace name or numeric id."), + typer.Argument(help=NAMESPACE_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/project.py b/src/dsctl/commands/project.py index 767cce6..ac2ae73 100644 --- a/src/dsctl/commands/project.py +++ b/src/dsctl/commands/project.py @@ -20,6 +20,10 @@ no_args_is_help=True, ) +PROJECT_HELP = ( + "Project name or numeric code. Run `dsctl project list` to discover values." +) + def register_project_commands(app: typer.Typer) -> None: """Register the `project` command group.""" @@ -81,7 +85,7 @@ def get_command( ctx: typer.Context, project: Annotated[ str, - typer.Argument(help="Project name or numeric code."), + typer.Argument(help=PROJECT_HELP), ], ) -> None: """Get one project by name or code.""" @@ -130,7 +134,7 @@ def update_command( ctx: typer.Context, project: Annotated[ str, - typer.Argument(help="Project name or numeric code."), + typer.Argument(help=PROJECT_HELP), ], *, name: Annotated[ @@ -193,7 +197,7 @@ def delete_command( ctx: typer.Context, project: Annotated[ str, - typer.Argument(help="Project name or numeric code."), + typer.Argument(help=PROJECT_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/project_parameter.py b/src/dsctl/commands/project_parameter.py index 0d616d2..2611ec6 100644 --- a/src/dsctl/commands/project_parameter.py +++ b/src/dsctl/commands/project_parameter.py @@ -19,6 +19,11 @@ no_args_is_help=True, ) +PROJECT_PARAMETER_HELP = ( + "Project parameter name or numeric code. Run `dsctl project-parameter list` " + "in the selected project to discover values." +) + def register_project_parameter_commands(app: typer.Typer) -> None: """Register the `project-parameter` command group.""" @@ -33,7 +38,10 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, search: Annotated[ @@ -47,7 +55,10 @@ def list_command( str | None, typer.Option( "--data-type", - help="Filter project parameters by DS projectParameterDataType.", + help=( + "Filter by DS projectParameterDataType. Run `dsctl enum list " + "data-type` to discover values." + ), ), ] = None, page_no: Annotated[ @@ -96,14 +107,17 @@ def get_command( ctx: typer.Context, project_parameter: Annotated[ str, - typer.Argument(help="Project parameter name or numeric code."), + typer.Argument(help=PROJECT_PARAMETER_HELP), ], *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, ) -> None: @@ -128,7 +142,10 @@ def create_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, name: Annotated[ @@ -149,7 +166,10 @@ def create_command( str, typer.Option( "--data-type", - help="DS projectParameterDataType value.", + help=( + "DS projectParameterDataType value. Run `dsctl enum list " + "data-type` to discover values." + ), ), ] = "VARCHAR", ) -> None: @@ -173,14 +193,17 @@ def update_command( ctx: typer.Context, project_parameter: Annotated[ str, - typer.Argument(help="Project parameter name or numeric code."), + typer.Argument(help=PROJECT_PARAMETER_HELP), ], *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, name: Annotated[ @@ -201,7 +224,10 @@ def update_command( str | None, typer.Option( "--data-type", - help="Updated DS projectParameterDataType value.", + help=( + "Updated DS projectParameterDataType value. Run `dsctl enum " + "list data-type` to discover values." + ), ), ] = None, ) -> None: @@ -228,14 +254,17 @@ def delete_command( ctx: typer.Context, project_parameter: Annotated[ str, - typer.Argument(help="Project parameter name or numeric code."), + typer.Argument(help=PROJECT_PARAMETER_HELP), ], *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, force: Annotated[ diff --git a/src/dsctl/commands/project_preference.py b/src/dsctl/commands/project_preference.py index 6cb365a..fb3083d 100644 --- a/src/dsctl/commands/project_preference.py +++ b/src/dsctl/commands/project_preference.py @@ -33,7 +33,10 @@ def get_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, ) -> None: @@ -57,7 +60,10 @@ def update_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, preferences_json: Annotated[ @@ -102,7 +108,10 @@ def enable_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, ) -> None: @@ -126,7 +135,10 @@ def disable_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, ) -> None: diff --git a/src/dsctl/commands/project_worker_group.py b/src/dsctl/commands/project_worker_group.py index d884614..ad99f4e 100644 --- a/src/dsctl/commands/project_worker_group.py +++ b/src/dsctl/commands/project_worker_group.py @@ -28,7 +28,10 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, ) -> None: @@ -52,14 +55,20 @@ def set_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, worker_groups: Annotated[ list[str] | None, typer.Option( "--worker-group", - help="Worker group to keep assigned to this project. Repeat as needed.", + help=( + "Worker group to keep assigned to this project. Repeat as " + "needed; run `dsctl worker-group list` to discover values." + ), ), ] = None, ) -> None: @@ -84,7 +93,10 @@ def clear_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Run `dsctl project list` to discover " + "values; falls back to stored project context." + ), ), ] = None, force: Annotated[ diff --git a/src/dsctl/commands/queue.py b/src/dsctl/commands/queue.py index 9c57b9f..bd6bb6e 100644 --- a/src/dsctl/commands/queue.py +++ b/src/dsctl/commands/queue.py @@ -16,6 +16,8 @@ no_args_is_help=True, ) +QUEUE_HELP = "Queue name or numeric id. Run `dsctl queue list` to discover values." + def register_queue_commands(app: typer.Typer) -> None: """Register the `queue` command group.""" @@ -77,7 +79,7 @@ def get_command( ctx: typer.Context, queue: Annotated[ str, - typer.Argument(help="Queue name or numeric id."), + typer.Argument(help=QUEUE_HELP), ], ) -> None: """Get one queue by name or id.""" @@ -97,14 +99,14 @@ def create_command( str, typer.Option( "--queue-name", - help="Human-facing queue name.", + help="Human-facing DS queue name used as the selector label.", ), ], queue: Annotated[ str, typer.Option( "--queue", - help="Underlying DolphinScheduler queue value.", + help="Underlying YARN queue value stored in DolphinScheduler.", ), ], ) -> None: @@ -126,21 +128,30 @@ def update_command( ctx: typer.Context, queue_identifier: Annotated[ str, - typer.Argument(help="Queue name or numeric id."), + typer.Argument( + help=QUEUE_HELP, + metavar="QUEUE", + ), ], *, queue_name: Annotated[ str | None, typer.Option( "--queue-name", - help="Updated queue name. Omit to keep the current queue name.", + help=( + "Updated human-facing DS queue name. Omit to keep the current " + "queue name." + ), ), ] = None, queue: Annotated[ str | None, typer.Option( "--queue", - help="Updated queue value. Omit to keep the current queue value.", + help=( + "Updated underlying YARN queue value. Omit to keep the current " + "queue value." + ), ), ] = None, ) -> None: @@ -163,7 +174,7 @@ def delete_command( ctx: typer.Context, queue: Annotated[ str, - typer.Argument(help="Queue name or numeric id."), + typer.Argument(help=QUEUE_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/resource.py b/src/dsctl/commands/resource.py index d500501..7e24c08 100644 --- a/src/dsctl/commands/resource.py +++ b/src/dsctl/commands/resource.py @@ -34,7 +34,10 @@ def list_command( str | None, typer.Option( "--dir", - help="DS directory fullName path. Defaults to the upstream base directory.", + help=( + "DS directory fullName path. Defaults to the upstream base " + "directory; run `dsctl resource list` to discover paths." + ), ), ] = None, search: Annotated[ @@ -89,7 +92,12 @@ def view_command( ctx: typer.Context, resource: Annotated[ str, - typer.Argument(help="DS resource fullName path."), + typer.Argument( + help=( + "DS resource fullName path. Run `dsctl resource list --dir DIR` " + "to discover paths." + ), + ), ], *, skip_line_num: Annotated[ @@ -145,7 +153,8 @@ def upload_command( "--dir", help=( "Destination DS directory fullName path. Defaults to the " - "upstream base directory." + "upstream base directory; run `dsctl resource list` to " + "discover paths." ), ), ] = None, @@ -186,7 +195,10 @@ def create_command( str, typer.Option( "--content", - help="Inline text content to write into the remote resource file.", + help=( + "Inline text content to write into the remote resource file. " + "For local files, use `dsctl resource upload --file PATH`." + ), ), ], directory: Annotated[ @@ -195,7 +207,8 @@ def create_command( "--dir", help=( "Destination DS directory fullName path. Defaults to the " - "upstream base directory." + "upstream base directory; run `dsctl resource list` to " + "discover paths." ), ), ] = None, @@ -228,7 +241,7 @@ def mkdir_command( "--dir", help=( "Parent DS directory fullName path. Defaults to the upstream " - "base directory." + "base directory; run `dsctl resource list` to discover paths." ), ), ] = None, @@ -251,7 +264,12 @@ def download_command( ctx: typer.Context, resource: Annotated[ str, - typer.Argument(help="DS resource fullName path."), + typer.Argument( + help=( + "DS resource fullName path. Run `dsctl resource list --dir DIR` " + "to discover paths." + ), + ), ], *, output: Annotated[ @@ -294,7 +312,12 @@ def delete_command( ctx: typer.Context, resource: Annotated[ str, - typer.Argument(help="DS resource fullName path."), + typer.Argument( + help=( + "DS resource fullName path. Run `dsctl resource list --dir DIR` " + "to discover paths." + ), + ), ], *, force: Annotated[ diff --git a/src/dsctl/commands/schedule.py b/src/dsctl/commands/schedule.py index 9c30702..b2d854a 100644 --- a/src/dsctl/commands/schedule.py +++ b/src/dsctl/commands/schedule.py @@ -20,6 +20,16 @@ no_args_is_help=True, ) +PROJECT_HELP = ( + "Project name or code. Run `dsctl project list` to discover values; falls " + "back to stored project context." +) +SCHEDULE_ID_HELP = "Schedule id. Use `dsctl schedule list` to discover values." +WORKFLOW_HELP = ( + "Workflow name or code. Run `dsctl workflow list` in the selected project " + "to discover values; falls back to workflow context." +) + def register_schedule_commands(app: typer.Typer) -> None: """Register the `schedule` command group.""" @@ -34,14 +44,18 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, workflow: Annotated[ str | None, typer.Option( "--workflow", - help="Exact workflow name or code to narrow the project schedule list.", + help=( + "Exact workflow name or code to narrow the project schedule " + "list. Run `dsctl workflow list` in the selected project to " + "discover values." + ), ), ] = None, search: Annotated[ @@ -100,7 +114,7 @@ def get_command( ctx: typer.Context, schedule_id: Annotated[ int, - typer.Argument(help="Schedule id."), + typer.Argument(help=SCHEDULE_ID_HELP), ], ) -> None: """Get one schedule by id.""" @@ -117,14 +131,19 @@ def preview_command( ctx: typer.Context, schedule_id: Annotated[ int | None, - typer.Argument(help="Existing schedule id to preview."), + typer.Argument( + help=( + "Existing schedule id to preview. Use `dsctl schedule list` to " + "discover values." + ) + ), ] = None, *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, cron: Annotated[ @@ -181,7 +200,12 @@ def explain_command( ctx: typer.Context, schedule_id: Annotated[ int | None, - typer.Argument(help="Existing schedule id to explain as an update."), + typer.Argument( + help=( + "Existing schedule id to explain as an update. Use `dsctl " + "schedule list` to discover values." + ) + ), ] = None, *, workflow: Annotated[ @@ -189,8 +213,9 @@ def explain_command( typer.Option( "--workflow", help=( - "Workflow name or code. Falls back to workflow context for " - "create explain." + "Workflow name or code. Run `dsctl workflow list` in the " + "selected project to discover values; falls back to workflow " + "context for create explain." ), ), ] = None, @@ -198,10 +223,7 @@ def explain_command( str | None, typer.Option( "--project", - help=( - "Project name or code. Falls back to stored project context for " - "create explain." - ), + help=PROJECT_HELP, ), ] = None, cron: Annotated[ @@ -254,7 +276,8 @@ def explain_command( help=( "Warning group id for create explain or updated value for " "update explain. Create explain can also inherit enabled " - "project preference when omitted." + "project preference when omitted; run `dsctl alert-group list` " + "to discover ids." ), ), ] = None, @@ -272,7 +295,8 @@ def explain_command( help=( "Worker group for create explain or updated value for update " "explain. Create explain can also inherit enabled project " - "preference when omitted." + "preference when omitted; run `dsctl worker-group list` to " + "discover values." ), ), ] = None, @@ -282,7 +306,8 @@ def explain_command( "--tenant-code", help=( "Tenant code for create explain. Create explain can also " - "inherit enabled project preference when omitted." + "inherit enabled project preference when omitted; run `dsctl " + "tenant list` to discover values." ), ), ] = None, @@ -294,7 +319,8 @@ def explain_command( help=( "Environment code for create explain or updated value for " "update explain. Create explain can also inherit enabled " - "project preference when omitted." + "project preference when omitted; run `dsctl environment list` " + "to discover values." ), ), ] = None, @@ -332,14 +358,14 @@ def create_command( str | None, typer.Option( "--workflow", - help="Workflow name or code. Falls back to workflow context.", + help=WORKFLOW_HELP, ), ] = None, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, cron: Annotated[ @@ -391,7 +417,8 @@ def create_command( min=0, help=( "Warning group id. Omit to keep the CLI fallback chain, " - "including enabled project preference." + "including enabled project preference; run `dsctl alert-group " + "list` to discover ids." ), ), ] = None, @@ -406,14 +433,20 @@ def create_command( str | None, typer.Option( "--worker-group", - help="Worker group. Omit to allow enabled project preference.", + help=( + "Worker group. Omit to allow enabled project preference; run " + "`dsctl worker-group list` to discover values." + ), ), ] = None, tenant_code: Annotated[ str | None, typer.Option( "--tenant-code", - help="Tenant code. Omit to allow enabled project preference.", + help=( + "Tenant code. Omit to allow enabled project preference; run " + "`dsctl tenant list` to discover values." + ), ), ] = None, environment_code: Annotated[ @@ -423,7 +456,8 @@ def create_command( min=0, help=( "Environment code. Omit to keep the CLI fallback chain, " - "including enabled project preference." + "including enabled project preference; run `dsctl environment " + "list` to discover values." ), ), ] = None, @@ -465,7 +499,7 @@ def update_command( ctx: typer.Context, schedule_id: Annotated[ int, - typer.Argument(help="Schedule id."), + typer.Argument(help=SCHEDULE_ID_HELP), ], *, cron: Annotated[ @@ -518,7 +552,10 @@ def update_command( typer.Option( "--warning-group-id", min=0, - help="Updated warning group id. Omit to keep the current value.", + help=( + "Updated warning group id. Run `dsctl alert-group list` to " + "discover ids; omit to keep the current value." + ), ), ] = None, priority: Annotated[ @@ -532,7 +569,10 @@ def update_command( str | None, typer.Option( "--worker-group", - help="Updated worker group. Omit to keep the current value.", + help=( + "Updated worker group. Run `dsctl worker-group list` to " + "discover values; omit to keep the current value." + ), ), ] = None, environment_code: Annotated[ @@ -540,7 +580,10 @@ def update_command( typer.Option( "--environment-code", min=0, - help="Updated environment code. Omit to keep the current value.", + help=( + "Updated environment code. Run `dsctl environment list` to " + "discover values; omit to keep the current value." + ), ), ] = None, confirm_risk: Annotated[ @@ -579,7 +622,7 @@ def delete_command( ctx: typer.Context, schedule_id: Annotated[ int, - typer.Argument(help="Schedule id."), + typer.Argument(help=SCHEDULE_ID_HELP), ], *, force: Annotated[ @@ -608,7 +651,7 @@ def online_command( ctx: typer.Context, schedule_id: Annotated[ int, - typer.Argument(help="Schedule id."), + typer.Argument(help=SCHEDULE_ID_HELP), ], ) -> None: """Bring one schedule online.""" @@ -625,7 +668,7 @@ def offline_command( ctx: typer.Context, schedule_id: Annotated[ int, - typer.Argument(help="Schedule id."), + typer.Argument(help=SCHEDULE_ID_HELP), ], ) -> None: """Bring one schedule offline.""" diff --git a/src/dsctl/commands/schema.py b/src/dsctl/commands/schema.py index eb2131b..e723bf3 100644 --- a/src/dsctl/commands/schema.py +++ b/src/dsctl/commands/schema.py @@ -1,6 +1,8 @@ +from typing import Annotated + import typer -from dsctl.cli_runtime import emit_result +from dsctl.cli_runtime import emit_result, get_app_state from dsctl.services.schema import get_schema_result @@ -9,6 +11,54 @@ def register_schema_commands(app: typer.Typer) -> None: app.command("schema")(schema_command) -def schema_command() -> None: +def schema_command( + ctx: typer.Context, + *, + group: Annotated[ + str | None, + typer.Option( + "--group", + help=( + "Return schema for one command group. Discover values with " + "`dsctl schema --list-groups`." + ), + ), + ] = None, + command: Annotated[ + str | None, + typer.Option( + "--command", + help=( + "Return schema for one stable command action. Discover values " + "with `dsctl schema --list-commands`." + ), + ), + ] = None, + list_groups: Annotated[ + bool, + typer.Option( + "--list-groups", + help="List valid values for --group.", + ), + ] = False, + list_commands: Annotated[ + bool, + typer.Option( + "--list-commands", + help="List valid values for --command.", + ), + ] = False, +) -> None: """Print the stable machine-readable CLI schema.""" - emit_result("schema", get_schema_result) + state = get_app_state(ctx) + env_file = None if state.env_file is None else str(state.env_file) + emit_result( + "schema", + lambda: get_schema_result( + env_file=env_file, + group=group, + command_action=command, + list_groups=list_groups, + list_commands=list_commands, + ), + ) diff --git a/src/dsctl/commands/task.py b/src/dsctl/commands/task.py index e4d61a1..6012470 100644 --- a/src/dsctl/commands/task.py +++ b/src/dsctl/commands/task.py @@ -10,6 +10,16 @@ no_args_is_help=True, ) +PROJECT_HELP = ( + "Project name or code. Run `dsctl project list` to discover values; falls " + "back to stored project context." +) +TASK_HELP = "Task name or numeric code. Use `dsctl task list` to discover values." +WORKFLOW_HELP = ( + "Workflow name or code. Run `dsctl workflow list` in the selected project " + "to discover values; falls back to workflow context." +) + def register_task_commands(app: typer.Typer) -> None: """Register the `task` command group.""" @@ -24,14 +34,14 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, workflow: Annotated[ str | None, typer.Option( "--workflow", - help="Workflow name or code. Falls back to workflow context.", + help=WORKFLOW_HELP, ), ] = None, search: Annotated[ @@ -63,21 +73,21 @@ def get_command( ctx: typer.Context, task: Annotated[ str, - typer.Argument(help="Task name or numeric code."), + typer.Argument(help=TASK_HELP), ], *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, workflow: Annotated[ str | None, typer.Option( "--workflow", - help="Workflow name or code. Falls back to workflow context.", + help=WORKFLOW_HELP, ), ] = None, ) -> None: @@ -100,21 +110,21 @@ def update_command( ctx: typer.Context, task: Annotated[ str, - typer.Argument(help="Task name or numeric code."), + typer.Argument(help=TASK_HELP), ], *, project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, workflow: Annotated[ str | None, typer.Option( "--workflow", - help="Workflow name or code. Falls back to workflow context.", + help=WORKFLOW_HELP, ), ] = None, set_values: Annotated[ @@ -123,7 +133,8 @@ def update_command( "--set", help=( "Inline task update in KEY=VALUE form. Repeat for multiple " - "fields. Inspect `dsctl schema` for supported keys and examples." + "fields. Inspect `dsctl schema --command task.update` for " + "supported keys and examples." ), ), ] = None, diff --git a/src/dsctl/commands/task_group.py b/src/dsctl/commands/task_group.py index faa529d..ccefb63 100644 --- a/src/dsctl/commands/task_group.py +++ b/src/dsctl/commands/task_group.py @@ -27,6 +27,10 @@ no_args_is_help=True, ) +TASK_GROUP_HELP = ( + "Task-group name or numeric id. Run `dsctl task-group list` to discover values." +) + def register_task_group_commands(app: typer.Typer) -> None: """Register the `task-group` command group.""" @@ -42,7 +46,10 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Use only for project-scoped listing.", + help=( + "Project name or code. Use only for project-scoped listing; " + "run `dsctl project list` to discover values." + ), ), ] = None, search: Annotated[ @@ -56,7 +63,7 @@ def list_command( str | None, typer.Option( "--status", - help="Filter task groups by status: open or closed.", + help="Filter task groups by status: open, closed, 1, or 0.", ), ] = None, page_no: Annotated[ @@ -94,7 +101,7 @@ def get_command( ctx: typer.Context, task_group: Annotated[ str, - typer.Argument(help="Task-group name or numeric id."), + typer.Argument(help=TASK_GROUP_HELP), ], ) -> None: """Get one task group by name or id.""" @@ -114,7 +121,10 @@ def create_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=( + "Project name or code. Falls back to stored project context; " + "run `dsctl project list` to discover values." + ), ), ] = None, name: Annotated[ @@ -150,7 +160,7 @@ def update_command( ctx: typer.Context, task_group: Annotated[ str, - typer.Argument(help="Task-group name or numeric id."), + typer.Argument(help=TASK_GROUP_HELP), ], *, name: Annotated[ @@ -198,7 +208,7 @@ def close_command( ctx: typer.Context, task_group: Annotated[ str, - typer.Argument(help="Task-group name or numeric id."), + typer.Argument(help=TASK_GROUP_HELP), ], ) -> None: """Close one task group.""" @@ -215,7 +225,7 @@ def start_command( ctx: typer.Context, task_group: Annotated[ str, - typer.Argument(help="Task-group name or numeric id."), + typer.Argument(help=TASK_GROUP_HELP), ], ) -> None: """Start one task group.""" @@ -232,7 +242,7 @@ def list_queue_command( ctx: typer.Context, task_group: Annotated[ str, - typer.Argument(help="Task-group name or numeric id."), + typer.Argument(help=TASK_GROUP_HELP), ], *, task_instance: Annotated[ @@ -250,7 +260,10 @@ def list_queue_command( str | None, typer.Option( "--status", - help="Filter by queue status: WAIT_QUEUE, ACQUIRE_SUCCESS, or RELEASE.", + help=( + "Filter by queue status: WAIT_QUEUE, ACQUIRE_SUCCESS, RELEASE, " + "-1, 1, or 2." + ), ), ] = None, page_no: Annotated[ @@ -289,7 +302,12 @@ def force_start_queue_command( ctx: typer.Context, queue_id: Annotated[ int, - typer.Argument(help="Numeric task-group queue id."), + typer.Argument( + help=( + "Numeric task-group queue id. Run " + "`dsctl task-group queue list TASK_GROUP` to discover ids." + ), + ), ], ) -> None: """Force-start one waiting task-group queue row.""" @@ -306,7 +324,12 @@ def set_priority_queue_command( ctx: typer.Context, queue_id: Annotated[ int, - typer.Argument(help="Numeric task-group queue id."), + typer.Argument( + help=( + "Numeric task-group queue id. Run " + "`dsctl task-group queue list TASK_GROUP` to discover ids." + ), + ), ], *, priority: Annotated[ diff --git a/src/dsctl/commands/task_instance.py b/src/dsctl/commands/task_instance.py index 48179a3..77e7a07 100644 --- a/src/dsctl/commands/task_instance.py +++ b/src/dsctl/commands/task_instance.py @@ -21,6 +21,32 @@ no_args_is_help=True, ) +TASK_INSTANCE_HELP = "Task instance id. Run `dsctl task-instance list` to discover ids." +WORKFLOW_INSTANCE_OPTION_HELP = ( + "Workflow instance id used to resolve the owning project. Run `dsctl " + "workflow-instance list` to discover ids." +) +WORKFLOW_INSTANCE_FILTER_HELP = ( + "Workflow instance id used to narrow the project-scoped task-instance query. " + "Run `dsctl workflow-instance list` to discover ids." +) +PROJECT_FILTER_HELP = ( + "Project name or code for the project-scoped query. Run `dsctl project list` " + "to discover values; required via flag or context when --workflow-instance " + "is omitted." +) +TASK_STATE_HELP = ( + "Filter by DS task execution status name. Run `dsctl enum list " + "task-execution-status` to discover values." +) +TASK_CODE_HELP = ( + "Filter by task definition code. Run `dsctl task list` to discover values." +) +TASK_EXECUTE_TYPE_HELP = ( + "Filter by DS task execute type: BATCH or STREAM. Run `dsctl enum list " + "task-execute-type` to discover values." +) + def register_task_instance_commands(app: typer.Typer) -> None: """Register the `task-instance` command group.""" @@ -32,12 +58,37 @@ def list_command( ctx: typer.Context, *, workflow_instance: Annotated[ - int, + int | None, typer.Option( "--workflow-instance", - help="Workflow instance id used to scope the task-instance query.", + help=WORKFLOW_INSTANCE_FILTER_HELP, ), - ], + ] = None, + project: Annotated[ + str | None, + typer.Option( + "--project", + help=PROJECT_FILTER_HELP, + ), + ] = None, + workflow: Annotated[ + str | None, + typer.Option( + "--workflow", + help=( + "Reserved compatibility option. DS 3.4.1 task-instance list " + "does not reliably filter by workflow definition." + ), + hidden=True, + ), + ] = None, + workflow_instance_name: Annotated[ + str | None, + typer.Option( + "--workflow-instance-name", + help="Filter by workflow instance name.", + ), + ] = None, page_no: Annotated[ int, typer.Option("--page-no", help="Remote page number."), @@ -57,29 +108,91 @@ def list_command( str | None, typer.Option( "--search", - help="Filter task instances by upstream searchVal.", + help=( + "Free-text upstream searchVal filter. Use --task for an exact " + "task instance name filter." + ), + ), + ] = None, + task: Annotated[ + str | None, + typer.Option( + "--task", + help="Filter by exact task instance name.", + ), + ] = None, + task_code: Annotated[ + int | None, + typer.Option( + "--task-code", + help=TASK_CODE_HELP, + ), + ] = None, + executor: Annotated[ + str | None, + typer.Option( + "--executor", + help="Filter by executor user name.", ), ] = None, state: Annotated[ str | None, typer.Option( "--state", - help="Filter by DS task execution status name.", + help=TASK_STATE_HELP, + ), + ] = None, + host: Annotated[ + str | None, + typer.Option( + "--host", + help="Filter by worker host.", + ), + ] = None, + start: Annotated[ + str | None, + typer.Option( + "--start", + help="Task start-time lower bound, e.g. '2026-04-11 10:00:00'.", + ), + ] = None, + end: Annotated[ + str | None, + typer.Option( + "--end", + help="Task start-time upper bound, e.g. '2026-04-11 11:00:00'.", + ), + ] = None, + execute_type: Annotated[ + str | None, + typer.Option( + "--execute-type", + help=TASK_EXECUTE_TYPE_HELP, ), ] = None, ) -> None: - """List task instances inside one workflow instance.""" + """List task instances with project-scoped runtime filters.""" state_obj = get_app_state(ctx) env_file = None if state_obj.env_file is None else str(state_obj.env_file) emit_result( "task-instance.list", lambda: list_task_instances_result( workflow_instance=workflow_instance, + project=project, + workflow=workflow, + workflow_instance_name=workflow_instance_name, page_no=page_no, page_size=page_size, all_pages=all_pages, search=search, + task=task, + task_code=task_code, + executor=executor, state=state, + host=host, + start=start, + end=end, + execute_type=execute_type, env_file=env_file, ), ) @@ -90,14 +203,14 @@ def get_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to resolve the owning project.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], ) -> None: @@ -119,14 +232,14 @@ def watch_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to resolve the owning project.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], interval_seconds: Annotated[ @@ -164,14 +277,14 @@ def sub_workflow_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to scope the task-instance relation.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], ) -> None: @@ -193,7 +306,7 @@ def log_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, tail: Annotated[ @@ -222,14 +335,14 @@ def force_success_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to resolve the owning project.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], ) -> None: @@ -251,14 +364,14 @@ def savepoint_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to resolve the owning project.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], ) -> None: @@ -280,14 +393,14 @@ def stop_command( ctx: typer.Context, task_instance: Annotated[ int, - typer.Argument(help="Task instance id."), + typer.Argument(help=TASK_INSTANCE_HELP), ], *, workflow_instance: Annotated[ int, typer.Option( "--workflow-instance", - help="Workflow instance id used to resolve the owning project.", + help=WORKFLOW_INSTANCE_OPTION_HELP, ), ], ) -> None: diff --git a/src/dsctl/commands/task_type.py b/src/dsctl/commands/task_type.py index dc1ef41..284db96 100644 --- a/src/dsctl/commands/task_type.py +++ b/src/dsctl/commands/task_type.py @@ -5,8 +5,12 @@ from dsctl.cli_runtime import emit_result, get_app_state from dsctl.services.task_type import list_task_types_result +TASK_TYPE_LIST_HELP = ( + "List live DS task types, categories, favourite flags, and CLI authoring coverage." +) + task_type_app = typer.Typer( - help="Discover DolphinScheduler task types for the current runtime.", + help="List live DS task-type catalog for the configured cluster and current user.", no_args_is_help=True, ) @@ -16,9 +20,9 @@ def register_task_type_commands(app: typer.Typer) -> None: app.add_typer(task_type_app, name="task-type") -@task_type_app.command("list") +@task_type_app.command("list", help=TASK_TYPE_LIST_HELP) def list_command(ctx: typer.Context) -> None: - """List DS task types plus the current user's favourite flags.""" + """List the live DS task-type catalog.""" state = get_app_state(ctx) env_file = None if state.env_file is None else str(state.env_file) emit_result( diff --git a/src/dsctl/commands/template.py b/src/dsctl/commands/template.py index ee1b304..d76d642 100644 --- a/src/dsctl/commands/template.py +++ b/src/dsctl/commands/template.py @@ -4,7 +4,11 @@ from dsctl.cli_runtime import emit_result from dsctl.services.template import ( + cluster_config_template_result, + datasource_template_result, + environment_config_template_result, parameter_syntax_result, + supported_datasource_types, supported_parameter_syntax_topics, task_template_result, task_template_types_result, @@ -12,7 +16,7 @@ ) template_app = typer.Typer( - help="Emit stable YAML templates for workflow authoring.", + help="Emit stable templates for workflow authoring and DS-native payloads.", no_args_is_help=True, ) @@ -57,12 +61,49 @@ def params_command( emit_result("template.params", lambda: parameter_syntax_result(topic=topic)) +@template_app.command("environment") +def environment_command() -> None: + """Emit a DS environment shell/export config template.""" + emit_result("template.environment", environment_config_template_result) + + +@template_app.command("cluster") +def cluster_command() -> None: + """Emit a DS cluster config JSON template.""" + emit_result("template.cluster", cluster_config_template_result) + + +@template_app.command("datasource") +def datasource_command( + datasource_type: Annotated[ + str | None, + typer.Option( + "--type", + help=( + "Datasource type to template. Omit for compact type discovery. " + "Run `dsctl template datasource` or `dsctl enum list db-type` " + "for all values. Common: " + f"{', '.join(supported_datasource_types()[:6])}." + ), + ), + ] = None, +) -> None: + """Emit datasource JSON payload-template type discovery or one template.""" + emit_result( + "template.datasource", + lambda: datasource_template_result(datasource_type=datasource_type), + ) + + @template_app.command("task") def task_command( task_type: Annotated[ str | None, typer.Argument( - help="Task type to template, for example SHELL, PYTHON, SQL, or HTTP.", + help=( + "Task type to template. Required unless --list. Run " + "`dsctl template task --list` to inspect supported values." + ), ), ] = None, list_types: Annotated[ @@ -77,15 +118,18 @@ def task_command( typer.Option( "--variant", help=( - "Task template scenario, for example minimal, resource, " - "post-json, or branching." + "Task template scenario. Valid choices depend on the selected " + "task type. Known variants include minimal, params, resource, " + "post-json, pre-post-statements, branching, condition-routing, " + "workflow-dependency, child-workflow, and datasource; inspect " + "per-type values with `dsctl template task --list`." ), ), ] = None, ) -> None: - """Emit one task YAML template.""" + """Emit one task YAML template or list supported task types.""" if list_types: - emit_result("template.task_types", task_template_types_result) + emit_result("template.task", task_template_types_result) return emit_result( "template.task", diff --git a/src/dsctl/commands/tenant.py b/src/dsctl/commands/tenant.py index ec2a888..ca2d20a 100644 --- a/src/dsctl/commands/tenant.py +++ b/src/dsctl/commands/tenant.py @@ -20,6 +20,8 @@ no_args_is_help=True, ) +TENANT_HELP = "Tenant code or numeric id. Run `dsctl tenant list` to discover values." + def register_tenant_commands(app: typer.Typer) -> None: """Register the `tenant` command group.""" @@ -81,7 +83,7 @@ def get_command( ctx: typer.Context, tenant: Annotated[ str, - typer.Argument(help="Tenant code or numeric id."), + typer.Argument(help=TENANT_HELP), ], ) -> None: """Get one tenant by code or id.""" @@ -108,7 +110,10 @@ def create_command( str, typer.Option( "--queue", - help="Queue name or numeric id to bind to this tenant.", + help=( + "Queue name or numeric id to bind to this tenant. Run " + "`dsctl queue list` to discover values." + ), ), ], description: Annotated[ @@ -138,7 +143,7 @@ def update_command( ctx: typer.Context, tenant: Annotated[ str, - typer.Argument(help="Tenant code or numeric id."), + typer.Argument(help=TENANT_HELP), ], *, tenant_code: Annotated[ @@ -152,7 +157,10 @@ def update_command( str | None, typer.Option( "--queue", - help="Updated queue name or numeric id. Omit to keep the current queue.", + help=( + "Updated queue name or numeric id. Run `dsctl queue list` to " + "discover values; omit to keep the current queue." + ), ), ] = None, description: Annotated[ @@ -203,7 +211,7 @@ def delete_command( ctx: typer.Context, tenant: Annotated[ str, - typer.Argument(help="Tenant code or numeric id."), + typer.Argument(help=TENANT_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/use.py b/src/dsctl/commands/use.py index 9d32390..89e864b 100644 --- a/src/dsctl/commands/use.py +++ b/src/dsctl/commands/use.py @@ -54,7 +54,10 @@ def use_project_command( name: Annotated[ str | None, typer.Argument( - help="Project name to persist for later commands.", + help=( + "Project name to persist for later commands. Run `dsctl " + "project list` to discover values." + ), ), ] = None, *, @@ -96,7 +99,10 @@ def use_workflow_command( name: Annotated[ str | None, typer.Argument( - help="Workflow name to persist for later commands.", + help=( + "Workflow name to persist for later commands. Run `dsctl " + "workflow list` in the selected project to discover values." + ), ), ] = None, *, diff --git a/src/dsctl/commands/user.py b/src/dsctl/commands/user.py index 6f07f83..438ea9a 100644 --- a/src/dsctl/commands/user.py +++ b/src/dsctl/commands/user.py @@ -35,6 +35,8 @@ no_args_is_help=True, ) +USER_HELP = "User name or numeric id. Run `dsctl user list` to discover values." + def register_user_commands(app: typer.Typer) -> None: """Register the `user` command group.""" @@ -98,7 +100,7 @@ def get_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], ) -> None: """Get one user by name or id.""" @@ -139,7 +141,9 @@ def create_command( str, typer.Option( "--tenant", - help="Tenant code or numeric id.", + help=( + "Tenant code or numeric id. Run `dsctl tenant list` to discover values." + ), ), ], state_value: Annotated[ @@ -162,7 +166,10 @@ def create_command( str | None, typer.Option( "--queue", - help="Optional queue-name override stored on the user.", + help=( + "Optional queue-name override stored on the user. Run " + "`dsctl queue list` to discover queue names." + ), ), ] = None, ) -> None: @@ -189,7 +196,7 @@ def update_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, user_name: Annotated[ @@ -217,7 +224,10 @@ def update_command( str | None, typer.Option( "--tenant", - help="Updated tenant code or numeric id.", + help=( + "Updated tenant code or numeric id. Run `dsctl tenant list` to " + "discover values." + ), ), ] = None, state_value: Annotated[ @@ -247,7 +257,10 @@ def update_command( str | None, typer.Option( "--queue", - help="Updated queue-name override stored on the user.", + help=( + "Updated queue-name override stored on the user. Run " + "`dsctl queue list` to discover queue names." + ), ), ] = None, clear_queue: Annotated[ @@ -315,7 +328,7 @@ def delete_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, force: Annotated[ @@ -344,11 +357,16 @@ def grant_project_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], project: Annotated[ str, - typer.Argument(help="Project name or numeric code."), + typer.Argument( + help=( + "Project name or numeric code. Run `dsctl project list` " + "to discover values." + ) + ), ], ) -> None: """Grant one project to one user with write permission.""" @@ -369,14 +387,17 @@ def grant_datasource_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, datasource: Annotated[ list[str], typer.Option( "--datasource", - help="Datasource name or numeric id. Repeat to grant multiple datasources.", + help=( + "Datasource name or numeric id. Repeat to grant multiple " + "datasources; run `dsctl datasource list` to discover values." + ), ), ], ) -> None: @@ -398,14 +419,17 @@ def grant_namespace_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, namespace: Annotated[ list[str], typer.Option( "--namespace", - help="Namespace name or numeric id. Repeat to grant multiple namespaces.", + help=( + "Namespace name or numeric id. Repeat to grant multiple " + "namespaces; run `dsctl namespace list` to discover values." + ), ), ], ) -> None: @@ -427,11 +451,16 @@ def revoke_project_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], project: Annotated[ str, - typer.Argument(help="Project name or numeric code."), + typer.Argument( + help=( + "Project name or numeric code. Run `dsctl project list` " + "to discover values." + ) + ), ], ) -> None: """Revoke one project from one user.""" @@ -452,7 +481,7 @@ def revoke_datasource_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, datasource: Annotated[ @@ -460,7 +489,8 @@ def revoke_datasource_command( typer.Option( "--datasource", help=( - "Datasource name or numeric id. Repeat to revoke multiple datasources." + "Datasource name or numeric id. Repeat to revoke multiple " + "datasources; run `dsctl datasource list` to discover values." ), ), ], @@ -483,14 +513,17 @@ def revoke_namespace_command( ctx: typer.Context, user: Annotated[ str, - typer.Argument(help="User name or numeric id."), + typer.Argument(help=USER_HELP), ], *, namespace: Annotated[ list[str], typer.Option( "--namespace", - help="Namespace name or numeric id. Repeat to revoke multiple namespaces.", + help=( + "Namespace name or numeric id. Repeat to revoke multiple " + "namespaces; run `dsctl namespace list` to discover values." + ), ), ], ) -> None: diff --git a/src/dsctl/commands/worker_group.py b/src/dsctl/commands/worker_group.py index 4173816..aa579cd 100644 --- a/src/dsctl/commands/worker_group.py +++ b/src/dsctl/commands/worker_group.py @@ -21,6 +21,10 @@ no_args_is_help=True, ) +WORKER_GROUP_HELP = ( + "Worker-group name or numeric id. Run `dsctl worker-group list` to discover values." +) + def register_worker_group_commands(app: typer.Typer) -> None: """Register the `worker-group` command group.""" @@ -82,7 +86,7 @@ def get_command( ctx: typer.Context, worker_group: Annotated[ str, - typer.Argument(help="Worker-group name or numeric id."), + typer.Argument(help=WORKER_GROUP_HELP), ], ) -> None: """Get one worker group by name or id.""" @@ -109,7 +113,10 @@ def create_command( list[str] | None, typer.Option( "--addr", - help="Worker address to include in addrList. Repeat as needed.", + help=( + "Worker server address to include in addrList. Repeat as needed; " + "run `dsctl monitor server worker` to discover workers." + ), ), ] = None, description: Annotated[ @@ -139,7 +146,7 @@ def update_command( ctx: typer.Context, worker_group: Annotated[ str, - typer.Argument(help="Worker-group name or numeric id."), + typer.Argument(help=WORKER_GROUP_HELP), ], *, name: Annotated[ @@ -155,7 +162,8 @@ def update_command( "--addr", help=( "Replacement worker address list. Repeat as needed. Omit to keep " - "the current addrList." + "the current addrList; run `dsctl monitor server worker` to " + "discover workers." ), ), ] = None, @@ -225,7 +233,7 @@ def delete_command( ctx: typer.Context, worker_group: Annotated[ str, - typer.Argument(help="Worker-group name or numeric id."), + typer.Argument(help=WORKER_GROUP_HELP), ], *, force: Annotated[ diff --git a/src/dsctl/commands/workflow.py b/src/dsctl/commands/workflow.py index 927a3e4..48e4925 100644 --- a/src/dsctl/commands/workflow.py +++ b/src/dsctl/commands/workflow.py @@ -34,6 +34,40 @@ ) workflow_app.add_typer(workflow_lineage_app, name="lineage") +ENVIRONMENT_CODE_HELP = ( + "Environment code. Run `dsctl environment list` to discover values; omit to " + "allow enabled project preference." +) +PARAM_HELP = ( + "Workflow start parameter in KEY=VALUE form. Repeat for multiple parameters." +) +PROJECT_HELP = ( + "Project name or code. Run `dsctl project list` to discover values; falls " + "back to stored project context." +) +TASK_HELP = ( + "Task name or numeric code inside the selected workflow. Run `dsctl task " + "list` to discover values." +) +TENANT_HELP = ( + "Override the tenant code used to start the workflow instance. Run `dsctl " + "tenant list` to discover values; omit to allow enabled project preference " + "before the DS fallback `default` tenant." +) +WARNING_GROUP_HELP = ( + "Warning group id. Run `dsctl alert-group list` to discover ids; omit to " + "allow enabled project preference." +) +WORKER_GROUP_HELP = ( + "Override the worker group used to start the workflow instance. Run `dsctl " + "worker-group list` to discover values; omit to allow enabled project " + "preference before the DS fallback `default` worker group." +) +WORKFLOW_HELP = ( + "Workflow name or numeric code. Run `dsctl workflow list` in the selected " + "project to discover values; falls back to workflow context when omitted." +) + def register_workflow_commands(app: typer.Typer) -> None: """Register the `workflow` command group.""" @@ -48,7 +82,7 @@ def list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, search: Annotated[ @@ -78,10 +112,7 @@ def get_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -89,7 +120,7 @@ def get_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, output_format: Annotated[ @@ -121,10 +152,7 @@ def describe_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -132,7 +160,7 @@ def describe_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -155,10 +183,7 @@ def digest_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -166,7 +191,7 @@ def digest_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -191,7 +216,7 @@ def lineage_list_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -213,10 +238,7 @@ def lineage_get_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context " - "when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -224,7 +246,7 @@ def lineage_get_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -247,10 +269,7 @@ def lineage_dependent_tasks_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context " - "when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -258,14 +277,14 @@ def lineage_dependent_tasks_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, task: Annotated[ str | None, typer.Option( "--task", - help="Task name or numeric code inside the selected workflow.", + help=TASK_HELP, ), ] = None, ) -> None: @@ -306,7 +325,10 @@ def create_command( str | None, typer.Option( "--project", - help="Override workflow.project from the YAML file.", + help=( + "Override workflow.project from the YAML file. Run `dsctl " + "project list` to discover values." + ), ), ] = None, dry_run: Annotated[ @@ -348,10 +370,7 @@ def edit_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context " - "when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -374,7 +393,7 @@ def edit_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, dry_run: Annotated[ @@ -406,10 +425,7 @@ def online_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -417,7 +433,7 @@ def online_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -440,10 +456,7 @@ def offline_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -451,7 +464,7 @@ def offline_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, ) -> None: @@ -474,10 +487,7 @@ def run_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -485,29 +495,21 @@ def run_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, worker_group: Annotated[ str | None, typer.Option( "--worker-group", - help=( - "Override the worker group used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` worker group." - ), + help=WORKER_GROUP_HELP, ), ] = None, tenant: Annotated[ str | None, typer.Option( "--tenant", - help=( - "Override the tenant code used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` tenant." - ), + help=TENANT_HELP, ), ] = None, failure_strategy: Annotated[ @@ -541,24 +543,21 @@ def run_command( int | None, typer.Option( "--warning-group-id", - help="Warning group id. Omit to allow enabled project preference.", + help=WARNING_GROUP_HELP, ), ] = None, environment_code: Annotated[ int | None, typer.Option( "--environment-code", - help="Environment code. Omit to allow enabled project preference.", + help=ENVIRONMENT_CODE_HELP, ), ] = None, params: Annotated[ list[str] | None, typer.Option( "--param", - help=( - "Workflow start parameter in KEY=VALUE form. Repeat for multiple " - "parameters." - ), + help=PARAM_HELP, ), ] = None, dry_run: Annotated[ @@ -608,10 +607,7 @@ def run_task_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -619,14 +615,14 @@ def run_task_command( str, typer.Option( "--task", - help="Task name or task code within the workflow definition.", + help=TASK_HELP, ), ], project: Annotated[ str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, scope: Annotated[ @@ -640,22 +636,14 @@ def run_task_command( str | None, typer.Option( "--worker-group", - help=( - "Override the worker group used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` worker group." - ), + help=WORKER_GROUP_HELP, ), ] = None, tenant: Annotated[ str | None, typer.Option( "--tenant", - help=( - "Override the tenant code used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` tenant." - ), + help=TENANT_HELP, ), ] = None, failure_strategy: Annotated[ @@ -689,24 +677,21 @@ def run_task_command( int | None, typer.Option( "--warning-group-id", - help="Warning group id. Omit to allow enabled project preference.", + help=WARNING_GROUP_HELP, ), ] = None, environment_code: Annotated[ int | None, typer.Option( "--environment-code", - help="Environment code. Omit to allow enabled project preference.", + help=ENVIRONMENT_CODE_HELP, ), ] = None, params: Annotated[ list[str] | None, typer.Option( "--param", - help=( - "Workflow start parameter in KEY=VALUE form. Repeat for multiple " - "parameters." - ), + help=PARAM_HELP, ), ] = None, dry_run: Annotated[ @@ -758,10 +743,7 @@ def backfill_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -769,7 +751,7 @@ def backfill_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, start: Annotated[ @@ -800,7 +782,7 @@ def backfill_command( str | None, typer.Option( "--task", - help="Optional task name or task code to backfill from.", + help=TASK_HELP, ), ] = None, scope: Annotated[ @@ -849,22 +831,14 @@ def backfill_command( str | None, typer.Option( "--worker-group", - help=( - "Override the worker group used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` worker group." - ), + help=WORKER_GROUP_HELP, ), ] = None, tenant: Annotated[ str | None, typer.Option( "--tenant", - help=( - "Override the tenant code used to start the workflow instance. " - "Omit to allow enabled project preference before the DS " - "fallback `default` tenant." - ), + help=TENANT_HELP, ), ] = None, failure_strategy: Annotated[ @@ -898,24 +872,21 @@ def backfill_command( int | None, typer.Option( "--warning-group-id", - help="Warning group id. Omit to allow enabled project preference.", + help=WARNING_GROUP_HELP, ), ] = None, environment_code: Annotated[ int | None, typer.Option( "--environment-code", - help="Environment code. Omit to allow enabled project preference.", + help=ENVIRONMENT_CODE_HELP, ), ] = None, params: Annotated[ list[str] | None, typer.Option( "--param", - help=( - "Workflow start parameter in KEY=VALUE form. Repeat for multiple " - "parameters." - ), + help=PARAM_HELP, ), ] = None, dry_run: Annotated[ @@ -975,10 +946,7 @@ def delete_command( workflow: Annotated[ str | None, typer.Argument( - help=( - "Workflow name or numeric code. Falls back to workflow context" - " when omitted." - ), + help=WORKFLOW_HELP, ), ] = None, *, @@ -986,7 +954,7 @@ def delete_command( str | None, typer.Option( "--project", - help="Project name or code. Falls back to stored project context.", + help=PROJECT_HELP, ), ] = None, force: Annotated[ diff --git a/src/dsctl/commands/workflow_instance.py b/src/dsctl/commands/workflow_instance.py index 5e5e214..62613be 100644 --- a/src/dsctl/commands/workflow_instance.py +++ b/src/dsctl/commands/workflow_instance.py @@ -24,6 +24,32 @@ no_args_is_help=True, ) +PROJECT_FILTER_HELP = ( + "Project name or code for project-scoped filters. Run `dsctl project list` " + "to discover values." +) +WORKFLOW_FILTER_HELP = ( + "Workflow name or code filter. With --project, resolved inside that project; " + "run `dsctl workflow list` to discover values." +) +WORKFLOW_INSTANCE_HELP = ( + "Workflow instance id. Run `dsctl workflow-instance list` to discover ids." +) +SUB_WORKFLOW_INSTANCE_HELP = ( + "Sub-workflow instance id. Run `dsctl workflow-instance list` to discover ids." +) +FINISHED_WORKFLOW_INSTANCE_HELP = ( + "Finished workflow instance id. Run `dsctl workflow-instance list` to discover ids." +) +WORKFLOW_STATE_HELP = ( + "Filter by DS workflow execution status name. Run `dsctl enum list " + "workflow-execution-status` to discover values." +) +INSTANCE_TASK_HELP = ( + "Task name or task code within the workflow instance. Run `dsctl " + "task-instance list --workflow-instance WORKFLOW_INSTANCE` to discover values." +) + def register_workflow_instance_commands(app: typer.Typer) -> None: """Register the `workflow-instance` command group.""" @@ -53,21 +79,56 @@ def list_command( str | None, typer.Option( "--project", - help="Filter by project name.", + help=PROJECT_FILTER_HELP, ), ] = None, workflow: Annotated[ str | None, typer.Option( "--workflow", - help="Filter by workflow name.", + help=WORKFLOW_FILTER_HELP, + ), + ] = None, + search: Annotated[ + str | None, + typer.Option( + "--search", + help="Filter workflow instances by upstream searchVal; requires --project.", + ), + ] = None, + executor: Annotated[ + str | None, + typer.Option( + "--executor", + help="Filter by executor user name; requires --project.", + ), + ] = None, + host: Annotated[ + str | None, + typer.Option( + "--host", + help="Filter by workflow instance host.", + ), + ] = None, + start: Annotated[ + str | None, + typer.Option( + "--start", + help="Filter by start time lower bound, e.g. '2026-04-11 10:00:00'.", + ), + ] = None, + end: Annotated[ + str | None, + typer.Option( + "--end", + help="Filter by start time upper bound, e.g. '2026-04-11 11:00:00'.", ), ] = None, state: Annotated[ str | None, typer.Option( "--state", - help="Filter by DS workflow execution status name.", + help=WORKFLOW_STATE_HELP, ), ] = None, ) -> None: @@ -82,6 +143,11 @@ def list_command( all_pages=all_pages, project=project, workflow=workflow, + search=search, + executor=executor, + host=host, + start=start, + end=end, state=state, env_file=env_file, ), @@ -93,7 +159,7 @@ def get_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], ) -> None: """Get one workflow instance by id.""" @@ -113,7 +179,7 @@ def parent_command( ctx: typer.Context, sub_workflow_instance: Annotated[ int, - typer.Argument(help="Sub-workflow instance id."), + typer.Argument(help=SUB_WORKFLOW_INSTANCE_HELP), ], ) -> None: """Return the parent workflow instance for one sub-workflow instance.""" @@ -133,7 +199,7 @@ def digest_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], ) -> None: """Return one compact workflow-instance runtime digest.""" @@ -153,7 +219,7 @@ def update_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Finished workflow instance id."), + typer.Argument(help=FINISHED_WORKFLOW_INSTANCE_HELP), ], *, patch: Annotated[ @@ -206,7 +272,7 @@ def watch_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], interval_seconds: Annotated[ int, @@ -242,7 +308,7 @@ def stop_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], ) -> None: """Request stop for one workflow instance.""" @@ -262,7 +328,7 @@ def rerun_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], ) -> None: """Request rerun for one finished workflow instance.""" @@ -282,7 +348,7 @@ def recover_failed_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], ) -> None: """Recover one failed workflow instance from failed tasks.""" @@ -302,13 +368,13 @@ def execute_task_command( ctx: typer.Context, workflow_instance: Annotated[ int, - typer.Argument(help="Workflow instance id."), + typer.Argument(help=WORKFLOW_INSTANCE_HELP), ], task: Annotated[ str, typer.Option( "--task", - help="Task name or task code within the workflow definition.", + help=INSTANCE_TASK_HELP, ), ], scope: Annotated[ diff --git a/src/dsctl/errors.py b/src/dsctl/errors.py index 660d021..07beea4 100644 --- a/src/dsctl/errors.py +++ b/src/dsctl/errors.py @@ -39,7 +39,7 @@ "in the selected project", ), WORKFLOW_RESOURCE: ("workflow list", "code", "in the selected project"), - ENV_RESOURCE: ("env list", "code", None), + ENV_RESOURCE: ("environment list", "code", None), CLUSTER_RESOURCE: ("cluster list", "code", None), DATASOURCE_RESOURCE: ("datasource list", "id", None), NAMESPACE_RESOURCE: ("namespace list", "id", None), @@ -128,6 +128,12 @@ class InvalidStateError(DsctlError): error_type = "invalid_state" +class TaskNotDispatchedError(DsctlError): + """Raised when DS has not dispatched a task instance and no log exists yet.""" + + error_type = "task_not_dispatched" + + class ConfirmationRequiredError(DsctlError): """Raised when a risky mutation requires one explicit confirmation token.""" diff --git a/src/dsctl/output_formats.py b/src/dsctl/output_formats.py new file mode 100644 index 0000000..724d5bd --- /dev/null +++ b/src/dsctl/output_formats.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, TypeAlias, TypeGuard, cast + +from dsctl.errors import UserInputError +from dsctl.services._data_shapes import DataShape, data_shape_for_action + +if TYPE_CHECKING: + from dsctl.support.json_types import JsonObject, JsonValue + +OutputFormat: TypeAlias = Literal["json", "table", "tsv"] +OUTPUT_FORMAT_CHOICES: tuple[OutputFormat, ...] = ("json", "table", "tsv") + + +@dataclass(frozen=True) +class RenderOptions: + """Resolved display settings for one command invocation.""" + + output_format: OutputFormat = "json" + columns: tuple[str, ...] = () + + +def parse_columns(value: str | None) -> tuple[str, ...]: + """Parse a comma-separated display-column option.""" + if value is None: + return () + columns = tuple(item.strip() for item in value.split(",") if item.strip()) + if not columns: + message = "--columns must include at least one column name" + raise UserInputError( + message, + suggestion="Pass a comma-separated list such as `--columns id,name,state`.", + ) + return columns + + +def validate_render_options(options: RenderOptions) -> RenderOptions: + """Reject ambiguous global display settings before running a command.""" + if options.columns and "*" in options.columns: + _validate_wildcard_columns(options.columns) + return options + + +def render_payload( + payload: JsonObject, + *, + action: str, + options: RenderOptions, +) -> str: + """Render one standard output envelope using the requested format.""" + if options.output_format == "json": + if options.columns and payload.get("ok"): + payload = _project_json_payload( + payload, + action=action, + columns=options.columns, + ) + return _render_json(payload) + if not payload.get("ok"): + return _render_error_payload(payload, output_format=options.output_format) + return _render_success_payload(payload, action=action, options=options) + + +def _render_json(payload: JsonValue) -> str: + return json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=True) + + +def _render_success_payload( + payload: JsonObject, + *, + action: str, + options: RenderOptions, +) -> str: + data = payload.get("data") + shape = data_shape_for_action(action) + rows = _extract_rows(data, shape=shape) + if rows is not None: + columns = _resolve_columns( + rows, + requested=options.columns, + defaults=() if shape is None else shape.default_columns, + action=action, + ) + return _render_rows(rows, columns=columns, output_format=options.output_format) + + if isinstance(data, Mapping): + if options.columns: + row = _mapping_to_json_object(data) + rows = (row,) + columns = _resolve_columns( + rows, + requested=options.columns, + defaults=(), + action=action, + ) + return _render_rows( + rows, + columns=columns, + output_format=options.output_format, + ) + key_value_rows = _object_rows(data) + return _render_rows( + key_value_rows, + columns=("field", "value"), + output_format=options.output_format, + ) + + scalar_rows: tuple[JsonObject, ...] = ( + {"field": "data", "value": _format_cell(data)}, + ) + return _render_rows( + scalar_rows, + columns=("field", "value"), + output_format=options.output_format, + ) + + +def _render_error_payload(payload: JsonObject, *, output_format: OutputFormat) -> str: + rows: list[JsonObject] = [ + {"field": "ok", "value": "false"}, + {"field": "action", "value": _format_cell(payload.get("action"))}, + ] + error = payload.get("error") + if isinstance(error, Mapping): + rows.extend( + {"field": f"error.{key}", "value": _format_cell(error[key])} + for key in ("type", "message", "suggestion") + if key in error + ) + source = error.get("source") + if isinstance(source, Mapping): + for key, value in source.items(): + rows.append( + { + "field": f"error.source.{key}", + "value": _format_cell(value), + } + ) + return _render_rows(rows, columns=("field", "value"), output_format=output_format) + + +def _project_json_payload( + payload: JsonObject, + *, + action: str, + columns: tuple[str, ...], +) -> JsonObject: + """Return a copy of a success envelope with row/object data projected.""" + projected = deepcopy(payload) + shape = data_shape_for_action(action) + if shape is not None and shape.row_path is not None: + value = _value_at_path(projected, shape.row_path) + replacement = _project_json_value(value, columns=columns, action=action) + if _replace_value_at_path(projected, shape.row_path, replacement): + return projected + raise _projection_not_supported_error(action=action, columns=columns) + + data = projected.get("data") + if isinstance(data, Mapping): + total_list = data.get("totalList") + if _is_sequence_like(total_list): + replacement = _project_json_value( + total_list, + columns=columns, + action=action, + ) + data_copy = dict(data) + data_copy["totalList"] = replacement + projected["data"] = data_copy + return projected + projected["data"] = _project_json_value( + data, + columns=columns, + action=action, + ) + return projected + + projected["data"] = _project_json_value(data, columns=columns, action=action) + return projected + + +def _project_json_value( + value: JsonValue | None, + *, + columns: tuple[str, ...], + action: str, +) -> JsonValue: + if isinstance(value, Mapping): + row = _mapping_to_json_object(value) + resolved = _resolve_columns( + (row,), + requested=columns, + defaults=(), + action=action, + ) + return _project_row(row, resolved) + + if _is_sequence_like(value): + rows = _json_rows_for_projection(value, action=action, columns=columns) + resolved = _resolve_columns(rows, requested=columns, defaults=(), action=action) + return [_project_row(row, resolved) for row in rows] + + raise _projection_not_supported_error(action=action, columns=columns) + + +def _json_rows_for_projection( + value: JsonValue | None, + *, + action: str, + columns: tuple[str, ...], +) -> tuple[JsonObject, ...]: + if not _is_sequence_like(value): + raise _projection_not_supported_error(action=action, columns=columns) + rows: list[JsonObject] = [] + for item in value: + if not isinstance(item, Mapping): + raise _projection_not_supported_error(action=action, columns=columns) + rows.append(_mapping_to_json_object(item)) + return tuple(rows) + + +def _project_row(row: JsonObject, columns: tuple[str, ...]) -> JsonObject: + return {column: row[column] for column in columns if column in row} + + +def _replace_value_at_path(root: JsonObject, path: str, value: JsonValue) -> bool: + current: JsonValue = root + parts = path.split(".") + for part in parts[:-1]: + if not isinstance(current, dict): + return False + current = current.get(part) + if not isinstance(current, dict): + return False + current[parts[-1]] = value + return True + + +def _is_sequence_like(value: JsonValue | None) -> TypeGuard[Sequence[JsonValue]]: + return isinstance(value, Sequence) and not isinstance( + value, + (str, bytes, bytearray), + ) + + +def _projection_not_supported_error( + *, + action: str, + columns: tuple[str, ...], +) -> UserInputError: + message = f"--columns can only project object or row-oriented output for {action}" + return UserInputError( + message, + details={"action": action, "columns": list(columns)}, + suggestion=( + f"Run `dsctl schema --command {action}` and inspect data_shape, " + "or omit --columns for this command." + ), + ) + + +def _extract_rows( + data: JsonValue | None, + *, + shape: DataShape | None, +) -> tuple[JsonObject, ...] | None: + if shape is not None and shape.row_path is not None: + value = _value_at_path({"data": data}, shape.row_path) + if shape.kind == "object" and isinstance(value, Mapping): + return (_mapping_to_json_object(value),) + rows = _coerce_rows(value) + if rows is not None: + return rows + + if isinstance(data, Mapping): + total_list = data.get("totalList") + rows = _coerce_rows(total_list) + if rows is not None: + return rows + return None + return _coerce_rows(data) + + +def _value_at_path(root: Mapping[str, JsonValue | None], path: str) -> JsonValue | None: + current: JsonValue | None = cast("JsonValue | None", root) + for part in path.split("."): + if not isinstance(current, Mapping): + return None + current = current.get(part) + return current + + +def _coerce_rows(value: JsonValue | None) -> tuple[JsonObject, ...] | None: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)): + return None + rows: list[JsonObject] = [] + for item in value: + if isinstance(item, Mapping): + rows.append(_mapping_to_json_object(item)) + else: + rows.append({"value": _format_cell(item)}) + return tuple(rows) + + +def _mapping_to_json_object(value: Mapping[str, JsonValue]) -> JsonObject: + return {str(key): item for key, item in value.items()} + + +def _object_rows(data: Mapping[str, JsonValue]) -> tuple[JsonObject, ...]: + return tuple( + {"field": key, "value": _format_cell(value)} for key, value in data.items() + ) + + +def _resolve_columns( + rows: Sequence[JsonObject], + *, + requested: tuple[str, ...], + defaults: tuple[str, ...], + action: str, +) -> tuple[str, ...]: + if requested: + if "*" in requested: + _validate_wildcard_columns(requested) + return _infer_columns(rows) + _validate_requested_columns(rows, requested, action=action) + return requested + if defaults and _any_column_present(rows, defaults): + return defaults + return _infer_columns(rows) + + +def _validate_wildcard_columns(columns: tuple[str, ...]) -> None: + if columns == ("*",): + return + message = "--columns '*' cannot be combined with explicit columns" + raise UserInputError( + message, + details={"columns": list(columns), "wildcard": "*"}, + suggestion=( + "Use `--columns '*'` for all row fields, or pass explicit columns " + "such as `--columns id,name,state`." + ), + ) + + +def _validate_requested_columns( + rows: Sequence[JsonObject], + columns: tuple[str, ...], + *, + action: str, +) -> None: + if not rows: + return + missing = [column for column in columns if not any(column in row for row in rows)] + if not missing: + return + message = f"Unknown display column for {action}: {', '.join(missing)}" + raise UserInputError( + message, + details={ + "action": action, + "columns": list(columns), + "unknown_columns": missing, + "available_columns": _infer_columns(rows), + }, + suggestion=( + f"Run `dsctl schema --command {action}` and inspect data_shape, " + "or retry with columns present in the JSON row payload." + ), + ) + + +def _any_column_present(rows: Sequence[JsonObject], columns: tuple[str, ...]) -> bool: + if not rows: + return True + return any(column in row for row in rows for column in columns) + + +def _infer_columns(rows: Sequence[JsonObject]) -> tuple[str, ...]: + columns: list[str] = [] + for row in rows: + for key in row: + if key not in columns: + columns.append(key) + return tuple(columns) + + +def _render_rows( + rows: Sequence[JsonObject], + *, + columns: tuple[str, ...], + output_format: OutputFormat, +) -> str: + if output_format == "tsv": + return _render_tsv(rows, columns=columns) + return _render_table(rows, columns=columns) + + +def _render_tsv(rows: Sequence[JsonObject], *, columns: tuple[str, ...]) -> str: + lines = ["\t".join(columns)] + lines.extend( + "\t".join(_format_tsv_cell(row.get(column)) for column in columns) + for row in rows + ) + return "\n".join(lines) + + +def _render_table(rows: Sequence[JsonObject], *, columns: tuple[str, ...]) -> str: + if not columns: + return "(no rows)" + rendered_rows = [ + [_format_cell(row.get(column)) for column in columns] for row in rows + ] + widths = [ + max((len(column), *(len(row[index]) for row in rendered_rows))) + for index, column in enumerate(columns) + ] + header = _render_table_line(columns, widths=widths) + separator = "-+-".join("-" * width for width in widths) + lines = [header, separator] + lines.extend(_render_table_line(tuple(row), widths=widths) for row in rendered_rows) + return "\n".join(lines) + + +def _render_table_line(values: tuple[str, ...], *, widths: Sequence[int]) -> str: + padded: list[str] = [] + for index, value in enumerate(values): + if index == len(values) - 1: + padded.append(value) + else: + padded.append(value.ljust(widths[index])) + return " | ".join(padded) + + +def _format_tsv_cell(value: JsonValue | None) -> str: + return _format_cell(value).replace("\t", " ").replace("\n", " ") + + +def _format_cell(value: JsonValue | None) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + return json.dumps(value, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + + +__all__ = [ + "OUTPUT_FORMAT_CHOICES", + "OutputFormat", + "RenderOptions", + "parse_columns", + "render_payload", + "validate_render_options", +] diff --git a/src/dsctl/services/_data_shapes.py b/src/dsctl/services/_data_shapes.py new file mode 100644 index 0000000..8a70b4c --- /dev/null +++ b/src/dsctl/services/_data_shapes.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, TypedDict + +DataShapeKind = Literal["page", "collection", "object", "summary"] + + +class DataShapeSchema(TypedDict, total=False): + """Schema payload describing how callers can read command data rows.""" + + kind: str + row_path: str + default_columns: list[str] + column_discovery: str + + +@dataclass(frozen=True) +class DataShape: + """One low-entropy row model shared by schema and display rendering.""" + + kind: DataShapeKind + row_path: str | None = None + default_columns: tuple[str, ...] = () + column_discovery: str = "runtime_row_keys" + + def to_schema(self) -> DataShapeSchema: + """Return the JSON-safe schema representation for this shape.""" + payload = DataShapeSchema(kind=self.kind) + if self.row_path is not None: + payload["row_path"] = self.row_path + if self.default_columns: + payload["default_columns"] = list(self.default_columns) + payload["column_discovery"] = self.column_discovery + return payload + + +PAGE_LIST_DEFAULTS: dict[str, tuple[str, ...]] = { + "access-token.list": ("id", "userName", "expireTime"), + "alert-group.list": ("id", "groupName", "description"), + "alert-plugin.list": ("id", "pluginInstanceName", "pluginDefineName"), + "audit.list": ("id", "modelName", "operationType", "userName", "createTime"), + "cluster.list": ("code", "name", "config"), + "datasource.list": ("id", "name", "type", "createTime"), + "environment.list": ("code", "name", "workerGroups", "description"), + "namespace.list": ("id", "namespace", "clusterName"), + "project-parameter.list": ("code", "paramName", "paramValue", "paramDataType"), + "project.list": ("code", "name", "description"), + "queue.list": ("id", "queueName", "queue"), + "resource.list": ("fullName", "fileName", "type", "size"), + "schedule.list": ( + "id", + "workflowDefinitionName", + "releaseState", + "startTime", + "endTime", + ), + "task-group.list": ("id", "name", "groupSize", "status"), + "task-group.queue.list": ("id", "taskName", "workflowInstanceName", "status"), + "task-instance.list": ( + "id", + "name", + "state", + "taskType", + "startTime", + "endTime", + "duration", + "host", + ), + "tenant.list": ("id", "tenantCode", "queueName", "description"), + "user.list": ("id", "userName", "userType", "tenantCode", "state"), + "worker-group.list": ("id", "name", "addrList", "description"), + "workflow-instance.list": ( + "id", + "name", + "state", + "scheduleTime", + "startTime", + "endTime", + "duration", + "host", + ), +} + +COLLECTION_DEFAULTS: dict[str, tuple[str, ...]] = { + "audit.model-types": ("name",), + "audit.operation-types": ("name",), + "enum.names": ("name", "list_command"), + "monitor.database": ("dbType", "state", "threadsConnections", "date"), + "monitor.server": ("id", "host", "port", "lastHeartbeatTime"), + "namespace.available": ("id", "namespace", "clusterName"), + "project-worker-group.list": ("id", "workerGroup", "projectCode"), + "task.list": ("code", "name", "version"), + "workflow.lineage.dependent-tasks": ( + "workflowDefinitionName", + "taskDefinitionName", + "projectCode", + ), + "workflow.list": ("code", "name", "version"), +} + +OBJECT_DEFAULTS: dict[str, tuple[str, ...]] = { + **{ + action.removesuffix(".list") + ".get": columns + for action, columns in PAGE_LIST_DEFAULTS.items() + }, + **{ + action.removesuffix(".list") + ".get": columns + for action, columns in COLLECTION_DEFAULTS.items() + if action.endswith(".list") + }, + "datasource.get": ("id", "name", "type", "host", "port", "database"), + "project-preference.get": (), + "workflow.lineage.get": (), + "workflow.describe": ("workflow", "tasks", "relations"), + "workflow.digest": ( + "taskCount", + "relationCount", + "taskTypeCounts", + "rootTasks", + "leafTasks", + ), + "workflow-instance.digest": ( + "taskCount", + "progress", + "taskStateCounts", + "runningTasks", + "failedTasks", + ), +} + +NESTED_ROW_SHAPES: dict[str, DataShape] = { + "alert-plugin.definition.list": DataShape( + kind="summary", + row_path="data.definitions", + default_columns=("id", "pluginName", "pluginType"), + ), + "doctor": DataShape( + kind="summary", + row_path="data.checks", + default_columns=("name", "status", "message", "suggestion"), + ), + "enum.list": DataShape( + kind="summary", + row_path="data.members", + default_columns=("name", "value", "attributes"), + ), + "schema": DataShape( + kind="summary", + row_path="data.rows", + ), + "task-type.list": DataShape( + kind="summary", + row_path="data.taskTypes", + default_columns=("taskType", "taskCategory", "isCollection"), + ), + "template.environment": DataShape( + kind="summary", + row_path="data.lines", + default_columns=("line", "purpose"), + ), + "template.cluster": DataShape( + kind="summary", + row_path="data.fields", + default_columns=("name", "required", "value_type", "description"), + ), + "template.datasource": DataShape( + kind="summary", + row_path="data.rows", + ), + "template.task": DataShape( + kind="summary", + row_path="data.rows", + ), + "template.workflow": DataShape( + kind="summary", + row_path="data.lines", + default_columns=("line_no", "line"), + ), + "workflow.lineage.list": DataShape( + kind="summary", + row_path="data.workFlowRelationDetailList", + default_columns=("workFlowCode", "workFlowName", "workFlowPublishStatus"), + ), +} + +DATA_SHAPES: dict[str, DataShape] = { + **{ + action: DataShape( + kind="page", + row_path="data.totalList", + default_columns=columns, + ) + for action, columns in PAGE_LIST_DEFAULTS.items() + }, + **{ + action: DataShape( + kind="collection", + row_path="data", + default_columns=columns, + ) + for action, columns in COLLECTION_DEFAULTS.items() + }, + **{ + action: DataShape( + kind="object", + row_path="data", + default_columns=columns, + ) + for action, columns in OBJECT_DEFAULTS.items() + }, + **NESTED_ROW_SHAPES, +} + + +def data_shape_for_action(action: str) -> DataShape | None: + """Return display/schema row metadata for one stable command action.""" + return DATA_SHAPES.get(action) + + +def data_shape_schema_for_action(action: str) -> DataShapeSchema | None: + """Return one JSON-safe schema data-shape payload when available.""" + shape = data_shape_for_action(action) + if shape is None: + return None + return shape.to_schema() + + +__all__ = [ + "DataShape", + "DataShapeSchema", + "data_shape_for_action", + "data_shape_schema_for_action", +] diff --git a/src/dsctl/services/_schema_groups_context.py b/src/dsctl/services/_schema_groups_context.py index 682521a..c7854a0 100644 --- a/src/dsctl/services/_schema_groups_context.py +++ b/src/dsctl/services/_schema_groups_context.py @@ -46,9 +46,13 @@ def use_group() -> dict[str, object]: argument( "name", value_type="string", - description="Project name to persist. Required unless --clear.", + description=( + "Project name to persist. Run `dsctl project list` " + "to discover values. Required unless --clear." + ), required=False, selector="opaque_name", + discovery_command="dsctl project list", ) ], options=use_target_options( @@ -67,10 +71,13 @@ def use_group() -> dict[str, object]: "name", value_type="string", description=( - "Workflow name to persist. Required unless --clear." + "Workflow name to persist. Run `dsctl workflow list` " + "in the selected project to discover values. Required " + "unless --clear." ), required=False, selector="opaque_name", + discovery_command="dsctl workflow list", ) ], options=use_target_options( @@ -129,8 +136,12 @@ def project_group() -> dict[str, object]: argument( "project", value_type="string", - description="Project name or numeric code.", + description=( + "Project name or numeric code. Run `dsctl project " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ) ], ), @@ -160,8 +171,12 @@ def project_group() -> dict[str, object]: argument( "project", value_type="string", - description="Project name or numeric code.", + description=( + "Project name or numeric code. Run `dsctl project " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ) ], options=[ @@ -193,8 +208,12 @@ def project_group() -> dict[str, object]: argument( "project", value_type="string", - description="Project name or numeric code.", + description=( + "Project name or numeric code. Run `dsctl project " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ) ], options=[ @@ -234,8 +253,10 @@ def project_parameter_group() -> dict[str, object]: "data-type", value_type="string", description=( - "Filter project parameters by DS projectParameterDataType." + "Filter by DS projectParameterDataType. Run `dsctl " + "enum list data-type` to discover values." ), + discovery_command="dsctl enum list data-type", ), option( "page-no", @@ -265,8 +286,13 @@ def project_parameter_group() -> dict[str, object]: argument( "project-parameter", value_type="string", - description="Project parameter name or numeric code.", + description=( + "Project parameter name or numeric code. Run `dsctl " + "project-parameter list` in the selected project to " + "discover values." + ), selector="name_or_code", + discovery_command="dsctl project-parameter list", ) ], options=[project_option()], @@ -292,8 +318,12 @@ def project_parameter_group() -> dict[str, object]: option( "data-type", value_type="string", - description="DS projectParameterDataType value.", + description=( + "DS projectParameterDataType value. Run `dsctl enum " + "list data-type` to discover values." + ), default="VARCHAR", + discovery_command="dsctl enum list data-type", ), ], ), @@ -305,8 +335,13 @@ def project_parameter_group() -> dict[str, object]: argument( "project-parameter", value_type="string", - description="Project parameter name or numeric code.", + description=( + "Project parameter name or numeric code. Run `dsctl " + "project-parameter list` in the selected project to " + "discover values." + ), selector="name_or_code", + discovery_command="dsctl project-parameter list", ) ], options=[ @@ -328,7 +363,11 @@ def project_parameter_group() -> dict[str, object]: option( "data-type", value_type="string", - description="Updated DS projectParameterDataType value.", + description=( + "Updated DS projectParameterDataType value. Run " + "`dsctl enum list data-type` to discover values." + ), + discovery_command="dsctl enum list data-type", ), ], ), @@ -340,8 +379,13 @@ def project_parameter_group() -> dict[str, object]: argument( "project-parameter", value_type="string", - description="Project parameter name or numeric code.", + description=( + "Project parameter name or numeric code. Run `dsctl " + "project-parameter list` in the selected project to " + "discover values." + ), selector="name_or_code", + discovery_command="dsctl project-parameter list", ) ], options=[ @@ -448,9 +492,11 @@ def project_worker_group_group() -> dict[str, object]: value_type="string", description=( "Worker group to keep assigned to this project. Repeat " - "as needed." + "as needed; run `dsctl worker-group list` to discover " + "values." ), multiple=True, + discovery_command="dsctl worker-group list", ), ], ), diff --git a/src/dsctl/services/_schema_groups_design.py b/src/dsctl/services/_schema_groups_design.py index 9e687da..877d138 100644 --- a/src/dsctl/services/_schema_groups_design.py +++ b/src/dsctl/services/_schema_groups_design.py @@ -11,6 +11,7 @@ ) from dsctl.services.pagination import DEFAULT_PAGE_SIZE from dsctl.services.template import ( + supported_datasource_types, supported_parameter_syntax_topics, supported_task_template_variants, ) @@ -70,8 +71,11 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Schedule id.", + description=( + "Schedule id. Use `dsctl schedule list` to discover values." + ), selector="id", + discovery_command="dsctl schedule list", ) ], ), @@ -83,9 +87,13 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Existing schedule id to preview.", + description=( + "Existing schedule id to preview. Use `dsctl schedule " + "list` to discover values." + ), required=False, selector="id", + discovery_command="dsctl schedule list", ) ], options=[ @@ -123,9 +131,13 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Existing schedule id to explain as an update.", + description=( + "Existing schedule id to explain as an update. Use " + "`dsctl schedule list` to discover values." + ), required=False, selector="id", + discovery_command="dsctl schedule list", ) ], options=[ @@ -162,11 +174,15 @@ def schedule_group() -> dict[str, object]: "failure-strategy", value_type="string", description="Failure strategy: CONTINUE or END.", + choices=["CONTINUE", "END"], + discovery_command="dsctl enum list failure-strategy", ), option( "warning-type", value_type="string", description="Warning type: NONE, SUCCESS, FAILURE, or ALL.", + choices=["NONE", "SUCCESS", "FAILURE", "ALL"], + discovery_command="dsctl enum list warning-type", ), option( "warning-group-id", @@ -176,6 +192,7 @@ def schedule_group() -> dict[str, object]: "for update explain. Create explain can also inherit an " "enabled project preference when omitted." ), + discovery_command="dsctl alert-group list", ), option( "priority", @@ -184,6 +201,8 @@ def schedule_group() -> dict[str, object]: "Workflow instance priority: HIGHEST, HIGH, MEDIUM, LOW, " "or LOWEST." ), + choices=["HIGHEST", "HIGH", "MEDIUM", "LOW", "LOWEST"], + discovery_command="dsctl enum list priority", ), option( "worker-group", @@ -193,6 +212,7 @@ def schedule_group() -> dict[str, object]: "update explain. Create explain can also inherit an " "enabled project preference when omitted." ), + discovery_command="dsctl worker-group list", ), option( "tenant-code", @@ -201,6 +221,7 @@ def schedule_group() -> dict[str, object]: "Tenant code for create explain. Create explain can " "also inherit an enabled project preference when omitted." ), + discovery_command="dsctl tenant list", ), option( "environment-code", @@ -210,6 +231,7 @@ def schedule_group() -> dict[str, object]: "for update explain. Create explain can also inherit " "an enabled project preference when omitted." ), + discovery_command="dsctl environment list", ), ], ), @@ -254,11 +276,15 @@ def schedule_group() -> dict[str, object]: "failure-strategy", value_type="string", description="Failure strategy: CONTINUE or END.", + choices=["CONTINUE", "END"], + discovery_command="dsctl enum list failure-strategy", ), option( "warning-type", value_type="string", description="Warning type: NONE, SUCCESS, FAILURE, or ALL.", + choices=["NONE", "SUCCESS", "FAILURE", "ALL"], + discovery_command="dsctl enum list warning-type", ), option( "warning-group-id", @@ -267,6 +293,7 @@ def schedule_group() -> dict[str, object]: "Warning group id. Omit to keep the CLI fallback " "chain, including enabled project preference." ), + discovery_command="dsctl alert-group list", ), option( "priority", @@ -275,6 +302,8 @@ def schedule_group() -> dict[str, object]: "Workflow instance priority: HIGHEST, HIGH, MEDIUM, LOW, " "or LOWEST." ), + choices=["HIGHEST", "HIGH", "MEDIUM", "LOW", "LOWEST"], + discovery_command="dsctl enum list priority", ), option( "worker-group", @@ -282,6 +311,7 @@ def schedule_group() -> dict[str, object]: description=( "Worker group. Omit to allow enabled project preference." ), + discovery_command="dsctl worker-group list", ), option( "tenant-code", @@ -289,6 +319,7 @@ def schedule_group() -> dict[str, object]: description=( "Tenant code. Omit to allow enabled project preference." ), + discovery_command="dsctl tenant list", ), option( "environment-code", @@ -297,6 +328,7 @@ def schedule_group() -> dict[str, object]: "Environment code. Omit to keep the CLI fallback " "chain, including enabled project preference." ), + discovery_command="dsctl environment list", ), confirm_risk_option(), ], @@ -309,8 +341,11 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Schedule id.", + description=( + "Schedule id. Use `dsctl schedule list` to discover values." + ), selector="id", + discovery_command="dsctl schedule list", ) ], options=[ @@ -348,11 +383,15 @@ def schedule_group() -> dict[str, object]: "failure-strategy", value_type="string", description="Failure strategy: CONTINUE or END.", + choices=["CONTINUE", "END"], + discovery_command="dsctl enum list failure-strategy", ), option( "warning-type", value_type="string", description="Warning type: NONE, SUCCESS, FAILURE, or ALL.", + choices=["NONE", "SUCCESS", "FAILURE", "ALL"], + discovery_command="dsctl enum list warning-type", ), option( "warning-group-id", @@ -360,6 +399,7 @@ def schedule_group() -> dict[str, object]: description=( "Updated warning group id. Omit to keep the current value." ), + discovery_command="dsctl alert-group list", ), option( "priority", @@ -368,6 +408,8 @@ def schedule_group() -> dict[str, object]: "Workflow instance priority: HIGHEST, HIGH, MEDIUM, LOW, " "or LOWEST." ), + choices=["HIGHEST", "HIGH", "MEDIUM", "LOW", "LOWEST"], + discovery_command="dsctl enum list priority", ), option( "worker-group", @@ -375,6 +417,7 @@ def schedule_group() -> dict[str, object]: description=( "Updated worker group. Omit to keep the current value." ), + discovery_command="dsctl worker-group list", ), option( "environment-code", @@ -382,6 +425,7 @@ def schedule_group() -> dict[str, object]: description=( "Updated environment code. Omit to keep the current value." ), + discovery_command="dsctl environment list", ), confirm_risk_option(), ], @@ -394,8 +438,11 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Schedule id.", + description=( + "Schedule id. Use `dsctl schedule list` to discover values." + ), selector="id", + discovery_command="dsctl schedule list", ) ], options=[ @@ -415,8 +462,11 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Schedule id.", + description=( + "Schedule id. Use `dsctl schedule list` to discover values." + ), selector="id", + discovery_command="dsctl schedule list", ) ], ), @@ -428,8 +478,11 @@ def schedule_group() -> dict[str, object]: argument( "schedule_id", value_type="integer", - description="Schedule id.", + description=( + "Schedule id. Use `dsctl schedule list` to discover values." + ), selector="id", + discovery_command="dsctl schedule list", ) ], ), @@ -441,7 +494,7 @@ def template_group(task_types: list[str]) -> dict[str, object]: """Build the template command group schema.""" return group( "template", - summary="Emit stable YAML templates for workflow authoring.", + summary="Emit stable templates for workflow authoring and DS-native payloads.", commands=[ command( "workflow", @@ -471,6 +524,37 @@ def template_group(task_types: list[str]) -> dict[str, object]: "Parameter syntax topic. Omit for compact discovery." ), choices=supported_parameter_syntax_topics(), + discovery_command="dsctl template params", + ) + ], + ), + command( + "environment", + action="template.environment", + summary="Emit a DS environment shell/export config template.", + ), + command( + "cluster", + action="template.cluster", + summary="Emit a DS cluster config JSON template.", + ), + command( + "datasource", + action="template.datasource", + summary=( + "Emit datasource JSON payload-template type discovery or one " + "template." + ), + options=[ + option( + "type", + value_type="string", + description=( + "Datasource type. Omit for discovery; full values via " + "`dsctl enum list db-type`." + ), + choices=supported_datasource_types(), + discovery_command="dsctl template datasource", ) ], ), @@ -485,6 +569,7 @@ def template_group(task_types: list[str]) -> dict[str, object]: description="Task type to template. Required unless --list.", required=False, choices=task_types, + discovery_command="dsctl template task --list", ) ], options=[ @@ -502,10 +587,15 @@ def template_group(task_types: list[str]) -> dict[str, object]: value_type="string", description=( "Task template scenario. Valid choices depend on " - "the selected task type and are discoverable through " - "`template task --list`." + "the selected task type. Known variants include " + "minimal, params, resource, post-json, " + "pre-post-statements, branching, condition-routing, " + "workflow-dependency, child-workflow, and datasource; " + "inspect per-type values with `dsctl template task " + "--list`." ), choices=supported_task_template_variants(), + discovery_command="dsctl template task --list", ), ], ), @@ -549,6 +639,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -576,6 +667,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[project_option()], @@ -594,6 +686,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[project_option()], @@ -616,8 +709,12 @@ def workflow_group() -> dict[str, object]: option( "project", value_type="string", - description="Override workflow.project from the YAML file.", + description=( + "Override workflow.project from the YAML file. Run " + "`dsctl project list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ), option( "dry-run", @@ -645,6 +742,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -683,6 +781,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[project_option()], @@ -701,6 +800,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[project_option()], @@ -722,6 +822,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -735,6 +836,7 @@ def workflow_group() -> dict[str, object]: "preference before the DS fallback `default` worker " "group." ), + discovery_command="dsctl worker-group list", ), option( "tenant", @@ -744,6 +846,7 @@ def workflow_group() -> dict[str, object]: "workflow instance. Omit to allow enabled project " "preference before the DS fallback `default` tenant." ), + discovery_command="dsctl tenant list", ), option( "failure-strategy", @@ -773,6 +876,7 @@ def workflow_group() -> dict[str, object]: "Warning group id. Omit to allow enabled project " "preference." ), + discovery_command="dsctl alert-group list", ), option( "environment-code", @@ -781,6 +885,7 @@ def workflow_group() -> dict[str, object]: "Environment code. Omit to allow enabled project " "preference." ), + discovery_command="dsctl environment list", ), option( "param", @@ -825,6 +930,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -832,10 +938,12 @@ def workflow_group() -> dict[str, object]: "task", value_type="string", description=( - "Task name or task code within the workflow definition." + "Task name or task code within the workflow " + "definition. Run `dsctl task list` to discover values." ), required=True, selector="name_or_code", + discovery_command="dsctl task list", ), project_option(), option( @@ -854,6 +962,7 @@ def workflow_group() -> dict[str, object]: "preference before the DS fallback `default` worker " "group." ), + discovery_command="dsctl worker-group list", ), option( "tenant", @@ -863,6 +972,7 @@ def workflow_group() -> dict[str, object]: "workflow instance. Omit to allow enabled project " "preference before the DS fallback `default` tenant." ), + discovery_command="dsctl tenant list", ), option( "failure-strategy", @@ -892,6 +1002,7 @@ def workflow_group() -> dict[str, object]: "Warning group id. Omit to allow enabled project " "preference." ), + discovery_command="dsctl alert-group list", ), option( "environment-code", @@ -900,6 +1011,7 @@ def workflow_group() -> dict[str, object]: "Environment code. Omit to allow enabled project " "preference." ), + discovery_command="dsctl environment list", ), option( "param", @@ -947,6 +1059,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -974,8 +1087,12 @@ def workflow_group() -> dict[str, object]: option( "task", value_type="string", - description="Optional task name or task code to backfill from.", + description=( + "Optional task name or task code to backfill from. " + "Run `dsctl task list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl task list", ), option( "scope", @@ -1032,6 +1149,7 @@ def workflow_group() -> dict[str, object]: "preference before the DS fallback `default` worker " "group." ), + discovery_command="dsctl worker-group list", ), option( "tenant", @@ -1041,6 +1159,7 @@ def workflow_group() -> dict[str, object]: "workflow instance. Omit to allow enabled project " "preference before the DS fallback `default` tenant." ), + discovery_command="dsctl tenant list", ), option( "failure-strategy", @@ -1070,6 +1189,7 @@ def workflow_group() -> dict[str, object]: "Warning group id. Omit to allow enabled project " "preference." ), + discovery_command="dsctl alert-group list", ), option( "environment-code", @@ -1078,6 +1198,7 @@ def workflow_group() -> dict[str, object]: "Environment code. Omit to allow enabled project " "preference." ), + discovery_command="dsctl environment list", ), option( "param", @@ -1123,6 +1244,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -1159,6 +1281,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[project_option()], @@ -1180,6 +1303,7 @@ def workflow_group() -> dict[str, object]: ), required=False, selector="name_or_code", + discovery_command="dsctl workflow list", ) ], options=[ @@ -1192,6 +1316,7 @@ def workflow_group() -> dict[str, object]: "workflow." ), selector="name_or_code", + discovery_command="dsctl task list", ), ], ), @@ -1236,8 +1361,12 @@ def task_group() -> dict[str, object]: argument( "task", value_type="string", - description="Task name or numeric code.", + description=( + "Task name or numeric code. Use `dsctl task list` to " + "discover values." + ), selector="name_or_code", + discovery_command="dsctl task list", ) ], options=[ @@ -1257,8 +1386,12 @@ def task_group() -> dict[str, object]: argument( "task", value_type="string", - description="Task name or numeric code.", + description=( + "Task name or numeric code. Use `dsctl task list` to " + "discover values." + ), selector="name_or_code", + discovery_command="dsctl task list", ) ], options=[ @@ -1273,11 +1406,12 @@ def task_group() -> dict[str, object]: value_type="string", description=( "Inline task update in KEY=VALUE form. Repeat for " - "multiple fields. Inspect `dsctl schema` for " - "supported keys and examples." + "multiple fields. Inspect `dsctl schema --command " + "task.update` for supported keys and examples." ), multiple=True, required=True, + discovery_command="dsctl schema --command task.update", examples=[ "command=python v2.py", "retry.times=5", diff --git a/src/dsctl/services/_schema_groups_governance.py b/src/dsctl/services/_schema_groups_governance.py index 5b3dd40..87f420f 100644 --- a/src/dsctl/services/_schema_groups_governance.py +++ b/src/dsctl/services/_schema_groups_governance.py @@ -1,18 +1,19 @@ from __future__ import annotations from dsctl.services._schema_primitives import argument, command, group, option +from dsctl.services.datasource_payload import datasource_payload_command_data from dsctl.services.pagination import DEFAULT_PAGE_SIZE def env_group() -> dict[str, object]: """Build the environment command group schema.""" return group( - "env", + "environment", summary="Manage DolphinScheduler environments.", commands=[ command( "list", - action="env.list", + action="environment.list", summary=( "List environments with optional filtering and pagination controls." ), @@ -47,21 +48,25 @@ def env_group() -> dict[str, object]: ), command( "get", - action="env.get", + action="environment.get", summary="Get one environment by name or code.", arguments=[ argument( "environment", value_type="string", - description="Environment name or numeric code.", + description=( + "Environment name or numeric code. Run `dsctl " + "environment list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl environment list", ) ], ), command( "create", - action="env.create", - summary="Create one environment.", + action="environment.create", + summary="Create one environment; pass --config or --config-file.", options=[ option( "name", @@ -72,8 +77,20 @@ def env_group() -> dict[str, object]: option( "config", value_type="string", - description="Environment config payload.", - required=True, + description=( + "Inline DS environment shell/export config. Prefer " + "--config-file for multiline configs." + ), + examples=["export JAVA_HOME=/opt/java"], + discovery_command="dsctl template environment", + ), + option( + "config-file", + value_type="path", + description=( + "Path to a DS environment shell/export config file." + ), + discovery_command="dsctl template environment", ), option( "description", @@ -85,22 +102,31 @@ def env_group() -> dict[str, object]: value_type="string", description=( "Worker group to bind to this environment. Repeat as " - "needed." + "needed; run `dsctl worker-group list` to discover " + "values." ), multiple=True, + discovery_command="dsctl worker-group list", ), ], ), command( "update", - action="env.update", - summary="Update one environment by name or code.", + action="environment.update", + summary=( + "Update one environment by name or code; config may come " + "from --config-file." + ), arguments=[ argument( "environment", value_type="string", - description="Environment name or numeric code.", + description=( + "Environment name or numeric code. Run `dsctl " + "environment list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl environment list", ) ], options=[ @@ -115,9 +141,22 @@ def env_group() -> dict[str, object]: "config", value_type="string", description=( - "Updated environment config. Omit to keep the current " + "Updated inline DS environment shell/export config. " + "Omit to keep the current config; prefer --config-file " + "for multiline configs." + ), + examples=["export JAVA_HOME=/opt/java"], + discovery_command="dsctl template environment", + ), + option( + "config-file", + value_type="path", + description=( + "Path to an updated DS environment shell/export config " + "file. Omit both config options to keep the current " "config." ), + discovery_command="dsctl template environment", ), option( "description", @@ -135,9 +174,11 @@ def env_group() -> dict[str, object]: value_type="string", description=( "Worker group to bind to this environment. Repeat as " - "needed." + "needed; run `dsctl worker-group list` to discover " + "values." ), multiple=True, + discovery_command="dsctl worker-group list", ), option( "clear-worker-groups", @@ -149,14 +190,18 @@ def env_group() -> dict[str, object]: ), command( "delete", - action="env.delete", + action="environment.delete", summary="Delete one environment by name or code.", arguments=[ argument( "environment", value_type="string", - description="Environment name or numeric code.", + description=( + "Environment name or numeric code. Run `dsctl " + "environment list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl environment list", ) ], options=[ @@ -220,15 +265,19 @@ def cluster_group() -> dict[str, object]: argument( "cluster", value_type="string", - description="Cluster name or numeric code.", + description=( + "Cluster name or numeric code. Run `dsctl cluster " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl cluster list", ) ], ), command( "create", action="cluster.create", - summary="Create one cluster.", + summary="Create one cluster; pass --config or --config-file.", options=[ option( "name", @@ -239,8 +288,17 @@ def cluster_group() -> dict[str, object]: option( "config", value_type="string", - description="Cluster config payload.", - required=True, + description=( + "Inline DS cluster config JSON. Prefer --config-file " + "for multiline Kubernetes configs." + ), + discovery_command="dsctl template cluster", + ), + option( + "config-file", + value_type="path", + description="Path to one DS cluster config JSON file.", + discovery_command="dsctl template cluster", ), option( "description", @@ -257,8 +315,12 @@ def cluster_group() -> dict[str, object]: argument( "cluster", value_type="string", - description="Cluster name or numeric code.", + description=( + "Cluster name or numeric code. Run `dsctl cluster " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl cluster list", ) ], options=[ @@ -273,8 +335,20 @@ def cluster_group() -> dict[str, object]: "config", value_type="string", description=( - "Updated cluster config. Omit to keep the current config." + "Updated inline DS cluster config JSON. Omit to keep " + "the current config; prefer --config-file for " + "multiline Kubernetes configs." + ), + discovery_command="dsctl template cluster", + ), + option( + "config-file", + value_type="path", + description=( + "Path to an updated DS cluster config JSON file. Omit " + "both config options to keep the current config." ), + discovery_command="dsctl template cluster", ), option( "description", @@ -297,8 +371,12 @@ def cluster_group() -> dict[str, object]: argument( "cluster", value_type="string", - description="Cluster name or numeric code.", + description=( + "Cluster name or numeric code. Run `dsctl cluster " + "list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl cluster list", ) ], options=[ @@ -318,14 +396,15 @@ def datasource_group() -> dict[str, object]: """Build the datasource command group schema.""" return group( "datasource", - summary="Manage DolphinScheduler datasources.", + summary=( + "Manage DolphinScheduler datasources. Create/update use DS-native " + "JSON payload files." + ), commands=[ command( "list", action="datasource.list", - summary=( - "List datasources with optional filtering and pagination controls." - ), + summary="List datasource identities and summary fields.", options=[ option( "search", @@ -363,8 +442,12 @@ def datasource_group() -> dict[str, object]: argument( "datasource", value_type="string", - description="Datasource name or numeric id.", + description=( + "Datasource name or numeric id. Run `dsctl " + "datasource list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl datasource list", ) ], ), @@ -372,6 +455,7 @@ def datasource_group() -> dict[str, object]: "create", action="datasource.create", summary="Create one datasource from a JSON payload file.", + payload=datasource_payload_command_data(), options=[ option( "file", @@ -381,6 +465,7 @@ def datasource_group() -> dict[str, object]: ), required=True, value_name="PATH", + discovery_command="dsctl template datasource", ) ], ), @@ -388,12 +473,17 @@ def datasource_group() -> dict[str, object]: "update", action="datasource.update", summary="Update one datasource from a JSON payload file.", + payload=datasource_payload_command_data(), arguments=[ argument( "datasource", value_type="string", - description="Datasource name or numeric id.", + description=( + "Datasource name or numeric id. Run `dsctl " + "datasource list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl datasource list", ) ], options=[ @@ -405,6 +495,7 @@ def datasource_group() -> dict[str, object]: ), required=True, value_name="PATH", + discovery_command="dsctl template datasource", ) ], ), @@ -416,8 +507,12 @@ def datasource_group() -> dict[str, object]: argument( "datasource", value_type="string", - description="Datasource name or numeric id.", + description=( + "Datasource name or numeric id. Run `dsctl " + "datasource list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl datasource list", ) ], options=[ @@ -432,13 +527,17 @@ def datasource_group() -> dict[str, object]: command( "test", action="datasource.test", - summary="Run one datasource connection test by name or id.", + summary="Run one datasource connection test after create or update.", arguments=[ argument( "datasource", value_type="string", - description="Datasource name or numeric id.", + description=( + "Datasource name or numeric id. Run `dsctl " + "datasource list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl datasource list", ) ], ), @@ -465,8 +564,10 @@ def resource_group() -> dict[str, object]: value_type="string", description=( "DS directory fullName path. Defaults to the upstream " - "base directory." + "base directory; run `dsctl resource list` to discover " + "paths." ), + discovery_command="dsctl resource list", ), option( "search", @@ -503,8 +604,12 @@ def resource_group() -> dict[str, object]: argument( "resource", value_type="string", - description="DS resource fullName path.", + description=( + "DS resource fullName path. Run " + "`dsctl resource list --dir DIR` to discover paths." + ), selector="resource_path", + discovery_command="dsctl resource list --dir DIR", ) ], options=[ @@ -539,8 +644,10 @@ def resource_group() -> dict[str, object]: value_type="string", description=( "Destination DS directory fullName path. Defaults to the " - "upstream base directory." + "upstream base directory; run `dsctl resource list` to " + "discover paths." ), + discovery_command="dsctl resource list", ), option( "name", @@ -570,7 +677,8 @@ def resource_group() -> dict[str, object]: value_type="string", description=( "Inline text content to write into the remote resource " - "file." + "file. For local files, use " + "`dsctl resource upload --file PATH`." ), required=True, ), @@ -579,8 +687,10 @@ def resource_group() -> dict[str, object]: value_type="string", description=( "Destination DS directory fullName path. Defaults to the " - "upstream base directory." + "upstream base directory; run `dsctl resource list` to " + "discover paths." ), + discovery_command="dsctl resource list", ), ], ), @@ -601,8 +711,10 @@ def resource_group() -> dict[str, object]: value_type="string", description=( "Parent DS directory fullName path. Defaults to the " - "upstream base directory." + "upstream base directory; run `dsctl resource list` to " + "discover paths." ), + discovery_command="dsctl resource list", ) ], ), @@ -614,8 +726,12 @@ def resource_group() -> dict[str, object]: argument( "resource", value_type="string", - description="DS resource fullName path.", + description=( + "DS resource fullName path. Run " + "`dsctl resource list --dir DIR` to discover paths." + ), selector="resource_path", + discovery_command="dsctl resource list --dir DIR", ) ], options=[ @@ -645,8 +761,12 @@ def resource_group() -> dict[str, object]: argument( "resource", value_type="string", - description="DS resource fullName path.", + description=( + "DS resource fullName path. Run " + "`dsctl resource list --dir DIR` to discover paths." + ), selector="resource_path", + discovery_command="dsctl resource list --dir DIR", ) ], options=[ @@ -711,8 +831,12 @@ def namespace_group() -> dict[str, object]: argument( "namespace", value_type="string", - description="Namespace name or numeric id.", + description=( + "Namespace name or numeric id. Run `dsctl namespace " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl namespace list", ) ], ), @@ -735,8 +859,12 @@ def namespace_group() -> dict[str, object]: option( "cluster-code", value_type="integer", - description="Owning cluster code.", + description=( + "Owning cluster code. Run `dsctl cluster list` " + "to discover codes." + ), required=True, + discovery_command="dsctl cluster list", ), ], ), @@ -748,8 +876,12 @@ def namespace_group() -> dict[str, object]: argument( "namespace", value_type="string", - description="Namespace name or numeric id.", + description=( + "Namespace name or numeric id. Run `dsctl namespace " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl namespace list", ) ], options=[ @@ -812,8 +944,12 @@ def queue_group() -> dict[str, object]: argument( "queue", value_type="string", - description="Queue name or numeric id.", + description=( + "Queue name or numeric id. Run `dsctl queue list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl queue list", ) ], ), @@ -825,13 +961,15 @@ def queue_group() -> dict[str, object]: option( "queue-name", value_type="string", - description="Human-facing queue name.", + description=( + "Human-facing DS queue name used as the selector label." + ), required=True, ), option( "queue", value_type="string", - description="Underlying DolphinScheduler queue value.", + description="Underlying YARN queue value stored in DS.", required=True, ), ], @@ -844,8 +982,12 @@ def queue_group() -> dict[str, object]: argument( "queue", value_type="string", - description="Queue name or numeric id.", + description=( + "Queue name or numeric id. Run `dsctl queue list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl queue list", ) ], options=[ @@ -853,14 +995,16 @@ def queue_group() -> dict[str, object]: "queue-name", value_type="string", description=( - "Updated queue name. Omit to keep the current queue name." + "Updated human-facing DS queue name. Omit to keep the " + "current queue name." ), ), option( "queue", value_type="string", description=( - "Updated queue value. Omit to keep the current queue value." + "Updated underlying YARN queue value. Omit to keep the " + "current queue value." ), ), ], @@ -873,8 +1017,12 @@ def queue_group() -> dict[str, object]: argument( "queue", value_type="string", - description="Queue name or numeric id.", + description=( + "Queue name or numeric id. Run `dsctl queue list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl queue list", ) ], options=[ @@ -940,8 +1088,12 @@ def worker_group_group() -> dict[str, object]: argument( "worker_group", value_type="string", - description="Worker-group name or numeric id.", + description=( + "Worker-group name or numeric id. Run `dsctl " + "worker-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl worker-group list", ) ], ), @@ -960,9 +1112,12 @@ def worker_group_group() -> dict[str, object]: "addr", value_type="string", description=( - "Worker address to include in addrList. Repeat as needed." + "Worker server address to include in addrList. Repeat " + "as needed; run `dsctl monitor server worker` to " + "discover workers." ), multiple=True, + discovery_command="dsctl monitor server worker", ), option( "description", @@ -979,8 +1134,12 @@ def worker_group_group() -> dict[str, object]: argument( "worker_group", value_type="string", - description="Worker-group name or numeric id.", + description=( + "Worker-group name or numeric id. Run `dsctl " + "worker-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl worker-group list", ) ], options=[ @@ -995,9 +1154,11 @@ def worker_group_group() -> dict[str, object]: "addr", value_type="string", description=( - "Replacement worker address list. Repeat as needed." + "Replacement worker address list. Repeat as needed; " + "run `dsctl monitor server worker` to discover workers." ), multiple=True, + discovery_command="dsctl monitor server worker", ), option( "clear-addrs", @@ -1026,8 +1187,12 @@ def worker_group_group() -> dict[str, object]: argument( "worker_group", value_type="string", - description="Worker-group name or numeric id.", + description=( + "Worker-group name or numeric id. Run `dsctl " + "worker-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl worker-group list", ) ], options=[ @@ -1059,7 +1224,11 @@ def task_group_group() -> dict[str, object]: option( "project", value_type="string", - description="Project name or code for project-scoped listing.", + description=( + "Project name or code for project-scoped listing. " + "Run `dsctl project list` to discover values." + ), + discovery_command="dsctl project list", ), option( "search", @@ -1069,7 +1238,8 @@ def task_group_group() -> dict[str, object]: option( "status", value_type="string", - description="Filter task groups by status: open or closed.", + description="Filter task groups by status.", + choices=["open", "closed", "1", "0"], ), option( "page-no", @@ -1099,8 +1269,12 @@ def task_group_group() -> dict[str, object]: argument( "task_group", value_type="string", - description="Task-group name or numeric id.", + description=( + "Task-group name or numeric id. Run `dsctl " + "task-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl task-group list", ) ], ), @@ -1114,8 +1288,10 @@ def task_group_group() -> dict[str, object]: value_type="string", description=( "Project name or code. Falls back to stored " - "project context." + "project context; run `dsctl project list` to " + "discover values." ), + discovery_command="dsctl project list", ), option( "name", @@ -1144,8 +1320,12 @@ def task_group_group() -> dict[str, object]: argument( "task_group", value_type="string", - description="Task-group name or numeric id.", + description=( + "Task-group name or numeric id. Run `dsctl " + "task-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl task-group list", ) ], options=[ @@ -1180,8 +1360,12 @@ def task_group_group() -> dict[str, object]: argument( "task_group", value_type="string", - description="Task-group name or numeric id.", + description=( + "Task-group name or numeric id. Run `dsctl " + "task-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl task-group list", ) ], ), @@ -1193,8 +1377,12 @@ def task_group_group() -> dict[str, object]: argument( "task_group", value_type="string", - description="Task-group name or numeric id.", + description=( + "Task-group name or numeric id. Run `dsctl " + "task-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl task-group list", ) ], ), @@ -1210,8 +1398,12 @@ def task_group_group() -> dict[str, object]: argument( "task_group", value_type="string", - description="Task-group name or numeric id.", + description=( + "Task-group name or numeric id. Run `dsctl " + "task-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl task-group list", ) ], options=[ @@ -1228,10 +1420,15 @@ def task_group_group() -> dict[str, object]: option( "status", value_type="string", - description=( - "Filter by queue status: WAIT_QUEUE, " - "ACQUIRE_SUCCESS, or RELEASE." - ), + description="Filter by task-group queue status.", + choices=[ + "WAIT_QUEUE", + "ACQUIRE_SUCCESS", + "RELEASE", + "-1", + "1", + "2", + ], ), option( "page-no", @@ -1267,8 +1464,15 @@ def task_group_group() -> dict[str, object]: argument( "queue_id", value_type="integer", - description="Numeric task-group queue id.", + description=( + "Numeric task-group queue id. Run " + "`dsctl task-group queue list TASK_GROUP` " + "to discover ids." + ), selector="id", + discovery_command=( + "dsctl task-group queue list TASK_GROUP" + ), ) ], ), @@ -1280,8 +1484,15 @@ def task_group_group() -> dict[str, object]: argument( "queue_id", value_type="integer", - description="Numeric task-group queue id.", + description=( + "Numeric task-group queue id. Run " + "`dsctl task-group queue list TASK_GROUP` " + "to discover ids." + ), selector="id", + discovery_command=( + "dsctl task-group queue list TASK_GROUP" + ), ) ], options=[ @@ -1349,8 +1560,12 @@ def alert_plugin_group() -> dict[str, object]: argument( "alert-plugin", value_type="string", - description="Alert-plugin instance name or numeric id.", + description=( + "Alert-plugin instance name or numeric id. Run " + "`dsctl alert-plugin list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-plugin list", ) ], ), @@ -1362,8 +1577,13 @@ def alert_plugin_group() -> dict[str, object]: argument( "plugin", value_type="string", - description="Alert UI plugin definition name or numeric id.", + description=( + "Alert UI plugin definition name or numeric id. Run " + "`dsctl alert-plugin definition list` to discover " + "values." + ), selector="name_or_id", + discovery_command="dsctl alert-plugin definition list", ) ], ), @@ -1381,22 +1601,46 @@ def alert_plugin_group() -> dict[str, object]: option( "plugin", value_type="string", - description="Alert UI plugin definition name or numeric id.", + description=( + "Alert UI plugin definition name or numeric id. Run " + "`dsctl alert-plugin definition list` to discover " + "values." + ), required=True, selector="name_or_id", + discovery_command="dsctl alert-plugin definition list", ), option( "params-json", value_type="string", - description="DS-native alert-plugin UI params JSON array.", + description=( + "DS-native alert-plugin UI params JSON array. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect fields." + ), + discovery_command="dsctl alert-plugin schema PLUGIN", + ), + option( + "param", + value_type="string", + description=( + "Alert-plugin UI param in KEY=VALUE form. Repeat for " + "multiple fields; run " + "`dsctl alert-plugin schema PLUGIN` to inspect keys." + ), + value_name="KEY=VALUE", + multiple=True, + discovery_command="dsctl alert-plugin schema PLUGIN", ), option( "file", value_type="path", description=( - "Path to one DS-native alert-plugin UI params JSON file." + "Path to one DS-native alert-plugin UI params JSON file. " + "Run `dsctl alert-plugin schema PLUGIN` to inspect " + "fields." ), value_name="PATH", + discovery_command="dsctl alert-plugin schema PLUGIN", ), ], ), @@ -1408,8 +1652,12 @@ def alert_plugin_group() -> dict[str, object]: argument( "alert-plugin", value_type="string", - description="Alert-plugin instance name or numeric id.", + description=( + "Alert-plugin instance name or numeric id. Run " + "`dsctl alert-plugin list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-plugin list", ) ], options=[ @@ -1424,15 +1672,31 @@ def alert_plugin_group() -> dict[str, object]: description=( "Replacement DS-native alert-plugin UI params JSON array." ), + discovery_command="dsctl alert-plugin schema PLUGIN", + ), + option( + "param", + value_type="string", + description=( + "Replacement alert-plugin UI param in KEY=VALUE form. " + "Repeat for multiple fields; omitted fields keep " + "current values. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect keys." + ), + value_name="KEY=VALUE", + multiple=True, + discovery_command="dsctl alert-plugin schema PLUGIN", ), option( "file", value_type="path", description=( "Path to one replacement DS-native alert-plugin UI " - "params JSON file." + "params JSON file. Run " + "`dsctl alert-plugin schema PLUGIN` to inspect fields." ), value_name="PATH", + discovery_command="dsctl alert-plugin schema PLUGIN", ), ], ), @@ -1444,8 +1708,12 @@ def alert_plugin_group() -> dict[str, object]: argument( "alert-plugin", value_type="string", - description="Alert-plugin instance name or numeric id.", + description=( + "Alert-plugin instance name or numeric id. Run " + "`dsctl alert-plugin list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-plugin list", ) ], options=[ @@ -1465,8 +1733,29 @@ def alert_plugin_group() -> dict[str, object]: argument( "alert-plugin", value_type="string", - description="Alert-plugin instance name or numeric id.", + description=( + "Alert-plugin instance name or numeric id. Run " + "`dsctl alert-plugin list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-plugin list", + ) + ], + ), + group( + "definition", + summary=( + "Discover supported alert-plugin definitions, not configured " + "alert-plugin instances." + ), + commands=[ + command( + "list", + action="alert-plugin.definition.list", + summary=( + "List alert-plugin definitions supported by the " + "current DolphinScheduler runtime." + ), ) ], ), @@ -1523,8 +1812,12 @@ def alert_group_group() -> dict[str, object]: argument( "alert-group", value_type="string", - description="Alert-group name or numeric id.", + description=( + "Alert-group name or numeric id. Run `dsctl " + "alert-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-group list", ) ], ), @@ -1544,9 +1837,11 @@ def alert_group_group() -> dict[str, object]: value_type="integer", description=( "Alert plugin instance id to bind to this group. " - "Repeat as needed." + "Repeat as needed; run `dsctl alert-plugin list` " + "to discover ids." ), multiple=True, + discovery_command="dsctl alert-plugin list", ), option( "description", @@ -1563,8 +1858,12 @@ def alert_group_group() -> dict[str, object]: argument( "alert-group", value_type="string", - description="Alert-group name or numeric id.", + description=( + "Alert-group name or numeric id. Run `dsctl " + "alert-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-group list", ) ], options=[ @@ -1580,9 +1879,11 @@ def alert_group_group() -> dict[str, object]: value_type="integer", description=( "Alert plugin instance id to bind to this group. " - "Repeat as needed." + "Repeat as needed; run `dsctl alert-plugin list` " + "to discover ids." ), multiple=True, + discovery_command="dsctl alert-plugin list", ), option( "clear-instance-ids", @@ -1611,8 +1912,12 @@ def alert_group_group() -> dict[str, object]: argument( "alert-group", value_type="string", - description="Alert-group name or numeric id.", + description=( + "Alert-group name or numeric id. Run `dsctl " + "alert-group list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl alert-group list", ) ], options=[ @@ -1675,8 +1980,12 @@ def tenant_group() -> dict[str, object]: argument( "tenant", value_type="string", - description="Tenant code or numeric id.", + description=( + "Tenant code or numeric id. Run `dsctl tenant list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl tenant list", ) ], ), @@ -1694,8 +2003,12 @@ def tenant_group() -> dict[str, object]: option( "queue", value_type="string", - description="Queue name or numeric id to bind to this tenant.", + description=( + "Queue name or numeric id to bind to this tenant. " + "Run `dsctl queue list` to discover values." + ), required=True, + discovery_command="dsctl queue list", ), option( "description", @@ -1712,8 +2025,12 @@ def tenant_group() -> dict[str, object]: argument( "tenant", value_type="string", - description="Tenant code or numeric id.", + description=( + "Tenant code or numeric id. Run `dsctl tenant list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl tenant list", ) ], options=[ @@ -1728,9 +2045,11 @@ def tenant_group() -> dict[str, object]: "queue", value_type="string", description=( - "Updated queue name or numeric id. Omit to keep the " - "current queue." + "Updated queue name or numeric id. Run `dsctl queue " + "list` to discover values; omit to keep the current " + "queue." ), + discovery_command="dsctl queue list", ), option( "description", @@ -1753,8 +2072,12 @@ def tenant_group() -> dict[str, object]: argument( "tenant", value_type="string", - description="Tenant code or numeric id.", + description=( + "Tenant code or numeric id. Run `dsctl tenant list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl tenant list", ) ], options=[ @@ -1816,8 +2139,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], ), @@ -1847,8 +2174,12 @@ def user_group() -> dict[str, object]: option( "tenant", value_type="string", - description="Tenant code or numeric id.", + description=( + "Tenant code or numeric id. Run `dsctl tenant list` " + "to discover values." + ), required=True, + discovery_command="dsctl tenant list", ), option( "state", @@ -1865,7 +2196,11 @@ def user_group() -> dict[str, object]: option( "queue", value_type="string", - description="Optional queue-name override stored on the user.", + description=( + "Optional queue-name override stored on the user. " + "Run `dsctl queue list` to discover queue names." + ), + discovery_command="dsctl queue list", ), ], ), @@ -1877,8 +2212,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -1900,7 +2239,11 @@ def user_group() -> dict[str, object]: option( "tenant", value_type="string", - description="Updated tenant code or numeric id.", + description=( + "Updated tenant code or numeric id. Run `dsctl tenant " + "list` to discover values." + ), + discovery_command="dsctl tenant list", ), option( "state", @@ -1924,7 +2267,11 @@ def user_group() -> dict[str, object]: option( "queue", value_type="string", - description="Updated queue-name override stored on the user.", + description=( + "Updated queue-name override stored on the user. " + "Run `dsctl queue list` to discover queue names." + ), + discovery_command="dsctl queue list", ), option( "clear-queue", @@ -1947,8 +2294,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user list` " + "to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -1972,14 +2323,22 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ), argument( "project", value_type="string", - description="Project name or numeric code.", + description=( + "Project name or numeric code. Run `dsctl " + "project list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ), ], ), @@ -1991,8 +2350,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -2001,11 +2364,13 @@ def user_group() -> dict[str, object]: value_type="string", description=( "Datasource name or numeric id. Repeat to grant " - "multiple datasources." + "multiple datasources; run `dsctl datasource " + "list` to discover values." ), selector="name_or_id", multiple=True, required=True, + discovery_command="dsctl datasource list", ) ], ), @@ -2017,8 +2382,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -2027,11 +2396,13 @@ def user_group() -> dict[str, object]: value_type="string", description=( "Namespace name or numeric id. Repeat to grant " - "multiple namespaces." + "multiple namespaces; run `dsctl namespace " + "list` to discover values." ), selector="name_or_id", multiple=True, required=True, + discovery_command="dsctl namespace list", ) ], ), @@ -2049,14 +2420,22 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ), argument( "project", value_type="string", - description="Project name or numeric code.", + description=( + "Project name or numeric code. Run `dsctl " + "project list` to discover values." + ), selector="name_or_code", + discovery_command="dsctl project list", ), ], ), @@ -2068,8 +2447,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -2078,11 +2461,13 @@ def user_group() -> dict[str, object]: value_type="string", description=( "Datasource name or numeric id. Repeat to revoke " - "multiple datasources." + "multiple datasources; run `dsctl datasource " + "list` to discover values." ), selector="name_or_id", multiple=True, required=True, + discovery_command="dsctl datasource list", ) ], ), @@ -2094,8 +2479,12 @@ def user_group() -> dict[str, object]: argument( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ) ], options=[ @@ -2104,11 +2493,13 @@ def user_group() -> dict[str, object]: value_type="string", description=( "Namespace name or numeric id. Repeat to revoke " - "multiple namespaces." + "multiple namespaces; run `dsctl namespace " + "list` to discover values." ), selector="name_or_id", multiple=True, required=True, + discovery_command="dsctl namespace list", ) ], ), @@ -2167,8 +2558,12 @@ def access_token_group() -> dict[str, object]: argument( "access-token", value_type="integer", - description="Access-token id.", + description=( + "Access-token id. Run `dsctl access-token list` " + "to discover values." + ), selector="id", + discovery_command="dsctl access-token list", ) ], ), @@ -2180,14 +2575,20 @@ def access_token_group() -> dict[str, object]: option( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user list` to " + "discover values." + ), selector="name_or_id", required=True, + discovery_command="dsctl user list", ), option( "expire-time", value_type="string", - description="Token expiration time.", + description=( + "Token expiration time, for example '2027-01-01 00:00:00'." + ), required=True, ), option( @@ -2207,21 +2608,32 @@ def access_token_group() -> dict[str, object]: argument( "access-token", value_type="integer", - description="Access-token id.", + description=( + "Access-token id. Run `dsctl access-token list` " + "to discover values." + ), selector="id", + discovery_command="dsctl access-token list", ) ], options=[ option( "user", value_type="string", - description="Updated user name or numeric id.", + description=( + "Updated user name or numeric id. Run `dsctl user " + "list` to discover values." + ), selector="name_or_id", + discovery_command="dsctl user list", ), option( "expire-time", value_type="string", - description="Updated token expiration time.", + description=( + "Updated token expiration time, for example " + "'2027-01-01 00:00:00'." + ), ), option( "token", @@ -2244,8 +2656,12 @@ def access_token_group() -> dict[str, object]: argument( "access-token", value_type="integer", - description="Access-token id.", + description=( + "Access-token id. Run `dsctl access-token list` " + "to discover values." + ), selector="id", + discovery_command="dsctl access-token list", ) ], options=[ @@ -2265,14 +2681,20 @@ def access_token_group() -> dict[str, object]: option( "user", value_type="string", - description="User name or numeric id.", + description=( + "User name or numeric id. Run `dsctl user list` to " + "discover values." + ), selector="name_or_id", required=True, + discovery_command="dsctl user list", ), option( "expire-time", value_type="string", - description="Token expiration time.", + description=( + "Token expiration time, for example '2027-01-01 00:00:00'." + ), required=True, ), ], diff --git a/src/dsctl/services/_schema_groups_meta.py b/src/dsctl/services/_schema_groups_meta.py index 0541060..7c08bbf 100644 --- a/src/dsctl/services/_schema_groups_meta.py +++ b/src/dsctl/services/_schema_groups_meta.py @@ -10,6 +10,11 @@ def enum_group() -> dict[str, object]: "enum", summary="Discover generated DolphinScheduler enums.", commands=[ + command( + "names", + action="enum.names", + summary="List supported generated enum discovery names.", + ), command( "list", action="enum.list", @@ -20,9 +25,10 @@ def enum_group() -> dict[str, object]: value_type="string", description="Stable enum discovery name.", choices=supported_enum_choices(), + discovery_command="dsctl enum names", ) ], - ) + ), ], ) @@ -31,12 +37,18 @@ def task_type_group() -> dict[str, object]: """Build the task-type command group schema.""" return group( "task-type", - summary="Discover DolphinScheduler task types for the current runtime.", + summary=( + "List live DS task-type catalog for the configured cluster and " + "current user." + ), commands=[ command( "list", action="task-type.list", - summary=("List DS task types plus the current user's favourite flags."), + summary=( + "List live DS task types, categories, favourite flags, and " + "CLI authoring coverage." + ), ) ], ) diff --git a/src/dsctl/services/_schema_groups_runtime.py b/src/dsctl/services/_schema_groups_runtime.py index 92c0dbe..53e55ba 100644 --- a/src/dsctl/services/_schema_groups_runtime.py +++ b/src/dsctl/services/_schema_groups_runtime.py @@ -27,14 +27,22 @@ def audit_group() -> dict[str, object]: option( "model-type", value_type="string", - description="Audit model type filter. Repeat as needed.", + description=( + "Audit model type filter. Repeat as needed; run " + "`dsctl audit model-types` to discover values." + ), multiple=True, + discovery_command="dsctl audit model-types", ), option( "operation-type", value_type="string", - description=("Audit operation type filter. Repeat as needed."), + description=( + "Audit operation type filter. Repeat as needed; run " + "`dsctl audit operation-types` to discover values." + ), multiple=True, + discovery_command="dsctl audit operation-types", ), option( "start", @@ -159,19 +167,65 @@ def workflow_instance_group() -> dict[str, object]: option( "project", value_type="string", - description="Filter by project name.", - selector="opaque_name", + description=( + "Project name or code for project-scoped filters. " + "Run `dsctl project list` to discover values." + ), + selector="name_or_code", + discovery_command="dsctl project list", ), option( "workflow", value_type="string", - description="Filter by workflow name.", + description=( + "Workflow name or code filter. With --project, " + "resolved inside that project; run `dsctl workflow " + "list` to discover values." + ), selector="opaque_name", + discovery_command="dsctl workflow list", + ), + option( + "search", + value_type="string", + description=( + "Filter workflow instances by upstream searchVal; " + "requires --project." + ), + ), + option( + "executor", + value_type="string", + description="Filter by executor user name; requires --project.", + ), + option( + "host", + value_type="string", + description="Filter by workflow instance host.", + ), + option( + "start", + value_type="string", + description=( + "Start datetime in DS format 'YYYY-MM-DD HH:MM:SS'." + ), + ), + option( + "end", + value_type="string", + description=( + "End datetime in DS format 'YYYY-MM-DD HH:MM:SS'." + ), ), option( "state", value_type="string", - description="Filter by DS workflow execution status name.", + description=( + "Filter by DS workflow execution status name. Run " + "`dsctl enum list workflow-execution-status` to " + "discover values." + ), + discovery_command="dsctl enum list workflow-execution-status", ), ], ), @@ -183,8 +237,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -198,8 +256,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "sub_workflow_instance", value_type="integer", - description="Sub-workflow instance id.", + description=( + "Sub-workflow instance id. Run `dsctl " + "workflow-instance list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -211,8 +273,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -224,8 +290,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Finished workflow instance id.", + description=( + "Finished workflow instance id. Run `dsctl " + "workflow-instance list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], options=[ @@ -263,8 +333,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], options=[ @@ -292,8 +366,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -305,8 +383,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -318,8 +400,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], ), @@ -331,8 +417,12 @@ def workflow_instance_group() -> dict[str, object]: argument( "workflow_instance", value_type="integer", - description="Workflow instance id.", + description=( + "Workflow instance id. Run `dsctl workflow-instance " + "list` to discover ids." + ), selector="id", + discovery_command="dsctl workflow-instance list", ) ], options=[ @@ -340,10 +430,17 @@ def workflow_instance_group() -> dict[str, object]: "task", value_type="string", description=( - "Task name or task code within the workflow definition." + "Task name or task code within the workflow " + "instance. Run `dsctl task-instance list " + "--workflow-instance WORKFLOW_INSTANCE` to discover " + "values." ), required=True, selector="name_or_code", + discovery_command=( + "dsctl task-instance list --workflow-instance " + "WORKFLOW_INSTANCE" + ), ), option( "scope", @@ -367,16 +464,34 @@ def task_instance_group() -> dict[str, object]: command( "list", action="task-instance.list", - summary="List task instances inside one workflow instance.", + summary="List task instances with project-scoped runtime filters.", options=[ option( "workflow-instance", value_type="integer", description=( - "Workflow instance id used to scope the " - "task-instance query." + "Workflow instance id used to narrow the " + "task-instance query. Run `dsctl workflow-instance " + "list` to discover ids." ), - required=True, + discovery_command="dsctl workflow-instance list", + ), + option( + "project", + value_type="string", + description=( + "Project name or code for the project-scoped query. " + "Run `dsctl project list` to discover values; " + "required via flag or context when " + "--workflow-instance is omitted." + ), + selector="name_or_code", + discovery_command="dsctl project list", + ), + option( + "workflow-instance-name", + value_type="string", + description="Filter by workflow instance name.", ), option( "page-no", @@ -399,12 +514,71 @@ def task_instance_group() -> dict[str, object]: option( "search", value_type="string", - description="Filter task instances by upstream searchVal.", + description=( + "Free-text upstream searchVal filter. Use --task for " + "an exact task instance name filter." + ), + ), + option( + "task", + value_type="string", + description="Filter by exact task instance name.", + ), + option( + "task-code", + value_type="integer", + description=( + "Filter by task definition code. Run `dsctl task " + "list` to discover values." + ), + discovery_command="dsctl task list", + ), + option( + "executor", + value_type="string", + description="Filter by executor user name.", ), option( "state", value_type="string", - description="Filter by DS task execution status name.", + description=( + "Filter by DS task execution status name. Run " + "`dsctl enum list task-execution-status` to discover " + "values." + ), + discovery_command="dsctl enum list task-execution-status", + ), + option( + "host", + value_type="string", + description="Filter by worker host.", + ), + option( + "start", + value_type="string", + description=( + "Task start-time lower bound in DS format " + "'YYYY-MM-DD HH:MM:SS'." + ), + ), + option( + "end", + value_type="string", + description=( + "Task start-time upper bound in DS format " + "'YYYY-MM-DD HH:MM:SS'." + ), + ), + option( + "execute-type", + value_type="string", + description=( + "Filter by DS task execute type: BATCH or STREAM. " + "Run `dsctl enum list task-execute-type` to discover " + "values." + ), + choices=["BATCH", "STREAM"], + discovery_command="dsctl enum list task-execute-type", ), ], ), @@ -416,8 +590,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -425,9 +603,12 @@ def task_instance_group() -> dict[str, object]: "workflow-instance", value_type="integer", description=( - "Workflow instance id used to resolve the owning project." + "Workflow instance id used to resolve the owning " + "project. Run `dsctl workflow-instance list` to " + "discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ) ], ), @@ -439,8 +620,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -448,9 +633,12 @@ def task_instance_group() -> dict[str, object]: "workflow-instance", value_type="integer", description=( - "Workflow instance id used to resolve the owning project." + "Workflow instance id used to resolve the owning " + "project. Run `dsctl workflow-instance list` to " + "discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ), option( "interval-seconds", @@ -479,8 +667,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -489,9 +681,11 @@ def task_instance_group() -> dict[str, object]: value_type="integer", description=( "Workflow instance id used to scope the " - "task-instance relation." + "task-instance relation. Run `dsctl " + "workflow-instance list` to discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ) ], ), @@ -503,8 +697,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -527,8 +725,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -536,9 +738,12 @@ def task_instance_group() -> dict[str, object]: "workflow-instance", value_type="integer", description=( - "Workflow instance id used to resolve the owning project." + "Workflow instance id used to resolve the owning " + "project. Run `dsctl workflow-instance list` to " + "discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ) ], ), @@ -550,8 +755,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -559,9 +768,12 @@ def task_instance_group() -> dict[str, object]: "workflow-instance", value_type="integer", description=( - "Workflow instance id used to resolve the owning project." + "Workflow instance id used to resolve the owning " + "project. Run `dsctl workflow-instance list` to " + "discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ) ], ), @@ -573,8 +785,12 @@ def task_instance_group() -> dict[str, object]: argument( "task_instance", value_type="integer", - description="Task instance id.", + description=( + "Task instance id. Run `dsctl task-instance list` " + "to discover ids." + ), selector="id", + discovery_command="dsctl task-instance list", ) ], options=[ @@ -582,9 +798,12 @@ def task_instance_group() -> dict[str, object]: "workflow-instance", value_type="integer", description=( - "Workflow instance id used to resolve the owning project." + "Workflow instance id used to resolve the owning " + "project. Run `dsctl workflow-instance list` to " + "discover ids." ), required=True, + discovery_command="dsctl workflow-instance list", ) ], ), diff --git a/src/dsctl/services/_schema_primitives.py b/src/dsctl/services/_schema_primitives.py index c53b787..a5bd7c5 100644 --- a/src/dsctl/services/_schema_primitives.py +++ b/src/dsctl/services/_schema_primitives.py @@ -1,10 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from collections.abc import Sequence + from dsctl.support.yaml_io import JsonObject, JsonValue + def use_target_options(*, clear_help: str) -> list[dict[str, object]]: """Build the shared `use` command options.""" @@ -30,8 +32,12 @@ def project_option() -> dict[str, object]: return option( "project", value_type="string", - description="Project name or code. Falls back to stored project context.", + description=( + "Project name or code. Run `dsctl project list` to discover values; " + "falls back to stored project context." + ), selector="name_or_code", + discovery_command="dsctl project list", ) @@ -42,6 +48,7 @@ def workflow_option(*, description: str) -> dict[str, object]: value_type="string", description=description, selector="name_or_code", + discovery_command="dsctl workflow list", ) @@ -83,16 +90,23 @@ def command( summary: str, arguments: list[dict[str, object]] | None = None, options: list[dict[str, object]] | None = None, + payload: JsonObject | None = None, + payload_schema: JsonObject | None = None, ) -> dict[str, object]: """Build one schema command payload.""" - return { + data: JsonObject = { "kind": "command", "name": name, "action": action, "summary": summary, - "arguments": arguments or [], - "options": options or [], + "arguments": cast("JsonValue", arguments or []), + "options": cast("JsonValue", options or []), } + if payload is not None: + data["payload"] = payload + if payload_schema is not None: + data["payload_schema"] = payload_schema + return cast("dict[str, object]", data) def argument( @@ -103,6 +117,7 @@ def argument( required: bool = True, selector: str | None = None, choices: Sequence[object] | None = None, + discovery_command: str | None = None, ) -> dict[str, object]: """Build one schema positional-argument payload.""" data: dict[str, object] = { @@ -116,6 +131,8 @@ def argument( data["selector"] = selector if choices is not None: data["choices"] = list(choices) + if discovery_command is not None: + data["discovery_command"] = discovery_command return data @@ -132,6 +149,7 @@ def option( multiple: bool = False, examples: Sequence[str] | None = None, supported_keys: Sequence[str] | None = None, + discovery_command: str | None = None, ) -> dict[str, object]: """Build one schema option payload.""" data: dict[str, object] = { @@ -156,4 +174,6 @@ def option( data["examples"] = list(examples) if supported_keys is not None: data["supported_keys"] = list(supported_keys) + if discovery_command is not None: + data["discovery_command"] = discovery_command return data diff --git a/src/dsctl/services/_surface_metadata.py b/src/dsctl/services/_surface_metadata.py index 8806e1e..d50eab6 100644 --- a/src/dsctl/services/_surface_metadata.py +++ b/src/dsctl/services/_surface_metadata.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypedDict from dsctl.cli_surface import ( AUDIT_RESOURCE, @@ -66,6 +66,16 @@ } +class SelfDescriptionData(TypedDict): + """Machine-readable self-description capabilities emitted by the CLI.""" + + schema: bool + template: bool + capabilities: bool + command_invocation_source: str + capabilities_scope: str + + def selection_schema_data() -> dict[str, object]: """Return the schema-scoped selection contract.""" return { @@ -133,6 +143,10 @@ def error_capabilities_data() -> dict[str, object]: def output_schema_data() -> dict[str, object]: """Return the schema-scoped standard output envelope contract.""" return { + "formats": ["json", "table", "tsv"], + "default_format": "json", + "format_option": "--output-format", + "columns_option": "--columns", "success_fields": list(OUTPUT_SUCCESS_FIELDS), "error_fields": list(OUTPUT_ERROR_FIELDS), "ok_values": { @@ -140,6 +154,8 @@ def output_schema_data() -> dict[str, object]: "error": False, }, "warning_details_aligned": True, + "data_shape_metadata": True, + "json_column_projection": True, } @@ -147,6 +163,11 @@ def output_capabilities_data() -> dict[str, object]: """Return the capabilities-scoped standard output support flags.""" return { "standard_envelope": True, + "formats": ["json", "table", "tsv"], + "default_format": "json", + "data_shape_metadata": True, + "display_columns": True, + "json_column_projection": True, "resolved_metadata": True, "warnings": True, "warning_details_alignment": True, @@ -154,12 +175,14 @@ def output_capabilities_data() -> dict[str, object]: } -def self_description_data() -> dict[str, bool]: +def self_description_data() -> SelfDescriptionData: """Return stable self-description capability flags.""" return { "schema": True, "template": True, "capabilities": True, + "command_invocation_source": "schema", + "capabilities_scope": "feature_discovery", } diff --git a/src/dsctl/services/_validation.py b/src/dsctl/services/_validation.py index f175a9d..13edae4 100644 --- a/src/dsctl/services/_validation.py +++ b/src/dsctl/services/_validation.py @@ -1,5 +1,7 @@ from __future__ import annotations +from time import strptime, struct_time + from dsctl.errors import UserInputError from dsctl.support.quartz import ( QUARTZ_CRON_FIELD_COUNTS, @@ -8,6 +10,8 @@ quartz_cron_suggestion, ) +DS_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + def require_non_empty_text(value: str, *, label: str) -> str: """Normalize one required CLI text field.""" @@ -44,6 +48,55 @@ def require_quartz_cron_text(value: str, *, label: str) -> str: ) from exc +def optional_ds_datetime(value: str | None, *, label: str) -> str | None: + """Normalize one optional DS datetime string.""" + if value is None: + return None + normalized = value.strip() + if not normalized: + return None + try: + parse_ds_datetime(normalized) + except ValueError as error: + message = f"{label} must match DS datetime format {DS_DATETIME_FORMAT!r}" + raise UserInputError( + message, + suggestion=( + f"Pass --{label} in '{DS_DATETIME_FORMAT}' format, for " + "example '2026-04-11 10:00:00'." + ), + ) from error + return normalized + + +def validate_ds_datetime_range( + start: str | None, + end: str | None, + *, + start_label: str = "start", + end_label: str = "end", +) -> None: + """Require an optional DS datetime range to be ordered when both ends exist.""" + if start is None or end is None: + return + start_value = parse_ds_datetime(start) + end_value = parse_ds_datetime(end) + if end_value < start_value: + message = f"{end_label} must be greater than or equal to {start_label}" + raise UserInputError( + message, + suggestion=( + f"Pass an --{end_label} value that is later than or equal " + f"to --{start_label}." + ), + ) + + +def parse_ds_datetime(value: str) -> struct_time: + """Parse one DS datetime string using the CLI-supported wire format.""" + return strptime(value, DS_DATETIME_FORMAT) + + def require_positive_int(value: int, *, label: str) -> int: """Require one positive CLI integer value.""" if value < 1: diff --git a/src/dsctl/services/alert_plugin.py b/src/dsctl/services/alert_plugin.py index d259e80..dbb30e2 100644 --- a/src/dsctl/services/alert_plugin.py +++ b/src/dsctl/services/alert_plugin.py @@ -1,7 +1,8 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, TypeAlias, TypedDict +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, TypeAlias, TypedDict, cast from dsctl.cli_surface import ALERT_PLUGIN_RESOURCE from dsctl.errors import ( @@ -16,6 +17,7 @@ from dsctl.output import CommandResult, require_json_object from dsctl.services._serialization import ( AlertPluginData, + StructuredDataValue, optional_text, serialize_alert_plugin_list_item, serialize_plugin_define, @@ -46,6 +48,8 @@ QUERY_PLUGINS_ERROR = 110003 +QUERY_PLUGINS_RESULT_IS_NULL = 110002 +QUERY_PLUGIN_DETAIL_RESULT_IS_NULL = 110004 UPDATE_ALERT_PLUGIN_INSTANCE_ERROR = 110005 DELETE_ALERT_PLUGIN_INSTANCE_ERROR = 110006 GET_ALERT_PLUGIN_INSTANCE_ERROR = 110007 @@ -84,6 +88,28 @@ class PluginDefineSelectionData(TypedDict): pluginType: str | None +class PluginDefineListItemData(TypedDict): + """Alert-plugin definition list item emitted by `definition list`.""" + + id: int + pluginName: str + pluginType: str | None + createTime: str | None + updateTime: str | None + + +class PluginDefinitionListData(TypedDict): + """Alert-plugin definition discovery payload.""" + + definitions: list[PluginDefineListItemData] + count: int + schema_command: str + + +PluginParamItem: TypeAlias = dict[str, StructuredDataValue] +PluginParamFieldData: TypeAlias = dict[str, StructuredDataValue] + + def list_alert_plugins_result( *, search: str | None = None, @@ -133,34 +159,48 @@ def get_alert_plugin_schema_result( ) +def list_alert_plugin_definitions_result( + *, + env_file: str | None = None, +) -> CommandResult: + """List alert-plugin definitions supported by the current DS runtime.""" + return run_with_service_runtime( + env_file, + _list_alert_plugin_definitions_result, + ) + + def create_alert_plugin_result( *, name: str, plugin: str, params_json: str | None = None, file: Path | None = None, + params: list[str] | None = None, env_file: str | None = None, ) -> CommandResult: - """Create one alert-plugin instance from DS-native UI param-list JSON.""" + """Create one alert-plugin instance from UI params or inline key/value fields.""" normalized_name = require_non_empty_text(name, label="alert-plugin name") normalized_plugin = require_non_empty_text(plugin, label="alert plugin") - plugin_instance_params = _plugin_instance_params_input( + inline_params = _inline_param_items(params) + _validate_plugin_param_sources( params_json=params_json, file=file, + inline_params=inline_params, required=True, ) - if plugin_instance_params is None: - message = "Alert-plugin params require exactly one input source" - raise UserInputError( - message, - suggestion="Pass exactly one of --params-json or --file.", - ) + plugin_instance_params = _plugin_instance_params_input( + params_json=params_json, + file=file, + required=False, + ) return run_with_service_runtime( env_file, _create_alert_plugin_result, name=normalized_name, plugin=normalized_plugin, plugin_instance_params=plugin_instance_params, + inline_params=inline_params, ) @@ -170,15 +210,23 @@ def update_alert_plugin_result( name: str | None = None, params_json: str | None = None, file: Path | None = None, + params: list[str] | None = None, env_file: str | None = None, ) -> CommandResult: """Update one alert-plugin instance while preserving omitted fields.""" - if name is None and params_json is None and file is None: + inline_params = _inline_param_items(params) + if name is None and params_json is None and file is None and not inline_params: message = "Alert-plugin update requires at least one field change" raise UserInputError( message, - suggestion="Pass --name or one of --params-json/--file.", + suggestion="Pass --name, --param, --params-json, or --file.", ) + _validate_plugin_param_sources( + params_json=params_json, + file=file, + inline_params=inline_params, + required=False, + ) normalized_name = ( require_non_empty_text(name, label="alert-plugin name") @@ -196,6 +244,7 @@ def update_alert_plugin_result( alert_plugin=alert_plugin, name=normalized_name, plugin_instance_params=plugin_instance_params, + inline_params=inline_params, ) @@ -300,7 +349,7 @@ def _get_alert_plugin_schema_result( plugin_define = _resolve_plugin_define(runtime, plugin=plugin) return CommandResult( data=require_json_object( - serialize_plugin_define(plugin_define), + _serialize_plugin_define_schema(plugin_define), label="alert-plugin schema data", ), resolved={ @@ -312,14 +361,48 @@ def _get_alert_plugin_schema_result( ) +def _list_alert_plugin_definitions_result(runtime: ServiceRuntime) -> CommandResult: + try: + plugin_defines = runtime.upstream.ui_plugins.list(plugin_type=ALERT_PLUGIN_TYPE) + except ApiResultError as error: + raise _translate_ui_plugin_api_error(error) from error + definitions = [ + _serialize_plugin_define_list_item(plugin_define) + for plugin_define in plugin_defines + ] + return CommandResult( + data=require_json_object( + PluginDefinitionListData( + definitions=definitions, + count=len(definitions), + schema_command="alert-plugin schema PLUGIN", + ), + label="alert-plugin definition list data", + ), + resolved={ + "pluginDefinitions": { + "pluginType": ALERT_PLUGIN_TYPE, + "source": "ui-plugins/query-by-type", + } + }, + ) + + def _create_alert_plugin_result( runtime: ServiceRuntime, *, name: str, plugin: str, - plugin_instance_params: str, + plugin_instance_params: str | None, + inline_params: Sequence[str], ) -> CommandResult: plugin_define = _resolve_plugin_define(runtime, plugin=plugin) + if plugin_instance_params is None: + plugin_instance_params = _plugin_instance_params_from_inline( + plugin_define, + inline_params=inline_params, + existing_params=None, + ) try: created_alert_plugin = runtime.upstream.alert_plugins.create( plugin_define_id=_plugin_define_id(plugin_define), @@ -367,6 +450,7 @@ def _update_alert_plugin_result( alert_plugin: str, name: str | None, plugin_instance_params: str | None, + inline_params: Sequence[str], ) -> CommandResult: resolved_alert_plugin = resolve_alert_plugin( alert_plugin, @@ -377,11 +461,21 @@ def _update_alert_plugin_result( alert_plugin_id=resolved_alert_plugin.id, ) next_name = current_alert_plugin.instanceName if name is None else name - next_params = ( - current_alert_plugin.pluginInstanceParams - if plugin_instance_params is None - else plugin_instance_params - ) + next_params: str | None + if plugin_instance_params is not None: + next_params = plugin_instance_params + elif inline_params: + plugin_define = _resolve_plugin_define( + runtime, + plugin=str(current_alert_plugin.pluginDefineId), + ) + next_params = _plugin_instance_params_from_inline( + plugin_define, + inline_params=inline_params, + existing_params=current_alert_plugin.pluginInstanceParams, + ) + else: + next_params = current_alert_plugin.pluginInstanceParams if next_name is None or next_params is None: message = "Alert-plugin payload was missing required fields" raise ApiTransportError( @@ -528,6 +622,14 @@ def _resolve_plugin_define( for plugin_define in plugin_defines if plugin_define.pluginName == plugin ] + if not matches: + normalized_plugin = plugin.casefold() + matches = [ + plugin_define + for plugin_define in plugin_defines + if plugin_define.pluginName is not None + and plugin_define.pluginName.casefold() == normalized_plugin + ] if not matches: message = f"Alert plugin {plugin!r} was not found" raise NotFoundError( @@ -544,7 +646,17 @@ def _resolve_plugin_define( "ids": [_plugin_define_id(plugin_define) for plugin_define in matches], }, ) - return matches[0] + plugin_define_id = _plugin_define_id(matches[0]) + try: + detailed_plugin_define = runtime.upstream.ui_plugins.get( + plugin_id=plugin_define_id + ) + except ApiResultError as error: + raise _translate_ui_plugin_api_error( + error, + plugin_id=plugin_define_id, + ) from error + return _require_alert_plugin_define_type(detailed_plugin_define) def _require_alert_plugin_list_item( @@ -654,6 +766,66 @@ def _plugin_define_name(plugin_define: PluginDefineRecord) -> str: return plugin_name +def _serialize_plugin_define_schema( + plugin_define: PluginDefineRecord, +) -> dict[str, StructuredDataValue]: + data = cast( + "dict[str, StructuredDataValue]", + dict(serialize_plugin_define(plugin_define)), + ) + data["pluginParamFields"] = _plugin_param_field_summaries( + _plugin_param_template(plugin_define) + ) + return data + + +def _serialize_plugin_define_list_item( + plugin_define: PluginDefineRecord, +) -> PluginDefineListItemData: + return { + "id": _plugin_define_id(plugin_define), + "pluginName": _plugin_define_name(plugin_define), + "pluginType": plugin_define.pluginType, + "createTime": plugin_define.createTime, + "updateTime": plugin_define.updateTime, + } + + +def _inline_param_items(params: list[str] | None) -> list[str]: + if params is None: + return [] + return [item for item in params if item is not None] + + +def _validate_plugin_param_sources( + *, + params_json: str | None, + file: Path | None, + inline_params: Sequence[str], + required: bool, +) -> None: + source_count = 0 + if params_json is not None: + source_count += 1 + if file is not None: + source_count += 1 + if inline_params: + source_count += 1 + + if required and source_count != 1: + message = "Alert-plugin params require exactly one input source" + raise UserInputError( + message, + suggestion="Pass exactly one of --param, --params-json, or --file.", + ) + if not required and source_count > 1: + message = "Alert-plugin params require at most one input source" + raise UserInputError( + message, + suggestion="Pass only one of --param, --params-json, or --file.", + ) + + def _plugin_instance_params_input( *, params_json: str | None, @@ -712,10 +884,245 @@ def _plugin_instance_params_input( return json.dumps(parsed, ensure_ascii=False, separators=(",", ":")) +def _plugin_instance_params_from_inline( + plugin_define: PluginDefineRecord, + *, + inline_params: Sequence[str], + existing_params: str | None, +) -> str: + template = _plugin_param_template(plugin_define) + if not template: + message = "Alert-plugin definition does not expose configurable params" + raise UserInputError( + message, + suggestion=( + "Use --params-json or --file only when the upstream plugin " + "requires raw params." + ), + ) + + existing_values = _plugin_param_values(existing_params) + inline_values = _canonical_inline_param_values( + inline_params, + template=template, + plugin_define=plugin_define, + ) + merged_params: list[PluginParamItem] = [] + missing_required_fields: list[str] = [] + + for item in template: + copied = dict(item) + field = _plugin_param_field(copied) + if field in existing_values: + copied["value"] = existing_values[field] + if field in inline_values: + copied["value"] = inline_values[field] + if _plugin_param_required(copied) and not _has_plugin_param_value( + copied.get("value") + ): + missing_required_fields.append(field) + merged_params.append(copied) + + if missing_required_fields: + message = "Alert-plugin params are missing required fields" + raise UserInputError( + message, + details={ + "resource": ALERT_PLUGIN_RESOURCE, + "plugin": _plugin_define_name(plugin_define), + "missing": missing_required_fields, + }, + suggestion=( + "Pass required fields with repeated --param KEY=VALUE options, " + "or submit the full DS UI params with --params-json/--file." + ), + ) + + return json.dumps(merged_params, ensure_ascii=False, separators=(",", ":")) + + +def _plugin_param_template( + plugin_define: PluginDefineRecord, +) -> list[PluginParamItem]: + plugin_params = plugin_define.pluginParams + if plugin_params is None: + return [] + return _parse_plugin_param_array( + plugin_params, + label="alert-plugin definition params", + plugin_define=plugin_define, + ) + + +def _plugin_param_values(existing_params: str | None) -> dict[str, StructuredDataValue]: + if existing_params is None: + return {} + values: dict[str, StructuredDataValue] = {} + for item in _parse_plugin_param_array( + existing_params, + label="current alert-plugin params", + plugin_define=None, + ): + field = _plugin_param_field(item) + if "value" in item: + values[field] = item["value"] + return values + + +def _parse_plugin_param_array( + value: str, + *, + label: str, + plugin_define: PluginDefineRecord | None, +) -> list[PluginParamItem]: + try: + parsed = json.loads(value) + except json.JSONDecodeError as error: + message = f"{label} must be valid JSON" + raise ApiTransportError( + message, + details=_plugin_param_error_details(plugin_define), + ) from error + if not isinstance(parsed, list) or any( + not isinstance(item, Mapping) for item in parsed + ): + message = f"{label} must be a JSON array of objects" + raise ApiTransportError( + message, + details=_plugin_param_error_details(plugin_define), + ) + return [dict(item) for item in parsed] + + +def _canonical_inline_param_values( + inline_params: Sequence[str], + *, + template: Sequence[Mapping[str, StructuredDataValue]], + plugin_define: PluginDefineRecord, +) -> dict[str, str]: + fields = [_plugin_param_field(item) for item in template] + exact_fields = {field: field for field in fields} + lower_fields: dict[str, list[str]] = {} + for field in fields: + lower_fields.setdefault(field.casefold(), []).append(field) + + values: dict[str, str] = {} + for item in inline_params: + raw_key, separator, raw_value = item.partition("=") + key = raw_key.strip() + if not separator or not key: + message = f"Invalid --param value {item!r}; expected KEY=VALUE" + raise UserInputError( + message, + suggestion=( + "Pass alert-plugin params as `--param WebHook=https://...`; " + "repeat the option for multiple fields." + ), + ) + canonical_key = exact_fields.get(key) + if canonical_key is None: + case_matches = lower_fields.get(key.casefold(), []) + if len(case_matches) == 1: + canonical_key = case_matches[0] + elif len(case_matches) > 1: + message = f"Alert-plugin param field {key!r} is ambiguous" + raise ConflictError( + message, + details={ + "resource": ALERT_PLUGIN_RESOURCE, + "plugin": _plugin_define_name(plugin_define), + "matches": case_matches, + }, + ) + if canonical_key is None: + message = f"Alert-plugin param field {key!r} is not supported" + raise UserInputError( + message, + details={ + "resource": ALERT_PLUGIN_RESOURCE, + "plugin": _plugin_define_name(plugin_define), + "supportedFields": fields, + }, + suggestion=( + "Run `dsctl alert-plugin schema " + f"{_plugin_define_name(plugin_define)}` to inspect supported " + "fields." + ), + ) + if canonical_key in values: + message = ( + f"Alert-plugin param field {canonical_key!r} was specified more " + "than once" + ) + raise UserInputError( + message, + suggestion="Pass each alert-plugin param field only once.", + ) + values[canonical_key] = raw_value + return values + + +def _plugin_param_field_summaries( + params: Sequence[Mapping[str, StructuredDataValue]], +) -> list[PluginParamFieldData]: + return [ + { + "field": _plugin_param_field(item), + "name": item.get("name"), + "title": item.get("title"), + "type": item.get("type"), + "required": _plugin_param_required(item), + "defaultValue": item.get("value"), + "options": item.get("options"), + } + for item in params + ] + + +def _plugin_param_field(item: Mapping[str, StructuredDataValue]) -> str: + field = item.get("field") + if not isinstance(field, str) or not field: + message = "Alert-plugin param schema item was missing field" + raise ApiTransportError(message, details={"resource": ALERT_PLUGIN_RESOURCE}) + return field + + +def _plugin_param_required(item: Mapping[str, StructuredDataValue]) -> bool: + validate = item.get("validate") + if not isinstance(validate, Sequence) or isinstance(validate, (str, bytes)): + return False + for rule in validate: + if not isinstance(rule, Mapping): + continue + required = rule.get("required") + if required is True: + return True + if isinstance(required, str) and required.casefold() == "true": + return True + return False + + +def _has_plugin_param_value(value: StructuredDataValue) -> bool: + if value is None: + return False + return not (isinstance(value, str) and value == "") + + +def _plugin_param_error_details( + plugin_define: PluginDefineRecord | None, +) -> dict[str, StructuredDataValue]: + details: dict[str, StructuredDataValue] = {"resource": ALERT_PLUGIN_RESOURCE} + if plugin_define is not None: + details["plugin"] = _plugin_define_name(plugin_define) + details["pluginDefineId"] = _plugin_define_id(plugin_define) + return details + + def _require_alert_plugin_define_type( plugin_define: PluginDefineRecord, ) -> PluginDefineRecord: - if plugin_define.pluginType == ALERT_PLUGIN_TYPE: + plugin_type = plugin_define.pluginType + if plugin_type is not None and plugin_type.casefold() == "alert": return plugin_define message = "Resolved plugin is not an alert plugin definition" raise UserInputError( @@ -796,15 +1203,21 @@ def _translate_ui_plugin_api_error( details: dict[str, object] = {"resource": ALERT_PLUGIN_RESOURCE} if plugin_id is not None: details["plugin_id"] = plugin_id - return NotFoundError( - f"Alert plugin id {plugin_id} was not found", - details=details, - ) + if error.result_code == QUERY_PLUGIN_DETAIL_RESULT_IS_NULL: + return NotFoundError( + f"Alert plugin id {plugin_id} was not found", + details=details, + ) if error.result_code == QUERY_PLUGINS_ERROR: return ConflictError( "Alert-plugin schema discovery was rejected by the upstream API", details=details, ) + if error.result_code == QUERY_PLUGINS_RESULT_IS_NULL: + return NotFoundError( + "No alert plugin definitions were returned by the upstream API", + details=details, + ) return error diff --git a/src/dsctl/services/audit.py b/src/dsctl/services/audit.py index be386a9..563c2b2 100644 --- a/src/dsctl/services/audit.py +++ b/src/dsctl/services/audit.py @@ -1,6 +1,5 @@ from __future__ import annotations -from time import strptime, struct_time from typing import TypeAlias from dsctl.cli_surface import AUDIT_RESOURCE @@ -13,7 +12,11 @@ serialize_audit_model_type, serialize_audit_operation_type, ) -from dsctl.services._validation import require_positive_int +from dsctl.services._validation import ( + optional_ds_datetime, + require_positive_int, + validate_ds_datetime_range, +) from dsctl.services.pagination import ( DEFAULT_PAGE_SIZE, MAX_AUTO_EXHAUST_PAGES, @@ -22,7 +25,6 @@ ) from dsctl.services.runtime import ServiceRuntime, run_with_service_runtime -AUDIT_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" AuditPageData: TypeAlias = PageData[AuditData] @@ -45,9 +47,9 @@ def list_audit_logs_result( operation_types, label="operation type", ) - normalized_start = _audit_datetime(start, label="start") - normalized_end = _audit_datetime(end, label="end") - _validate_audit_range(normalized_start, normalized_end) + normalized_start = optional_ds_datetime(start, label="start") + normalized_end = optional_ds_datetime(end, label="end") + validate_ds_datetime_range(normalized_start, normalized_end) return run_with_service_runtime( env_file, _list_audit_logs_result, @@ -182,41 +184,6 @@ def _filter_values( return tuple(normalized_values) -def _audit_datetime(value: str | None, *, label: str) -> str | None: - normalized = optional_text(value) - if normalized is None: - return None - try: - _parse_audit_datetime(normalized) - except ValueError as error: - message = f"{label} must match DS datetime format {AUDIT_DATETIME_FORMAT!r}" - raise UserInputError( - message, - suggestion=( - f"Pass --{label} in '{AUDIT_DATETIME_FORMAT}' format, for " - "example '2026-04-11 10:00:00'." - ), - ) from error - return normalized - - -def _validate_audit_range(start: str | None, end: str | None) -> None: - if start is None or end is None: - return - start_value = _parse_audit_datetime(start) - end_value = _parse_audit_datetime(end) - if end_value < start_value: - message = "end must be greater than or equal to start" - raise UserInputError( - message, - suggestion="Pass an --end value that is later than or equal to --start.", - ) - - -def _parse_audit_datetime(value: str) -> struct_time: - return strptime(value, AUDIT_DATETIME_FORMAT) - - __all__ = [ "list_audit_logs_result", "list_audit_model_types_result", diff --git a/src/dsctl/services/capabilities.py b/src/dsctl/services/capabilities.py index 475871a..2cb588d 100644 --- a/src/dsctl/services/capabilities.py +++ b/src/dsctl/services/capabilities.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TypedDict +from collections.abc import Mapping +from typing import TYPE_CHECKING, TypedDict from dsctl import __version__ from dsctl.cli_surface import ( @@ -9,6 +10,7 @@ WORKFLOW_INSTANCE_RESOURCE, ) from dsctl.config import load_selected_ds_version +from dsctl.errors import UserInputError from dsctl.models.task_spec import supported_typed_task_types from dsctl.output import CommandResult, require_json_object from dsctl.services._surface_metadata import ( @@ -21,9 +23,11 @@ selection_capabilities_data, self_description_data, ) +from dsctl.services.datasource_payload import datasource_template_index_data from dsctl.services.enums import enum_capabilities_data from dsctl.services.monitor import MONITOR_SERVER_TYPE_CHOICES from dsctl.services.template import ( + cluster_config_template_capability_data, generic_task_template_types, parameter_syntax_index_data, supported_task_template_types, @@ -40,6 +44,9 @@ upstream_default_task_types_by_category, ) +if TYPE_CHECKING: + from dsctl.support.yaml_io import JsonObject + class DsCapabilitiesData(TypedDict): """Selected DS version support metadata emitted by capabilities.""" @@ -54,21 +61,91 @@ class DsCapabilitiesData(TypedDict): versions: list[VersionSupportData] -def get_capabilities_result(*, env_file: str | None = None) -> CommandResult: +CAPABILITIES_HEADER_KEYS = ("cli", "ds", "self_description") +CAPABILITIES_SECTION_CHOICES = ( + "selection", + "output", + "errors", + "resources", + "planes", + "authoring", + "schedule", + "monitor", + "enums", + "runtime", +) +CAPABILITIES_SUMMARY_SECTIONS = ( + "resources", + "planes", + "runtime", + "schedule", + "monitor", + "enums", +) +AUTHORING_SUMMARY_KEYS = ( + "workflow_yaml_create", + "workflow_yaml_export", + "workflow_yaml_lint", + "workflow_digest", + "workflow_schedule_block", + "workflow_dry_run", + "cluster_config_template", + "task_template_types", + "datasource_payload_templates", + "datasource_template_types", + "typed_task_specs", + "generic_task_template_types", + "untemplated_upstream_task_types", +) + + +def get_capabilities_result( + *, + env_file: str | None = None, + summary: bool = False, + section: str | None = None, +) -> CommandResult: """Return stable capability discovery for the current CLI surface.""" + if summary and section is not None: + message = "--summary and --section are mutually exclusive" + raise UserInputError( + message, + suggestion="Pass either --summary or --section SECTION, not both.", + ) selected_version = load_selected_ds_version(env_file) support = get_version_support(selected_version) - return CommandResult( - data=require_json_object( - _capabilities_data(support), - label="capabilities data", + data = require_json_object( + _capabilities_data(support), + label="capabilities data", + ) + if summary: + return CommandResult( + data=_capabilities_summary_data(data), + resolved={"capabilities": {"view": "summary"}}, + ) + if section is not None: + normalized_section = section.strip() + return CommandResult( + data=_capabilities_section_data(data, normalized_section), + resolved={ + "capabilities": { + "view": "section", + "section": normalized_section, + } + }, ) + return CommandResult( + data=data, ) -def schema_capabilities_data() -> dict[str, object]: +def schema_capabilities_data(*, ds_version: str | None = None) -> dict[str, object]: """Return the schema-scoped capabilities subset.""" - support = get_default_version_support() + support = ( + get_default_version_support() + if ds_version is None + else get_version_support(ds_version) + ) task_types = list(supported_task_template_types()) typed_task_specs = list(supported_typed_task_types()) generic_task_templates = list(generic_task_template_types()) @@ -87,6 +164,16 @@ def schema_capabilities_data() -> dict[str, object]: "with_schedule_option": True, }, "parameters": parameter_syntax_index_data(), + "environment": { + "command": "dsctl template environment", + "source_options": ["--config TEXT", "--config-file PATH"], + "target_commands": [ + "dsctl environment create --name NAME --config-file env.sh", + "dsctl environment update ENVIRONMENT --config-file env.sh", + ], + }, + "cluster": cluster_config_template_capability_data(), + "datasource": datasource_template_index_data(), "task": { "supported_types": task_types, "typed_types": typed_task_specs, @@ -101,6 +188,12 @@ def schema_capabilities_data() -> dict[str, object]: "workflow_digest": True, "workflow_schedule_block": True, "workflow_dry_run": True, + "environment_config_template": True, + "cluster_config_template": True, + "datasource_payload_templates": True, + "datasource_template_types": datasource_template_index_data()[ + "supported_types" + ], "typed_task_specs": typed_task_specs, "generic_task_template_types": generic_task_templates, "upstream_default_task_types": upstream_task_types, @@ -164,7 +257,13 @@ def _capabilities_data(support: VersionSupport) -> dict[str, object]: "workflow_schedule_block": True, "workflow_dry_run": True, "parameter_syntax": parameter_syntax_index_data(), + "environment_config_template": True, + "cluster_config_template": True, "task_template_types": task_types, + "datasource_payload_templates": True, + "datasource_template_types": datasource_template_index_data()[ + "supported_types" + ], "task_templates": task_template_metadata(), "typed_task_specs": typed_task_specs, "generic_task_template_types": generic_task_templates, @@ -200,3 +299,53 @@ def _ds_capabilities_data(support: VersionSupport) -> DsCapabilitiesData: "supported_versions": list(SUPPORTED_VERSIONS), "versions": list(supported_version_metadata()), } + + +def _capabilities_summary_data(capabilities: JsonObject) -> JsonObject: + summary = _capabilities_header(capabilities) + for section in CAPABILITIES_SUMMARY_SECTIONS: + summary[section] = capabilities[section] + summary["authoring"] = _authoring_summary(capabilities) + return summary + + +def _capabilities_section_data( + capabilities: JsonObject, + section: str, +) -> JsonObject: + if section not in CAPABILITIES_SECTION_CHOICES: + message = f"Unknown capabilities section: {section}" + raise UserInputError( + message, + details={ + "section": section, + "available_sections": list(CAPABILITIES_SECTION_CHOICES), + }, + suggestion=( + "Run `dsctl capabilities --summary` or pass one section name " + "from the available_sections list." + ), + ) + section_data = capabilities.get(section) + if section_data is None: + message = f"Capabilities section is not available: {section}" + raise UserInputError(message, details={"section": section}) + data = _capabilities_header(capabilities) + data[section] = section_data + return data + + +def _capabilities_header(capabilities: JsonObject) -> JsonObject: + return {key: capabilities[key] for key in CAPABILITIES_HEADER_KEYS} + + +def _authoring_summary(capabilities: JsonObject) -> JsonObject: + authoring_value = capabilities.get("authoring") + if not isinstance(authoring_value, Mapping): + message = "capabilities data is missing authoring" + raise TypeError(message) + return { + key: authoring_value[key] + for key in AUTHORING_SUMMARY_KEYS + if key in authoring_value + } diff --git a/src/dsctl/services/datasource.py b/src/dsctl/services/datasource.py index 4a3d945..90bf3b1 100644 --- a/src/dsctl/services/datasource.py +++ b/src/dsctl/services/datasource.py @@ -22,6 +22,10 @@ require_non_empty_text, require_positive_int, ) +from dsctl.services.datasource_payload import ( + DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, + require_datasource_payload_type, +) from dsctl.services.pagination import ( DEFAULT_PAGE_SIZE, MAX_AUTO_EXHAUST_PAGES, @@ -40,10 +44,6 @@ MASKED_PASSWORD = "*" * 6 -_DATASOURCE_PAYLOAD_REVIEW_SUGGESTION = ( - "Review the DS-native JSON payload, then retry. A good baseline is the " - "output of `dsctl datasource get DATASOURCE`." -) DATASOURCE_EXISTS = 10015 CREATE_DATASOURCE_ERROR = 10033 @@ -127,7 +127,7 @@ def create_datasource_result( """Create one datasource from one DS-native JSON payload file.""" payload = _load_datasource_payload_or_error(file) _require_datasource_name(payload, operation="create") - _require_datasource_type(payload, operation="create") + payload["type"] = _require_datasource_type(payload, operation="create") if "id" in payload: message = "Datasource create payload must not include id" raise UserInputError( @@ -163,7 +163,7 @@ def update_datasource_result( """Update one datasource from one DS-native JSON payload file.""" payload = _load_datasource_payload_or_error(file) _require_datasource_name(payload, operation="update") - _require_datasource_type(payload, operation="update") + payload["type"] = _require_datasource_type(payload, operation="update") return run_with_service_runtime( env_file, @@ -434,8 +434,9 @@ def _load_datasource_payload_or_error(path: Path) -> dict[str, object]: message, details={"file": str(path)}, suggestion=( - "Fix the JSON syntax, or regenerate a DS-native payload with " - "`dsctl datasource get DATASOURCE`." + "Fix the JSON syntax, or run `dsctl template datasource` to choose " + "a type and `dsctl template datasource --type TYPE` to regenerate " + "a payload skeleton." ), ) from exc return dict(require_json_object(parsed, label="datasource payload")) @@ -506,7 +507,7 @@ def _require_datasource_name( message = f"Datasource {operation} payload requires string field 'name'" raise UserInputError( message, - suggestion=_DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, + suggestion=DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, ) return require_non_empty_text(name, label="datasource name") @@ -521,9 +522,10 @@ def _require_datasource_type( message = f"Datasource {operation} payload requires string field 'type'" raise UserInputError( message, - suggestion=_DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, + suggestion=DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, ) - return require_non_empty_text(datasource_type, label="datasource type") + normalized_type = require_non_empty_text(datasource_type, label="datasource type") + return require_datasource_payload_type(normalized_type) def _payload_json(payload: Mapping[str, object]) -> str: @@ -594,7 +596,7 @@ def _translate_datasource_api_error( return UserInputError( error.message, details=details, - suggestion=_DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, + suggestion=DATASOURCE_PAYLOAD_REVIEW_SUGGESTION, ) return error diff --git a/src/dsctl/services/datasource_payload.py b/src/dsctl/services/datasource_payload.py new file mode 100644 index 0000000..ee4502e --- /dev/null +++ b/src/dsctl/services/datasource_payload.py @@ -0,0 +1,488 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from dsctl.errors import UserInputError +from dsctl.upstream import ( + DATASOURCE_CONTRACT_VERSION, + datasource_base_payload_fields, + datasource_type_names, + normalize_datasource_type, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from dsctl.support.yaml_io import JsonObject, JsonValue + + +DATASOURCE_TYPE_ENUM = "db-type" +DEFAULT_DATASOURCE_TEMPLATE_TYPE = "MYSQL" +DATASOURCE_TEMPLATE_DISCOVERY_COMMAND = "dsctl template datasource" +DATASOURCE_TEMPLATE_COMMAND = "dsctl template datasource --type MYSQL" +DATASOURCE_TEMPLATE_COMMAND_PATTERN = "dsctl template datasource --type TYPE" +DATASOURCE_TEMPLATE_TARGET_COMMANDS = ( + "dsctl datasource create --file FILE", + "dsctl datasource update DATASOURCE --file FILE", +) +DATASOURCE_TYPE_DISCOVERY_COMMAND = f"dsctl enum list {DATASOURCE_TYPE_ENUM}" +DATASOURCE_PAYLOAD_REVIEW_SUGGESTION = ( + "Review the DS-native JSON payload, or run `dsctl template datasource` " + "to choose a type and `dsctl template datasource --type TYPE` to generate " + "a skeleton." +) + + +class DataSourcePayloadTemplateData(TypedDict): + """One concrete datasource payload template.""" + + type: str + target_commands: list[str] + source_option: str + payload: JsonObject + json: str + fields: list[JsonObject] + rules: list[str] + + +class DataSourcePayloadTemplateIndexData(TypedDict): + """Compact datasource payload-template discovery.""" + + default_type: str + template_command: str + template_command_pattern: str + target_commands: list[str] + type_enum: str + type_discovery_command: str + supported_types: list[str] + + +@dataclass(frozen=True) +class _DataSourceTypeExtraField: + name: str + value_type: str + description: str + example: JsonValue + choices: tuple[str, ...] = () + + def to_data(self) -> JsonObject: + data: JsonObject = { + "name": self.name, + "value_type": self.value_type, + "description": self.description, + "example": self.example, + } + if self.choices: + data["choices"] = list(self.choices) + return data + + +def supported_datasource_template_types() -> tuple[str, ...]: + """Return datasource types supported by local payload templates.""" + return datasource_type_names(DATASOURCE_CONTRACT_VERSION) + + +def normalize_datasource_payload_type(datasource_type: str) -> str | None: + """Normalize one user-provided datasource type against generated DbType.""" + return normalize_datasource_type(DATASOURCE_CONTRACT_VERSION, datasource_type) + + +def require_datasource_payload_type(datasource_type: str) -> str: + """Normalize one datasource type or raise a stable user-input error.""" + normalized = normalize_datasource_payload_type(datasource_type) + if normalized is not None: + return normalized + supported = list(supported_datasource_template_types()) + message = f"Unsupported datasource type {datasource_type!r}" + raise UserInputError( + message, + details={ + "type": datasource_type, + "supported_types": supported, + }, + suggestion=( + "Run `dsctl template datasource` to choose a supported datasource " + "type, then `dsctl template datasource --type TYPE`." + ), + ) + + +def datasource_payload_command_data() -> JsonObject: + """Return compact datasource payload metadata for command schema.""" + return { + "format": "json", + "source_option": "--file", + "target_commands": list(DATASOURCE_TEMPLATE_TARGET_COMMANDS), + "ds_model": "BaseDataSourceParamDTO", + "upstream_request_shape": "DataSourceController request body String jsonStr", + "template_command": DATASOURCE_TEMPLATE_COMMAND, + "template_command_pattern": DATASOURCE_TEMPLATE_COMMAND_PATTERN, + "template_discovery_command": DATASOURCE_TEMPLATE_DISCOVERY_COMMAND, + "template_json_path": "data.json", + "template_payload_path": "data.payload", + "type_enum": DATASOURCE_TYPE_ENUM, + "type_discovery_command": DATASOURCE_TYPE_DISCOVERY_COMMAND, + "rules": datasource_payload_rules(), + } + + +def datasource_template_index_data() -> DataSourcePayloadTemplateIndexData: + """Return compact datasource template discovery metadata.""" + return DataSourcePayloadTemplateIndexData( + default_type=DEFAULT_DATASOURCE_TEMPLATE_TYPE, + template_command=DATASOURCE_TEMPLATE_COMMAND, + template_command_pattern=DATASOURCE_TEMPLATE_COMMAND_PATTERN, + target_commands=list(DATASOURCE_TEMPLATE_TARGET_COMMANDS), + type_enum=DATASOURCE_TYPE_ENUM, + type_discovery_command=DATASOURCE_TYPE_DISCOVERY_COMMAND, + supported_types=list(supported_datasource_template_types()), + ) + + +def datasource_template_data(datasource_type: str) -> DataSourcePayloadTemplateData: + """Return one DS-native datasource JSON payload template.""" + normalized_type = require_datasource_payload_type(datasource_type) + payload = _datasource_payload_template(normalized_type) + return DataSourcePayloadTemplateData( + type=normalized_type, + target_commands=list(DATASOURCE_TEMPLATE_TARGET_COMMANDS), + source_option="--file", + payload=payload, + json=json.dumps(payload, indent=2, ensure_ascii=False) + "\n", + fields=_payload_fields_for_type(normalized_type), + rules=datasource_payload_rules(), + ) + + +def datasource_payload_rules() -> list[str]: + """Return stable datasource payload authoring rules.""" + return [ + "Create payloads must not include id; DS assigns it.", + "Update payloads may omit id or set it to the selected datasource id.", + "Create payloads must include the real password when the type uses one.", + ( + "Update payloads may use the masked password ****** to preserve " + "the stored password." + ), + "Use DS-native field names exactly, including userName and type.", + "Use `dsctl datasource test DATASOURCE` after create or update.", + ] + + +def _base_payload_fields_data() -> list[JsonObject]: + return [ + field.to_data() + for field in datasource_base_payload_fields(DATASOURCE_CONTRACT_VERSION) + ] + + +def _payload_fields_for_type(datasource_type: str) -> list[JsonObject]: + fields = _base_payload_fields_data() + for field in fields: + if field.get("name") == "type": + field.pop("choices", None) + fields.extend( + field.to_data() for field in _EXTRA_FIELDS_BY_TYPE.get(datasource_type, ()) + ) + return fields + + +def _datasource_payload_template(datasource_type: str) -> JsonObject: + profile = _TEMPLATE_PROFILES.get(datasource_type) + if profile is not None: + return dict(profile) + + payload: JsonObject = { + "type": datasource_type, + "name": f"{datasource_type.lower()}_example", + "note": "", + "host": "db.example.com", + "port": _DEFAULT_PORT_BY_TYPE.get(datasource_type, 0), + "database": "default", + "userName": "user", + "password": "change-me", + "other": _default_other_params(datasource_type), + } + payload.update(_extra_payload_defaults(datasource_type)) + return payload + + +def _default_other_params(datasource_type: str) -> dict[str, str]: + if datasource_type == "MYSQL": + return {"serverTimezone": "UTC"} + return {} + + +def _extra_payload_defaults(datasource_type: str) -> JsonObject: + return { + field.name: field.example + for field in _EXTRA_FIELDS_BY_TYPE.get(datasource_type, ()) + } + + +def _profile_payload( + datasource_type: str, + payload: Mapping[str, JsonValue], +) -> JsonObject: + base: JsonObject = { + "type": datasource_type, + "name": f"{datasource_type.lower()}_example", + "note": "", + } + base.update(payload) + return base + + +_DEFAULT_PORT_BY_TYPE: dict[str, int] = { + "MYSQL": 3306, + "POSTGRESQL": 5432, + "HIVE": 10000, + "SPARK": 10000, + "CLICKHOUSE": 8123, + "ORACLE": 1521, + "SQLSERVER": 1433, + "DB2": 50000, + "PRESTO": 8080, + "H2": 9092, + "REDSHIFT": 5439, + "TRINO": 8080, + "STARROCKS": 9030, + "AZURESQL": 1433, + "DAMENG": 5236, + "OCEANBASE": 2881, + "SSH": 22, + "KYUUBI": 10009, + "DATABEND": 8000, + "SNOWFLAKE": 443, + "VERTICA": 5433, + "HANA": 30015, + "DORIS": 9030, + "DOLPHINDB": 8848, +} +_HDFS_EXTRA_FIELDS = ( + _DataSourceTypeExtraField( + "principal", + "string", + "Kerberos principal for HDFS-style datasource plugins.", + "", + ), + _DataSourceTypeExtraField( + "javaSecurityKrb5Conf", + "string", + "Path to krb5.conf for Kerberos-enabled HDFS-style datasource plugins.", + "", + ), + _DataSourceTypeExtraField( + "loginUserKeytabUsername", + "string", + "Keytab login user for Kerberos-enabled HDFS-style datasource plugins.", + "", + ), + _DataSourceTypeExtraField( + "loginUserKeytabPath", + "string", + "Keytab path for Kerberos-enabled HDFS-style datasource plugins.", + "", + ), +) +_EXTRA_FIELDS_BY_TYPE: dict[str, tuple[_DataSourceTypeExtraField, ...]] = { + "ALIYUN_SERVERLESS_SPARK": ( + _DataSourceTypeExtraField( + "accessKeyId", + "string", + "Aliyun access key id.", + "change-me", + ), + _DataSourceTypeExtraField( + "accessKeySecret", + "string", + "Aliyun access key secret.", + "change-me", + ), + _DataSourceTypeExtraField( + "regionId", + "string", + "Aliyun region id.", + "cn-hangzhou", + ), + _DataSourceTypeExtraField( + "endpoint", + "string", + "Optional Aliyun endpoint override.", + "", + ), + ), + "ATHENA": ( + _DataSourceTypeExtraField( + "awsRegion", + "string", + "AWS region used by the Athena datasource plugin.", + "us-east-1", + ), + ), + "AZURESQL": ( + _DataSourceTypeExtraField( + "mode", + "enum", + "Azure SQL authentication mode.", + "SqlPassword", + choices=( + "SqlPassword", + "ActiveDirectoryPassword", + "ActiveDirectoryMSI", + "ActiveDirectoryServicePrincipal", + "accessToken", + ), + ), + _DataSourceTypeExtraField( + "MSIClientId", + "string", + "Azure managed-identity client id for ActiveDirectoryMSI mode.", + "", + ), + _DataSourceTypeExtraField( + "endpoint", + "string", + "Access-token endpoint for accessToken mode.", + "", + ), + ), + "HIVE": _HDFS_EXTRA_FIELDS, + "K8S": ( + _DataSourceTypeExtraField( + "kubeConfig", + "string", + "Kubernetes kubeconfig content.", + "change-me", + ), + _DataSourceTypeExtraField( + "namespace", + "string", + "Kubernetes namespace.", + "default", + ), + ), + "OCEANBASE": ( + _DataSourceTypeExtraField( + "compatibleMode", + "string", + "OceanBase compatibility mode.", + "mysql", + ), + ), + "ORACLE": ( + _DataSourceTypeExtraField( + "connectType", + "enum", + "Oracle connection type.", + "ORACLE_SERVICE_NAME", + choices=("ORACLE_SERVICE_NAME", "ORACLE_SID"), + ), + ), + "REDSHIFT": ( + _DataSourceTypeExtraField( + "mode", + "enum", + "Redshift authentication mode.", + "password", + choices=("password", "IAM-accessKey"), + ), + _DataSourceTypeExtraField( + "dbUser", + "string", + "Redshift IAM database user for IAM-accessKey mode.", + "", + ), + ), + "SAGEMAKER": ( + _DataSourceTypeExtraField( + "awsRegion", + "string", + "AWS region used by the SageMaker datasource plugin.", + "us-east-1", + ), + ), + "SPARK": _HDFS_EXTRA_FIELDS, + "SSH": ( + _DataSourceTypeExtraField( + "privateKey", + "string", + "Optional SSH private key content.", + "", + ), + ), + "ZEPPELIN": ( + _DataSourceTypeExtraField( + "restEndpoint", + "string", + "Zeppelin REST endpoint.", + "https://zeppelin.example.com", + ), + ), +} +_TEMPLATE_PROFILES: dict[str, JsonObject] = { + "ALIYUN_SERVERLESS_SPARK": _profile_payload( + "ALIYUN_SERVERLESS_SPARK", + _extra_payload_defaults("ALIYUN_SERVERLESS_SPARK"), + ), + "ATHENA": _profile_payload( + "ATHENA", + { + "database": "default", + "userName": "access-key-id", + "password": "secret-access-key", + **_extra_payload_defaults("ATHENA"), + }, + ), + "K8S": _profile_payload( + "K8S", + _extra_payload_defaults("K8S"), + ), + "SAGEMAKER": _profile_payload( + "SAGEMAKER", + { + "userName": "access-key-id", + "password": "secret-access-key", + **_extra_payload_defaults("SAGEMAKER"), + }, + ), + "SSH": _profile_payload( + "SSH", + { + "host": "ssh.example.com", + "port": _DEFAULT_PORT_BY_TYPE["SSH"], + "userName": "user", + "password": "change-me", + **_extra_payload_defaults("SSH"), + }, + ), + "ZEPPELIN": _profile_payload( + "ZEPPELIN", + { + "userName": "user", + "password": "change-me", + **_extra_payload_defaults("ZEPPELIN"), + }, + ), +} + + +__all__ = [ + "DATASOURCE_PAYLOAD_REVIEW_SUGGESTION", + "DATASOURCE_TEMPLATE_COMMAND", + "DATASOURCE_TEMPLATE_COMMAND_PATTERN", + "DATASOURCE_TEMPLATE_DISCOVERY_COMMAND", + "DATASOURCE_TEMPLATE_TARGET_COMMANDS", + "DATASOURCE_TYPE_DISCOVERY_COMMAND", + "DATASOURCE_TYPE_ENUM", + "DEFAULT_DATASOURCE_TEMPLATE_TYPE", + "datasource_payload_command_data", + "datasource_payload_rules", + "datasource_template_data", + "datasource_template_index_data", + "normalize_datasource_payload_type", + "require_datasource_payload_type", + "supported_datasource_template_types", +] diff --git a/src/dsctl/services/enums.py b/src/dsctl/services/enums.py index 4dc22b1..8f670c1 100644 --- a/src/dsctl/services/enums.py +++ b/src/dsctl/services/enums.py @@ -4,7 +4,7 @@ from dsctl.config import load_selected_ds_version from dsctl.errors import UserInputError -from dsctl.output import CommandResult, require_json_object +from dsctl.output import CommandResult, require_json_object, require_json_value from dsctl.upstream import ( SUPPORTED_VERSIONS, get_enum_spec, @@ -36,6 +36,13 @@ class EnumData(TypedDict): members: list[EnumMemberData] +class EnumNameData(TypedDict): + """One supported enum discovery name.""" + + name: str + list_command: str + + class ResolvedEnumData(TypedDict): """Resolved enum selector metadata.""" @@ -44,6 +51,28 @@ class ResolvedEnumData(TypedDict): ds_version: str +def list_enum_names_result(*, env_file: str | None = None) -> CommandResult: + """Return supported generated enum discovery names.""" + support = get_version_support(load_selected_ds_version(env_file)) + names = supported_enum_names(support.contract_version) + rows = [ + EnumNameData(name=name, list_command=f"dsctl enum list {name}") + for name in names + ] + return CommandResult( + data=require_json_value(rows, label="enum names data"), + resolved={ + "enum": require_json_object( + { + "ds_version": support.server_version, + "count": len(rows), + }, + label="resolved enum names", + ) + }, + ) + + def list_enum_result(enum_name: str, *, env_file: str | None = None) -> CommandResult: """Return one generated enum and its members.""" requested_name = enum_name.strip() @@ -58,10 +87,7 @@ def list_enum_result(enum_name: str, *, env_file: str | None = None) -> CommandR "enum": enum_name, "supported_enums": supported, }, - suggestion=( - "Run `capabilities` and inspect `data.enums.names` to choose " - "a supported enum name." - ), + suggestion="Run `dsctl enum names` to choose a supported enum name.", ) return CommandResult( @@ -120,4 +146,9 @@ def _enum_member_data(member: EnumMemberSpec) -> EnumMemberData: } -__all__ = ["enum_capabilities_data", "list_enum_result", "supported_enum_choices"] +__all__ = [ + "enum_capabilities_data", + "list_enum_names_result", + "list_enum_result", + "supported_enum_choices", +] diff --git a/src/dsctl/services/schema.py b/src/dsctl/services/schema.py index 84909ea..5dbabc2 100644 --- a/src/dsctl/services/schema.py +++ b/src/dsctl/services/schema.py @@ -1,6 +1,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING from dsctl import __version__ from dsctl.cli_surface import ( @@ -36,7 +37,10 @@ WORKFLOW_INSTANCE_RESOURCE, WORKFLOW_RESOURCE, ) +from dsctl.config import load_selected_ds_version +from dsctl.errors import UserInputError from dsctl.output import CommandResult, require_json_object +from dsctl.services._data_shapes import data_shape_schema_for_action from dsctl.services._schema_groups_context import ( project_group as _project_group, ) @@ -109,21 +113,120 @@ output_schema_data, selection_schema_data, ) -from dsctl.services.capabilities import schema_capabilities_data +from dsctl.services.capabilities import ( + CAPABILITIES_SECTION_CHOICES, + schema_capabilities_data, +) from dsctl.services.template import supported_task_template_types from dsctl.upstream import SUPPORTED_VERSIONS, supported_version_metadata +if TYPE_CHECKING: + from dsctl.support.yaml_io import JsonObject, JsonValue + SchemaGroupBuilder = Callable[[list[str]], dict[str, object]] +SCOPED_SCHEMA_HEADER_KEYS = ( + "schema_version", + "cli", + "supported_ds_versions", + "ds_versions", + "global_options", + "selection", + "output", + "errors", + "confirmation", +) -def get_schema_result() -> CommandResult: +def get_schema_result( + *, + env_file: str | None = None, + group: str | None = None, + command_action: str | None = None, + list_groups: bool = False, + list_commands: bool = False, +) -> CommandResult: """Return the stable machine-readable CLI schema for the current surface.""" - return CommandResult(data=require_json_object(_schema_data(), label="schema data")) + scope_count = sum( + ( + group is not None, + command_action is not None, + list_groups, + list_commands, + ) + ) + if scope_count > 1: + message = ( + "--group, --command, --list-groups, and --list-commands are " + "mutually exclusive" + ) + raise UserInputError( + message, + suggestion=( + "Pass only one schema scope option, or omit them for the full schema." + ), + ) + selected_ds_version = load_selected_ds_version(env_file) + data = require_json_object( + _schema_data(ds_version=selected_ds_version), + label="schema data", + ) + if group is not None: + normalized_group = group.strip() + scoped_data = _schema_group_data(data, normalized_group) + return CommandResult( + data=scoped_data, + resolved={ + "schema": { + "view": "group", + "group": normalized_group, + } + }, + ) + if list_groups: + return CommandResult( + data=_schema_group_discovery_rows(data), + resolved={ + "schema": { + "view": "groups", + "next": "dsctl schema --group GROUP", + } + }, + ) + if list_commands: + return CommandResult( + data=_schema_command_discovery_rows(data), + resolved={ + "schema": { + "view": "commands", + "next": "dsctl schema --command ACTION", + } + }, + ) + if command_action is not None: + normalized_action = command_action.strip() + scoped_data = _schema_command_data(data, normalized_action) + return CommandResult( + data=scoped_data, + resolved={ + "schema": { + "view": "command", + "command": normalized_action, + } + }, + ) + return CommandResult(data=data) -def _schema_data() -> dict[str, object]: +def _schema_data(*, ds_version: str) -> dict[str, object]: task_types = list(supported_task_template_types()) command_groups = _command_groups(task_types) + commands = [ + require_json_object(command_data, label="schema command data") + for command_data in ( + *(_top_level_command_schema(name) for name in TOP_LEVEL_COMMANDS), + *(command_groups[name] for name in COMMAND_GROUPS), + ) + ] return { "schema_version": 1, "cli": { @@ -141,24 +244,35 @@ def _schema_data() -> dict[str, object]: "environment." ), value_name="PATH", - ) + ), + _option( + "output-format", + value_type="string", + description=( + "Render output as json, table, or tsv. json keeps the full " + "standard envelope unless --columns is used for explicit " + "data projection." + ), + default="json", + choices=["json", "table", "tsv"], + value_name="FORMAT", + ), + _option( + "columns", + value_type="string", + description=( + "Comma-separated row/object fields to render or project. " + "In json mode this narrows the standard envelope data payload." + ), + value_name="CSV", + ), ], "selection": selection_schema_data(), "output": output_schema_data(), "errors": error_schema_data(), "confirmation": confirmation_schema_data(), - "capabilities": schema_capabilities_data(), - "commands": [ - *( - _command( - name, - action=name, - summary=TOP_LEVEL_COMMAND_SUMMARIES[name], - ) - for name in TOP_LEVEL_COMMANDS - ), - *(command_groups[name] for name in COMMAND_GROUPS), - ], + "capabilities": schema_capabilities_data(ds_version=ds_version), + "commands": _annotate_command_data_shapes(commands), } @@ -168,6 +282,548 @@ def _command_groups(task_types: list[str]) -> dict[str, dict[str, object]]: } +def _schema_group_data(schema_data: JsonObject, group_name: str) -> JsonObject: + group = _find_schema_group(schema_data, group_name) + scoped = _schema_header(schema_data) + scoped["commands"] = [group] + scoped["rows"] = _schema_group_summary_rows(group) + return scoped + + +def _schema_command_data(schema_data: JsonObject, command_action: str) -> JsonObject: + command = _find_schema_command(schema_data, command_action) + scoped = _schema_header(schema_data) + scoped["commands"] = [command] + scoped["rows"] = _schema_command_detail_rows(command, action=command_action) + return scoped + + +def _schema_header(schema_data: JsonObject) -> JsonObject: + return {key: schema_data[key] for key in SCOPED_SCHEMA_HEADER_KEYS} + + +def _schema_group_discovery_rows(schema_data: JsonObject) -> list[JsonObject]: + rows: list[JsonObject] = [] + for item in _schema_command_nodes(schema_data): + if item.get("kind") != "group": + continue + name = item.get("name") + if not isinstance(name, str): + continue + rows.append( + { + "name": name, + "summary": str(item.get("summary", "")), + "command_count": len(_schema_group_commands(item)), + "schema_command": f"dsctl schema --group {name}", + } + ) + return rows + + +def _schema_command_discovery_rows(schema_data: JsonObject) -> list[JsonObject]: + rows: list[JsonObject] = [] + for item in _schema_command_nodes(schema_data): + rows.extend(_schema_command_discovery_rows_from_node(item, group_name=None)) + return rows + + +def _schema_command_discovery_rows_from_node( + node: JsonObject, + *, + group_name: str | None, +) -> list[JsonObject]: + if node.get("kind") == "command": + action = node.get("action") + if not isinstance(action, str): + return [] + return [ + { + "action": action, + "group": group_name, + "name": str(node.get("name", "")), + "summary": str(node.get("summary", "")), + "schema_command": f"dsctl schema --command {action}", + } + ] + if node.get("kind") != "group": + return [] + + current_group_name = group_name + node_name = node.get("name") + if current_group_name is None and isinstance(node_name, str): + current_group_name = node_name + + rows: list[JsonObject] = [] + group_action = node.get("group_action") + if isinstance(group_action, dict): + action_data = require_json_object(group_action, label="schema group action") + action = action_data.get("action") + if isinstance(action, str): + rows.append( + { + "action": action, + "group": current_group_name, + "name": str(node.get("name", "")), + "summary": str(action_data.get("summary", "")), + "schema_command": f"dsctl schema --command {action}", + } + ) + for child in _schema_group_commands(node): + rows.extend( + _schema_command_discovery_rows_from_node( + child, + group_name=current_group_name, + ) + ) + return rows + + +def _schema_group_summary_rows(group_data: JsonObject) -> list[JsonObject]: + rows: list[JsonObject] = [] + group_action = group_data.get("group_action") + if isinstance(group_action, Mapping): + action_data = require_json_object(group_action, label="schema group action") + action = action_data.get("action") + if isinstance(action, str): + rows.append( + { + "kind": "group_action", + "action": action, + "name": str(group_data.get("name", "")), + "summary": str(action_data.get("summary", "")), + "schema_command": f"dsctl schema --command {action}", + } + ) + for command_node in _schema_group_commands(group_data): + action = command_node.get("action") + if not isinstance(action, str): + continue + rows.append( + { + "kind": "command", + "action": action, + "name": str(command_node.get("name", "")), + "summary": str(command_node.get("summary", "")), + "schema_command": f"dsctl schema --command {action}", + } + ) + return rows + + +def _schema_command_detail_rows( + command_node: JsonObject, + *, + action: str, +) -> list[JsonObject]: + command_data = _find_action_node(command_node, action) + if command_data is None: + return [] + rows: list[JsonObject] = [ + { + "kind": "command", + "name": action, + "description": str(command_data.get("summary", "")), + } + ] + rows.extend( + _schema_parameter_row("argument", require_json_object(item, label="argument")) + for item in _schema_command_items(command_data, "arguments") + ) + rows.extend( + _schema_parameter_row("option", require_json_object(item, label="option")) + for item in _schema_command_items(command_data, "options") + ) + payload = command_data.get("payload") + if isinstance(payload, Mapping): + rows.extend( + _schema_payload_rows( + require_json_object(payload, label="schema payload data") + ) + ) + data_shape = command_data.get("data_shape") + if isinstance(data_shape, Mapping): + rows.extend( + _schema_mapping_rows( + "data_shape", + require_json_object(data_shape, label="schema data shape"), + ) + ) + return rows + + +def _find_action_node(node: JsonObject, action: str) -> JsonObject | None: + if node.get("kind") == "command" and node.get("action") == action: + return node + if node.get("kind") != "group": + return None + group_action = node.get("group_action") + if isinstance(group_action, Mapping) and group_action.get("action") == action: + return require_json_object(group_action, label="schema group action") + for child in _schema_group_commands(node): + matched = _find_action_node(child, action) + if matched is not None: + return matched + return None + + +def _schema_command_items( + command_data: JsonObject, + key: str, +) -> list[JsonObject]: + value = command_data.get(key) + if not isinstance(value, list): + return [] + return [require_json_object(item, label=f"schema {key} item") for item in value] + + +def _schema_parameter_row(kind: str, item: JsonObject) -> JsonObject: + row: JsonObject = { + "kind": kind, + "name": str(item.get("name", "")), + "type": str(item.get("type", "")), + "required": bool(item.get("required", False)), + "description": str(item.get("description", "")), + } + flag = item.get("flag") + if isinstance(flag, str): + row["flag"] = flag + value = _schema_parameter_value(item) + if value: + row["value"] = value + discovery_command = item.get("discovery_command") + if isinstance(discovery_command, str): + row["discovery_command"] = discovery_command + return row + + +def _schema_parameter_value(item: JsonObject) -> str: + parts: list[str] = [] + selector = item.get("selector") + if isinstance(selector, str): + parts.append(f"selector={selector}") + if "default" in item: + parts.append(f"default={_compact_schema_value(item['default'])}") + choices = item.get("choices") + if isinstance(choices, list): + parts.append( + f"choices={_schema_choices_summary(choices, item=item)}", + ) + if item.get("multiple") is True: + parts.append("multiple=true") + value_name = item.get("value_name") + if isinstance(value_name, str): + parts.append(f"value_name={value_name}") + return "; ".join(parts) + + +def _schema_choices_summary(choices: list[JsonValue], *, item: JsonObject) -> str: + if len(choices) <= 8: + return _compact_schema_value(choices) + discovery_command = item.get("discovery_command") + if isinstance(discovery_command, str): + return f"{len(choices)} values; use discovery_command" + return f"{len(choices)} values" + + +def _schema_payload_rows(payload: JsonObject) -> list[JsonObject]: + interesting_keys = ( + "format", + "source_option", + "template_discovery_command", + "template_command", + "template_command_pattern", + "template_json_path", + "template_payload_path", + "type_discovery_command", + "type_enum", + "upstream_request_shape", + ) + rows: list[JsonObject] = [] + for key in interesting_keys: + if key not in payload: + continue + rows.append( + { + "kind": "payload", + "name": key, + "value": _compact_schema_value(payload[key]), + } + ) + return rows + + +def _schema_mapping_rows( + kind: str, + value: JsonObject, +) -> list[JsonObject]: + return [ + { + "kind": kind, + "name": str(key), + "value": _compact_schema_value(item), + } + for key, item in value.items() + ] + + +def _compact_schema_value(value: JsonValue) -> str: + if isinstance(value, list): + scalar_values: list[str] = [] + for item in value: + if isinstance(item, dict | list): + return ", ".join(str(nested) for nested in value) + scalar_values.append(_compact_schema_scalar(item)) + return ", ".join(scalar_values) + if isinstance(value, str | int | float | bool) or value is None: + return _compact_schema_scalar(value) + return str(value) + + +def _compact_schema_scalar(value: JsonValue) -> str: + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def _find_schema_group(schema_data: JsonObject, group_name: str) -> JsonObject: + for item in _schema_command_nodes(schema_data): + if item.get("kind") == "group" and item.get("name") == group_name: + return item + available_groups = [ + str(item["name"]) + for item in _schema_command_nodes(schema_data) + if item.get("kind") == "group" and isinstance(item.get("name"), str) + ] + message = f"Unknown schema group: {group_name}" + raise UserInputError( + message, + details={"group": group_name, "available_groups": available_groups}, + suggestion="Run `dsctl schema --list-groups` to choose a group name.", + ) + + +def _find_schema_command(schema_data: JsonObject, command_action: str) -> JsonObject: + for item in _schema_command_nodes(schema_data): + matched = _match_schema_command_node(item, command_action) + if matched is not None: + return matched + message = f"Unknown schema command: {command_action}" + raise UserInputError( + message, + details={ + "command": command_action, + "available_commands": _available_schema_command_actions(schema_data), + }, + suggestion="Run `dsctl schema --list-commands` to choose a command action.", + ) + + +def _match_schema_command_node( + node: JsonObject, + command_action: str, +) -> JsonObject | None: + if node.get("kind") == "command" and node.get("action") == command_action: + return node + if node.get("kind") != "group": + return None + group_action = node.get("group_action") + if isinstance(group_action, dict) and group_action.get("action") == command_action: + return _schema_group_with_single_action( + node, + group_action=require_json_object( + group_action, + label="schema group action", + ), + ) + for child in _schema_group_commands(node): + matched_child = _match_schema_command_node(child, command_action) + if matched_child is not None: + return _schema_group_with_single_action(node, command=matched_child) + return None + + +def _schema_group_with_single_action( + group_data: JsonObject, + *, + command: JsonObject | None = None, + group_action: JsonObject | None = None, +) -> JsonObject: + scoped = dict(group_data) + scoped["commands"] = [] if command is None else [command] + if group_action is None: + scoped.pop("group_action", None) + else: + scoped["group_action"] = group_action + return scoped + + +def _schema_command_nodes(schema_data: JsonObject) -> list[JsonObject]: + commands = schema_data.get("commands") + if not isinstance(commands, list): + message = "schema data is missing commands" + raise TypeError(message) + return [require_json_object(item, label="schema command") for item in commands] + + +def _schema_group_commands(group_data: JsonObject) -> list[JsonObject]: + commands = group_data.get("commands") + if not isinstance(commands, list): + return [] + return [ + require_json_object(item, label="schema group command") for item in commands + ] + + +def _annotate_command_data_shapes( + commands: list[JsonObject], +) -> list[JsonObject]: + return [ + _annotate_command_node_data_shape(command_node) for command_node in commands + ] + + +def _annotate_command_node_data_shape(command_node: JsonObject) -> JsonObject: + annotated = dict(command_node) + action = annotated.get("action") + if isinstance(action, str): + shape = data_shape_schema_for_action(action) + if shape is not None: + annotated["data_shape"] = require_json_object( + shape, + label="schema data shape", + ) + group_action = annotated.get("group_action") + if isinstance(group_action, dict): + group_action_data = require_json_object( + group_action, + label="schema group action", + ) + group_action_name = group_action_data.get("action") + if isinstance(group_action_name, str): + shape = data_shape_schema_for_action(group_action_name) + if shape is not None: + group_action_copy = dict(group_action_data) + group_action_copy["data_shape"] = require_json_object( + shape, + label="schema data shape", + ) + annotated["group_action"] = group_action_copy + commands_value = annotated.get("commands") + if isinstance(commands_value, list): + annotated["commands"] = [ + _annotate_command_node_data_shape( + require_json_object(item, label="schema nested command") + ) + for item in commands_value + ] + return annotated + + +def _top_level_command_schema(name: str) -> JsonObject: + if name == "schema": + return require_json_object( + _command( + name, + action=name, + summary=TOP_LEVEL_COMMAND_SUMMARIES[name], + options=[ + _option( + "group", + value_type="string", + description=( + "Return schema for one command group. Discover " + "values with `dsctl schema --list-groups`." + ), + discovery_command="dsctl schema --list-groups", + ), + _option( + "command", + value_type="string", + description=( + "Return schema for one stable command action. " + "Discover values with `dsctl schema --list-commands`." + ), + discovery_command="dsctl schema --list-commands", + ), + _option( + "list-groups", + value_type="boolean", + description="List valid values for --group.", + default=False, + ), + _option( + "list-commands", + value_type="boolean", + description="List valid values for --command.", + default=False, + ), + ], + ), + label="top-level command schema", + ) + if name == "capabilities": + return require_json_object( + _command( + name, + action=name, + summary=TOP_LEVEL_COMMAND_SUMMARIES[name], + options=[ + _option( + "summary", + value_type="boolean", + description="Return lightweight capability discovery.", + default=False, + ), + _option( + "section", + value_type="string", + description=( + "Return one top-level capability section. Supported: " + f"{', '.join(CAPABILITIES_SECTION_CHOICES)}. Discover " + "values with `dsctl schema --command capabilities`." + ), + choices=list(CAPABILITIES_SECTION_CHOICES), + discovery_command="dsctl schema --command capabilities", + ), + ], + ), + label="top-level command schema", + ) + return require_json_object( + _command( + name, + action=name, + summary=TOP_LEVEL_COMMAND_SUMMARIES[name], + ), + label="top-level command schema", + ) + + +def _available_schema_command_actions(schema_data: JsonObject) -> list[str]: + actions: list[str] = [] + for item in _schema_command_nodes(schema_data): + actions.extend(_schema_command_actions(item)) + return actions + + +def _schema_command_actions(node: JsonObject) -> list[str]: + actions: list[str] = [] + action = node.get("action") + if isinstance(action, str): + actions.append(action) + group_action = node.get("group_action") + if isinstance(group_action, dict): + group_action_name = group_action.get("action") + if isinstance(group_action_name, str): + actions.append(group_action_name) + for child in _schema_group_commands(node): + actions.extend(_schema_command_actions(child)) + return actions + + def _static_group_builder( factory: Callable[[], dict[str, object]], ) -> SchemaGroupBuilder: diff --git a/src/dsctl/services/task_instance.py b/src/dsctl/services/task_instance.py index a4c3a2c..fe2f23a 100644 --- a/src/dsctl/services/task_instance.py +++ b/src/dsctl/services/task_instance.py @@ -2,7 +2,7 @@ import time from collections import deque -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, TypeAlias, TypedDict from dsctl.cli_surface import TASK_INSTANCE_RESOURCE from dsctl.errors import ( @@ -10,6 +10,7 @@ InvalidStateError, NotFoundError, PermissionDeniedError, + TaskNotDispatchedError, UserInputError, WaitTimeoutError, ) @@ -25,17 +26,34 @@ optional_text, serialize_task_instance, ) -from dsctl.services._validation import require_non_negative_int, require_positive_int +from dsctl.services._validation import ( + optional_ds_datetime, + require_non_negative_int, + require_positive_int, + validate_ds_datetime_range, +) from dsctl.services.pagination import ( DEFAULT_PAGE_SIZE, MAX_AUTO_EXHAUST_PAGES, PageData, requested_page_data, ) +from dsctl.services.resolver import ( + ResolvedProject, + ResolvedWorkflow, +) +from dsctl.services.resolver import project as resolve_project +from dsctl.services.resolver import workflow as resolve_workflow from dsctl.services.runtime import ServiceRuntime, run_with_service_runtime +from dsctl.services.selection import ( + SelectedValue, + require_project_selection, + with_selection_source, +) from dsctl.upstream.runtime_enums import ( TASK_EXECUTION_FINISHED_STATES, TASK_EXECUTION_FORCE_SUCCESS_ALLOWED_STATES, + task_execute_type_value, task_execution_status_value, workflow_execution_status_is_final, ) @@ -43,12 +61,18 @@ if TYPE_CHECKING: from dsctl.upstream.protocol import TaskInstanceRecord, WorkflowInstanceRecord +ResolvedMetadataValue: TypeAlias = int | str | None +ResolvedMetadata: TypeAlias = dict[str, ResolvedMetadataValue] +TaskInstanceListResolvedValue: TypeAlias = int | str | bool | None | ResolvedMetadata +TaskInstanceListResolvedData: TypeAlias = dict[str, TaskInstanceListResolvedValue] + LOG_CHUNK_SIZE = 1000 MAX_LOG_CHUNKS = 200 DEFAULT_TASK_INSTANCE_WATCH_INTERVAL_SECONDS = 5 DEFAULT_TASK_INSTANCE_WATCH_TIMEOUT_SECONDS = 600 TASK_INSTANCE_NOT_FOUND = 10008 +TASK_INSTANCE_LOG_PATH_EMPTY = 10103 TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE = 10021 TASK_INSTANCE_STATE_OPERATION_ERROR = 10166 TASK_SAVEPOINT_ERROR = 10196 @@ -72,6 +96,43 @@ def _workflow_instance_get_command(workflow_instance_id: int) -> str: return f"dsctl workflow-instance get {workflow_instance_id}" +def _unsupported_task_instance_workflow_filter( + *, + workflow: str, + workflow_instance_id: int | None, +) -> UserInputError: + if workflow_instance_id is not None: + message = "`task-instance list` does not accept --workflow" + suggestion = ( + "Drop --workflow; --workflow-instance already scopes the " + "task-instance query to one workflow run." + ) + else: + message = ( + "`task-instance list` cannot reliably filter by workflow definition " + "in DolphinScheduler 3.4.1" + ) + suggestion = ( + "Run `dsctl workflow-instance list --project --workflow " + f"{workflow}` to find workflow instance ids, then run `dsctl " + "task-instance list --workflow-instance `." + ) + return UserInputError( + message, + details={ + "resource": TASK_INSTANCE_RESOURCE, + "workflow": workflow, + "workflow_instance_id": workflow_instance_id, + "upstream_filter": "workflowDefinitionName", + "reason": ( + "DS 3.4.1 ignores workflowDefinitionName for regular BATCH " + "task-instance paging queries." + ), + }, + suggestion=suggestion, + ) + + class WorkflowInstanceSelectionData(TypedDict): """Resolved workflow-instance selector emitted in JSON envelopes.""" @@ -102,32 +163,76 @@ class TaskInstanceSubWorkflowData(TypedDict): def list_task_instances_result( *, - workflow_instance: int, + workflow_instance: int | None = None, + project: str | None = None, + workflow: str | None = None, + workflow_instance_name: str | None = None, page_no: int = 1, page_size: int = DEFAULT_PAGE_SIZE, all_pages: bool = False, search: str | None = None, + task: str | None = None, + task_code: int | None = None, + executor: str | None = None, state: str | None = None, + host: str | None = None, + start: str | None = None, + end: str | None = None, + execute_type: str | None = None, env_file: str | None = None, ) -> CommandResult: - """List task instances inside one workflow instance.""" - normalized_workflow_instance = require_positive_int( - workflow_instance, - label="workflow_instance", + """List task instances with project-scoped DS runtime filters.""" + normalized_workflow_instance = ( + None + if workflow_instance is None + else require_positive_int( + workflow_instance, + label="workflow_instance", + ) ) normalized_page_no = require_positive_int(page_no, label="page_no") normalized_page_size = require_positive_int(page_size, label="page_size") + normalized_project = optional_text(project) + normalized_workflow = optional_text(workflow) + if normalized_workflow is not None: + raise _unsupported_task_instance_workflow_filter( + workflow=normalized_workflow, + workflow_instance_id=normalized_workflow_instance, + ) + normalized_workflow_instance_name = optional_text(workflow_instance_name) normalized_search = optional_text(search) + normalized_task = optional_text(task) + normalized_task_code = ( + None + if task_code is None + else require_positive_int(task_code, label="task_code") + ) + normalized_executor = optional_text(executor) normalized_state = _normalized_task_instance_state(state) + normalized_host = optional_text(host) + normalized_start = optional_ds_datetime(start, label="start") + normalized_end = optional_ds_datetime(end, label="end") + validate_ds_datetime_range(normalized_start, normalized_end) + normalized_execute_type = _normalized_task_execute_type(execute_type) return run_with_service_runtime( env_file, _list_task_instances_result, workflow_instance_id=normalized_workflow_instance, + project=normalized_project, + workflow=normalized_workflow, + workflow_instance_name=normalized_workflow_instance_name, page_no=normalized_page_no, page_size=normalized_page_size, all_pages=all_pages, search=normalized_search, + task=normalized_task, + task_code=normalized_task_code, + executor=normalized_executor, state=normalized_state, + host=normalized_host, + start=normalized_start, + end=normalized_end, + execute_type=normalized_execute_type, ) @@ -304,30 +409,53 @@ def stop_task_instance_result( def _list_task_instances_result( runtime: ServiceRuntime, *, - workflow_instance_id: int, + workflow_instance_id: int | None, + project: str | None, + workflow: str | None, + workflow_instance_name: str | None, page_no: int, page_size: int, all_pages: bool, search: str | None, + task: str | None, + task_code: int | None, + executor: str | None, state: str | None, + host: str | None, + start: str | None, + end: str | None, + execute_type: str | None, ) -> CommandResult: - workflow_instance = get_workflow_instance( - runtime, - workflow_instance_id=workflow_instance_id, + resolved_project, selected_project, resolved_workflow = ( + _resolve_task_instance_list_scope( + runtime, + workflow_instance_id=workflow_instance_id, + project=project, + workflow=workflow, + ) ) - project_code = require_workflow_instance_project_code( - workflow_instance.projectCode, + workflow_definition_name = ( + None if resolved_workflow is None else resolved_workflow.name ) data = require_json_object( requested_page_data( lambda current_page_no, current_page_size: ( runtime.upstream.task_instances.list( + project_code=resolved_project.code, workflow_instance_id=workflow_instance_id, - project_code=project_code, + workflow_instance_name=workflow_instance_name, + workflow_definition_name=workflow_definition_name, page_no=current_page_no, page_size=current_page_size, search=search, + task_name=task, + task_code=task_code, + executor=executor, state=state, + host=host, + start_time=start, + end_time=end, + task_execute_type=execute_type, ) ), page_no=page_no, @@ -339,25 +467,172 @@ def _list_task_instances_result( ), label="task-instance list data", ) - resolved: dict[str, int | str | None] = { - "workflow_instance": workflow_instance_id, - "page_no": page_no, - "page_size": page_size, - "all": all_pages, - } - if search is not None: - resolved["search"] = search - if state is not None: - resolved["state"] = state return CommandResult( data=data, resolved=require_json_object( - resolved, + _task_instance_list_resolved( + resolved_project=resolved_project, + selected_project=selected_project, + resolved_workflow=resolved_workflow, + workflow_instance_id=workflow_instance_id, + workflow_instance_name=workflow_instance_name, + page_no=page_no, + page_size=page_size, + all_pages=all_pages, + search=search, + task=task, + task_code=task_code, + executor=executor, + state=state, + host=host, + start=start, + end=end, + execute_type=execute_type, + ), label="task-instance list resolved", ), ) +def _resolve_task_instance_list_scope( + runtime: ServiceRuntime, + *, + workflow_instance_id: int | None, + project: str | None, + workflow: str | None, +) -> tuple[ResolvedProject, SelectedValue | None, ResolvedWorkflow | None]: + if workflow_instance_id is not None: + workflow_instance = get_workflow_instance( + runtime, + workflow_instance_id=workflow_instance_id, + ) + project_code = require_workflow_instance_project_code( + workflow_instance.projectCode, + ) + resolved_project = resolve_project( + str(project_code), + adapter=runtime.upstream.projects, + ) + selected_project: SelectedValue | None = None + if project is not None: + explicit_project = resolve_project( + project, + adapter=runtime.upstream.projects, + ) + if explicit_project.code != project_code: + message = ( + "Selected project does not match the workflow instance project" + ) + raise UserInputError( + message, + details={ + "project": project, + "project_code": explicit_project.code, + "workflow_instance_id": workflow_instance_id, + "workflow_instance_project_code": project_code, + }, + suggestion=( + "Use the project that owns the workflow instance, or omit " + "--project when --workflow-instance is already provided." + ), + ) + resolved_project = explicit_project + selected_project = SelectedValue(value=project, source="flag") + resolved_workflow = ( + None + if workflow is None + else resolve_workflow( + workflow, + adapter=runtime.upstream.workflows, + project_code=resolved_project.code, + ) + ) + return resolved_project, selected_project, resolved_workflow + + selected_project = require_project_selection(project, runtime=runtime) + resolved_project = resolve_project( + selected_project.value, + adapter=runtime.upstream.projects, + ) + resolved_workflow = ( + None + if workflow is None + else resolve_workflow( + workflow, + adapter=runtime.upstream.workflows, + project_code=resolved_project.code, + ) + ) + return resolved_project, selected_project, resolved_workflow + + +def _task_instance_list_resolved( + *, + resolved_project: ResolvedProject, + selected_project: SelectedValue | None, + resolved_workflow: ResolvedWorkflow | None, + workflow_instance_id: int | None, + workflow_instance_name: str | None, + page_no: int, + page_size: int, + all_pages: bool, + search: str | None, + task: str | None, + task_code: int | None, + executor: str | None, + state: str | None, + host: str | None, + start: str | None, + end: str | None, + execute_type: str | None, +) -> TaskInstanceListResolvedData: + project_data = _project_metadata(resolved_project) + if selected_project is not None: + project_data = dict(with_selection_source(project_data, selected_project)) + resolved: TaskInstanceListResolvedData = { + "project": project_data, + "page_no": page_no, + "page_size": page_size, + "all": all_pages, + } + optional_fields: dict[str, TaskInstanceListResolvedValue] = { + "workflow_instance": workflow_instance_id, + "workflow_instance_name": workflow_instance_name, + "workflow": ( + None if resolved_workflow is None else _workflow_metadata(resolved_workflow) + ), + "search": search, + "task": task, + "task_code": task_code, + "executor": executor, + "state": state, + "host": host, + "start": start, + "end": end, + "execute_type": execute_type, + } + resolved.update( + {key: value for key, value in optional_fields.items() if value is not None} + ) + return resolved + + +def _project_metadata(project: ResolvedProject) -> ResolvedMetadata: + return { + "code": project.code, + "name": project.name, + "description": project.description, + } + + +def _workflow_metadata(workflow: ResolvedWorkflow) -> ResolvedMetadata: + return { + "code": workflow.code, + "name": workflow.name, + "version": workflow.version, + } + + def _get_task_instance_result( runtime: ServiceRuntime, *, @@ -506,11 +781,17 @@ def _get_task_instance_log_result( lines: deque[str] = deque(maxlen=tail) skip_line_num = 0 for _ in range(MAX_LOG_CHUNKS): - chunk = runtime.upstream.task_instances.log_chunk( - task_instance_id=task_instance_id, - skip_line_num=skip_line_num, - limit=LOG_CHUNK_SIZE, - ) + try: + chunk = runtime.upstream.task_instances.log_chunk( + task_instance_id=task_instance_id, + skip_line_num=skip_line_num, + limit=LOG_CHUNK_SIZE, + ) + except ApiResultError as exc: + raise _task_instance_log_error( + exc, + task_instance_id=task_instance_id, + ) from exc chunk_lines = (chunk.message or "").splitlines() lines.extend(chunk_lines) if chunk.lineNum < LOG_CHUNK_SIZE: @@ -737,12 +1018,31 @@ def _normalized_task_instance_state(value: str | None) -> str | None: message, details={"state": value}, suggestion=( - "Run `dsctl enum list task_execution_status` to inspect the " + "Run `dsctl enum list task-execution-status` to inspect the " "supported DS task-instance states." ), ) from exc +def _normalized_task_execute_type(value: str | None) -> str | None: + normalized = optional_text(value) + if normalized is None: + return None + candidate = normalized.upper() + try: + return task_execute_type_value(candidate) + except KeyError as exc: + message = "Task execute type must be one of the DS task execute-type names" + raise UserInputError( + message, + details={"execute_type": value}, + suggestion=( + "Run `dsctl enum list task-execute-type` to inspect the " + "supported DS task execute-type names." + ), + ) from exc + + def _task_instance_context( runtime: ServiceRuntime, *, @@ -956,6 +1256,31 @@ def _task_instance_action_error( return error +def _task_instance_log_error( + error: ApiResultError, + *, + task_instance_id: int, +) -> ApiResultError | TaskNotDispatchedError: + if error.result_code != TASK_INSTANCE_LOG_PATH_EMPTY: + return error + return TaskNotDispatchedError( + "Task instance log is not available because the task has not been dispatched.", + details={ + "resource": TASK_INSTANCE_RESOURCE, + "id": task_instance_id, + "result_code": error.result_code, + "result_message": error.result_message, + }, + source=error.source, + suggestion=( + "Inspect the task instance state with `task-instance list " + "--workflow-instance ` or run " + "`workflow-instance digest ` to confirm " + "whether DolphinScheduler has dispatched the task." + ), + ) + + def _task_instance_sub_workflow_error( error: ApiResultError, *, diff --git a/src/dsctl/services/template.py b/src/dsctl/services/template.py index 2e94939..bbda314 100644 --- a/src/dsctl/services/template.py +++ b/src/dsctl/services/template.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from textwrap import dedent from typing import TYPE_CHECKING, TypeAlias, TypedDict @@ -8,6 +9,12 @@ from dsctl.models.task_spec import canonical_task_type from dsctl.output import CommandResult, require_json_object from dsctl.services import _task_templates +from dsctl.services.datasource_payload import ( + datasource_template_data, + datasource_template_index_data, + require_datasource_payload_type, + supported_datasource_template_types, +) if TYPE_CHECKING: from dsctl.services._task_templates import TaskTemplateMetadata @@ -22,6 +29,7 @@ class TaskTemplateTypesData(TypedDict): generic_task_types: list[str] task_types_by_category: dict[str, list[str]] task_templates: dict[str, TaskTemplateMetadata] + rows: list[TaskTemplateTypeRowData] class ParameterFieldData(TypedDict): @@ -132,6 +140,74 @@ class ParameterTopicData(TypedDict): summary: str +class EnvironmentConfigLineData(TypedDict): + """One line in the DS environment config shell template.""" + + line: str + purpose: str + + +class ClusterConfigFieldData(TypedDict): + """One field in the DS cluster config JSON object.""" + + name: str + required: bool + value_type: str + description: str + + +class TextLineData(TypedDict): + """One rendered text-template line for table and tsv output.""" + + line_no: int + line: str + + +class EnvironmentConfigTemplateData(TypedDict): + """Stable discovery payload for `dsctl template environment`.""" + + filename: str + config: str + lines: list[EnvironmentConfigLineData] + target_commands: list[str] + source_options: list[str] + upstream_request_shape: str + rules: list[str] + + +class ClusterConfigTemplateData(TypedDict): + """Stable discovery payload for `dsctl template cluster`.""" + + filename: str + config: str + payload: dict[str, str] + fields: list[ClusterConfigFieldData] + rows: list[ClusterConfigFieldData] + target_commands: list[str] + source_options: list[str] + upstream_request_shape: str + upstream_ui_shape: str + rules: list[str] + + +class ClusterConfigTemplateCapabilityData(TypedDict): + """Compact capability metadata for cluster config templates.""" + + command: str + source_options: list[str] + target_commands: list[str] + + +class TaskTemplateTypeRowData(TypedDict): + """One compact task-template type row.""" + + task_type: str + kind: str + category: str + default_variant: str + variants: str + + class ParameterSyntaxIndexData(TypedDict): """Compact discovery payload for `dsctl template params`.""" @@ -199,10 +275,10 @@ def parameter_syntax_index_data() -> ParameterSyntaxIndexData: }, ], "recommended_flow": [ - "Run `template params` first and select only the needed topic.", - "Run `template task TYPE --variant params` for task-specific YAML.", - "Run `lint workflow FILE` before sending the workflow to DS.", - "Run `workflow create --file FILE --dry-run` before mutation.", + "Run `dsctl template params` first and select only the needed topic.", + "Run `dsctl template task TYPE --variant params` for task-specific YAML.", + "Run `dsctl lint workflow FILE` before sending the workflow to DS.", + "Run `dsctl workflow create --file FILE --dry-run` before mutation.", ], "rules": [ "The CLI preserves DS parameter expressions as strings.", @@ -266,7 +342,7 @@ def _normalize_parameter_syntax_topic(topic: str) -> str: raise UserInputError( message, details={"topic": topic}, - suggestion="Run `template params` to inspect available topics.", + suggestion="Run `dsctl template params` to inspect available topics.", ) @@ -603,6 +679,11 @@ def supported_task_template_types() -> tuple[str, ...]: return _task_templates.supported_task_template_types() +def supported_datasource_types() -> tuple[str, ...]: + """Return datasource types supported by local payload templates.""" + return supported_datasource_template_types() + + def typed_task_template_types() -> tuple[str, ...]: """Return task types backed by typed `task_params` models.""" return _task_templates.typed_task_template_types() @@ -620,15 +701,189 @@ def task_template_metadata() -> dict[str, TaskTemplateMetadata]: def workflow_template_result(*, with_schedule: bool = False) -> CommandResult: """Return the stable workflow YAML template.""" + yaml_text = _workflow_template_yaml(with_schedule=with_schedule) return CommandResult( data=require_json_object( - {"yaml": _workflow_template_yaml(with_schedule=with_schedule)}, + { + "yaml": yaml_text, + "lines": _text_lines(yaml_text), + }, label="workflow template data", ), resolved={"with_schedule": with_schedule}, ) +def environment_config_template_result() -> CommandResult: + """Return one DS environment shell/export config template.""" + lines = [ + EnvironmentConfigLineData( + line="export JAVA_HOME=/opt/java", + purpose="Java runtime used by shell, Java, and JVM-based task types.", + ), + EnvironmentConfigLineData( + line="export HADOOP_HOME=/opt/hadoop", + purpose="Hadoop client installation used by Hadoop ecosystem tasks.", + ), + EnvironmentConfigLineData( + line="export HADOOP_CONF_DIR=/etc/hadoop/conf", + purpose="Hadoop/YARN configuration directory visible to workers.", + ), + EnvironmentConfigLineData( + line="export SPARK_HOME=/opt/spark", + purpose="Spark client installation used by Spark tasks.", + ), + EnvironmentConfigLineData( + line="export PYTHON_LAUNCHER=/opt/python/bin/python3", + purpose="Python interpreter path used by Python-style tasks.", + ), + EnvironmentConfigLineData( + line=("export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH"), + purpose="Expose selected runtimes on PATH without replacing worker PATH.", + ), + ] + config = "\n".join(item["line"] for item in lines) + "\n" + return CommandResult( + data=require_json_object( + EnvironmentConfigTemplateData( + filename="env.sh", + config=config, + lines=lines, + target_commands=[ + "dsctl environment create --name NAME --config-file env.sh", + "dsctl environment update ENVIRONMENT --config-file env.sh", + ], + source_options=["--config TEXT", "--config-file PATH"], + upstream_request_shape=( + "EnvironmentController form field `config` stores raw " + "shell/export text." + ), + rules=[ + "Use shell/export syntax, not JSON.", + "Prefer --config-file for multiline environment configs.", + "The paths must exist on DolphinScheduler worker hosts.", + "Keep secrets out of environment configs when possible.", + "Bind worker groups with repeated --worker-group values.", + ], + ), + label="environment config template data", + ), + resolved={"template": "environment.config"}, + ) + + +def cluster_config_template_result() -> CommandResult: + """Return one DS cluster config JSON template.""" + payload = { + "k8s": _cluster_k8s_config_placeholder(), + "yarn": "", + } + fields = [ + ClusterConfigFieldData( + name="k8s", + required=True, + value_type="string", + description=( + "Kubernetes kubeconfig content. DS currently reads this field " + "when resolving a cluster's Kubernetes config." + ), + ), + ClusterConfigFieldData( + name="yarn", + required=False, + value_type="string", + description=( + "Reserved by the DS UI shape; DS 3.4.1 does not actively use " + "this field." + ), + ), + ] + return CommandResult( + data=require_json_object( + ClusterConfigTemplateData( + filename="cluster-config.json", + config=_cluster_config_json(payload), + payload=payload, + fields=fields, + rows=fields, + target_commands=[ + ( + "dsctl cluster create --name NAME " + "--config-file cluster-config.json" + ), + "dsctl cluster update CLUSTER --config-file cluster-config.json", + ], + source_options=["--config TEXT", "--config-file PATH"], + upstream_request_shape=( + "ClusterController form field `config` stores a raw string; " + "DS 3.4.1 expects a JSON object for cluster config usage." + ), + upstream_ui_shape=( + "The DS 3.4.1 UI submits JSON.stringify({k8s, yarn})." + ), + rules=[ + "Use JSON object syntax, not a bare kubeconfig string.", + "Prefer --config-file for multiline Kubernetes kubeconfigs.", + "Keep the k8s value as the full kubeconfig text.", + "Keep yarn as an empty string unless your DS deployment uses it.", + "The kubeconfig must be usable from DolphinScheduler API/workers.", + ], + ), + label="cluster config template data", + ), + resolved={"template": "cluster.config"}, + ) + + +def cluster_config_template_capability_data() -> ClusterConfigTemplateCapabilityData: + """Return compact capability metadata for cluster config templates.""" + return { + "command": "dsctl template cluster", + "source_options": ["--config TEXT", "--config-file PATH"], + "target_commands": [ + "dsctl cluster create --name NAME --config-file cluster-config.json", + "dsctl cluster update CLUSTER --config-file cluster-config.json", + ], + } + + +def datasource_template_result(datasource_type: str | None = None) -> CommandResult: + """Return datasource payload-template discovery or one JSON template.""" + if datasource_type is None: + index_data = datasource_template_index_data() + data = dict(index_data) + data["rows"] = [ + { + "type": datasource_type_name, + "template_command": ( + f"dsctl template datasource --type {datasource_type_name}" + ), + } + for datasource_type_name in index_data["supported_types"] + ] + return CommandResult( + data=require_json_object( + data, + label="datasource template index data", + ), + resolved={"view": "list"}, + ) + normalized_type = require_datasource_payload_type(datasource_type) + template_data = datasource_template_data(normalized_type) + data = dict(template_data) + data["rows"] = template_data["fields"] + return CommandResult( + data=require_json_object( + data, + label="datasource template data", + ), + resolved={ + "view": "template", + "datasource_type": normalized_type, + }, + ) + + def task_template_result( task_type: str, *, @@ -638,13 +893,15 @@ def task_template_result( normalized = _normalize_task_type(task_type) normalized_variant = _normalize_task_template_variant(normalized, variant) template_kind = _task_templates.task_template_kind(normalized) + yaml_text = _task_templates.task_template_yaml( + normalized, + variant=normalized_variant, + ) return CommandResult( data=require_json_object( { - "yaml": _task_templates.task_template_yaml( - normalized, - variant=normalized_variant, - ) + "yaml": yaml_text, + "rows": _text_lines(yaml_text), }, label="task template data", ), @@ -677,26 +934,37 @@ def task_template_types_result() -> CommandResult: generic_task_types=generic_task_types, task_types_by_category=task_types_by_category, task_templates=task_template_metadata(), + rows=_task_template_type_rows(), ), label="task template types data", - ) + ), + resolved={"mode": "list"}, ) def _normalize_task_type(task_type: str) -> str: normalized = canonical_task_type(task_type) - suggestion = "Run `template task --list` to inspect supported task types." + suggestion = "Run `dsctl template task --list` to inspect supported task types." if not normalized: - supported = ", ".join(_SUPPORTED_TASK_TEMPLATE_TYPES) - message = f"TASK_TYPE is required. Supported: {supported}" - raise UserInputError(message, suggestion=suggestion) + message = "TASK_TYPE is required." + raise UserInputError( + message, + details={ + "available_task_types_count": len(_SUPPORTED_TASK_TEMPLATE_TYPES), + "discovery_command": "dsctl template task --list", + }, + suggestion=suggestion, + ) if normalized in _SUPPORTED_TASK_TEMPLATE_TYPES: return normalized - supported = ", ".join(_SUPPORTED_TASK_TEMPLATE_TYPES) - message = f"Unsupported task template type '{task_type}'. Supported: {supported}" + message = f"Unsupported task template type '{task_type}'." raise UserInputError( message, - details={"task_type": task_type}, + details={ + "task_type": task_type, + "available_task_types_count": len(_SUPPORTED_TASK_TEMPLATE_TYPES), + "discovery_command": "dsctl template task --list", + }, suggestion=suggestion, ) @@ -708,15 +976,18 @@ def _normalize_task_template_variant(task_type: str, variant: str | None) -> str supported_variants = _task_templates.task_template_variants(task_type) if normalized in supported_variants: return normalized - supported = ", ".join(supported_variants) message = ( - f"Unsupported task template variant '{variant}' for task type " - f"'{task_type}'. Supported: {supported}" + f"Unsupported task template variant '{variant}' for task type '{task_type}'." ) raise UserInputError( message, - details={"task_type": task_type, "variant": variant}, - suggestion="Run `template task --list` to inspect supported variants.", + details={ + "task_type": task_type, + "variant": variant, + "available_variants": list(supported_variants), + "discovery_command": "dsctl template task --list", + }, + suggestion="Run `dsctl template task --list` to inspect supported variants.", ) @@ -856,6 +1127,35 @@ def _workflow_template_yaml(*, with_schedule: bool) -> str: return f"{base}{schedule}" +def _cluster_k8s_config_placeholder() -> str: + return dedent( + """\ + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority-data: CHANGE_ME_BASE64_CA + server: https://KUBERNETES_API_SERVER:6443 + name: kubernetes + contexts: + - context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes + current-context: kubernetes-admin@kubernetes + users: + - name: kubernetes-admin + user: + client-certificate-data: CHANGE_ME_BASE64_CERT + client-key-data: CHANGE_ME_BASE64_KEY + """ + ) + + +def _cluster_config_json(payload: dict[str, str]) -> str: + return json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + + def _task_template_types_data( *, task_types: list[str], @@ -863,6 +1163,7 @@ def _task_template_types_data( generic_task_types: list[str], task_types_by_category: dict[str, list[str]], task_templates: dict[str, TaskTemplateMetadata], + rows: list[TaskTemplateTypeRowData], ) -> TaskTemplateTypesData: return { "task_types": task_types, @@ -871,9 +1172,24 @@ def _task_template_types_data( "generic_task_types": generic_task_types, "task_types_by_category": task_types_by_category, "task_templates": task_templates, + "rows": rows, } +def _task_template_type_rows() -> list[TaskTemplateTypeRowData]: + metadata = task_template_metadata() + return [ + TaskTemplateTypeRowData( + task_type=task_type, + kind=metadata[task_type]["kind"], + category=metadata[task_type]["category"], + default_variant=metadata[task_type]["default_variant"], + variants=",".join(metadata[task_type]["variants"]), + ) + for task_type in supported_task_template_types() + ] + + def _task_template_types_by_category() -> dict[str, tuple[str, ...]]: metadata = task_template_metadata() categories: dict[str, list[str]] = {} @@ -883,6 +1199,13 @@ def _task_template_types_by_category() -> dict[str, tuple[str, ...]]: return {category: tuple(task_types) for category, task_types in categories.items()} +def _text_lines(text: str) -> list[TextLineData]: + return [ + TextLineData(line_no=index, line=line) + for index, line in enumerate(text.splitlines(), start=1) + ] + + _PARAMETER_SYNTAX_TOPICS = ( "overview", "property", diff --git a/src/dsctl/services/workflow_instance.py b/src/dsctl/services/workflow_instance.py index 8ae4650..eeb8fb9 100644 --- a/src/dsctl/services/workflow_instance.py +++ b/src/dsctl/services/workflow_instance.py @@ -1,7 +1,7 @@ from __future__ import annotations import time -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, TypeAlias, TypedDict from dsctl.cli_surface import TASK_RESOURCE, WORKFLOW_INSTANCE_RESOURCE from dsctl.errors import ( @@ -26,9 +26,11 @@ serialize_workflow_instance, ) from dsctl.services._validation import ( + optional_ds_datetime, require_non_empty_text, require_non_negative_int, require_positive_int, + validate_ds_datetime_range, ) from dsctl.services._workflow_instance_digest import ( digest_workflow_instance as _digest_workflow_instance, @@ -44,9 +46,15 @@ collect_all_pages, requested_page_data, ) -from dsctl.services.resolver import ResolvedProjectData, ResolvedTaskData +from dsctl.services.resolver import ( + ResolvedProject, + ResolvedProjectData, + ResolvedTaskData, + ResolvedWorkflow, +) from dsctl.services.resolver import project as resolve_project from dsctl.services.resolver import task as resolve_task +from dsctl.services.resolver import workflow as resolve_workflow from dsctl.services.runtime import ServiceRuntime, run_with_service_runtime from dsctl.upstream.runtime_enums import ( WORKFLOW_EXECUTION_FAILURE_STATE, @@ -85,6 +93,7 @@ class WorkflowInstanceSelectionData(TypedDict): WORKFLOW_NODE_HAS_CYCLE = 50019 WORKFLOW_NODE_S_PARAMETER_INVALID = 50020 CHECK_WORKFLOW_TASK_RELATION_ERROR = 50036 +QUERY_WORKFLOW_INSTANCE_LIST_PAGING_ERROR = 10113 WORKFLOW_INSTANCE_PATCH_SUPPORTED_WORKFLOW_FIELDS = frozenset( {"global_params", "timeout"} ) @@ -134,6 +143,9 @@ class WorkflowInstanceUpdateNoChangeWarningDetail(TypedDict): request_sent: bool +WorkflowInstanceListResolvedValue: TypeAlias = int | str | bool + + def list_workflow_instances_result( *, page_no: int = 1, @@ -141,12 +153,28 @@ def list_workflow_instances_result( all_pages: bool = False, project: str | None = None, workflow: str | None = None, + search: str | None = None, + executor: str | None = None, + host: str | None = None, + start: str | None = None, + end: str | None = None, state: str | None = None, env_file: str | None = None, ) -> CommandResult: """List workflow instances using explicit runtime filters.""" normalized_project = optional_text(project) normalized_workflow = optional_text(workflow) + normalized_search = optional_text(search) + normalized_executor = optional_text(executor) + normalized_host = optional_text(host) + normalized_start = optional_ds_datetime(start, label="start") + normalized_end = optional_ds_datetime(end, label="end") + validate_ds_datetime_range(normalized_start, normalized_end) + _validate_workflow_instance_list_filters( + project=normalized_project, + search=normalized_search, + executor=normalized_executor, + ) normalized_state = _normalized_workflow_instance_state(state) normalized_page_no = require_positive_int(page_no, label="page_no") normalized_page_size = require_positive_int(page_size, label="page_size") @@ -158,6 +186,11 @@ def list_workflow_instances_result( all_pages=all_pages, project=normalized_project, workflow=normalized_workflow, + search=normalized_search, + executor=normalized_executor, + host=normalized_host, + start=normalized_start, + end=normalized_end, state=normalized_state, ) @@ -342,6 +375,58 @@ def update_workflow_instance_result( ) +def _validate_workflow_instance_list_filters( + *, + project: str | None, + search: str | None, + executor: str | None, +) -> None: + if project is not None: + return + project_scoped_filters = [ + name + for name, value in (("search", search), ("executor", executor)) + if value is not None + ] + if not project_scoped_filters: + return + message = "These workflow-instance list filters require --project" + raise UserInputError( + message, + details={"filters": project_scoped_filters}, + suggestion=( + "Pass --project PROJECT with --search or --executor, or use " + "--workflow, --state, --host, --start, and --end for global " + "workflow-instance filtering." + ), + ) + + +def _resolve_workflow_instance_list_project( + runtime: ServiceRuntime, + *, + project: str | None, +) -> ResolvedProject | None: + if project is None: + return None + return resolve_project(project, adapter=runtime.upstream.projects) + + +def _resolve_workflow_instance_list_workflow( + runtime: ServiceRuntime, + *, + workflow: str | None, + project: ResolvedProject | None, +) -> ResolvedWorkflow | None: + if workflow is None or project is None: + return None + return resolve_workflow( + workflow, + adapter=runtime.upstream.workflows, + project_code=project.code, + ) + + def _list_workflow_instances_result( runtime: ServiceRuntime, *, @@ -350,16 +435,41 @@ def _list_workflow_instances_result( all_pages: bool, project: str | None, workflow: str | None, + search: str | None, + executor: str | None, + host: str | None, + start: str | None, + end: str | None, state: str | None, ) -> CommandResult: + resolved_project = _resolve_workflow_instance_list_project( + runtime, + project=project, + ) + resolved_workflow = _resolve_workflow_instance_list_workflow( + runtime, + workflow=workflow, + project=resolved_project, + ) data = require_json_object( requested_page_data( lambda current_page_no, current_page_size: ( runtime.upstream.workflow_instances.list( page_no=current_page_no, page_size=current_page_size, + project_code=( + None if resolved_project is None else resolved_project.code + ), + workflow_code=( + None if resolved_workflow is None else resolved_workflow.code + ), project_name=project, workflow_name=workflow, + search=search, + executor=executor, + host=host, + start_time=start, + end_time=end, state=state, ) ), @@ -369,20 +479,35 @@ def _list_workflow_instances_result( resource=WORKFLOW_INSTANCE_RESOURCE, serialize_item=serialize_workflow_instance, max_pages=MAX_AUTO_EXHAUST_PAGES, + translate_error=lambda exc: _translate_workflow_instance_list_error( + exc, + project=project, + workflow=workflow, + search=search, + executor=executor, + host=host, + start=start, + end=end, + state=state, + ), ), label="workflow-instance list data", ) - resolved: dict[str, int | str | None] = { - "page_no": page_no, - "page_size": page_size, - "all": all_pages, - } - if project is not None: - resolved["project"] = project - if workflow is not None: - resolved["workflow"] = workflow - if state is not None: - resolved["state"] = state + resolved = _workflow_instance_list_resolved( + page_no=page_no, + page_size=page_size, + all_pages=all_pages, + project=project, + resolved_project=resolved_project, + workflow=workflow, + resolved_workflow=resolved_workflow, + search=search, + executor=executor, + host=host, + start=start, + end=end, + state=state, + ) return CommandResult( data=data, resolved=require_json_object( @@ -392,6 +517,45 @@ def _list_workflow_instances_result( ) +def _workflow_instance_list_resolved( + *, + page_no: int, + page_size: int, + all_pages: bool, + project: str | None, + resolved_project: ResolvedProject | None, + workflow: str | None, + resolved_workflow: ResolvedWorkflow | None, + search: str | None, + executor: str | None, + host: str | None, + start: str | None, + end: str | None, + state: str | None, +) -> dict[str, WorkflowInstanceListResolvedValue]: + resolved: dict[str, WorkflowInstanceListResolvedValue] = { + "page_no": page_no, + "page_size": page_size, + "all": all_pages, + } + optional_values: dict[str, WorkflowInstanceListResolvedValue | None] = { + "project": project, + "project_code": None if resolved_project is None else resolved_project.code, + "workflow": workflow, + "workflow_code": None if resolved_workflow is None else resolved_workflow.code, + "search": search, + "executor": executor, + "host": host, + "start": start, + "end": end, + "state": state, + } + resolved.update( + {key: value for key, value in optional_values.items() if value is not None} + ) + return resolved + + def _get_workflow_instance_result( runtime: ServiceRuntime, *, @@ -1113,6 +1277,46 @@ def _raise_workflow_instance_action_error( raise exc +def _translate_workflow_instance_list_error( + exc: ApiResultError, + *, + project: str | None, + workflow: str | None, + search: str | None, + executor: str | None, + host: str | None, + start: str | None, + end: str | None, + state: str | None, +) -> ApiResultError | UserInputError: + if exc.result_code != QUERY_WORKFLOW_INSTANCE_LIST_PAGING_ERROR: + return exc + filters = { + key: value + for key, value in { + "project": project, + "workflow": workflow, + "search": search, + "executor": executor, + "host": host, + "start": start, + "end": end, + "state": state, + }.items() + if value is not None + } + message = "DolphinScheduler rejected the workflow-instance list filters." + return UserInputError( + message, + details={"filters": filters}, + suggestion=( + "Retry with a narrower workflow-instance filter, for example " + "`dsctl workflow-instance list --project PROJECT --start " + "'YYYY-MM-DD HH:MM:SS' --end 'YYYY-MM-DD HH:MM:SS'`." + ), + ) + + def _parent_workflow_instance_not_found( *, sub_workflow_instance_id: int, @@ -1356,7 +1560,7 @@ def _normalized_workflow_instance_state(value: str | None) -> str | None: message, details={"state": value}, suggestion=( - "Run `dsctl enum list workflow_execution_status` to inspect " + "Run `dsctl enum list workflow-execution-status` to inspect " "the supported state names." ), ) from exc diff --git a/src/dsctl/upstream/__init__.py b/src/dsctl/upstream/__init__.py index 4f63918..92bcfa1 100644 --- a/src/dsctl/upstream/__init__.py +++ b/src/dsctl/upstream/__init__.py @@ -1,3 +1,10 @@ +from dsctl.upstream.datasource_contracts import ( + DATASOURCE_CONTRACT_VERSION, + DataSourcePayloadFieldSpec, + datasource_base_payload_fields, + datasource_type_names, + normalize_datasource_type, +) from dsctl.upstream.enums import get_enum_spec, supported_enum_names from dsctl.upstream.protocol import UpstreamAdapter from dsctl.upstream.registry import ( @@ -16,14 +23,19 @@ ) __all__ = [ + "DATASOURCE_CONTRACT_VERSION", "SUPPORTED_VERSIONS", + "DataSourcePayloadFieldSpec", "UpstreamAdapter", "VersionSupport", "VersionSupportData", + "datasource_base_payload_fields", + "datasource_type_names", "get_adapter", "get_default_version_support", "get_enum_spec", "get_version_support", + "normalize_datasource_type", "normalize_version", "supported_enum_names", "supported_version_metadata", diff --git a/src/dsctl/upstream/adapters/ds_3_4_1.py b/src/dsctl/upstream/adapters/ds_3_4_1.py index 49e7865..1e36511 100644 --- a/src/dsctl/upstream/adapters/ds_3_4_1.py +++ b/src/dsctl/upstream/adapters/ds_3_4_1.py @@ -129,6 +129,9 @@ StartTaskGroupParams, UpdateTaskGroupParams, ) +from dsctl.generated.versions.ds_3_4_1.api.operations.task_instance import ( + QueryTaskListPagingParams, +) from dsctl.generated.versions.ds_3_4_1.api.operations.tenant import ( CreateTenantParams, QueryTenantListPagingParams, @@ -160,6 +163,7 @@ from dsctl.generated.versions.ds_3_4_1.api.operations.workflow_instance import ( QueryParentInstanceBySubIdParams, QuerySubWorkflowInstanceByTaskIdParams, + QueryWorkflowInstanceListParams, UpdateWorkflowInstanceParams, ) from dsctl.generated.versions.ds_3_4_1.api.operations.workflow_lineage import ( @@ -188,6 +192,9 @@ from dsctl.generated.versions.ds_3_4_1.common.enums.task_depend_type import ( TaskDependType, ) +from dsctl.generated.versions.ds_3_4_1.common.enums.task_execute_type import ( + TaskExecuteType, +) from dsctl.generated.versions.ds_3_4_1.common.enums.warning_type import WarningType from dsctl.generated.versions.ds_3_4_1.common.enums.workflow_execution_type_enum import ( # noqa: E501 WorkflowExecutionTypeEnum, @@ -294,9 +301,6 @@ from dsctl.generated.versions.ds_3_4_1.dao.entities.task_instance import ( TaskInstance, ) - from dsctl.generated.versions.ds_3_4_1.dao.entities.task_instance_dependent_details import ( # noqa: E501 - TaskInstanceDependentDetailsAbstractTaskInstanceContext, - ) from dsctl.generated.versions.ds_3_4_1.dao.entities.user import User from dsctl.generated.versions.ds_3_4_1.dao.entities.work_flow_lineage import ( WorkFlowLineage, @@ -2191,16 +2195,47 @@ def list( *, page_no: int, page_size: int, + project_code: int | None = None, + workflow_code: int | None = None, project_name: str | None = None, workflow_name: str | None = None, + search: str | None = None, + executor: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, state: str | None = None, ) -> PageInfoWorkflowInstance: + if project_code is not None: + return self.client.workflow_instance.query_workflow_instance_list( + project_code, + QueryWorkflowInstanceListParams( + workflowDefinitionCode=workflow_code, + searchVal=search, + executorName=executor, + stateType=_workflow_execution_state(state), + host=host, + startDate=start_time, + endDate=end_time, + pageNo=page_no, + pageSize=page_size, + ), + ) + if workflow_code is not None or search is not None or executor is not None: + message = ( + "Project-scoped workflow-instance filters require project_code " + "in the DS 3.4.1 adapter." + ) + raise ValueError(message) return self.client.workflow_instance_v2.query_workflow_instance_list_paging( workflow_instance_contracts.WorkflowInstanceQueryRequest( pageNo=page_no, pageSize=page_size, projectName=project_name, workflowName=workflow_name, + host=host, + startDate=start_time, + endDate=end_time, state=_workflow_execution_state_code(state), ) ) @@ -2304,28 +2339,40 @@ class _DS341TaskInstanceOperations: def list( self, *, - workflow_instance_id: int, project_code: int, page_no: int, page_size: int, + workflow_instance_id: int | None = None, + workflow_instance_name: str | None = None, + workflow_definition_name: str | None = None, search: str | None = None, + task_name: str | None = None, + task_code: int | None = None, + executor: str | None = None, state: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, + task_execute_type: str | None = None, ) -> TaskInstancePageRecord: - task_list = ( - self.client.workflow_instance.query_task_list_by_workflow_instance_id( - project_code, - workflow_instance_id, - ) - ) - items = _filtered_task_instance_items( - task_list.taskList or [], - search=search, - state=state, - ) - return _TaskInstanceListPage.from_items( - items, - page_no=page_no, - page_size=page_size, + return self.client.task_instance.query_task_list_paging( + project_code, + QueryTaskListPagingParams( + workflowInstanceId=workflow_instance_id, + workflowInstanceName=workflow_instance_name, + workflowDefinitionName=workflow_definition_name, + searchVal=search, + taskName=task_name, + taskCode=task_code, + executorName=executor, + stateType=_task_execution_state(state), + host=host, + startDate=start_time, + endDate=end_time, + taskExecuteType=_task_execute_type(task_execute_type), + pageNo=page_no, + pageSize=page_size, + ), ) def get( @@ -2531,6 +2578,14 @@ def _workflow_execution_state_code(value: str | None) -> int | None: return workflow_execution_status_enums.WorkflowExecutionStatus[value].code +def _workflow_execution_state( + value: str | None, +) -> workflow_execution_status_enums.WorkflowExecutionStatus | None: + if value is None: + return None + return workflow_execution_status_enums.WorkflowExecutionStatus[value] + + def _task_execution_state( value: str | None, ) -> task_execution_status_enums.TaskExecutionStatus | None: @@ -2539,83 +2594,10 @@ def _task_execution_state( return task_execution_status_enums.TaskExecutionStatus[value] -@dataclass(frozen=True) -class _TaskInstanceListPage: - """Client-side page wrapper for workflow-instance task lists.""" - - total_list_value: list[TaskInstanceDependentDetailsAbstractTaskInstanceContext] - total: int - total_page_value: int - page_size_value: int - current_page_value: int - - @classmethod - def from_items( - cls, - items: Sequence[TaskInstanceDependentDetailsAbstractTaskInstanceContext], - *, - page_no: int, - page_size: int, - ) -> _TaskInstanceListPage: - total = len(items) - total_pages = 0 if total == 0 else ((total - 1) // page_size) + 1 - start = (page_no - 1) * page_size - stop = start + page_size - return cls( - total_list_value=list(items[start:stop]), - total=total, - total_page_value=total_pages, - page_size_value=page_size, - current_page_value=page_no, - ) - - @property - def totalList( # noqa: N802 - self, - ) -> list[TaskInstanceDependentDetailsAbstractTaskInstanceContext] | None: - return self.total_list_value - - @property - def totalPage(self) -> int | None: # noqa: N802 - return self.total_page_value - - @property - def pageSize(self) -> int | None: # noqa: N802 - return self.page_size_value - - @property - def currentPage(self) -> int | None: # noqa: N802 - return self.current_page_value - - @property - def pageNo(self) -> int | None: # noqa: N802 - return self.current_page_value - - -def _filtered_task_instance_items( - items: Sequence[TaskInstanceDependentDetailsAbstractTaskInstanceContext], - *, - search: str | None, - state: str | None, -) -> list[TaskInstanceDependentDetailsAbstractTaskInstanceContext]: - filtered: list[TaskInstanceDependentDetailsAbstractTaskInstanceContext] = [] - normalized_search = None if search is None else search.lower() - for item in items: - if normalized_search is not None: - name = item.name - if name is None or normalized_search not in name.lower(): - continue - if state is not None and _enum_member_value(item.state) != state: - continue - filtered.append(item) - return filtered - - -def _enum_member_value(value: object) -> str | None: - member_value = getattr(value, "value", None) - if isinstance(member_value, str): - return member_value - return None +def _task_execute_type(value: str | None) -> TaskExecuteType | None: + if value is None: + return None + return TaskExecuteType[value] def _task_depend_type(scope: str) -> TaskDependType: diff --git a/src/dsctl/upstream/datasource_contracts.py b/src/dsctl/upstream/datasource_contracts.py new file mode 100644 index 0000000..620b0dd --- /dev/null +++ b/src/dsctl/upstream/datasource_contracts.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import cache +from typing import TYPE_CHECKING + +from dsctl.errors import ConfigError +from dsctl.generated.versions.ds_3_4_1.plugin.datasource_api.datasource import ( + BaseDataSourceParamDTO, +) +from dsctl.generated.versions.ds_3_4_1.spi.enums.db_type import DbType +from dsctl.upstream.registry import normalize_version + +if TYPE_CHECKING: + from dsctl.support.json_types import JsonObject + +DATASOURCE_CONTRACT_VERSION = "3.4.1" + +_FIELD_VALUE_TYPES: dict[str, str] = { + "id": "integer", + "name": "string", + "note": "string", + "host": "string", + "port": "integer", + "database": "string", + "userName": "string", + "password": "string", + "other": "object", + "type": "enum", +} +_FIELD_DESCRIPTIONS: dict[str, str] = { + "id": "Datasource id. Omit on create; update may include the selected id.", + "name": "Datasource display name.", + "note": "Optional datasource description.", + "host": "Datasource host or DS plugin address field.", + "port": "Datasource port.", + "database": "Database, schema, service, or catalog value used by the plugin.", + "userName": "Datasource login user where the plugin requires one.", + "password": "Datasource password or secret where the plugin requires one.", + "other": "JDBC or plugin-specific key/value options.", + "type": "Datasource type. Values come from DS DbType.", +} +_CLI_REQUIRED_FIELDS = frozenset({"name", "type"}) + + +@dataclass(frozen=True) +class DataSourcePayloadFieldSpec: + """One generated datasource payload field exposed above upstream.""" + + name: str + value_type: str + cli_required: bool + description: str + choices: tuple[str, ...] = () + + def to_data(self) -> JsonObject: + """Return a JSON-safe field schema payload.""" + data: JsonObject = { + "name": self.name, + "value_type": self.value_type, + "required_by_cli": self.cli_required, + "description": self.description, + } + if self.choices: + data["choices"] = list(self.choices) + return data + + +def datasource_type_names(version: str) -> tuple[str, ...]: + """Return DS datasource type wire values for one contract version.""" + _require_datasource_contract_version(version) + return tuple(member.value for member in DbType) + + +def normalize_datasource_type(version: str, datasource_type: str) -> str | None: + """Return the canonical DS datasource type, accepting common enum aliases.""" + _require_datasource_contract_version(version) + requested = _normalize_datasource_type_key(datasource_type) + if not requested: + return None + for member in DbType: + aliases = ( + member.name, + member.value, + member.name_field, + member.descp, + ) + if requested in {_normalize_datasource_type_key(alias) for alias in aliases}: + return member.value + return None + + +@cache +def datasource_base_payload_fields( + version: str, +) -> tuple[DataSourcePayloadFieldSpec, ...]: + """Return generated BaseDataSourceParamDTO field metadata.""" + _require_datasource_contract_version(version) + fields = BaseDataSourceParamDTO.model_fields + type_names = datasource_type_names(version) + return tuple( + DataSourcePayloadFieldSpec( + name=field_name, + value_type=_FIELD_VALUE_TYPES.get(field_name, "json"), + cli_required=field_name in _CLI_REQUIRED_FIELDS, + description=_FIELD_DESCRIPTIONS.get( + field_name, + "Datasource payload field.", + ), + choices=type_names if field_name == "type" else (), + ) + for field_name in fields + ) + + +def _require_datasource_contract_version(version: str) -> None: + normalized = normalize_version(version) + if normalized == DATASOURCE_CONTRACT_VERSION: + return + supported_versions = [DATASOURCE_CONTRACT_VERSION] + message = f"Unsupported datasource contract version {version!r}" + raise ConfigError( + message, + details={ + "version": version, + "supported_versions": supported_versions, + }, + ) + + +def _normalize_datasource_type_key(value: str) -> str: + return value.strip().upper().replace("-", "_") + + +__all__ = [ + "DATASOURCE_CONTRACT_VERSION", + "DataSourcePayloadFieldSpec", + "datasource_base_payload_fields", + "datasource_type_names", + "normalize_datasource_type", +] diff --git a/src/dsctl/upstream/protocol.py b/src/dsctl/upstream/protocol.py index c6f9243..719662f 100644 --- a/src/dsctl/upstream/protocol.py +++ b/src/dsctl/upstream/protocol.py @@ -2928,8 +2928,15 @@ def list( *, page_no: int, page_size: int, + project_code: int | None = None, + workflow_code: int | None = None, project_name: str | None = None, workflow_name: str | None = None, + search: str | None = None, + executor: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, state: str | None = None, ) -> WorkflowInstancePageRecord: """Return one page of workflow instances.""" @@ -3162,14 +3169,23 @@ class TaskInstanceOperations(Protocol): def list( self, *, - workflow_instance_id: int, project_code: int, page_no: int, page_size: int, + workflow_instance_id: int | None = None, + workflow_instance_name: str | None = None, + workflow_definition_name: str | None = None, search: str | None = None, + task_name: str | None = None, + task_code: int | None = None, + executor: str | None = None, state: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, + task_execute_type: str | None = None, ) -> TaskInstancePageRecord: - """Return one page of task instances inside one workflow instance.""" + """Return one project-scoped page of task instances.""" def get( self, diff --git a/src/dsctl/upstream/runtime_enums.py b/src/dsctl/upstream/runtime_enums.py index bf38b4c..9acf95c 100644 --- a/src/dsctl/upstream/runtime_enums.py +++ b/src/dsctl/upstream/runtime_enums.py @@ -115,6 +115,11 @@ def task_execution_status_value(name: str) -> str: return _task_execution_status_value(name) +def task_execute_type_value(name: str) -> str: + """Return the DS task execute-type wire value for one enum name.""" + return TaskExecuteType[name].value + + def _enum_wire_value(value: StringEnumValue | str | None) -> str | None: if isinstance(value, str): return value @@ -134,6 +139,7 @@ def _enum_wire_value(value: StringEnumValue | str | None) -> str | None: "WORKFLOW_EXECUTION_FAILURE_STATE", "WORKFLOW_EXECUTION_STOP_STATE", "WorkflowExecutionStatusInfo", + "task_execute_type_value", "task_execution_status_value", "workflow_execution_status_info", "workflow_execution_status_is_final", diff --git a/tests/commands/test_access_token.py b/tests/commands/test_access_token.py index 8a1ae73..6fea0ae 100644 --- a/tests/commands/test_access_token.py +++ b/tests/commands/test_access_token.py @@ -121,6 +121,14 @@ def test_access_token_get_command_reports_not_found_suggestion() -> None: ) +def test_access_token_get_help_points_to_list_for_selector() -> None: + result = runner.invoke(app, ["access-token", "get", "--help"]) + + assert result.exit_code == 0 + assert "access-token" in result.stdout + assert "list" in result.stdout + + def test_access_token_create_command_returns_created_token() -> None: result = runner.invoke( app, @@ -141,6 +149,15 @@ def test_access_token_create_command_returns_created_token() -> None: assert payload["data"]["userId"] == 1 +def test_access_token_create_help_points_to_user_list_and_time_format() -> None: + result = runner.invoke(app, ["access-token", "create", "--help"]) + + assert result.exit_code == 0 + assert "dsctl user list" in result.stdout + assert "2027-01-01" in result.stdout + assert "00:00:00" in result.stdout + + def test_access_token_update_command_can_regenerate_token() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_alert_group.py b/tests/commands/test_alert_group.py index 88decfd..58cfd5d 100644 --- a/tests/commands/test_alert_group.py +++ b/tests/commands/test_alert_group.py @@ -77,6 +77,14 @@ def test_alert_group_get_command_resolves_name() -> None: assert payload["data"]["alertInstanceIds"] == "7,8" +def test_alert_group_get_help_points_to_list_for_selector() -> None: + result = runner.invoke(app, ["alert-group", "get", "--help"]) + + assert result.exit_code == 0 + assert "alert-group" in result.stdout + assert "list" in result.stdout + + def test_alert_group_create_command_returns_created_group() -> None: result = runner.invoke( app, @@ -101,6 +109,13 @@ def test_alert_group_create_command_returns_created_group() -> None: assert payload["data"]["alertInstanceIds"] == "7,8" +def test_alert_group_create_help_points_to_alert_plugin_list() -> None: + result = runner.invoke(app, ["alert-group", "create", "--help"]) + + assert result.exit_code == 0 + assert "alert-plugin list" in result.stdout + + def test_alert_group_update_command_returns_updated_group() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_alert_plugin.py b/tests/commands/test_alert_plugin.py index c8c920a..623ecec 100644 --- a/tests/commands/test_alert_plugin.py +++ b/tests/commands/test_alert_plugin.py @@ -102,6 +102,14 @@ def test_alert_plugin_get_command_resolves_name() -> None: assert payload["data"]["alertPluginName"] == "Slack" +def test_alert_plugin_selector_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["alert-plugin", "get", "--help"]) + + assert result.exit_code == 0 + assert "alert-plugin" in result.stdout + assert "list" in result.stdout + + def test_alert_plugin_schema_command_returns_plugin_definition() -> None: result = runner.invoke(app, ["alert-plugin", "schema", "Slack"]) @@ -111,6 +119,22 @@ def test_alert_plugin_schema_command_returns_plugin_definition() -> None: assert payload["data"]["pluginName"] == "Slack" +def test_alert_plugin_schema_help_points_to_definition_discovery() -> None: + result = runner.invoke(app, ["alert-plugin", "schema", "--help"]) + + assert result.exit_code == 0 + assert "definition list" in result.stdout + + +def test_alert_plugin_definition_list_command_returns_supported_definitions() -> None: + result = runner.invoke(app, ["alert-plugin", "definition", "list"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "alert-plugin.definition.list" + assert payload["data"]["definitions"][0]["pluginName"] == "Slack" + + def test_alert_plugin_create_command_reads_params_from_file( tmp_path: Path, ) -> None: @@ -137,6 +161,36 @@ def test_alert_plugin_create_command_reads_params_from_file( assert payload["data"]["instanceName"] == "slack-nightly" +def test_alert_plugin_create_help_points_to_schema_discovery() -> None: + result = runner.invoke(app, ["alert-plugin", "create", "--help"]) + + assert result.exit_code == 0 + assert "definition list" in result.stdout + assert "alert-plugin schema PLUGIN" in result.stdout + + +def test_alert_plugin_create_command_accepts_inline_params() -> None: + result = runner.invoke( + app, + [ + "alert-plugin", + "create", + "--name", + "slack-nightly", + "--plugin", + "slack", + "--param", + "url=https://hooks.example.test/nightly", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + params = json.loads(payload["data"]["pluginInstanceParams"]) + assert payload["action"] == "alert-plugin.create" + assert params[0]["value"] == "https://hooks.example.test/nightly" + + def test_alert_plugin_update_command_returns_updated_payload() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_audit.py b/tests/commands/test_audit.py index 17456f3..7879c77 100644 --- a/tests/commands/test_audit.py +++ b/tests/commands/test_audit.py @@ -58,6 +58,14 @@ def test_audit_list_command_returns_paginated_payload() -> None: assert payload["data"]["totalList"][0]["modelName"] == "daily-etl" +def test_audit_list_help_points_to_filter_discovery_commands() -> None: + result = runner.invoke(app, ["audit", "list", "--help"]) + + assert result.exit_code == 0 + assert "audit model-types" in result.stdout + assert "audit operation-types" in result.stdout + + def test_audit_model_types_command_returns_tree() -> None: result = runner.invoke(app, ["audit", "model-types"]) diff --git a/tests/commands/test_capabilities.py b/tests/commands/test_capabilities.py index 6509b6e..1924816 100644 --- a/tests/commands/test_capabilities.py +++ b/tests/commands/test_capabilities.py @@ -4,7 +4,9 @@ from typer.testing import CliRunner from dsctl.app import app +from dsctl.services.datasource_payload import datasource_template_index_data from dsctl.services.template import parameter_syntax_index_data +from tests.support import normalize_cli_help runner = CliRunner() @@ -61,11 +63,17 @@ def test_capabilities_command_returns_surface_discovery() -> None: "project", "workflow", ] - assert payload["data"]["resources"]["groups"]["enum"]["commands"] == ["list"] + assert payload["data"]["resources"]["groups"]["enum"]["commands"] == [ + "names", + "list", + ] assert payload["data"]["resources"]["groups"]["task-type"]["commands"] == ["list"] assert payload["data"]["resources"]["groups"]["template"]["commands"] == [ "workflow", "params", + "environment", + "cluster", + "datasource", "task", ] assert payload["data"]["resources"]["groups"]["monitor"]["commands"] == [ @@ -91,6 +99,11 @@ def test_capabilities_command_returns_surface_discovery() -> None: } assert payload["data"]["output"] == { "standard_envelope": True, + "formats": ["json", "table", "tsv"], + "default_format": "json", + "data_shape_metadata": True, + "display_columns": True, + "json_column_projection": True, "resolved_metadata": True, "warnings": True, "warning_details_alignment": True, @@ -100,14 +113,34 @@ def test_capabilities_command_returns_surface_discovery() -> None: "schema": True, "template": True, "capabilities": True, + "command_invocation_source": "schema", + "capabilities_scope": "feature_discovery", } assert payload["data"]["authoring"]["parameter_syntax"] == ( parameter_syntax_index_data() ) + assert payload["data"]["authoring"]["environment_config_template"] is True + assert payload["data"]["authoring"]["cluster_config_template"] is True + assert payload["data"]["authoring"]["datasource_payload_templates"] is True + assert ( + payload["data"]["authoring"]["datasource_template_types"] + == (datasource_template_index_data()["supported_types"]) + ) assert payload["data"]["enums"]["discovery"] is True assert "priority" in payload["data"]["enums"]["names"] +def test_capabilities_help_points_to_section_discovery() -> None: + result = runner.invoke(app, ["capabilities", "--help"]) + + assert result.exit_code == 0 + help_text = normalize_cli_help(result.stdout) + assert "dsctl schema --command" in help_text + assert "capabilities" in help_text + assert "selection" in help_text + assert "runtime" in help_text + + def test_capabilities_command_honors_env_file_ds_version() -> None: with runner.isolated_filesystem(): Path("cluster.env").write_text("DS_VERSION=3.3.2\n", encoding="utf-8") @@ -121,3 +154,54 @@ def test_capabilities_command_honors_env_file_ds_version() -> None: assert payload["data"]["ds"]["contract_version"] == "3.4.1" assert payload["data"]["ds"]["tested"] is False assert "priority" in payload["data"]["enums"]["names"] + + +def test_capabilities_command_returns_summary() -> None: + result = runner.invoke(app, ["capabilities", "--summary"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "capabilities" + assert payload["resolved"] == {"capabilities": {"view": "summary"}} + assert "resources" in payload["data"] + assert "runtime" in payload["data"] + assert "authoring" in payload["data"] + assert "parameter_syntax" not in payload["data"]["authoring"] + + +def test_capabilities_command_returns_section() -> None: + result = runner.invoke(app, ["capabilities", "--section", "runtime"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "capabilities" + assert payload["resolved"] == { + "capabilities": { + "view": "section", + "section": "runtime", + } + } + assert set(payload["data"]) == {"cli", "ds", "self_description", "runtime"} + assert payload["data"]["runtime"]["task-instance"]["commands"] == [ + "list", + "get", + "watch", + "sub-workflow", + "log", + "force-success", + "savepoint", + "stop", + ] + + +def test_capabilities_command_rejects_conflicting_scope_options() -> None: + result = runner.invoke( + app, + ["capabilities", "--summary", "--section", "runtime"], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "capabilities" + assert payload["error"]["type"] == "user_input_error" + assert "mutually exclusive" in payload["error"]["message"] diff --git a/tests/commands/test_cluster.py b/tests/commands/test_cluster.py index 5f83290..72f122a 100644 --- a/tests/commands/test_cluster.py +++ b/tests/commands/test_cluster.py @@ -1,4 +1,5 @@ import json +from pathlib import Path import pytest from typer.testing import CliRunner @@ -63,6 +64,14 @@ def test_cluster_get_command_resolves_name() -> None: assert payload["data"]["config"] == "kube-prod" +def test_cluster_selector_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["cluster", "get", "--help"]) + + assert result.exit_code == 0 + assert "cluster" in result.stdout + assert "list" in result.stdout + + def test_cluster_create_command_returns_created_cluster() -> None: result = runner.invoke( app, @@ -83,6 +92,69 @@ def test_cluster_create_command_returns_created_cluster() -> None: assert payload["data"]["config"] == "kube-ops" +def test_cluster_create_command_accepts_config_file() -> None: + with runner.isolated_filesystem(): + config = '{"k8s":"config","yarn":""}' + with Path("cluster-config.json").open("w", encoding="utf-8") as handle: + handle.write(config) + + result = runner.invoke( + app, + [ + "cluster", + "create", + "--name", + "ops", + "--config-file", + "cluster-config.json", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["data"]["config"] == config + + +def test_cluster_create_command_requires_one_config_source() -> None: + result = runner.invoke(app, ["cluster", "create", "--name", "ops"]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "cluster.create" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Pass --config TEXT or --config-file PATH. Run " + "`dsctl template cluster` for an example JSON config." + ) + + +def test_cluster_create_command_rejects_conflicting_config_sources() -> None: + with runner.isolated_filesystem(): + with Path("cluster-config.json").open("w", encoding="utf-8") as handle: + handle.write('{"k8s":"","yarn":""}') + + result = runner.invoke( + app, + [ + "cluster", + "create", + "--name", + "ops", + "--config", + "{}", + "--config-file", + "cluster-config.json", + ], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Pass inline config with --config or read it from --config-file." + ) + + def test_cluster_update_command_returns_updated_cluster() -> None: result = runner.invoke( app, @@ -102,6 +174,28 @@ def test_cluster_update_command_returns_updated_cluster() -> None: assert payload["data"]["config"] == "kube-changed" +def test_cluster_update_command_accepts_config_file() -> None: + with runner.isolated_filesystem(): + config = '{"k8s":"changed","yarn":""}' + with Path("cluster-config.json").open("w", encoding="utf-8") as handle: + handle.write(config) + + result = runner.invoke( + app, + [ + "cluster", + "update", + "k8s-prod", + "--config-file", + "cluster-config.json", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["data"]["config"] == config + + def test_cluster_update_command_rejects_conflicting_description_flags() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_datasource.py b/tests/commands/test_datasource.py index c01254c..fe9997f 100644 --- a/tests/commands/test_datasource.py +++ b/tests/commands/test_datasource.py @@ -80,6 +80,29 @@ def test_datasource_get_command_resolves_name() -> None: assert payload["data"]["host"] == "db.example" +def test_datasource_mutating_selector_help_points_to_list() -> None: + for command in ("delete", "test"): + result = runner.invoke(app, ["datasource", command, "--help"]) + + assert result.exit_code == 0 + assert "datasource" in result.stdout + assert "list" in result.stdout + + +def test_datasource_payload_file_help_points_to_template_flow() -> None: + create_result = runner.invoke(app, ["datasource", "create", "--help"]) + update_result = runner.invoke(app, ["datasource", "update", "--help"]) + + assert create_result.exit_code == 0 + assert "Start with `dsctl template datasource`" in create_result.stdout + assert "pass the saved" in create_result.stdout + assert "write data.json to this file" not in create_result.stdout + assert update_result.exit_code == 0 + assert "Start from `dsctl datasource get DATASOURCE`" in update_result.stdout + assert "pass" in update_result.stdout + assert "the saved JSON path here" in update_result.stdout + + def test_datasource_create_command_returns_created_payload(tmp_path: Path) -> None: file = _write_json( tmp_path / "create.json", @@ -99,6 +122,28 @@ def test_datasource_create_command_returns_created_payload(tmp_path: Path) -> No assert payload["resolved"]["datasource"]["id"] == 8 +def test_datasource_create_command_rejects_unknown_type(tmp_path: Path) -> None: + file = _write_json( + tmp_path / "unknown.json", + { + "name": "analytics", + "type": "UNKNOWN", + "password": "secret", + }, + ) + + result = runner.invoke(app, ["datasource", "create", "--file", str(file)]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "datasource.create" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Run `dsctl template datasource` to choose a supported datasource type, " + "then `dsctl template datasource --type TYPE`." + ) + + def test_datasource_create_command_rejects_payload_with_id(tmp_path: Path) -> None: file = _write_json( tmp_path / "with-id.json", diff --git a/tests/commands/test_enum.py b/tests/commands/test_enum.py index 21b6413..c0657e9 100644 --- a/tests/commands/test_enum.py +++ b/tests/commands/test_enum.py @@ -7,6 +7,27 @@ runner = CliRunner() +def test_enum_names_command_returns_supported_enum_names() -> None: + result = runner.invoke(app, ["enum", "names"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "enum.names" + assert "view" not in payload["resolved"]["enum"] + assert {"name": "priority", "list_command": "dsctl enum list priority"} in ( + payload["data"] + ) + + +def test_enum_names_command_can_render_table_rows() -> None: + result = runner.invoke(app, ["--output-format", "table", "enum", "names"]) + + assert result.exit_code == 0 + assert "name" in result.stdout + assert "list_command" in result.stdout + assert "dsctl enum list priority" in result.stdout + + def test_enum_list_command_returns_enum_members() -> None: result = runner.invoke(app, ["enum", "list", "priority"]) @@ -35,6 +56,5 @@ def test_enum_list_command_rejects_unknown_enum() -> None: assert payload["action"] == "enum.list" assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == ( - "Run `capabilities` and inspect `data.enums.names` to choose a " - "supported enum name." + "Run `dsctl enum names` to choose a supported enum name." ) diff --git a/tests/commands/test_env.py b/tests/commands/test_env.py index 54f40f1..5c91563 100644 --- a/tests/commands/test_env.py +++ b/tests/commands/test_env.py @@ -1,4 +1,5 @@ import json +from pathlib import Path import pytest from typer.testing import CliRunner @@ -12,7 +13,7 @@ FakeProjectAdapter, fake_service_runtime, ) -from tests.support import make_profile +from tests.support import make_profile, normalize_cli_help runner = CliRunner() @@ -44,32 +45,49 @@ def patch_env_service( def test_env_list_command_returns_paginated_payload() -> None: - result = runner.invoke(app, ["env", "list", "--page-size", "1"]) + result = runner.invoke(app, ["environment", "list", "--page-size", "1"]) assert result.exit_code == 0 payload = json.loads(result.stdout) assert payload["ok"] is True - assert payload["action"] == "env.list" + assert payload["action"] == "environment.list" assert payload["data"]["total"] == 2 assert payload["data"]["pageSize"] == 1 assert payload["data"]["totalList"][0]["name"] == "prod" def test_env_get_command_resolves_name() -> None: - result = runner.invoke(app, ["env", "get", "prod"]) + result = runner.invoke(app, ["environment", "get", "prod"]) assert result.exit_code == 0 payload = json.loads(result.stdout) - assert payload["action"] == "env.get" + assert payload["action"] == "environment.get" assert payload["resolved"]["environment"]["code"] == 7 assert payload["data"]["name"] == "prod" +def test_env_selector_help_points_to_list() -> None: + result = runner.invoke(app, ["environment", "get", "--help"]) + + assert result.exit_code == 0 + assert "environment" in result.stdout + assert "list" in result.stdout + + +def test_env_create_help_points_to_config_template() -> None: + result = runner.invoke(app, ["environment", "create", "--help"]) + + assert result.exit_code == 0 + help_text = normalize_cli_help(result.stdout) + assert "--config-file" in help_text + assert "dsctl template environment" in help_text + + def test_env_create_command_returns_created_environment() -> None: result = runner.invoke( app, [ - "env", + "environment", "create", "--name", "qa", @@ -82,16 +100,81 @@ def test_env_create_command_returns_created_environment() -> None: assert result.exit_code == 0 payload = json.loads(result.stdout) - assert payload["action"] == "env.create" + assert payload["action"] == "environment.create" assert payload["data"]["name"] == "qa" assert payload["data"]["workerGroups"] == ["default"] +def test_env_create_command_accepts_config_file() -> None: + with runner.isolated_filesystem(): + Path("env.sh").write_text( + "export JAVA_HOME=/opt/java\nexport PATH=$JAVA_HOME/bin:$PATH\n", + encoding="utf-8", + ) + + result = runner.invoke( + app, + [ + "environment", + "create", + "--name", + "qa", + "--config-file", + "env.sh", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "environment.create" + assert payload["data"]["config"] == ( + "export JAVA_HOME=/opt/java\nexport PATH=$JAVA_HOME/bin:$PATH" + ) + + +def test_env_create_command_requires_config_source() -> None: + result = runner.invoke(app, ["environment", "create", "--name", "qa"]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "environment.create" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Pass --config TEXT or --config-file PATH. Run `dsctl template environment` " + "for an example shell/export config." + ) + + +def test_env_create_command_rejects_multiple_config_sources() -> None: + with runner.isolated_filesystem(): + Path("env.sh").write_text("export JAVA_HOME=/opt/java\n", encoding="utf-8") + + result = runner.invoke( + app, + [ + "environment", + "create", + "--name", + "qa", + "--config", + "export JAVA_HOME=/other", + "--config-file", + "env.sh", + ], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "environment.create" + assert payload["error"]["type"] == "user_input_error" + assert "mutually exclusive" in payload["error"]["message"] + + def test_env_update_command_can_clear_description_and_worker_groups() -> None: result = runner.invoke( app, [ - "env", + "environment", "update", "prod", "--config", @@ -103,17 +186,43 @@ def test_env_update_command_can_clear_description_and_worker_groups() -> None: assert result.exit_code == 0 payload = json.loads(result.stdout) - assert payload["action"] == "env.update" + assert payload["action"] == "environment.update" assert payload["data"]["description"] is None assert payload["data"]["workerGroups"] == [] +def test_env_update_command_accepts_config_file() -> None: + with runner.isolated_filesystem(): + Path("env.sh").write_text( + "export JAVA_HOME=/opt/java-21\nexport PATH=$JAVA_HOME/bin:$PATH\n", + encoding="utf-8", + ) + + result = runner.invoke( + app, + [ + "environment", + "update", + "prod", + "--config-file", + "env.sh", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "environment.update" + assert payload["data"]["config"] == ( + "export JAVA_HOME=/opt/java-21\nexport PATH=$JAVA_HOME/bin:$PATH" + ) + + def test_env_update_command_requires_one_change_suggestion() -> None: - result = runner.invoke(app, ["env", "update", "prod"]) + result = runner.invoke(app, ["environment", "update", "prod"]) assert result.exit_code == 1 payload = json.loads(result.stdout) - assert payload["action"] == "env.update" + assert payload["action"] == "environment.update" assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == ( "Pass at least one update flag such as --name, --config, " @@ -145,7 +254,7 @@ def reject_update( result = runner.invoke( app, [ - "env", + "environment", "update", "prod", "--config", @@ -155,7 +264,7 @@ def reject_update( assert result.exit_code == 1 payload = json.loads(result.stdout) - assert payload["action"] == "env.update" + assert payload["action"] == "environment.update" assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == ( "Verify --name, --config, and --worker-group values, then retry." @@ -163,19 +272,19 @@ def reject_update( def test_env_delete_command_requires_force() -> None: - result = runner.invoke(app, ["env", "delete", "prod"]) + result = runner.invoke(app, ["environment", "delete", "prod"]) assert result.exit_code == 1 payload = json.loads(result.stdout) - assert payload["action"] == "env.delete" + assert payload["action"] == "environment.delete" assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == "Retry the same command with --force." def test_env_delete_command_returns_deleted_confirmation() -> None: - result = runner.invoke(app, ["env", "delete", "prod", "--force"]) + result = runner.invoke(app, ["environment", "delete", "prod", "--force"]) assert result.exit_code == 0 payload = json.loads(result.stdout) - assert payload["action"] == "env.delete" + assert payload["action"] == "environment.delete" assert payload["data"]["deleted"] is True diff --git a/tests/commands/test_meta.py b/tests/commands/test_meta.py index cab08bb..8d9b184 100644 --- a/tests/commands/test_meta.py +++ b/tests/commands/test_meta.py @@ -3,7 +3,7 @@ from typer.testing import CliRunner -from dsctl.app import app +from dsctl.app import _misplaced_root_option, app runner = CliRunner() @@ -26,6 +26,33 @@ def test_version_command_reports_cli_and_ds_versions() -> None: } +def test_version_command_can_render_tsv_columns() -> None: + result = runner.invoke( + app, + ["--output-format", "tsv", "--columns", "cli,ds,family", "version"], + ) + + assert result.exit_code == 0 + assert result.stdout == ("cli\tds\tfamily\n0.1.0\t3.4.1\tworkflow-3.3-plus\n") + + +def test_version_command_can_project_json_columns() -> None: + result = runner.invoke(app, ["--columns", "cli,ds", "version"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["ok"] is True + assert payload["data"] == {"cli": "0.1.0", "ds": "3.4.1"} + + +def test_misplaced_root_option_detection() -> None: + assert ( + _misplaced_root_option(["worker-group", "list", "--output-format", "table"]) + == "--output-format" + ) + assert _misplaced_root_option(["--output-format", "table", "version"]) is None + + def test_version_command_honors_env_file_ds_version() -> None: with runner.isolated_filesystem(): Path("cluster.env").write_text("DS_VERSION=3.3.2\n", encoding="utf-8") diff --git a/tests/commands/test_namespace.py b/tests/commands/test_namespace.py index 2f445b1..3617d79 100644 --- a/tests/commands/test_namespace.py +++ b/tests/commands/test_namespace.py @@ -82,6 +82,14 @@ def test_namespace_get_command_resolves_name() -> None: assert payload["data"]["clusterCode"] == 9001 +def test_namespace_selector_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["namespace", "get", "--help"]) + + assert result.exit_code == 0 + assert "namespace" in result.stdout + assert "list" in result.stdout + + def test_namespace_available_command_returns_current_user_visible_set() -> None: result = runner.invoke(app, ["namespace", "available"]) @@ -112,6 +120,13 @@ def test_namespace_create_command_returns_created_namespace() -> None: assert payload["resolved"]["namespace"]["id"] == 23 +def test_namespace_create_help_points_to_cluster_discovery() -> None: + result = runner.invoke(app, ["namespace", "create", "--help"]) + + assert result.exit_code == 0 + assert "dsctl cluster list" in result.stdout + + def test_namespace_create_command_reports_upstream_input_suggestion( fake_namespace_adapter: FakeNamespaceAdapter, ) -> None: diff --git a/tests/commands/test_project.py b/tests/commands/test_project.py index 8e46b12..f212b93 100644 --- a/tests/commands/test_project.py +++ b/tests/commands/test_project.py @@ -75,6 +75,14 @@ def test_project_get_command_resolves_name() -> None: assert payload["data"]["name"] == "etl-prod" +def test_project_get_help_points_to_list_for_selector() -> None: + result = runner.invoke(app, ["project", "get", "--help"]) + + assert result.exit_code == 0 + assert "project" in result.stdout + assert "list" in result.stdout + + def test_project_get_command_reports_not_found_suggestion() -> None: result = runner.invoke(app, ["project", "get", "missing"]) diff --git a/tests/commands/test_project_parameter.py b/tests/commands/test_project_parameter.py index d13bcb5..6b7fd3b 100644 --- a/tests/commands/test_project_parameter.py +++ b/tests/commands/test_project_parameter.py @@ -75,6 +75,17 @@ def test_project_parameter_list_command_returns_paginated_payload() -> None: assert payload["resolved"]["project"]["source"] == "context" +def test_project_parameter_list_help_points_to_project_and_data_type_discovery() -> ( + None +): + result = runner.invoke(app, ["project-parameter", "list", "--help"]) + + assert result.exit_code == 0 + assert "project list" in result.stdout + assert "enum list" in result.stdout + assert "data-type" in result.stdout + + def test_project_parameter_get_command_reports_not_found_suggestion() -> None: result = runner.invoke(app, ["project-parameter", "get", "missing"]) @@ -88,6 +99,15 @@ def test_project_parameter_get_command_reports_not_found_suggestion() -> None: ) +def test_project_parameter_get_help_points_to_selected_project_list() -> None: + result = runner.invoke(app, ["project-parameter", "get", "--help"]) + + assert result.exit_code == 0 + assert "project-parameter" in result.stdout + assert "list" in result.stdout + assert "selected project" in result.stdout + + def test_project_parameter_create_command_returns_created_parameter() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_project_preference.py b/tests/commands/test_project_preference.py index ab4eeb0..cb2222f 100644 --- a/tests/commands/test_project_preference.py +++ b/tests/commands/test_project_preference.py @@ -67,6 +67,13 @@ def test_project_preference_get_command_returns_payload() -> None: assert payload["resolved"]["project"]["source"] == "context" +def test_project_preference_get_help_points_to_project_list() -> None: + result = runner.invoke(app, ["project-preference", "get", "--help"]) + + assert result.exit_code == 0 + assert "project list" in result.stdout + + def test_project_preference_update_command_accepts_inline_json() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_project_worker_group.py b/tests/commands/test_project_worker_group.py index 1cec03c..b318b9e 100644 --- a/tests/commands/test_project_worker_group.py +++ b/tests/commands/test_project_worker_group.py @@ -93,6 +93,13 @@ def test_project_worker_group_set_command_accepts_repeated_worker_group_flags() assert [item["workerGroup"] for item in payload["data"]] == ["default", "gpu"] +def test_project_worker_group_set_help_points_to_worker_group_list() -> None: + result = runner.invoke(app, ["project-worker-group", "set", "--help"]) + + assert result.exit_code == 0 + assert "worker-group list" in result.stdout + + def test_project_worker_group_clear_command_requires_force() -> None: result = runner.invoke(app, ["project-worker-group", "clear"]) diff --git a/tests/commands/test_queue.py b/tests/commands/test_queue.py index 3ee3b69..1e80c58 100644 --- a/tests/commands/test_queue.py +++ b/tests/commands/test_queue.py @@ -63,6 +63,14 @@ def test_queue_get_command_resolves_name() -> None: assert payload["data"]["queue"] == "root.default" +def test_queue_selector_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["queue", "get", "--help"]) + + assert result.exit_code == 0 + assert "queue" in result.stdout + assert "list" in result.stdout + + def test_queue_create_command_returns_created_queue() -> None: result = runner.invoke( app, @@ -83,6 +91,15 @@ def test_queue_create_command_returns_created_queue() -> None: assert payload["data"]["queue"] == "root.ops" +def test_queue_create_help_distinguishes_name_and_value() -> None: + result = runner.invoke(app, ["queue", "create", "--help"]) + + assert result.exit_code == 0 + assert "selector" in result.stdout + assert "label" in result.stdout + assert "YARN queue value" in result.stdout + + def test_queue_update_command_returns_updated_queue() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_resource.py b/tests/commands/test_resource.py index 37c9ddf..2b54038 100644 --- a/tests/commands/test_resource.py +++ b/tests/commands/test_resource.py @@ -12,7 +12,7 @@ FakeResourceItem, fake_service_runtime, ) -from tests.support import make_profile +from tests.support import make_profile, normalize_cli_help runner = CliRunner() @@ -85,6 +85,15 @@ def test_resource_view_command_returns_text_content() -> None: assert payload["data"]["content"] == "select 1;" +def test_resource_path_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["resource", "view", "--help"]) + + assert result.exit_code == 0 + help_text = normalize_cli_help(result.stdout) + assert "dsctl resource list" in help_text + assert "--dir DIR" in help_text + + def test_resource_upload_and_mkdir_commands_mutate_resources(tmp_path: Path) -> None: upload_path = tmp_path / "upload.sql" upload_path.write_text("select 3;\n", encoding="utf-8") @@ -119,6 +128,13 @@ def test_resource_create_command_rejects_empty_content() -> None: ) +def test_resource_create_help_points_to_upload_for_local_files() -> None: + result = runner.invoke(app, ["resource", "create", "--help"]) + + assert result.exit_code == 0 + assert "resource upload --file PATH" in normalize_cli_help(result.stdout) + + def test_resource_download_command_writes_output_file(tmp_path: Path) -> None: result = runner.invoke( app, diff --git a/tests/commands/test_schedule.py b/tests/commands/test_schedule.py index 2e51112..bc76484 100644 --- a/tests/commands/test_schedule.py +++ b/tests/commands/test_schedule.py @@ -92,6 +92,14 @@ def test_schedule_list_command_returns_paginated_payload() -> None: assert payload["data"]["totalList"][0]["workflowDefinitionName"] == "daily-sync" +def test_schedule_list_help_points_to_project_and_workflow_discovery() -> None: + result = runner.invoke(app, ["schedule", "list", "--help"]) + + assert result.exit_code == 0 + assert "project list" in result.stdout + assert "workflow list" in result.stdout + + def test_schedule_list_command_rejects_workflow_and_search_together() -> None: result = runner.invoke( app, @@ -125,6 +133,13 @@ def test_schedule_get_command_returns_schedule_payload() -> None: assert payload["data"]["crontab"] == "0 0 2 * * ?" +def test_schedule_get_help_points_to_schedule_list() -> None: + result = runner.invoke(app, ["schedule", "get", "--help"]) + + assert result.exit_code == 0 + assert "schedule list" in result.stdout + + def test_schedule_preview_command_returns_times_and_analysis() -> None: result = runner.invoke(app, ["schedule", "preview", "1"]) @@ -158,6 +173,18 @@ def test_schedule_preview_command_rejects_mixing_id_and_schedule_fields() -> Non ) +def test_schedule_create_help_points_to_related_discovery_commands() -> None: + result = runner.invoke(app, ["schedule", "create", "--help"]) + + assert result.exit_code == 0 + assert "workflow list" in result.stdout + assert "project list" in result.stdout + assert "alert-group" in result.stdout + assert "worker-group" in result.stdout + assert "tenant list" in result.stdout + assert "environment list" in result.stdout + + def test_schedule_explain_command_returns_create_explanation() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_schema.py b/tests/commands/test_schema.py index 00edd2c..746a5d0 100644 --- a/tests/commands/test_schema.py +++ b/tests/commands/test_schema.py @@ -1,10 +1,16 @@ import json +from pathlib import Path from typer.testing import CliRunner from dsctl.app import app from dsctl.models import supported_typed_task_types -from dsctl.services.template import parameter_syntax_index_data, task_template_metadata +from dsctl.services.datasource_payload import datasource_template_index_data +from dsctl.services.template import ( + cluster_config_template_capability_data, + parameter_syntax_index_data, + task_template_metadata, +) from dsctl.upstream import upstream_default_task_types runner = CliRunner() @@ -28,7 +34,7 @@ def test_schema_command_returns_machine_readable_cli_surface() -> None: "use", "enum", "lint", - "env", + "environment", "cluster", "datasource", "namespace", @@ -53,13 +59,29 @@ def test_schema_command_returns_machine_readable_cli_surface() -> None: "generic_types": expected_generic_types, "templates_by_type": task_template_metadata(), } + assert payload["data"]["capabilities"]["templates"]["datasource"] == ( + datasource_template_index_data() + ) assert payload["data"]["capabilities"]["templates"]["parameters"] == ( parameter_syntax_index_data() ) + assert payload["data"]["capabilities"]["templates"]["environment"] == { + "command": "dsctl template environment", + "source_options": ["--config TEXT", "--config-file PATH"], + "target_commands": [ + "dsctl environment create --name NAME --config-file env.sh", + "dsctl environment update ENVIRONMENT --config-file env.sh", + ], + } + assert payload["data"]["capabilities"]["templates"]["cluster"] == ( + cluster_config_template_capability_data() + ) assert payload["data"]["capabilities"]["self_description"] == { "schema": True, "template": True, "capabilities": True, + "command_invocation_source": "schema", + "capabilities_scope": "feature_discovery", } assert payload["data"]["errors"] == { "fields": ["type", "message", "details", "source", "suggestion"], @@ -89,6 +111,10 @@ def test_schema_command_returns_machine_readable_cli_surface() -> None: }, } assert payload["data"]["output"] == { + "formats": ["json", "table", "tsv"], + "default_format": "json", + "format_option": "--output-format", + "columns_option": "--columns", "success_fields": [ "ok", "action", @@ -111,9 +137,174 @@ def test_schema_command_returns_machine_readable_cli_surface() -> None: "error": False, }, "warning_details_aligned": True, + "data_shape_metadata": True, + "json_column_projection": True, } assert payload["data"]["capabilities"]["monitor"] == { "health": True, "database": True, "server_types": ["master", "worker", "alert-server"], } + + +def test_schema_command_honors_env_file_ds_version() -> None: + with runner.isolated_filesystem(): + Path("cluster.env").write_text("DS_VERSION=3.3.2\n", encoding="utf-8") + + result = runner.invoke(app, ["--env-file", "cluster.env", "schema"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["data"]["capabilities"]["ds"]["selected_version"] == "3.3.2" + assert payload["data"]["capabilities"]["ds"]["current_version"] == "3.3.2" + assert payload["data"]["capabilities"]["ds"]["tested"] is False + + +def test_schema_command_returns_group_scope() -> None: + result = runner.invoke(app, ["schema", "--group", "task-instance"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "schema" + assert payload["resolved"] == { + "schema": { + "view": "group", + "group": "task-instance", + } + } + assert "capabilities" not in payload["data"] + commands = payload["data"]["commands"] + assert len(commands) == 1 + assert commands[0]["name"] == "task-instance" + + +def test_schema_command_returns_command_scope() -> None: + result = runner.invoke(app, ["schema", "--command", "task-instance.list"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "schema" + assert payload["resolved"] == { + "schema": { + "view": "command", + "command": "task-instance.list", + } + } + commands = payload["data"]["commands"] + assert len(commands) == 1 + assert commands[0]["name"] == "task-instance" + assert [item["action"] for item in commands[0]["commands"]] == [ + "task-instance.list" + ] + + +def test_schema_command_can_list_group_and_command_values() -> None: + groups_result = runner.invoke(app, ["schema", "--list-groups"]) + + assert groups_result.exit_code == 0 + groups_payload = json.loads(groups_result.stdout) + assert groups_payload["resolved"]["schema"]["view"] == "groups" + assert groups_payload["data"][0]["schema_command"] == "dsctl schema --group use" + + commands_result = runner.invoke(app, ["schema", "--list-commands"]) + + assert commands_result.exit_code == 0 + commands_payload = json.loads(commands_result.stdout) + assert commands_payload["resolved"]["schema"]["view"] == "commands" + assert commands_payload["data"][0]["group"] is None + assert any( + item["action"] == "datasource.create" + and item["schema_command"] == "dsctl schema --command datasource.create" + for item in commands_payload["data"] + ) + + +def test_schema_command_list_values_render_as_table_rows() -> None: + result = runner.invoke( + app, + ["--output-format", "table", "schema", "--list-groups"], + ) + + assert result.exit_code == 0 + assert "name" in result.stdout + assert "schema_command" in result.stdout + assert "dsctl schema --group use" in result.stdout + + +def test_schema_command_datasource_create_uses_payload_reference() -> None: + result = runner.invoke(app, ["schema", "--command", "datasource.create"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + commands = payload["data"]["commands"] + datasource_create = commands[0]["commands"][0] + assert "payload_schema" not in datasource_create + assert datasource_create["payload"]["template_command"] == ( + "dsctl template datasource --type MYSQL" + ) + assert datasource_create["payload"]["template_command_pattern"] == ( + "dsctl template datasource --type TYPE" + ) + assert datasource_create["payload"]["template_json_path"] == "data.json" + assert datasource_create["payload"]["template_discovery_command"] == ( + "dsctl template datasource" + ) + + +def test_schema_command_datasource_create_table_output_is_compact() -> None: + result = runner.invoke( + app, + ["--output-format", "table", "schema", "--command", "datasource.create"], + ) + + assert result.exit_code == 0 + assert max(len(line) for line in result.stdout.splitlines()) < 240 + assert "dsctl template datasource --type MYSQL" in result.stdout + assert "template_discovery_command" in result.stdout + assert "additional_fields_by_type" not in result.stdout + + +def test_schema_command_long_choices_render_as_discovery_hint() -> None: + result = runner.invoke( + app, + ["--output-format", "table", "schema", "--command", "template.datasource"], + ) + + assert result.exit_code == 0 + assert max(len(line) for line in result.stdout.splitlines()) < 240 + assert "choices=29 values; use discovery_command" in result.stdout + assert "dsctl template datasource" in result.stdout + assert "ALIYUN_SERVERLESS_SPARK" not in result.stdout + + +def test_schema_command_table_output_supports_contract_columns() -> None: + result = runner.invoke( + app, + [ + "--output-format", + "table", + "--columns", + "flag,description,discovery_command", + "schema", + "--command", + "environment.create", + ], + ) + + assert result.exit_code == 0 + assert "--config" in result.stdout + assert "dsctl template environment" in result.stdout + assert "Unknown display column" not in result.stdout + + +def test_schema_command_rejects_conflicting_scope_options() -> None: + result = runner.invoke( + app, + ["schema", "--group", "workflow", "--command", "workflow.run"], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "schema" + assert payload["error"]["type"] == "user_input_error" + assert "mutually exclusive" in payload["error"]["message"] diff --git a/tests/commands/test_task.py b/tests/commands/test_task.py index 9e829fc..8ec4648 100644 --- a/tests/commands/test_task.py +++ b/tests/commands/test_task.py @@ -19,7 +19,7 @@ FakeWorkflowTaskRelation, fake_service_runtime, ) -from tests.support import make_profile +from tests.support import make_profile, normalize_cli_help runner = CliRunner() @@ -122,6 +122,14 @@ def test_task_list_command_returns_filtered_tasks() -> None: assert payload["data"] == [{"code": 201, "name": "extract", "version": 1}] +def test_task_list_help_points_to_project_and_workflow_discovery() -> None: + result = runner.invoke(app, ["task", "list", "--help"]) + + assert result.exit_code == 0 + assert "project list" in result.stdout + assert "workflow list" in result.stdout + + def test_task_get_command_returns_task_payload() -> None: result = runner.invoke(app, ["task", "get", "extract"]) @@ -132,6 +140,22 @@ def test_task_get_command_returns_task_payload() -> None: assert payload["data"]["taskType"] == "SHELL" +def test_task_get_help_points_to_task_list() -> None: + result = runner.invoke(app, ["task", "get", "--help"]) + + assert result.exit_code == 0 + assert "task list" in result.stdout + + +def test_task_update_help_points_to_command_schema() -> None: + result = runner.invoke(app, ["task", "update", "--help"]) + + assert result.exit_code == 0 + help_text = normalize_cli_help(result.stdout) + assert "schema --command" in help_text + assert "task.update" in help_text + + def test_task_update_command_can_dry_run_native_update() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_task_group.py b/tests/commands/test_task_group.py index 13b5d8a..3053e48 100644 --- a/tests/commands/test_task_group.py +++ b/tests/commands/test_task_group.py @@ -96,6 +96,18 @@ def test_task_group_list_command_reports_status_choices() -> None: assert payload["error"]["suggestion"] == "Pass `open`/`closed` or `1`/`0`." +def test_task_group_help_points_to_selector_and_status_values() -> None: + get_result = runner.invoke(app, ["task-group", "get", "--help"]) + list_result = runner.invoke(app, ["task-group", "list", "--help"]) + + assert get_result.exit_code == 0 + assert "task-group" in get_result.stdout + assert "list" in get_result.stdout + assert list_result.exit_code == 0 + assert "open," in list_result.stdout + assert "closed, 1, or 0" in list_result.stdout + + def test_task_group_create_command_uses_project_selection() -> None: result = runner.invoke( app, @@ -136,6 +148,15 @@ def test_task_group_queue_force_start_command_returns_confirmation() -> None: assert payload["data"] == {"queueId": 31, "forceStarted": True} +def test_task_group_queue_help_points_to_queue_id_discovery() -> None: + result = runner.invoke(app, ["task-group", "queue", "force-start", "--help"]) + + assert result.exit_code == 0 + assert "task-group queue list" in result.stdout + assert "discover" in result.stdout + assert "ids" in result.stdout + + def test_task_group_queue_force_start_command_reports_already_started( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/commands/test_task_instance.py b/tests/commands/test_task_instance.py index eaa699d..97d02fa 100644 --- a/tests/commands/test_task_instance.py +++ b/tests/commands/test_task_instance.py @@ -63,7 +63,12 @@ def patch_task_instance_service(monkeypatch: pytest.MonkeyPatch) -> None: project_code_value=7, task_code_value=201, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("RUNNING_EXECUTION"), + start_time_value="2026-04-11 10:00:00", + host="worker-1", + executor_name_value="alice", + task_execute_type_value=FakeEnumValue("BATCH"), ), FakeTaskInstance( id=3002, @@ -74,7 +79,12 @@ def patch_task_instance_service(monkeypatch: pytest.MonkeyPatch) -> None: project_code_value=7, task_code_value=202, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("FAILURE"), + start_time_value="2026-04-11 10:05:00", + host="worker-1", + executor_name_value="bob", + task_execute_type_value=FakeEnumValue("BATCH"), ), FakeTaskInstance( id=3003, @@ -85,7 +95,11 @@ def patch_task_instance_service(monkeypatch: pytest.MonkeyPatch) -> None: project_code_value=7, task_code_value=203, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("SUCCESS"), + start_time_value="2026-04-11 10:10:00", + executor_name_value="alice", + task_execute_type_value=FakeEnumValue("BATCH"), ), ], log_messages_by_task_instance_id={3001: ["line-1", "line-2", "line-3"]}, @@ -134,6 +148,84 @@ def test_task_instance_list_command_supports_all_pages() -> None: assert len(payload["data"]["totalList"]) == 2 +def test_task_instance_list_command_supports_project_filters() -> None: + result = runner.invoke( + app, + [ + "task-instance", + "list", + "--project", + "etl-prod", + "--host", + "worker-1", + "--executor", + "bob", + "--start", + "2026-04-11 10:00:00", + "--end", + "2026-04-11 10:10:00", + "--execute-type", + "BATCH", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "task-instance.list" + assert payload["resolved"]["project"]["code"] == 7 + assert payload["resolved"]["host"] == "worker-1" + assert payload["data"]["total"] == 1 + assert payload["data"]["totalList"][0]["id"] == 3002 + + +def test_task_instance_list_command_requires_project_without_workflow_instance() -> ( + None +): + result = runner.invoke(app, ["task-instance", "list"]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "task-instance.list" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Pass --project NAME or run `dsctl use project NAME`." + ) + + +def test_task_instance_list_command_rejects_workflow_definition_filter() -> None: + result = runner.invoke( + app, + [ + "task-instance", + "list", + "--project", + "etl-prod", + "--workflow", + "daily-sync", + ], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "task-instance.list" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["details"]["upstream_filter"] == "workflowDefinitionName" + assert "workflow-instance list" in payload["error"]["suggestion"] + + +def test_task_instance_list_help_points_to_filter_discovery() -> None: + result = runner.invoke(app, ["task-instance", "list", "--help"]) + + assert result.exit_code == 0 + assert "workflow-instance" in result.stdout + assert "project" in result.stdout + assert "task-execution-status" in result.stdout + assert "BATCH" in result.stdout + assert "STREAM" in result.stdout + assert "task-execute-type" in result.stdout + assert "list" in result.stdout + + def test_task_instance_get_command_returns_one_instance() -> None: result = runner.invoke( app, @@ -146,6 +238,15 @@ def test_task_instance_get_command_returns_one_instance() -> None: assert payload["data"]["workflowInstanceId"] == 901 +def test_task_instance_get_help_points_to_instance_discovery() -> None: + result = runner.invoke(app, ["task-instance", "get", "--help"]) + + assert result.exit_code == 0 + assert "task-instance" in result.stdout + assert "workflow-instance" in result.stdout + assert "list" in result.stdout + + def test_task_instance_watch_command_returns_finished_instance() -> None: result = runner.invoke( app, @@ -361,11 +462,34 @@ def test_task_instance_list_command_reports_supported_state_names() -> None: assert payload["action"] == "task-instance.list" assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == ( - "Run `dsctl enum list task_execution_status` to inspect the supported DS " + "Run `dsctl enum list task-execution-status` to inspect the supported DS " "task-instance states." ) +def test_task_instance_list_command_reports_supported_execute_types() -> None: + result = runner.invoke( + app, + [ + "task-instance", + "list", + "--workflow-instance", + "901", + "--execute-type", + "not-real", + ], + ) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["action"] == "task-instance.list" + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Run `dsctl enum list task-execute-type` to inspect the supported DS " + "task execute-type names." + ) + + def test_task_instance_stop_command_reports_running_state_suggestion() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_task_type.py b/tests/commands/test_task_type.py index 1c95ba6..7286c82 100644 --- a/tests/commands/test_task_type.py +++ b/tests/commands/test_task_type.py @@ -60,3 +60,15 @@ def test_task_type_list_command_returns_remote_discovery_payload() -> None: } assert "SPARK" in payload["data"]["cliCoverage"]["genericTaskTemplateTypes"] assert payload["data"]["cliCoverage"]["untemplatedTaskTypes"] == ["CUSTOM_PLUGIN"] + + +def test_task_type_help_distinguishes_live_catalog_from_template_catalog() -> None: + group_result = runner.invoke(app, ["task-type", "--help"]) + list_result = runner.invoke(app, ["task-type", "list", "--help"]) + + assert group_result.exit_code == 0 + assert list_result.exit_code == 0 + assert "live DS task-type catalog" in group_result.stdout + assert "configured cluster and current user" in group_result.stdout + assert "CLI authoring" in list_result.stdout + assert "coverage" in list_result.stdout diff --git a/tests/commands/test_template.py b/tests/commands/test_template.py index a9cbad4..a27baee 100644 --- a/tests/commands/test_template.py +++ b/tests/commands/test_template.py @@ -5,6 +5,7 @@ from dsctl.app import app from dsctl.models import supported_typed_task_types from dsctl.upstream import upstream_default_task_types +from tests.support import normalize_cli_help runner = CliRunner() @@ -18,6 +19,7 @@ def test_template_workflow_command_returns_yaml_document() -> None: assert payload["resolved"]["with_schedule"] is False assert "workflow:" in payload["data"]["yaml"] assert "tasks:" in payload["data"]["yaml"] + assert payload["data"]["lines"][0]["line"].startswith("# Workflow YAML") assert "schedule:" not in payload["data"]["yaml"] @@ -61,7 +63,123 @@ def test_template_params_command_rejects_unknown_topic() -> None: payload = json.loads(result.stdout) assert payload["error"]["type"] == "user_input_error" assert payload["error"]["suggestion"] == ( - "Run `template params` to inspect available topics." + "Run `dsctl template params` to inspect available topics." + ) + + +def test_template_environment_command_returns_environment_config_template() -> None: + result = runner.invoke(app, ["template", "environment"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "template.environment" + assert payload["resolved"] == {"template": "environment.config"} + assert payload["data"]["filename"] == "env.sh" + assert "export JAVA_HOME=/opt/java" in payload["data"]["config"] + assert payload["data"]["lines"][0]["line"] == "export JAVA_HOME=/opt/java" + + +def test_template_environment_command_can_render_table_rows() -> None: + result = runner.invoke( + app, + ["--output-format", "table", "template", "environment"], + ) + + assert result.exit_code == 0 + assert "line" in result.stdout + assert "purpose" in result.stdout + assert "export JAVA_HOME=/opt/java" in result.stdout + assert "target_commands" not in result.stdout + + +def test_template_cluster_command_returns_cluster_config_template() -> None: + result = runner.invoke(app, ["template", "cluster"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "template.cluster" + assert payload["resolved"] == {"template": "cluster.config"} + assert payload["data"]["filename"] == "cluster-config.json" + assert "apiVersion: v1" in payload["data"]["payload"]["k8s"] + assert json.loads(payload["data"]["config"]) == payload["data"]["payload"] + assert payload["data"]["rows"] == payload["data"]["fields"] + + +def test_template_cluster_command_can_render_table_rows() -> None: + result = runner.invoke( + app, + ["--output-format", "table", "template", "cluster"], + ) + + assert result.exit_code == 0 + assert "name" in result.stdout + assert "value_type" in result.stdout + assert "k8s" in result.stdout + assert "CHANGE_ME_BASE64" not in result.stdout + + +def test_template_datasource_command_returns_discovery() -> None: + result = runner.invoke(app, ["template", "datasource"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "template.datasource" + assert payload["resolved"] == {"view": "list"} + assert payload["data"]["default_type"] == "MYSQL" + assert payload["data"]["template_command"] == ( + "dsctl template datasource --type MYSQL" + ) + assert payload["data"]["template_command_pattern"] == ( + "dsctl template datasource --type TYPE" + ) + assert "POSTGRESQL" in payload["data"]["supported_types"] + assert { + "type": "MYSQL", + "template_command": "dsctl template datasource --type MYSQL", + } in payload["data"]["rows"] + assert "fields" not in payload["data"] + + +def test_template_datasource_help_points_to_type_discovery() -> None: + result = runner.invoke(app, ["template", "datasource", "--help"]) + + assert result.exit_code == 0 + assert "dsctl template datasource" in result.stdout + assert "enum list db-type" in result.stdout + + +def test_template_cluster_help_describes_json_config_template() -> None: + result = runner.invoke(app, ["template", "cluster", "--help"]) + + assert result.exit_code == 0 + assert "cluster config JSON template" in result.stdout + + +def test_template_datasource_command_returns_payload_for_type() -> None: + result = runner.invoke(app, ["template", "datasource", "--type", "mysql"]) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "template.datasource" + assert payload["resolved"]["view"] == "template" + assert payload["resolved"]["datasource_type"] == "MYSQL" + assert payload["data"]["type"] == "MYSQL" + assert payload["data"]["payload"]["type"] == "MYSQL" + assert payload["data"]["payload"]["port"] == 3306 + assert json.loads(payload["data"]["json"]) == payload["data"]["payload"] + assert payload["data"]["rows"] == payload["data"]["fields"] + assert "payload_schema" not in payload["data"] + + +def test_template_datasource_command_rejects_unknown_type() -> None: + result = runner.invoke(app, ["template", "datasource", "--type", "unknown"]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["suggestion"] == ( + "Run `dsctl template datasource` to choose a supported datasource type, " + "then `dsctl template datasource --type TYPE`." ) @@ -79,6 +197,7 @@ def test_template_task_command_normalizes_task_type() -> None: "resource", ] assert "type: SHELL" in payload["data"]["yaml"] + assert payload["data"]["rows"][0]["line"].startswith("# Task template") assert "# Optional task runtime controls:" in payload["data"]["yaml"] assert "# timeout_notify_strategy: WARN" in payload["data"]["yaml"] @@ -126,8 +245,13 @@ def test_template_task_command_rejects_unknown_task_type() -> None: assert result.exit_code == 1 payload = json.loads(result.stdout) assert payload["error"]["type"] == "user_input_error" + assert payload["error"]["message"] == "Unsupported task template type 'UNKNOWN'." + assert payload["error"]["details"]["task_type"] == "UNKNOWN" + assert payload["error"]["details"]["discovery_command"] == ( + "dsctl template task --list" + ) assert payload["error"]["suggestion"] == ( - "Run `template task --list` to inspect supported task types." + "Run `dsctl template task --list` to inspect supported task types." ) @@ -136,12 +260,14 @@ def test_template_task_command_can_list_supported_types() -> None: assert result.exit_code == 0 payload = json.loads(result.stdout) - assert payload["action"] == "template.task_types" + assert payload["action"] == "template.task" + assert payload["resolved"] == {"mode": "list"} assert payload["data"]["count"] == len(upstream_default_task_types()) assert payload["data"]["task_types"] == list(upstream_default_task_types()) assert payload["data"]["typed_task_types"] == list(supported_typed_task_types()) assert "SPARK" in payload["data"]["generic_task_types"] assert "Logic" in payload["data"]["task_types_by_category"] + assert payload["data"]["rows"][0]["task_type"] == "SHELL" assert payload["data"]["task_templates"]["SHELL"]["variants"] == [ "minimal", "params", @@ -149,13 +275,45 @@ def test_template_task_command_can_list_supported_types() -> None: ] +def test_template_task_help_points_to_type_and_variant_discovery() -> None: + result = runner.invoke(app, ["template", "task", "--help"]) + + assert result.exit_code == 0 + help_text = normalize_cli_help(result.stdout) + assert "Required unless --list" in help_text + assert "post-json" in help_text + assert "workflow-dependency" in help_text + assert "dsctl template task --list" in help_text + + def test_template_task_command_requires_type_without_list() -> None: result = runner.invoke(app, ["template", "task"]) assert result.exit_code == 1 payload = json.loads(result.stdout) assert payload["error"]["type"] == "user_input_error" - assert payload["error"]["message"].startswith("TASK_TYPE is required.") + assert payload["error"]["message"] == "TASK_TYPE is required." + assert payload["error"]["details"]["discovery_command"] == ( + "dsctl template task --list" + ) assert payload["error"]["suggestion"] == ( - "Run `template task --list` to inspect supported task types." + "Run `dsctl template task --list` to inspect supported task types." ) + + +def test_template_table_outputs_use_row_shapes() -> None: + cases = [ + (["template", "workflow"], "line_no | line"), + (["template", "environment"], "purpose"), + (["template", "cluster"], "name"), + (["template", "datasource"], "type"), + (["template", "datasource", "--type", "MYSQL"], "name"), + (["template", "task", "--list"], "task_type"), + (["template", "task", "SHELL"], "line_no | line"), + ] + for args, expected_header in cases: + result = runner.invoke(app, ["--output-format", "table", *args]) + + assert result.exit_code == 0 + assert expected_header in result.stdout.splitlines()[0] + assert max(len(line) for line in result.stdout.splitlines()) < 220 diff --git a/tests/commands/test_tenant.py b/tests/commands/test_tenant.py index f1d044b..c314965 100644 --- a/tests/commands/test_tenant.py +++ b/tests/commands/test_tenant.py @@ -92,6 +92,14 @@ def test_tenant_get_command_resolves_code() -> None: assert payload["data"]["queueName"] == "default" +def test_tenant_get_help_points_to_list_for_selector() -> None: + result = runner.invoke(app, ["tenant", "get", "--help"]) + + assert result.exit_code == 0 + assert "tenant" in result.stdout + assert "list" in result.stdout + + def test_tenant_create_command_returns_created_tenant() -> None: result = runner.invoke( app, @@ -112,6 +120,13 @@ def test_tenant_create_command_returns_created_tenant() -> None: assert payload["data"]["queueId"] == 11 +def test_tenant_create_help_points_to_queue_list() -> None: + result = runner.invoke(app, ["tenant", "create", "--help"]) + + assert result.exit_code == 0 + assert "dsctl queue list" in result.stdout + + def test_tenant_update_command_returns_updated_tenant() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_use.py b/tests/commands/test_use.py index ca89221..312bcbe 100644 --- a/tests/commands/test_use.py +++ b/tests/commands/test_use.py @@ -35,3 +35,15 @@ def test_use_clear_command_clears_project_scope() -> None: assert payload["action"] == "use.clear" assert payload["data"]["project"] is None assert payload["data"]["workflow"] is None + + +def test_use_help_points_to_context_target_discovery() -> None: + project_result = runner.invoke(app, ["use", "project", "--help"]) + workflow_result = runner.invoke(app, ["use", "workflow", "--help"]) + + assert project_result.exit_code == 0 + assert "project" in project_result.stdout + assert "list" in project_result.stdout + assert workflow_result.exit_code == 0 + assert "workflow" in workflow_result.stdout + assert "list" in workflow_result.stdout diff --git a/tests/commands/test_user.py b/tests/commands/test_user.py index 2e0010b..d04badc 100644 --- a/tests/commands/test_user.py +++ b/tests/commands/test_user.py @@ -213,6 +213,14 @@ def test_user_get_command_resolves_name() -> None: assert payload["data"]["timeZone"] == "Asia/Shanghai" +def test_user_get_help_points_to_list_for_selector() -> None: + result = runner.invoke(app, ["user", "get", "--help"]) + + assert result.exit_code == 0 + assert "user" in result.stdout + assert "list" in result.stdout + + def test_user_create_command_returns_created_user() -> None: result = runner.invoke( app, @@ -239,6 +247,14 @@ def test_user_create_command_returns_created_user() -> None: assert payload["data"]["tenantId"] == 11 +def test_user_create_help_points_to_tenant_and_queue_lists() -> None: + result = runner.invoke(app, ["user", "create", "--help"]) + + assert result.exit_code == 0 + assert "dsctl tenant list" in result.stdout + assert "dsctl queue list" in result.stdout + + def test_user_create_command_reports_upstream_input_suggestion( fake_user_adapter: FakeUserAdapter, ) -> None: @@ -340,6 +356,15 @@ def test_user_grant_project_command_returns_confirmation() -> None: assert payload["resolved"]["project"]["code"] == 701 +def test_user_grant_project_help_points_to_user_and_project_lists() -> None: + result = runner.invoke(app, ["user", "grant", "project", "--help"]) + + assert result.exit_code == 0 + assert "user" in result.stdout + assert "list" in result.stdout + assert "project" in result.stdout + + def test_user_revoke_project_command_returns_confirmation() -> None: result = runner.invoke(app, ["user", "revoke", "project", "alice", "etl-prod"]) @@ -372,6 +397,14 @@ def test_user_grant_datasource_command_returns_confirmation() -> None: assert [item["id"] for item in payload["data"]["datasources"]] == [7, 9] +def test_user_grant_datasource_help_points_to_datasource_list() -> None: + result = runner.invoke(app, ["user", "grant", "datasource", "--help"]) + + assert result.exit_code == 0 + assert "datasource" in result.stdout + assert "list" in result.stdout + + def test_user_revoke_datasource_command_returns_confirmation() -> None: result = runner.invoke( app, @@ -414,6 +447,13 @@ def test_user_grant_namespace_command_returns_confirmation() -> None: assert [item["id"] for item in payload["data"]["namespaces"]] == [21, 22] +def test_user_grant_namespace_help_points_to_namespace_list() -> None: + result = runner.invoke(app, ["user", "grant", "namespace", "--help"]) + + assert result.exit_code == 0 + assert "dsctl namespace list" in result.stdout + + def test_user_revoke_namespace_command_returns_confirmation() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_worker_group.py b/tests/commands/test_worker_group.py index fd55ac6..8ee8904 100644 --- a/tests/commands/test_worker_group.py +++ b/tests/commands/test_worker_group.py @@ -63,6 +63,14 @@ def test_worker_group_get_command_resolves_name() -> None: assert payload["data"]["addrList"] == "worker-a:1234" +def test_worker_group_selector_help_points_to_list_discovery() -> None: + result = runner.invoke(app, ["worker-group", "get", "--help"]) + + assert result.exit_code == 0 + assert "worker-group" in result.stdout + assert "list" in result.stdout + + def test_worker_group_create_command_returns_created_worker_group() -> None: result = runner.invoke( app, @@ -85,6 +93,14 @@ def test_worker_group_create_command_returns_created_worker_group() -> None: assert payload["data"]["addrList"] == "worker-a:1234,worker-b:1234" +def test_worker_group_create_help_points_to_worker_server_discovery() -> None: + result = runner.invoke(app, ["worker-group", "create", "--help"]) + + assert result.exit_code == 0 + assert "monitor server" in result.stdout + assert "worker" in result.stdout + + def test_worker_group_update_command_returns_updated_worker_group() -> None: result = runner.invoke( app, diff --git a/tests/commands/test_workflow.py b/tests/commands/test_workflow.py index 0467098..e88214a 100644 --- a/tests/commands/test_workflow.py +++ b/tests/commands/test_workflow.py @@ -207,6 +207,20 @@ def test_workflow_get_command_can_emit_yaml_inside_json_envelope() -> None: assert "workflow:" in payload["data"]["yaml"] +def test_workflow_list_help_points_to_project_discovery() -> None: + result = runner.invoke(app, ["workflow", "list", "--help"]) + + assert result.exit_code == 0 + assert "project list" in result.stdout + + +def test_workflow_get_help_points_to_workflow_discovery() -> None: + result = runner.invoke(app, ["workflow", "get", "--help"]) + + assert result.exit_code == 0 + assert "workflow list" in result.stdout + + def test_workflow_get_command_reports_supported_output_formats() -> None: result = runner.invoke(app, ["workflow", "get", "--format", "table"]) @@ -282,6 +296,13 @@ def test_workflow_lineage_dependent_tasks_command_can_filter_by_task() -> None: ] +def test_workflow_lineage_dependent_tasks_help_points_to_task_discovery() -> None: + result = runner.invoke(app, ["workflow", "lineage", "dependent-tasks", "--help"]) + + assert result.exit_code == 0 + assert "task list" in result.stdout + + def test_workflow_create_command_can_dry_run_yaml_spec( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -533,6 +554,19 @@ def test_workflow_run_command_returns_created_instance_ids() -> None: assert payload["data"]["workflowInstanceIds"] == [901] +def test_workflow_run_help_points_to_runtime_selector_discovery() -> None: + result = runner.invoke(app, ["workflow", "run", "--help"]) + + assert result.exit_code == 0 + assert "workflow" in result.stdout + assert "project" in result.stdout + assert "worker-group" in result.stdout + assert "tenant" in result.stdout + assert "alert-group" in result.stdout + assert "environment" in result.stdout + assert "list" in result.stdout + + def test_workflow_run_task_command_returns_created_instance_ids_and_warning() -> None: result = runner.invoke(app, ["workflow", "run-task", "--task", "extract"]) @@ -548,6 +582,14 @@ def test_workflow_run_task_command_returns_created_instance_ids_and_warning() -> ) +def test_workflow_run_task_help_points_to_task_discovery() -> None: + result = runner.invoke(app, ["workflow", "run-task", "--help"]) + + assert result.exit_code == 0 + assert "task" in result.stdout + assert "list" in result.stdout + + def test_workflow_run_command_can_dry_run_runtime_options() -> None: result = runner.invoke( app, @@ -602,6 +644,17 @@ def test_workflow_backfill_command_can_dry_run_task_scope() -> None: assert form["runMode"] == "RUN_MODE_SERIAL" +def test_workflow_backfill_help_points_to_task_and_runtime_discovery() -> None: + result = runner.invoke(app, ["workflow", "backfill", "--help"]) + + assert result.exit_code == 0 + assert "task" in result.stdout + assert "list" in result.stdout + assert "worker-group" in result.stdout + assert "tenant" in result.stdout + assert "environment" in result.stdout + + def test_workflow_backfill_command_reports_missing_time_selection() -> None: result = runner.invoke(app, ["workflow", "backfill"]) diff --git a/tests/commands/test_workflow_instance.py b/tests/commands/test_workflow_instance.py index 6ae3d1e..3f1a1c5 100644 --- a/tests/commands/test_workflow_instance.py +++ b/tests/commands/test_workflow_instance.py @@ -85,6 +85,7 @@ def patch_workflow_instance_service(monkeypatch: pytest.MonkeyPatch) -> None: state_value=FakeEnumValue("RUNNING_EXECUTION"), run_times_value=1, name="daily-sync-901", + host="master-1", executor_id_value=11, executor_name_value="alice", worker_group_value="default", @@ -98,6 +99,7 @@ def patch_workflow_instance_service(monkeypatch: pytest.MonkeyPatch) -> None: state_value=FakeEnumValue("SUCCESS"), run_times_value=1, name="child-workflow-903", + host="master-2", executor_id_value=12, executor_name_value="bob", worker_group_value="default", @@ -166,6 +168,35 @@ def test_workflow_instance_list_command_supports_all_pages() -> None: assert len(payload["data"]["totalList"]) == 2 +def test_workflow_instance_list_command_accepts_project_scoped_filters() -> None: + result = runner.invoke( + app, + [ + "workflow-instance", + "list", + "--project", + "etl-prod", + "--search", + "daily", + "--executor", + "alice", + "--host", + "master", + ], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["action"] == "workflow-instance.list" + assert payload["resolved"]["project"] == "etl-prod" + assert payload["resolved"]["project_code"] == 7 + assert payload["resolved"]["search"] == "daily" + assert payload["resolved"]["executor"] == "alice" + assert payload["resolved"]["host"] == "master" + assert payload["data"]["total"] == 1 + assert payload["data"]["totalList"][0]["id"] == 901 + + def test_workflow_instance_list_command_reports_supported_state_names() -> None: result = runner.invoke( app, @@ -180,11 +211,29 @@ def test_workflow_instance_list_command_reports_supported_state_names() -> None: "Workflow instance state must be one of the DS execution status names" ) assert payload["error"]["suggestion"] == ( - "Run `dsctl enum list workflow_execution_status` to inspect the " + "Run `dsctl enum list workflow-execution-status` to inspect the " "supported state names." ) +def test_workflow_instance_list_help_points_to_filter_discovery() -> None: + result = runner.invoke(app, ["workflow-instance", "list", "--help"]) + + assert result.exit_code == 0 + assert "project" in result.stdout + assert "workflow" in result.stdout + assert "workflow-execution-status" in result.stdout + assert "list" in result.stdout + + +def test_workflow_instance_get_help_points_to_instance_discovery() -> None: + result = runner.invoke(app, ["workflow-instance", "get", "--help"]) + + assert result.exit_code == 0 + assert "workflow-instance" in result.stdout + assert "list" in result.stdout + + def test_workflow_instance_get_command_returns_one_instance() -> None: result = runner.invoke(app, ["workflow-instance", "get", "901"]) @@ -753,3 +802,12 @@ def test_workflow_instance_execute_task_command_reports_scope_choices() -> None: assert payload["error"]["suggestion"] == ( "Pass `--scope self`, `--scope pre`, or `--scope post`." ) + + +def test_workflow_instance_execute_task_help_points_to_task_discovery() -> None: + result = runner.invoke(app, ["workflow-instance", "execute-task", "--help"]) + + assert result.exit_code == 0 + assert "task-instance" in result.stdout + assert "workflow-instance" in result.stdout + assert "list" in result.stdout diff --git a/tests/fakes.py b/tests/fakes.py index cb46e25..ac716b0 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -1155,10 +1155,12 @@ class FakeUiPluginAdapter: plugin_defines: list[FakePluginDefine] def list(self, *, plugin_type: str) -> Sequence[FakePluginDefine]: + normalized_plugin_type = plugin_type.casefold() return [ plugin_define for plugin_define in self.plugin_defines - if plugin_define.pluginType == plugin_type + if plugin_define.pluginType is not None + and plugin_define.pluginType.casefold() == normalized_plugin_type ] def get(self, *, plugin_id: int) -> FakePluginDefine: @@ -1166,7 +1168,7 @@ def get(self, *, plugin_id: int) -> FakePluginDefine: if plugin_define.id == plugin_id: return plugin_define raise ApiResultError( - result_code=110003, + result_code=110004, result_message=f"alert plugin define id {plugin_id} not found", ) @@ -5205,12 +5207,72 @@ def list( *, page_no: int, page_size: int, + project_code: int | None = None, + workflow_code: int | None = None, project_name: str | None = None, workflow_name: str | None = None, + search: str | None = None, + executor: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, state: str | None = None, ) -> FakeWorkflowInstancePage: - del project_name, workflow_name + del project_name filtered = list(self.workflow_instances) + if project_code is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.projectCode == project_code + ] + if workflow_code is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.workflowDefinitionCode == workflow_code + ] + if workflow_name is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.name is not None + and workflow_name.lower() in workflow_instance.name.lower() + ] + if search is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.name is not None + and search.lower() in workflow_instance.name.lower() + ] + if executor is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.executorName == executor + ] + if host is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.host is not None + and host.lower() in workflow_instance.host.lower() + ] + if start_time is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.startTime is not None + and workflow_instance.startTime >= start_time + ] + if end_time is not None: + filtered = [ + workflow_instance + for workflow_instance in filtered + if workflow_instance.startTime is not None + and workflow_instance.startTime <= end_time + ] if state is not None: filtered = [ workflow_instance @@ -5606,33 +5668,42 @@ class FakeTaskInstanceAdapter: def list( self, *, - workflow_instance_id: int, project_code: int, page_no: int, page_size: int, + workflow_instance_id: int | None = None, + workflow_instance_name: str | None = None, + workflow_definition_name: str | None = None, search: str | None = None, + task_name: str | None = None, + task_code: int | None = None, + executor: str | None = None, state: str | None = None, + host: str | None = None, + start_time: str | None = None, + end_time: str | None = None, + task_execute_type: str | None = None, ) -> FakeTaskInstancePage: filtered = [ task_instance for task_instance in self.task_instances - if task_instance.workflowInstanceId == workflow_instance_id - and task_instance.projectCode == project_code + if _matches_task_instance_query( + task_instance, + project_code=project_code, + workflow_instance_id=workflow_instance_id, + workflow_instance_name=workflow_instance_name, + workflow_definition_name=workflow_definition_name, + search=search, + task_name=task_name, + task_code=task_code, + executor=executor, + state=state, + host=host, + start_time=start_time, + end_time=end_time, + task_execute_type=task_execute_type, + ) ] - if search is not None: - filtered = [ - task_instance - for task_instance in filtered - if task_instance.name is not None - and search.lower() in task_instance.name.lower() - ] - if state is not None: - filtered = [ - task_instance - for task_instance in filtered - if task_instance.state is not None - and task_instance.state.value == state - ] start = (page_no - 1) * page_size stop = start + page_size total = len(filtered) @@ -5749,6 +5820,61 @@ def stop( ) +def _matches_task_instance_query( + task_instance: FakeTaskInstance, + *, + project_code: int, + workflow_instance_id: int | None, + workflow_instance_name: str | None, + workflow_definition_name: str | None, + search: str | None, + task_name: str | None, + task_code: int | None, + executor: str | None, + state: str | None, + host: str | None, + start_time: str | None, + end_time: str | None, + task_execute_type: str | None, +) -> bool: + start_value = task_instance.startTime + state_value = None if task_instance.state is None else task_instance.state.value + execute_type_value = ( + None + if task_instance.taskExecuteType is None + else task_instance.taskExecuteType.value + ) + return all( + ( + task_instance.projectCode == project_code, + workflow_instance_id is None + or task_instance.workflowInstanceId == workflow_instance_id, + workflow_instance_name is None + or task_instance.workflowInstanceName == workflow_instance_name, + workflow_definition_name is None + or task_instance.processDefinitionName == workflow_definition_name, + search is None + or ( + task_instance.name is not None + and search.lower() in task_instance.name.lower() + ), + task_name is None or task_instance.name == task_name, + task_code is None or task_instance.taskCode == task_code, + executor is None or task_instance.executorName == executor, + state is None or state_value == state, + host is None + or ( + task_instance.host is not None + and host.lower() in task_instance.host.lower() + ), + start_time is None + or (start_value is not None and start_value >= start_time), + end_time is None or (start_value is not None and start_value <= end_time), + task_execute_type is None or execute_type_value == task_execute_type, + ) + ) + + @dataclass class FakeScheduleAdapter: schedules: list[FakeSchedule] diff --git a/tests/live/test_governance_optional_surfaces.py b/tests/live/test_governance_optional_surfaces.py index d71fa50..1af163e 100644 --- a/tests/live/test_governance_optional_surfaces.py +++ b/tests/live/test_governance_optional_surfaces.py @@ -355,6 +355,29 @@ def test_admin_alert_plugin_and_group_lifecycle_round_trip( current_group_name = group_name alert_plugin_id: int | None = None + definition_list_payload = require_ok_payload( + run_dsctl( + live_repo_root, + ["alert-plugin", "definition", "list"], + env_file=live_admin_env_file, + ), + expected_action="alert-plugin.definition.list", + label="alert-plugin definition list", + ) + definition_list_data = require_mapping( + definition_list_payload["data"], + label="alert-plugin definition list data", + ) + definition_rows = require_list( + definition_list_data["definitions"], + label="alert-plugin definition rows", + ) + assert any( + require_mapping(row, label="alert-plugin definition row").get("pluginName") + == "Script" + for row in definition_rows + ) + schema_payload = require_ok_payload( run_dsctl( live_repo_root, diff --git a/tests/live/test_runtime_surfaces.py b/tests/live/test_runtime_surfaces.py index 86ad566..35f3bfb 100644 --- a/tests/live/test_runtime_surfaces.py +++ b/tests/live/test_runtime_surfaces.py @@ -249,7 +249,7 @@ def test_admin_environment_lifecycle_round_trips( run_dsctl( live_repo_root, [ - "env", + "environment", "create", "--name", initial_name, @@ -262,7 +262,7 @@ def test_admin_environment_lifecycle_round_trips( ], env_file=live_admin_env_file, ), - expected_action="env.create", + expected_action="environment.create", label="env create", ) create_data = require_mapping(create_payload["data"], label="env create data") @@ -276,10 +276,10 @@ def test_admin_environment_lifecycle_round_trips( get_payload = require_ok_payload( run_dsctl( live_repo_root, - ["env", "get", current_name], + ["environment", "get", current_name], env_file=live_admin_env_file, ), - expected_action="env.get", + expected_action="environment.get", label="env get", ) get_data = require_mapping(get_payload["data"], label="env get data") @@ -296,7 +296,7 @@ def test_admin_environment_lifecycle_round_trips( run_dsctl( live_repo_root, [ - "env", + "environment", "list", "--search", initial_name, @@ -305,7 +305,7 @@ def test_admin_environment_lifecycle_round_trips( ], env_file=live_admin_env_file, ), - expected_action="env.list", + expected_action="environment.list", label="env list", ) list_data = require_mapping(list_payload["data"], label="env list data") @@ -319,7 +319,7 @@ def test_admin_environment_lifecycle_round_trips( run_dsctl( live_repo_root, [ - "env", + "environment", "update", current_name, "--name", @@ -333,7 +333,7 @@ def test_admin_environment_lifecycle_round_trips( ], env_file=live_admin_env_file, ), - expected_action="env.update", + expected_action="environment.update", label="env update", ) update_data = require_mapping(update_payload["data"], label="env update data") @@ -346,10 +346,10 @@ def test_admin_environment_lifecycle_round_trips( get_by_code_payload = require_ok_payload( run_dsctl( live_repo_root, - ["env", "get", str(environment_code)], + ["environment", "get", str(environment_code)], env_file=live_admin_env_file, ), - expected_action="env.get", + expected_action="environment.get", label="env get by code", ) get_by_code_data = require_mapping( @@ -362,10 +362,10 @@ def test_admin_environment_lifecycle_round_trips( delete_payload = require_ok_payload( run_dsctl( live_repo_root, - ["env", "delete", current_name, "--force"], + ["environment", "delete", current_name, "--force"], env_file=live_admin_env_file, ), - expected_action="env.delete", + expected_action="environment.delete", label="env delete", ) delete_data = require_mapping(delete_payload["data"], label="env delete data") diff --git a/tests/live/test_schedule_surfaces.py b/tests/live/test_schedule_surfaces.py index a547baa..12617d3 100644 --- a/tests/live/test_schedule_surfaces.py +++ b/tests/live/test_schedule_surfaces.py @@ -109,7 +109,7 @@ def test_etl_schedule_lifecycle_round_trips_and_triggers_runtime( run_dsctl( live_repo_root, [ - "env", + "environment", "create", "--name", environment_name, @@ -122,7 +122,7 @@ def test_etl_schedule_lifecycle_round_trips_and_triggers_runtime( ], env_file=live_admin_env_file, ), - expected_action="env.create", + expected_action="environment.create", label="schedule environment create", ) environment_create_data = require_mapping( @@ -657,7 +657,7 @@ def test_etl_schedule_lifecycle_round_trips_and_triggers_runtime( if environment_created and environment_code is not None: run_dsctl( live_repo_root, - ["env", "delete", str(environment_code), "--force"], + ["environment", "delete", str(environment_code), "--force"], env_file=live_admin_env_file, ) if project_created: diff --git a/tests/live/test_workflow_runtime_surfaces.py b/tests/live/test_workflow_runtime_surfaces.py index 16f1d10..5a73c59 100644 --- a/tests/live/test_workflow_runtime_surfaces.py +++ b/tests/live/test_workflow_runtime_surfaces.py @@ -916,6 +916,53 @@ def test_etl_workflow_definition_and_runtime_surfaces_round_trip( label="extract task-instance id", ) + project_task_instance_list_result = wait_for_result( + live_repo_root, + [ + "task-instance", + "list", + "--project", + project_name, + "--task", + "extract", + "--state", + "SUCCESS", + "--execute-type", + "BATCH", + "--start", + "2020-01-01 00:00:00", + "--end", + "2099-01-01 00:00:00", + "--page-size", + "20", + ], + env_file=live_etl_env_file, + timeout_seconds=20.0, + interval_seconds=2.0, + accept=lambda result: _task_instance_list_has_rows( + result, + count=1, + ), + ) + project_task_instance_list_payload = require_ok_payload( + project_task_instance_list_result, + expected_action="task-instance.list", + label="project-scoped task-instance list", + ) + project_task_instance_list_data = require_mapping( + project_task_instance_list_payload["data"], + label="project-scoped task-instance list data", + ) + project_task_instance_rows = require_list( + project_task_instance_list_data["totalList"], + label="project-scoped task-instance rows", + ) + assert any( + require_mapping(item, label="project-scoped task-instance row").get("id") + == extract_task_instance_id + for item in project_task_instance_rows + ) + task_instance_watch_payload = require_ok_payload( run_dsctl( live_repo_root, diff --git a/tests/services/test_alert_plugin.py b/tests/services/test_alert_plugin.py index a7546ae..fecc0a9 100644 --- a/tests/services/test_alert_plugin.py +++ b/tests/services/test_alert_plugin.py @@ -38,7 +38,9 @@ "field": "url", "name": "url", "type": "input", + "value": None, "props": {"placeholder": "Webhook URL"}, + "validate": [{"required": True}], } ], ensure_ascii=False, @@ -81,7 +83,7 @@ def fake_ui_plugin_adapter() -> FakeUiPluginAdapter: FakePluginDefine( id=3, plugin_name_value="Slack", - plugin_type_value="ALERT", + plugin_type_value="alert", plugin_params_value=ALERT_PLUGIN_SCHEMA, ) ] @@ -189,10 +191,73 @@ def test_get_alert_plugin_schema_result_returns_plugin_definition( "pluginDefine": { "id": 3, "pluginName": "Slack", + "pluginType": "alert", + } + } + data = _mapping(result.data) + assert data["pluginParams"] == ALERT_PLUGIN_SCHEMA + assert data["pluginParamFields"] == [ + { + "field": "url", + "name": "url", + "title": None, + "type": "input", + "required": True, + "defaultValue": None, + "options": None, + } + ] + + +def test_list_alert_plugin_definitions_result_returns_supported_definitions( + monkeypatch: pytest.MonkeyPatch, + fake_ui_plugin_adapter: FakeUiPluginAdapter, + fake_alert_plugin_adapter: FakeAlertPluginAdapter, +) -> None: + _install_alert_plugin_service_fakes( + monkeypatch, + ui_plugin_adapter=fake_ui_plugin_adapter, + alert_plugin_adapter=fake_alert_plugin_adapter, + ) + + result = alert_plugin_service.list_alert_plugin_definitions_result() + + assert result.resolved == { + "pluginDefinitions": { "pluginType": "ALERT", + "source": "ui-plugins/query-by-type", } } - assert _mapping(result.data)["pluginParams"] == ALERT_PLUGIN_SCHEMA + data = _mapping(result.data) + assert data["count"] == 1 + assert data["schema_command"] == "alert-plugin schema PLUGIN" + assert data["definitions"] == [ + { + "id": 3, + "pluginName": "Slack", + "pluginType": "alert", + "createTime": None, + "updateTime": None, + } + ] + + +def test_get_alert_plugin_schema_result_accepts_id_and_case_insensitive_name( + monkeypatch: pytest.MonkeyPatch, + fake_ui_plugin_adapter: FakeUiPluginAdapter, + fake_alert_plugin_adapter: FakeAlertPluginAdapter, +) -> None: + _install_alert_plugin_service_fakes( + monkeypatch, + ui_plugin_adapter=fake_ui_plugin_adapter, + alert_plugin_adapter=fake_alert_plugin_adapter, + ) + + by_id = alert_plugin_service.get_alert_plugin_schema_result("3") + by_name = alert_plugin_service.get_alert_plugin_schema_result("slack") + + assert _mapping(by_id.data)["pluginParams"] == ALERT_PLUGIN_SCHEMA + assert _mapping(by_name.data)["pluginName"] == "Slack" def test_create_alert_plugin_result_returns_refreshed_payload( @@ -222,12 +287,36 @@ def test_create_alert_plugin_result_returns_refreshed_payload( "pluginDefine": { "id": 3, "pluginName": "Slack", - "pluginType": "ALERT", + "pluginType": "alert", }, } assert _mapping(result.data)["instanceName"] == "slack-nightly" +def test_create_alert_plugin_result_builds_params_from_inline_fields( + monkeypatch: pytest.MonkeyPatch, + fake_ui_plugin_adapter: FakeUiPluginAdapter, + fake_alert_plugin_adapter: FakeAlertPluginAdapter, +) -> None: + _install_alert_plugin_service_fakes( + monkeypatch, + ui_plugin_adapter=fake_ui_plugin_adapter, + alert_plugin_adapter=fake_alert_plugin_adapter, + ) + + result = alert_plugin_service.create_alert_plugin_result( + name="slack-nightly", + plugin="slack", + params=["URL=https://hooks.example.test/nightly"], + ) + + data = _mapping(result.data) + params = json.loads(str(data["pluginInstanceParams"])) + assert data["instanceName"] == "slack-nightly" + assert params[0]["field"] == "url" + assert params[0]["value"] == "https://hooks.example.test/nightly" + + def test_create_alert_plugin_result_rejects_non_array_params_payload( monkeypatch: pytest.MonkeyPatch, fake_ui_plugin_adapter: FakeUiPluginAdapter, @@ -287,6 +376,47 @@ def test_update_alert_plugin_result_preserves_omitted_params( assert data["pluginInstanceParams"] == ALERT_PLUGIN_PARAMS +def test_update_alert_plugin_result_overlays_inline_fields( + monkeypatch: pytest.MonkeyPatch, + fake_ui_plugin_adapter: FakeUiPluginAdapter, + fake_alert_plugin_adapter: FakeAlertPluginAdapter, +) -> None: + _install_alert_plugin_service_fakes( + monkeypatch, + ui_plugin_adapter=fake_ui_plugin_adapter, + alert_plugin_adapter=fake_alert_plugin_adapter, + ) + + result = alert_plugin_service.update_alert_plugin_result( + "slack-ops", + params=["url=https://hooks.example.test/updated"], + ) + + data = _mapping(result.data) + params = json.loads(str(data["pluginInstanceParams"])) + assert data["instanceName"] == "slack-ops" + assert params[0]["value"] == "https://hooks.example.test/updated" + + +def test_create_alert_plugin_result_requires_inline_required_fields( + monkeypatch: pytest.MonkeyPatch, + fake_ui_plugin_adapter: FakeUiPluginAdapter, + fake_alert_plugin_adapter: FakeAlertPluginAdapter, +) -> None: + _install_alert_plugin_service_fakes( + monkeypatch, + ui_plugin_adapter=fake_ui_plugin_adapter, + alert_plugin_adapter=fake_alert_plugin_adapter, + ) + + with pytest.raises(UserInputError, match="missing required fields"): + alert_plugin_service.create_alert_plugin_result( + name="slack-nightly", + plugin="Slack", + params=["url="], + ) + + def test_delete_alert_plugin_result_requires_force( monkeypatch: pytest.MonkeyPatch, fake_ui_plugin_adapter: FakeUiPluginAdapter, diff --git a/tests/services/test_capabilities.py b/tests/services/test_capabilities.py index 642a730..db398c4 100644 --- a/tests/services/test_capabilities.py +++ b/tests/services/test_capabilities.py @@ -1,7 +1,14 @@ +import pytest + from dsctl.cli_surface import SURFACE_PLANES +from dsctl.errors import UserInputError from dsctl.models import supported_typed_task_types from dsctl.services.capabilities import get_capabilities_result -from dsctl.services.template import parameter_syntax_index_data, task_template_metadata +from dsctl.services.datasource_payload import datasource_template_index_data +from dsctl.services.template import ( + parameter_syntax_index_data, + task_template_metadata, +) from dsctl.upstream import ( upstream_default_task_types, upstream_default_task_types_by_category, @@ -22,6 +29,7 @@ EXPECTED_UNTEMPLATED_UPSTREAM_TASK_TYPES: list[str] = [] EXPECTED_TASK_TEMPLATE_METADATA = task_template_metadata() EXPECTED_PARAMETER_SYNTAX = parameter_syntax_index_data() +EXPECTED_DATASOURCE_TEMPLATE_INDEX = datasource_template_index_data() EXPECTED_VERSION_METADATA = [ { "server_version": "3.3.2", @@ -68,7 +76,7 @@ def test_capabilities_result_describes_current_stable_surface() -> None: "precedence": ["flag", "context"], "name_first_resources": [ "project", - "env", + "environment", "cluster", "datasource", "namespace", @@ -96,6 +104,8 @@ def test_capabilities_result_describes_current_stable_surface() -> None: "schema": True, "template": True, "capabilities": True, + "command_invocation_source": "schema", + "capabilities_scope": "feature_discovery", } assert data["resources"]["top_level"] == [ "version", @@ -108,15 +118,18 @@ def test_capabilities_result_describes_current_stable_surface() -> None: "project", "workflow", ] - assert data["resources"]["groups"]["enum"]["commands"] == ["list"] + assert data["resources"]["groups"]["enum"]["commands"] == ["names", "list"] assert data["resources"]["groups"]["lint"]["commands"] == ["workflow"] assert data["resources"]["groups"]["task-type"]["commands"] == ["list"] assert data["resources"]["groups"]["template"]["commands"] == [ "workflow", "params", + "environment", + "cluster", + "datasource", "task", ] - assert data["resources"]["groups"]["env"]["commands"] == [ + assert data["resources"]["groups"]["environment"]["commands"] == [ "list", "get", "create", @@ -185,6 +198,7 @@ def test_capabilities_result_describes_current_stable_surface() -> None: "update", "delete", "test", + "definition", ] assert data["resources"]["groups"]["alert-group"]["commands"] == [ "list", @@ -291,6 +305,12 @@ def test_capabilities_result_describes_current_stable_surface() -> None: "workflow_schedule_block": True, "workflow_dry_run": True, "parameter_syntax": EXPECTED_PARAMETER_SYNTAX, + "environment_config_template": True, + "cluster_config_template": True, + "datasource_payload_templates": True, + "datasource_template_types": EXPECTED_DATASOURCE_TEMPLATE_INDEX[ + "supported_types" + ], "task_template_types": EXPECTED_TEMPLATE_TASK_TYPES, "task_templates": EXPECTED_TASK_TEMPLATE_METADATA, "typed_task_specs": EXPECTED_TYPED_TASK_TYPES, @@ -359,3 +379,58 @@ def test_capabilities_result_describes_current_stable_surface() -> None: assert data["planes"] == { name: list(resources) for name, resources in SURFACE_PLANES.items() } + + +def test_capabilities_result_can_return_summary() -> None: + result = get_capabilities_result(summary=True) + data = result.data + + assert isinstance(data, dict) + assert result.resolved == {"capabilities": {"view": "summary"}} + assert data["cli"] == {"name": "dsctl", "version": "0.1.0"} + assert data["ds"] == EXPECTED_DS_CAPABILITIES + assert "resources" in data + assert "runtime" in data + assert "authoring" in data + authoring = data["authoring"] + assert isinstance(authoring, dict) + assert authoring["workflow_yaml_create"] is True + assert authoring["task_template_types"] == EXPECTED_TEMPLATE_TASK_TYPES + assert "parameter_syntax" not in authoring + assert "task_templates" not in authoring + + +def test_capabilities_result_can_return_one_section() -> None: + result = get_capabilities_result(section="authoring") + data = result.data + + assert isinstance(data, dict) + assert result.resolved == { + "capabilities": { + "view": "section", + "section": "authoring", + } + } + assert set(data) == {"cli", "ds", "self_description", "authoring"} + authoring = data["authoring"] + assert isinstance(authoring, dict) + assert authoring["parameter_syntax"] == EXPECTED_PARAMETER_SYNTAX + assert authoring["task_templates"] == EXPECTED_TASK_TEMPLATE_METADATA + + +def test_capabilities_result_rejects_conflicting_scope_options() -> None: + with pytest.raises(UserInputError, match="mutually exclusive"): + get_capabilities_result(summary=True, section="runtime") + + +def test_capabilities_result_rejects_unknown_section() -> None: + with pytest.raises( + UserInputError, + match="Unknown capabilities section", + ) as exc_info: + get_capabilities_result(section="missing") + + assert exc_info.value.details["section"] == "missing" + available_sections = exc_info.value.details["available_sections"] + assert isinstance(available_sections, list) + assert "authoring" in available_sections diff --git a/tests/services/test_datasource.py b/tests/services/test_datasource.py index 5307d2b..c7b5f36 100644 --- a/tests/services/test_datasource.py +++ b/tests/services/test_datasource.py @@ -162,6 +162,53 @@ def test_create_datasource_result_returns_created_detail( assert data["password"] == payload["password"] +def test_create_datasource_result_normalizes_datasource_type( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + payload = { + "name": "warehouse", + "type": "mysql", + "host": "db.example", + "port": 3306, + "database": "warehouse", + "userName": "etl", + "password": "secret", + } + adapter = FakeDataSourceAdapter(datasources=[]) + _install_datasource_service_fakes(monkeypatch, adapter) + file = _write_json(tmp_path / "warehouse.json", payload) + + result = datasource_service.create_datasource_result(file=file) + + resolved_datasource = _mapping(result.resolved["datasource"]) + assert resolved_datasource["type"] == "MYSQL" + + +def test_create_datasource_result_rejects_unknown_datasource_type( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + adapter = FakeDataSourceAdapter(datasources=[]) + _install_datasource_service_fakes(monkeypatch, adapter) + file = _write_json( + tmp_path / "unknown.json", + { + "name": "warehouse", + "type": "UNKNOWN", + "password": "secret", + }, + ) + + with pytest.raises(UserInputError, match="Unsupported datasource type") as exc_info: + datasource_service.create_datasource_result(file=file) + + assert exc_info.value.suggestion == ( + "Run `dsctl template datasource` to choose a supported datasource type, " + "then `dsctl template datasource --type TYPE`." + ) + + def test_create_datasource_result_maps_duplicate_name_to_conflict( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -303,8 +350,9 @@ def test_create_datasource_result_rejects_invalid_json_payload( datasource_service.create_datasource_result(file=file) assert exc_info.value.suggestion == ( - "Fix the JSON syntax, or regenerate a DS-native payload with `dsctl " - "datasource get DATASOURCE`." + "Fix the JSON syntax, or run `dsctl template datasource` to choose a " + "type and `dsctl template datasource --type TYPE` to regenerate a " + "payload skeleton." ) diff --git a/tests/services/test_enums.py b/tests/services/test_enums.py index b454d50..aab6849 100644 --- a/tests/services/test_enums.py +++ b/tests/services/test_enums.py @@ -3,7 +3,11 @@ import pytest from dsctl.errors import UserInputError -from dsctl.services.enums import list_enum_result, supported_enum_choices +from dsctl.services.enums import ( + list_enum_names_result, + list_enum_result, + supported_enum_choices, +) def _mapping(value: object) -> dict[str, object]: @@ -59,6 +63,21 @@ def test_list_enum_result_returns_generated_enum_members() -> None: } +def test_list_enum_names_result_returns_supported_enum_names() -> None: + result = list_enum_names_result() + data = result.data + + assert isinstance(data, list) + + assert result.resolved == { + "enum": { + "ds_version": "3.4.1", + "count": len(data), + } + } + assert {"name": "priority", "list_command": "dsctl enum list priority"} in data + + def test_list_enum_result_resolves_class_name_alias() -> None: result = list_enum_result("ReleaseState") data = _mapping(result.data) diff --git a/tests/services/test_resolver.py b/tests/services/test_resolver.py index c60c204..6729857 100644 --- a/tests/services/test_resolver.py +++ b/tests/services/test_resolver.py @@ -165,7 +165,7 @@ def test_environment_resolver_uses_cli_resource_slug_in_not_found_details() -> N with pytest.raises(NotFoundError) as exc_info: resolver_service.environment("missing", adapter=adapter) - assert exc_info.value.details["resource"] == "env" + assert exc_info.value.details["resource"] == "environment" def test_environment_resolver_uses_search_pages_instead_of_full_scan( diff --git a/tests/services/test_schema.py b/tests/services/test_schema.py index 73a8169..eea695f 100644 --- a/tests/services/test_schema.py +++ b/tests/services/test_schema.py @@ -1,11 +1,21 @@ +from pathlib import Path + +import pytest + +from dsctl.errors import UserInputError from dsctl.models import supported_typed_task_types +from dsctl.services.datasource_payload import datasource_template_index_data from dsctl.services.pagination import DEFAULT_PAGE_SIZE from dsctl.services.schema import get_schema_result from dsctl.services.task_instance import ( DEFAULT_TASK_INSTANCE_WATCH_INTERVAL_SECONDS, DEFAULT_TASK_INSTANCE_WATCH_TIMEOUT_SECONDS, ) -from dsctl.services.template import parameter_syntax_index_data, task_template_metadata +from dsctl.services.template import ( + cluster_config_template_capability_data, + parameter_syntax_index_data, + task_template_metadata, +) from dsctl.services.workflow_instance import ( DEFAULT_WATCH_INTERVAL_SECONDS, DEFAULT_WATCH_TIMEOUT_SECONDS, @@ -85,7 +95,7 @@ def test_schema_result_describes_current_stable_surface() -> None: }, "name_first_resources": [ "project", - "env", + "environment", "cluster", "datasource", "namespace", @@ -109,6 +119,10 @@ def test_schema_result_describes_current_stable_surface() -> None: ], } assert data["output"] == { + "formats": ["json", "table", "tsv"], + "default_format": "json", + "format_option": "--output-format", + "columns_option": "--columns", "success_fields": [ "ok", "action", @@ -131,6 +145,8 @@ def test_schema_result_describes_current_stable_surface() -> None: "error": False, }, "warning_details_aligned": True, + "data_shape_metadata": True, + "json_column_projection": True, } commands = data["commands"] @@ -144,7 +160,7 @@ def test_schema_result_describes_current_stable_surface() -> None: "use", "enum", "lint", - "env", + "environment", "cluster", "datasource", "namespace", @@ -171,6 +187,44 @@ def test_schema_result_describes_current_stable_surface() -> None: "task", "task-instance", ] + schema_command = _find_command(commands, "schema") + schema_options = _require_list(schema_command["options"]) + assert _find_option(schema_options, "group")["description"] == ( + "Return schema for one command group. Discover values with " + "`dsctl schema --list-groups`." + ) + assert _find_option(schema_options, "group")["discovery_command"] == ( + "dsctl schema --list-groups" + ) + assert _find_option(schema_options, "command")["description"] == ( + "Return schema for one stable command action. Discover values with " + "`dsctl schema --list-commands`." + ) + assert _find_option(schema_options, "command")["discovery_command"] == ( + "dsctl schema --list-commands" + ) + assert _find_option(schema_options, "list-groups")["default"] is False + assert _find_option(schema_options, "list-commands")["default"] is False + global_options = _require_list(data["global_options"]) + assert _find_option(global_options, "output-format")["choices"] == [ + "json", + "table", + "tsv", + ] + assert _find_option(global_options, "columns")["value_name"] == "CSV" + capabilities_command = _find_command(commands, "capabilities") + capabilities_options = _require_list(capabilities_command["options"]) + assert _find_option(capabilities_options, "summary")["default"] is False + section_option = _find_option(capabilities_options, "section") + assert section_option["description"] == ( + "Return one top-level capability section. Supported: selection, output, " + "errors, resources, planes, authoring, schedule, monitor, enums, runtime. " + "Discover values with `dsctl schema --command capabilities`." + ) + assert section_option["discovery_command"] == "dsctl schema --command capabilities" + section_choices = section_option["choices"] + assert isinstance(section_choices, list) + assert "runtime" in section_choices template_group = _find_group(commands, "template") params_command = _find_command(template_group["commands"], "params") @@ -186,6 +240,23 @@ def test_schema_result_describes_current_stable_surface() -> None: "output", "all", ] + assert topic_option["discovery_command"] == "dsctl template params" + env_template_command = _find_command(template_group["commands"], "environment") + assert env_template_command["action"] == "template.environment" + assert env_template_command["data_shape"] == { + "kind": "summary", + "row_path": "data.lines", + "default_columns": ["line", "purpose"], + "column_discovery": "runtime_row_keys", + } + cluster_template_command = _find_command(template_group["commands"], "cluster") + assert cluster_template_command["action"] == "template.cluster" + assert cluster_template_command["data_shape"] == { + "kind": "summary", + "row_path": "data.fields", + "default_columns": ["name", "required", "value_type", "description"], + "column_discovery": "runtime_row_keys", + } task_command = _find_command(template_group["commands"], "task") task_arguments = _require_list(task_command["arguments"]) first_task_argument = _require_dict(task_arguments[0]) @@ -193,20 +264,37 @@ def test_schema_result_describes_current_stable_surface() -> None: variant_option = _find_option(task_options, "variant") assert task_command["action"] == "template.task" assert first_task_argument["choices"] == EXPECTED_TEMPLATE_TASK_TYPES + assert first_task_argument["discovery_command"] == "dsctl template task --list" variant_choices = variant_option["choices"] assert isinstance(variant_choices, list) assert "resource" in variant_choices assert "post-json" in variant_choices + variant_description = _require_str(variant_option["description"]) + assert "Known variants include" in variant_description + assert "workflow-dependency" in variant_description + assert "dsctl template task --list" in variant_description + assert variant_option["discovery_command"] == "dsctl template task --list" + datasource_template_command = _find_command( + template_group["commands"], + "datasource", + ) + datasource_template_options = _require_list(datasource_template_command["options"]) + datasource_type_option = _find_option(datasource_template_options, "type") + assert ( + datasource_type_option["choices"] + == datasource_template_index_data()["supported_types"] + ) + assert datasource_type_option["discovery_command"] == "dsctl template datasource" enum_group = _find_group(commands, "enum") enum_command_names = [ _require_dict(item)["name"] for item in _require_list(enum_group["commands"]) ] - assert enum_command_names == ["list"] + assert enum_command_names == ["names", "list"] enum_list = _find_command(enum_group["commands"], "list") - enum_arguments = _require_list(enum_list["arguments"]) - first_enum_argument = _require_dict(enum_arguments[0]) - enum_choices = _require_list(first_enum_argument["choices"]) + enum_argument = _require_dict(_require_list(enum_list["arguments"])[0]) + assert enum_argument["discovery_command"] == "dsctl enum names" + enum_choices = _require_list(enum_argument["choices"]) assert "priority" in enum_choices assert "resource-type" in enum_choices @@ -222,8 +310,16 @@ def test_schema_result_describes_current_stable_surface() -> None: for item in _require_list(task_type_group["commands"]) ] assert task_type_command_names == ["list"] + assert task_type_group["summary"] == ( + "List live DS task-type catalog for the configured cluster and current user." + ) + task_type_list = _find_command(task_type_group["commands"], "list") + assert task_type_list["summary"] == ( + "List live DS task types, categories, favourite flags, and CLI authoring " + "coverage." + ) - env_group = _find_group(commands, "env") + env_group = _find_group(commands, "environment") env_command_names = [ _require_dict(item)["name"] for item in _require_list(env_group["commands"]) ] @@ -234,6 +330,20 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + env_create = _find_command(env_group["commands"], "create") + env_create_options = _require_list(env_create["options"]) + env_create_config = _find_option(env_create_options, "config") + env_create_config_file = _find_option(env_create_options, "config-file") + assert env_create_config["discovery_command"] == "dsctl template environment" + assert env_create_config["examples"] == ["export JAVA_HOME=/opt/java"] + assert env_create_config["required"] is False + assert env_create_config_file["discovery_command"] == "dsctl template environment" + env_update = _find_command(env_group["commands"], "update") + env_update_options = _require_list(env_update["options"]) + assert ( + _find_option(env_update_options, "config-file")["discovery_command"] + == "dsctl template environment" + ) cluster_group = _find_group(commands, "cluster") cluster_command_names = [ @@ -246,6 +356,19 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + cluster_create = _find_command(cluster_group["commands"], "create") + cluster_create_options = _require_list(cluster_create["options"]) + cluster_create_config = _find_option(cluster_create_options, "config") + cluster_create_config_file = _find_option(cluster_create_options, "config-file") + assert cluster_create_config["discovery_command"] == "dsctl template cluster" + assert cluster_create_config["required"] is False + assert cluster_create_config_file["discovery_command"] == "dsctl template cluster" + cluster_update = _find_command(cluster_group["commands"], "update") + cluster_update_options = _require_list(cluster_update["options"]) + assert ( + _find_option(cluster_update_options, "config-file")["discovery_command"] + == "dsctl template cluster" + ) datasource_group = _find_group(commands, "datasource") datasource_command_names = [ @@ -260,6 +383,48 @@ def test_schema_result_describes_current_stable_surface() -> None: "delete", "test", ] + datasource_create = _find_command(datasource_group["commands"], "create") + datasource_payload = _require_dict(datasource_create["payload"]) + assert "payload_schema" not in datasource_create + assert datasource_payload == { + "format": "json", + "source_option": "--file", + "target_commands": [ + "dsctl datasource create --file FILE", + "dsctl datasource update DATASOURCE --file FILE", + ], + "ds_model": "BaseDataSourceParamDTO", + "upstream_request_shape": "DataSourceController request body String jsonStr", + "template_command": "dsctl template datasource --type MYSQL", + "template_command_pattern": "dsctl template datasource --type TYPE", + "template_discovery_command": "dsctl template datasource", + "template_json_path": "data.json", + "template_payload_path": "data.payload", + "type_enum": "db-type", + "type_discovery_command": "dsctl enum list db-type", + "rules": [ + "Create payloads must not include id; DS assigns it.", + "Update payloads may omit id or set it to the selected datasource id.", + "Create payloads must include the real password when the type uses one.", + ( + "Update payloads may use the masked password ****** to preserve " + "the stored password." + ), + "Use DS-native field names exactly, including userName and type.", + "Use `dsctl datasource test DATASOURCE` after create or update.", + ], + } + datasource_update = _find_command(datasource_group["commands"], "update") + assert _require_dict(datasource_update["payload"])["template_command"] == ( + "dsctl template datasource --type MYSQL" + ) + + project_group = _find_group(commands, "project") + project_get = _find_command(project_group["commands"], "get") + project_get_args = _require_list(project_get["arguments"]) + assert ( + _require_dict(project_get_args[0])["discovery_command"] == "dsctl project list" + ) schedule_group = _find_group(commands, "schedule") schedule_command_names = [ @@ -277,6 +442,44 @@ def test_schema_result_describes_current_stable_surface() -> None: "online", "offline", ] + schedule_list = _find_command(schedule_group["commands"], "list") + schedule_list_options = _require_list(schedule_list["options"]) + assert ( + _find_option(schedule_list_options, "project")["discovery_command"] + == "dsctl project list" + ) + assert ( + _find_option(schedule_list_options, "workflow")["discovery_command"] + == "dsctl workflow list" + ) + schedule_get = _find_command(schedule_group["commands"], "get") + schedule_get_args = _require_list(schedule_get["arguments"]) + assert ( + _require_dict(schedule_get_args[0])["discovery_command"] + == "dsctl schedule list" + ) + schedule_create = _find_command(schedule_group["commands"], "create") + schedule_create_options = _require_list(schedule_create["options"]) + assert _find_option(schedule_create_options, "failure-strategy")["choices"] == [ + "CONTINUE", + "END", + ] + assert ( + _find_option(schedule_create_options, "warning-group-id")["discovery_command"] + == "dsctl alert-group list" + ) + assert ( + _find_option(schedule_create_options, "worker-group")["discovery_command"] + == "dsctl worker-group list" + ) + assert ( + _find_option(schedule_create_options, "tenant-code")["discovery_command"] + == "dsctl tenant list" + ) + assert ( + _find_option(schedule_create_options, "environment-code")["discovery_command"] + == "dsctl environment list" + ) project_parameter_group = _find_group(commands, "project-parameter") project_parameter_command_names = [ @@ -290,6 +493,25 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + project_parameter_list = _find_command( + project_parameter_group["commands"], + "list", + ) + project_parameter_list_options = _require_list(project_parameter_list["options"]) + assert ( + _find_option(project_parameter_list_options, "project")["discovery_command"] + == "dsctl project list" + ) + assert ( + _find_option(project_parameter_list_options, "data-type")["discovery_command"] + == "dsctl enum list data-type" + ) + project_parameter_get = _find_command(project_parameter_group["commands"], "get") + project_parameter_get_args = _require_list(project_parameter_get["arguments"]) + assert ( + _require_dict(project_parameter_get_args[0])["discovery_command"] + == "dsctl project-parameter list" + ) project_preference_group = _find_group(commands, "project-preference") project_preference_command_names = [ @@ -302,6 +524,12 @@ def test_schema_result_describes_current_stable_surface() -> None: "enable", "disable", ] + project_preference_get = _find_command(project_preference_group["commands"], "get") + project_preference_get_options = _require_list(project_preference_get["options"]) + assert ( + _find_option(project_preference_get_options, "project")["discovery_command"] + == "dsctl project list" + ) project_worker_group_group = _find_group(commands, "project-worker-group") project_worker_group_command_names = [ @@ -313,6 +541,19 @@ def test_schema_result_describes_current_stable_surface() -> None: "set", "clear", ] + project_worker_group_set = _find_command( + project_worker_group_group["commands"], + "set", + ) + project_worker_group_set_options = _require_list( + project_worker_group_set["options"] + ) + assert ( + _find_option(project_worker_group_set_options, "worker-group")[ + "discovery_command" + ] + == "dsctl worker-group list" + ) access_token_group = _find_group(commands, "access-token") access_token_command_names = [ @@ -327,6 +568,18 @@ def test_schema_result_describes_current_stable_surface() -> None: "delete", "generate", ] + access_token_get = _find_command(access_token_group["commands"], "get") + access_token_get_args = _require_list(access_token_get["arguments"]) + assert ( + _require_dict(access_token_get_args[0])["discovery_command"] + == "dsctl access-token list" + ) + access_token_create = _find_command(access_token_group["commands"], "create") + access_token_create_options = _require_list(access_token_create["options"]) + assert ( + _find_option(access_token_create_options, "user")["discovery_command"] + == "dsctl user list" + ) namespace_group = _find_group(commands, "namespace") namespace_command_names = [ @@ -340,6 +593,12 @@ def test_schema_result_describes_current_stable_surface() -> None: "create", "delete", ] + namespace_create = _find_command(namespace_group["commands"], "create") + namespace_create_options = _require_list(namespace_create["options"]) + assert ( + _find_option(namespace_create_options, "cluster-code")["discovery_command"] + == "dsctl cluster list" + ) resource_group = _find_group(commands, "resource") resource_command_names = [ @@ -355,6 +614,16 @@ def test_schema_result_describes_current_stable_surface() -> None: "download", "delete", ] + resource_view = _find_command(resource_group["commands"], "view") + resource_view_args = _require_list(resource_view["arguments"]) + resource_view_arg = _require_dict(resource_view_args[0]) + assert resource_view_arg["discovery_command"] == "dsctl resource list --dir DIR" + resource_list = _find_command(resource_group["commands"], "list") + resource_list_options = _require_list(resource_list["options"]) + assert ( + _find_option(resource_list_options, "dir")["discovery_command"] + == "dsctl resource list" + ) queue_group = _find_group(commands, "queue") queue_command_names = [ @@ -367,6 +636,9 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + queue_get = _find_command(queue_group["commands"], "get") + queue_get_args = _require_list(queue_get["arguments"]) + assert _require_dict(queue_get_args[0])["discovery_command"] == "dsctl queue list" worker_group_group = _find_group(commands, "worker-group") worker_group_command_names = [ @@ -380,6 +652,18 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + worker_group_create = _find_command(worker_group_group["commands"], "create") + worker_group_create_options = _require_list(worker_group_create["options"]) + assert ( + _find_option(worker_group_create_options, "addr")["discovery_command"] + == "dsctl monitor server worker" + ) + worker_group_get = _find_command(worker_group_group["commands"], "get") + worker_group_get_args = _require_list(worker_group_get["arguments"]) + assert ( + _require_dict(worker_group_get_args[0])["discovery_command"] + == "dsctl worker-group list" + ) task_group_group = _find_group(commands, "task-group") task_group_command_names = [ @@ -395,6 +679,20 @@ def test_schema_result_describes_current_stable_surface() -> None: "start", "queue", ] + task_group_list = _find_command(task_group_group["commands"], "list") + task_group_list_options = _require_list(task_group_list["options"]) + assert _find_option(task_group_list_options, "status")["choices"] == [ + "open", + "closed", + "1", + "0", + ] + task_group_get = _find_command(task_group_group["commands"], "get") + task_group_get_args = _require_list(task_group_get["arguments"]) + assert ( + _require_dict(task_group_get_args[0])["discovery_command"] + == "dsctl task-group list" + ) task_group_queue_group = _find_group(task_group_group["commands"], "queue") task_group_queue_command_names = [ _require_dict(item)["name"] @@ -405,6 +703,25 @@ def test_schema_result_describes_current_stable_surface() -> None: "force-start", "set-priority", ] + task_group_queue_list = _find_command(task_group_queue_group["commands"], "list") + task_group_queue_list_options = _require_list(task_group_queue_list["options"]) + assert _find_option(task_group_queue_list_options, "status")["choices"] == [ + "WAIT_QUEUE", + "ACQUIRE_SUCCESS", + "RELEASE", + "-1", + "1", + "2", + ] + task_group_queue_force_start = _find_command( + task_group_queue_group["commands"], + "force-start", + ) + force_start_args = _require_list(task_group_queue_force_start["arguments"]) + assert ( + _require_dict(force_start_args[0])["discovery_command"] + == "dsctl task-group queue list TASK_GROUP" + ) alert_plugin_group = _find_group(commands, "alert-plugin") alert_plugin_command_names = [ @@ -419,7 +736,33 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", "test", + "definition", + ] + alert_plugin_create = _find_command(alert_plugin_group["commands"], "create") + alert_plugin_create_options = _require_list(alert_plugin_create["options"]) + assert ( + _find_option(alert_plugin_create_options, "plugin")["discovery_command"] + == "dsctl alert-plugin definition list" + ) + assert ( + _find_option(alert_plugin_create_options, "param")["discovery_command"] + == "dsctl alert-plugin schema PLUGIN" + ) + alert_plugin_get = _find_command(alert_plugin_group["commands"], "get") + alert_plugin_get_args = _require_list(alert_plugin_get["arguments"]) + assert ( + _require_dict(alert_plugin_get_args[0])["discovery_command"] + == "dsctl alert-plugin list" + ) + alert_plugin_definition_group = _find_group( + alert_plugin_group["commands"], + "definition", + ) + alert_plugin_definition_command_names = [ + _require_dict(item)["name"] + for item in _require_list(alert_plugin_definition_group["commands"]) ] + assert alert_plugin_definition_command_names == ["list"] alert_group_group = _find_group(commands, "alert-group") alert_group_command_names = [ @@ -433,6 +776,18 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + alert_group_get = _find_command(alert_group_group["commands"], "get") + alert_group_get_args = _require_list(alert_group_get["arguments"]) + assert ( + _require_dict(alert_group_get_args[0])["discovery_command"] + == "dsctl alert-group list" + ) + alert_group_create = _find_command(alert_group_group["commands"], "create") + alert_group_create_options = _require_list(alert_group_create["options"]) + assert ( + _find_option(alert_group_create_options, "instance-id")["discovery_command"] + == "dsctl alert-plugin list" + ) tenant_group = _find_group(commands, "tenant") tenant_command_names = [ @@ -445,6 +800,15 @@ def test_schema_result_describes_current_stable_surface() -> None: "update", "delete", ] + tenant_get = _find_command(tenant_group["commands"], "get") + tenant_get_args = _require_list(tenant_get["arguments"]) + assert _require_dict(tenant_get_args[0])["discovery_command"] == "dsctl tenant list" + tenant_create = _find_command(tenant_group["commands"], "create") + tenant_create_options = _require_list(tenant_create["options"]) + assert ( + _find_option(tenant_create_options, "queue")["discovery_command"] + == "dsctl queue list" + ) user_group = _find_group(commands, "user") user_command_names = [ @@ -459,11 +823,36 @@ def test_schema_result_describes_current_stable_surface() -> None: "grant", "revoke", ] + user_get = _find_command(user_group["commands"], "get") + user_get_args = _require_list(user_get["arguments"]) + assert _require_dict(user_get_args[0])["discovery_command"] == "dsctl user list" + user_create = _find_command(user_group["commands"], "create") + user_create_options = _require_list(user_create["options"]) + assert ( + _find_option(user_create_options, "tenant")["discovery_command"] + == "dsctl tenant list" + ) + assert ( + _find_option(user_create_options, "queue")["discovery_command"] + == "dsctl queue list" + ) user_grant_group = _find_group(_require_list(user_group["commands"]), "grant") assert [ _require_dict(item)["name"] for item in _require_list(user_grant_group["commands"]) ] == ["project", "datasource", "namespace"] + user_grant_project = _find_command(user_grant_group["commands"], "project") + user_grant_project_args = _require_list(user_grant_project["arguments"]) + assert ( + _require_dict(user_grant_project_args[1])["discovery_command"] + == "dsctl project list" + ) + user_grant_datasource = _find_command(user_grant_group["commands"], "datasource") + user_grant_datasource_options = _require_list(user_grant_datasource["options"]) + assert ( + _find_option(user_grant_datasource_options, "datasource")["discovery_command"] + == "dsctl datasource list" + ) user_revoke_group = _find_group(_require_list(user_group["commands"]), "revoke") assert [ _require_dict(item)["name"] @@ -494,6 +883,16 @@ def test_schema_result_describes_current_stable_surface() -> None: "model-types", "operation-types", ] + audit_list = _find_command(audit_group["commands"], "list") + audit_list_options = _require_list(audit_list["options"]) + assert ( + _find_option(audit_list_options, "model-type")["discovery_command"] + == "dsctl audit model-types" + ) + assert ( + _find_option(audit_list_options, "operation-type")["discovery_command"] + == "dsctl audit operation-types" + ) workflow_group = _find_group(commands, "workflow") workflow_command_names = [ @@ -529,12 +928,40 @@ def test_schema_result_describes_current_stable_surface() -> None: workflow_edit_options = _require_list(workflow_edit["options"]) assert _find_option(workflow_edit_options, "patch")["required"] is True assert _find_option(workflow_edit_options, "dry-run")["default"] is False + workflow_get = _find_command(workflow_group["commands"], "get") + workflow_get_args = _require_list(workflow_get["arguments"]) + assert ( + _require_dict(workflow_get_args[0])["discovery_command"] + == "dsctl workflow list" + ) workflow_delete = _find_command(workflow_group["commands"], "delete") workflow_delete_options = _require_list(workflow_delete["options"]) assert _find_option(workflow_delete_options, "force")["default"] is False + workflow_run = _find_command(workflow_group["commands"], "run") + workflow_run_options = _require_list(workflow_run["options"]) + assert ( + _find_option(workflow_run_options, "worker-group")["discovery_command"] + == "dsctl worker-group list" + ) + assert ( + _find_option(workflow_run_options, "tenant")["discovery_command"] + == "dsctl tenant list" + ) + assert ( + _find_option(workflow_run_options, "warning-group-id")["discovery_command"] + == "dsctl alert-group list" + ) + assert ( + _find_option(workflow_run_options, "environment-code")["discovery_command"] + == "dsctl environment list" + ) workflow_run_task = _find_command(workflow_group["commands"], "run-task") workflow_run_task_options = _require_list(workflow_run_task["options"]) assert _find_option(workflow_run_task_options, "task")["required"] is True + assert ( + _find_option(workflow_run_task_options, "task")["discovery_command"] + == "dsctl task list" + ) assert _find_option(workflow_run_task_options, "scope")["default"] == "self" assert _find_option(workflow_run_task_options, "scope")["choices"] == [ "self", @@ -548,6 +975,10 @@ def test_schema_result_describes_current_stable_surface() -> None: assert _find_option(workflow_run_task_options, "param")["multiple"] is True workflow_backfill = _find_command(workflow_group["commands"], "backfill") workflow_backfill_options = _require_list(workflow_backfill["options"]) + assert ( + _find_option(workflow_backfill_options, "task")["discovery_command"] + == "dsctl task list" + ) assert _find_option(workflow_backfill_options, "scope")["default"] == "self" assert _find_option(workflow_backfill_options, "run-mode")["default"] == "serial" assert ( @@ -575,6 +1006,29 @@ def test_schema_result_describes_current_stable_surface() -> None: "recover-failed", "execute-task", ] + workflow_instance_list = _find_command( + workflow_instance_group["commands"], + "list", + ) + workflow_instance_list_options = _require_list(workflow_instance_list["options"]) + assert ( + _find_option(workflow_instance_list_options, "project")["discovery_command"] + == "dsctl project list" + ) + assert ( + _find_option(workflow_instance_list_options, "workflow")["discovery_command"] + == "dsctl workflow list" + ) + assert ( + _find_option(workflow_instance_list_options, "state")["discovery_command"] + == "dsctl enum list workflow-execution-status" + ) + workflow_instance_get = _find_command(workflow_instance_group["commands"], "get") + workflow_instance_get_args = _require_list(workflow_instance_get["arguments"]) + assert ( + _require_dict(workflow_instance_get_args[0])["discovery_command"] + == "dsctl workflow-instance list" + ) workflow_instance_update = _find_command( workflow_instance_group["commands"], "update", @@ -594,6 +1048,12 @@ def test_schema_result_describes_current_stable_surface() -> None: workflow_instance_execute_task_options = _require_list( workflow_instance_execute_task["options"] ) + assert ( + _find_option(workflow_instance_execute_task_options, "task")[ + "discovery_command" + ] + == "dsctl task-instance list --workflow-instance WORKFLOW_INSTANCE" + ) assert _find_option(workflow_instance_execute_task_options, "scope")["choices"] == [ "self", "pre", @@ -638,12 +1098,56 @@ def test_schema_result_describes_current_stable_surface() -> None: "savepoint", "stop", ] + task_instance_list = _find_command(task_instance_group["commands"], "list") + task_instance_list_options = _require_list(task_instance_list["options"]) + assert ( + _find_option(task_instance_list_options, "workflow-instance")[ + "discovery_command" + ] + == "dsctl workflow-instance list" + ) + assert ( + _find_option(task_instance_list_options, "project")["discovery_command"] + == "dsctl project list" + ) + assert ( + _find_option(task_instance_list_options, "task-code")["discovery_command"] + == "dsctl task list" + ) + assert ( + _find_option(task_instance_list_options, "state")["discovery_command"] + == "dsctl enum list task-execution-status" + ) + execute_type_option = _find_option(task_instance_list_options, "execute-type") + execute_type_description = _require_str(execute_type_option["description"]) + assert "BATCH or STREAM" in execute_type_description + assert ( + execute_type_option["discovery_command"] == "dsctl enum list task-execute-type" + ) + task_instance_get = _find_command(task_instance_group["commands"], "get") + task_instance_get_args = _require_list(task_instance_get["arguments"]) + task_instance_get_options = _require_list(task_instance_get["options"]) + assert ( + _require_dict(task_instance_get_args[0])["discovery_command"] + == "dsctl task-instance list" + ) + assert ( + _find_option(task_instance_get_options, "workflow-instance")[ + "discovery_command" + ] + == "dsctl workflow-instance list" + ) capabilities = data["capabilities"] assert capabilities == { "ds": EXPECTED_DS_CAPABILITIES, "output": { "standard_envelope": True, + "formats": ["json", "table", "tsv"], + "default_format": "json", + "data_shape_metadata": True, + "display_columns": True, + "json_column_projection": True, "resolved_metadata": True, "warnings": True, "warning_details_alignment": True, @@ -661,10 +1165,22 @@ def test_schema_result_describes_current_stable_surface() -> None: "schema": True, "template": True, "capabilities": True, + "command_invocation_source": "schema", + "capabilities_scope": "feature_discovery", }, "templates": { "workflow": {"with_schedule_option": True}, "parameters": EXPECTED_PARAMETER_SYNTAX, + "environment": { + "command": "dsctl template environment", + "source_options": ["--config TEXT", "--config-file PATH"], + "target_commands": [ + "dsctl environment create --name NAME --config-file env.sh", + "dsctl environment update ENVIRONMENT --config-file env.sh", + ], + }, + "cluster": cluster_config_template_capability_data(), + "datasource": datasource_template_index_data(), "task": { "supported_types": EXPECTED_TEMPLATE_TASK_TYPES, "typed_types": EXPECTED_TYPED_TASK_TYPES, @@ -679,6 +1195,12 @@ def test_schema_result_describes_current_stable_surface() -> None: "workflow_digest": True, "workflow_schedule_block": True, "workflow_dry_run": True, + "environment_config_template": True, + "cluster_config_template": True, + "datasource_payload_templates": True, + "datasource_template_types": datasource_template_index_data()[ + "supported_types" + ], "typed_task_specs": EXPECTED_TYPED_TASK_TYPES, "generic_task_template_types": EXPECTED_GENERIC_TEMPLATE_TASK_TYPES, "upstream_default_task_types": EXPECTED_UPSTREAM_TASK_TYPES, @@ -709,6 +1231,373 @@ def test_schema_result_describes_current_stable_surface() -> None: } +def test_schema_result_honors_env_file_ds_version(tmp_path: Path) -> None: + env_file = tmp_path / "cluster.env" + env_file.write_text("DS_VERSION=3.3.2\n", encoding="utf-8") + + result = get_schema_result(env_file=str(env_file)) + data = result.data + + assert isinstance(data, dict) + capabilities = data["capabilities"] + assert isinstance(capabilities, dict) + ds_capabilities = capabilities["ds"] + assert isinstance(ds_capabilities, dict) + assert ds_capabilities["selected_version"] == "3.3.2" + assert ds_capabilities["current_version"] == "3.3.2" + assert ds_capabilities["tested"] is False + + +def test_schema_result_can_return_one_group() -> None: + result = get_schema_result(group="task-instance") + data = result.data + + assert isinstance(data, dict) + assert "capabilities" not in data + assert result.resolved == { + "schema": { + "view": "group", + "group": "task-instance", + } + } + commands = _require_list(data["commands"]) + assert len(commands) == 1 + task_instance_group = _require_dict(commands[0]) + assert task_instance_group["kind"] == "group" + assert task_instance_group["name"] == "task-instance" + task_instance_list = _find_command(task_instance_group["commands"], "list") + assert task_instance_list["action"] == "task-instance.list" + task_instance_options = _require_list(task_instance_list["options"]) + assert "workflow" not in { + _require_dict(item)["name"] for item in task_instance_options + } + rows = _require_list(data["rows"]) + assert rows[0] == { + "kind": "command", + "action": "task-instance.list", + "name": "list", + "summary": "List task instances with project-scoped runtime filters.", + "schema_command": "dsctl schema --command task-instance.list", + } + + +def test_schema_result_can_return_one_command() -> None: + result = get_schema_result(command_action="task-instance.list") + data = result.data + + assert isinstance(data, dict) + assert "capabilities" not in data + assert result.resolved == { + "schema": { + "view": "command", + "command": "task-instance.list", + } + } + commands = _require_list(data["commands"]) + assert len(commands) == 1 + task_instance_group = _require_dict(commands[0]) + task_instance_commands = _require_list(task_instance_group["commands"]) + assert len(task_instance_commands) == 1 + task_instance_list = _require_dict(task_instance_commands[0]) + assert task_instance_list["action"] == "task-instance.list" + assert task_instance_list["data_shape"] == { + "kind": "page", + "row_path": "data.totalList", + "default_columns": [ + "id", + "name", + "state", + "taskType", + "startTime", + "endTime", + "duration", + "host", + ], + "column_discovery": "runtime_row_keys", + } + rows = _require_list(data["rows"]) + assert rows[0] == { + "kind": "command", + "name": "task-instance.list", + "description": "List task instances with project-scoped runtime filters.", + } + assert any( + _require_dict(row).get("kind") == "data_shape" + and _require_dict(row).get("name") == "row_path" + and _require_dict(row).get("value") == "data.totalList" + for row in rows + ) + + workflow_instance_result = get_schema_result( + command_action="workflow-instance.list" + ) + workflow_instance_data = _require_dict(workflow_instance_result.data) + workflow_instance_group = _require_dict( + _require_list(workflow_instance_data["commands"])[0] + ) + workflow_instance_command = _require_dict( + _require_list(workflow_instance_group["commands"])[0] + ) + assert workflow_instance_command["data_shape"] == { + "kind": "page", + "row_path": "data.totalList", + "default_columns": [ + "id", + "name", + "state", + "scheduleTime", + "startTime", + "endTime", + "duration", + "host", + ], + "column_discovery": "runtime_row_keys", + } + + workflow_instance_get_result = get_schema_result( + command_action="workflow-instance.get" + ) + workflow_instance_get_data = _require_dict(workflow_instance_get_result.data) + workflow_instance_get_group = _require_dict( + _require_list(workflow_instance_get_data["commands"])[0] + ) + workflow_instance_get_command = _require_dict( + _require_list(workflow_instance_get_group["commands"])[0] + ) + assert workflow_instance_get_command["data_shape"] == { + "kind": "object", + "row_path": "data", + "default_columns": [ + "id", + "name", + "state", + "scheduleTime", + "startTime", + "endTime", + "duration", + "host", + ], + "column_discovery": "runtime_row_keys", + } + + datasource_list_result = get_schema_result(command_action="datasource.list") + datasource_list_data = _require_dict(datasource_list_result.data) + datasource_group = _require_dict(_require_list(datasource_list_data["commands"])[0]) + datasource_list_command = _require_dict( + _require_list(datasource_group["commands"])[0] + ) + assert datasource_list_command["data_shape"] == { + "kind": "page", + "row_path": "data.totalList", + "default_columns": ["id", "name", "type", "createTime"], + "column_discovery": "runtime_row_keys", + } + datasource_get_result = get_schema_result(command_action="datasource.get") + datasource_get_data = _require_dict(datasource_get_result.data) + datasource_get_group = _require_dict( + _require_list(datasource_get_data["commands"])[0] + ) + datasource_get_command = _require_dict( + _require_list(datasource_get_group["commands"])[0] + ) + assert datasource_get_command["data_shape"] == { + "kind": "object", + "row_path": "data", + "default_columns": ["id", "name", "type", "host", "port", "database"], + "column_discovery": "runtime_row_keys", + } + + +def test_schema_result_command_rows_expose_payload_discovery() -> None: + result = get_schema_result(command_action="datasource.create") + data = _require_dict(result.data) + rows = [_require_dict(row) for row in _require_list(data["rows"])] + + file_row = next(row for row in rows if row.get("name") == "file") + assert file_row["discovery_command"] == "dsctl template datasource" + assert file_row["description"] == ( + "Path to one DS-native datasource JSON payload file." + ) + assert { + "kind": "payload", + "name": "template_discovery_command", + "value": "dsctl template datasource", + } in rows + + +def test_schema_result_can_list_group_and_command_discovery_rows() -> None: + groups_result = get_schema_result(list_groups=True) + groups_data = _require_list(groups_result.data) + first_group = _require_dict(groups_data[0]) + + assert groups_result.resolved == { + "schema": { + "view": "groups", + "next": "dsctl schema --group GROUP", + } + } + assert first_group == { + "name": "use", + "summary": "Set or clear persisted CLI context.", + "command_count": 2, + "schema_command": "dsctl schema --group use", + } + + commands_result = get_schema_result(list_commands=True) + commands_data = _require_list(commands_result.data) + version_command = next( + _require_dict(item) + for item in commands_data + if _require_dict(item)["action"] == "version" + ) + datasource_create = next( + _require_dict(item) + for item in commands_data + if _require_dict(item)["action"] == "datasource.create" + ) + + assert commands_result.resolved == { + "schema": { + "view": "commands", + "next": "dsctl schema --command ACTION", + } + } + assert version_command == { + "action": "version", + "group": None, + "name": "version", + "summary": "Return CLI and supported DolphinScheduler version metadata.", + "schema_command": "dsctl schema --command version", + } + assert datasource_create == { + "action": "datasource.create", + "group": "datasource", + "name": "create", + "summary": "Create one datasource from a JSON payload file.", + "schema_command": "dsctl schema --command datasource.create", + } + + +def test_schema_result_exposes_collection_and_nested_data_shapes() -> None: + workflow_result = get_schema_result(command_action="workflow.list") + workflow_data = _require_dict(workflow_result.data) + workflow_group = _require_dict(_require_list(workflow_data["commands"])[0]) + workflow_command = _require_dict(_require_list(workflow_group["commands"])[0]) + assert workflow_command["data_shape"] == { + "kind": "collection", + "row_path": "data", + "default_columns": ["code", "name", "version"], + "column_discovery": "runtime_row_keys", + } + + task_type_result = get_schema_result(command_action="task-type.list") + task_type_data = _require_dict(task_type_result.data) + task_type_group = _require_dict(_require_list(task_type_data["commands"])[0]) + task_type_command = _require_dict(_require_list(task_type_group["commands"])[0]) + assert task_type_command["data_shape"] == { + "kind": "summary", + "row_path": "data.taskTypes", + "default_columns": ["taskType", "taskCategory", "isCollection"], + "column_discovery": "runtime_row_keys", + } + + alert_definition_result = get_schema_result( + command_action="alert-plugin.definition.list" + ) + alert_definition_data = _require_dict(alert_definition_result.data) + alert_definition_group = _require_dict( + _require_list(alert_definition_data["commands"])[0] + ) + alert_definition_subgroup = _require_dict( + _require_list(alert_definition_group["commands"])[0] + ) + alert_definition_command = _require_dict( + _require_list(alert_definition_subgroup["commands"])[0] + ) + assert alert_definition_command["data_shape"] == { + "kind": "summary", + "row_path": "data.definitions", + "default_columns": ["id", "pluginName", "pluginType"], + "column_discovery": "runtime_row_keys", + } + + digest_result = get_schema_result(command_action="workflow-instance.digest") + digest_data = _require_dict(digest_result.data) + digest_group = _require_dict(_require_list(digest_data["commands"])[0]) + digest_command = _require_dict(_require_list(digest_group["commands"])[0]) + assert digest_command["data_shape"] == { + "kind": "object", + "row_path": "data", + "default_columns": [ + "taskCount", + "progress", + "taskStateCounts", + "runningTasks", + "failedTasks", + ], + "column_discovery": "runtime_row_keys", + } + + +def test_schema_result_can_return_one_top_level_command() -> None: + result = get_schema_result(command_action="version") + data = result.data + + assert isinstance(data, dict) + commands = _require_list(data["commands"]) + assert len(commands) == 1 + version_command = _require_dict(commands[0]) + assert version_command["kind"] == "command" + assert version_command["action"] == "version" + + +def test_schema_result_can_return_group_action_command() -> None: + result = get_schema_result(command_action="use.clear") + data = result.data + + assert isinstance(data, dict) + commands = _require_list(data["commands"]) + assert len(commands) == 1 + use_group = _require_dict(commands[0]) + assert use_group["name"] == "use" + assert _require_dict(use_group["group_action"])["action"] == "use.clear" + assert _require_list(use_group["commands"]) == [] + + +def test_schema_result_rejects_conflicting_scope_options() -> None: + with pytest.raises(UserInputError, match="mutually exclusive"): + get_schema_result(group="workflow", command_action="workflow.run") + + with pytest.raises(UserInputError, match="mutually exclusive"): + get_schema_result(group="workflow", list_groups=True) + + +def test_schema_result_rejects_unknown_group() -> None: + with pytest.raises(UserInputError, match="Unknown schema group") as exc_info: + get_schema_result(group="missing") + + assert exc_info.value.details["group"] == "missing" + available_groups = exc_info.value.details["available_groups"] + assert isinstance(available_groups, list) + assert "task-instance" in available_groups + assert exc_info.value.suggestion == ( + "Run `dsctl schema --list-groups` to choose a group name." + ) + + +def test_schema_result_rejects_unknown_command() -> None: + with pytest.raises(UserInputError, match="Unknown schema command") as exc_info: + get_schema_result(command_action="missing.command") + + assert exc_info.value.details["command"] == "missing.command" + available_commands = exc_info.value.details["available_commands"] + assert isinstance(available_commands, list) + assert "task-instance.list" in available_commands + assert exc_info.value.suggestion == ( + "Run `dsctl schema --list-commands` to choose a command action." + ) + + def test_schema_result_describes_group_level_use_clear_action() -> None: result = get_schema_result() data = result.data @@ -744,6 +1633,17 @@ def test_schema_result_describes_group_level_use_clear_action() -> None: _require_dict(item)["name"] for item in _require_list(use_group["commands"]) ] assert use_command_names == ["project", "workflow"] + use_project = _find_command(use_group["commands"], "project") + use_project_args = _require_list(use_project["arguments"]) + assert ( + _require_dict(use_project_args[0])["discovery_command"] == "dsctl project list" + ) + use_workflow = _find_command(use_group["commands"], "workflow") + use_workflow_args = _require_list(use_workflow["arguments"]) + assert ( + _require_dict(use_workflow_args[0])["discovery_command"] + == "dsctl workflow list" + ) def test_schema_defaults_follow_runtime_constants() -> None: @@ -788,6 +1688,9 @@ def test_schema_task_update_set_option_exposes_supported_keys() -> None: task_update_options = _require_list(task_update["options"]) set_option = _find_option(task_update_options, "set") + task_update_args = _require_list(task_update["arguments"]) + assert _require_dict(task_update_args[0])["discovery_command"] == "dsctl task list" + assert set_option["discovery_command"] == "dsctl schema --command task.update" assert set_option["supported_keys"] == [ "command", "cpu_quota", diff --git a/tests/services/test_surface_quality.py b/tests/services/test_surface_quality.py index 48b6e67..531ca31 100644 --- a/tests/services/test_surface_quality.py +++ b/tests/services/test_surface_quality.py @@ -3,10 +3,15 @@ import ast import inspect import re +import shlex from itertools import product from pathlib import Path from typing import TypedDict +from tests.support import normalize_cli_help +from typer.testing import CliRunner + +from dsctl.app import app from dsctl.cli_surface import ( NAME_FIRST_RESOURCES, RESOURCE_COMMAND_TREE, @@ -38,9 +43,10 @@ class GroupNode(TypedDict): ) SERVICES_DIR = Path(__file__).resolve().parents[2] / "src" / "dsctl" / "services" REPO_ROOT = Path(__file__).resolve().parents[2] +COMMANDS_DIR = REPO_ROOT / "src" / "dsctl" / "commands" NAME_FIRST_RESOURCE_RESOLVERS = { "project": "project", - "env": "environment", + "environment": "environment", "cluster": "cluster", "datasource": "datasource", "namespace": "namespace", @@ -113,6 +119,7 @@ class GroupNode(TypedDict): "## Naming and Selection Rules", ), } +RUNNER = CliRunner() def test_services_do_not_inline_resource_slug_literals() -> None: @@ -201,6 +208,133 @@ def test_paginated_schema_commands_expose_standard_all_option() -> None: assert malformed == [] +def test_literal_emit_result_actions_are_declared_in_schema() -> None: + data = get_schema_result().data + assert isinstance(data, dict) + commands = data["commands"] + assert isinstance(commands, list) + declared_actions = _iter_schema_action_names(commands) + emitted_actions = _literal_emit_result_actions() + + missing = [ + f"{action} ({path.relative_to(REPO_ROOT)}:{line_number})" + for action, path, line_number in emitted_actions + if action not in declared_actions + ] + + assert missing == [] + + +def test_schema_declared_commands_expose_help() -> None: + data = get_schema_result().data + assert isinstance(data, dict) + commands = data["commands"] + assert isinstance(commands, list) + + failures: list[str] = [] + for path, command in _iter_schema_command_paths(commands): + result = RUNNER.invoke(app, [*path, "--help"]) + if result.exit_code != 0: + action = command.get("action") + failures.append(f"{' '.join(path)} ({action}) exited {result.exit_code}") + + assert failures == [] + + +def test_schema_selector_fields_expose_discovery_commands() -> None: + data = get_schema_result().data + assert isinstance(data, dict) + commands = data["commands"] + assert isinstance(commands, list) + + missing: list[str] = [] + for path, command in _iter_schema_command_paths(commands): + for field_kind, field in _iter_schema_command_fields(command): + if not field.get("selector"): + continue + discovery_command = field.get("discovery_command") + if isinstance(discovery_command, str) and discovery_command: + continue + missing.append( + f"{'.'.join(path)}:{field_kind}:{field.get('name', '')}" + ) + + assert missing == [] + + +def test_schema_choice_fields_are_discoverable_from_help_or_schema() -> None: + data = get_schema_result().data + assert isinstance(data, dict) + commands = data["commands"] + assert isinstance(commands, list) + + issues: list[str] = [] + for path, command in _iter_schema_command_paths(commands): + result = RUNNER.invoke(app, [*path, "--help"]) + if result.exit_code != 0: + issues.append(f"{' '.join(path)}: help exited {result.exit_code}") + continue + help_text = normalize_cli_help(result.stdout) + for field_kind, field in _iter_schema_command_fields(command): + choices = field.get("choices") + if not isinstance(choices, list) or not choices: + continue + field_label = f"{'.'.join(path)}:{field_kind}:{field.get('name')}" + if len(choices) <= 10: + missing_choices = [ + str(choice) for choice in choices if str(choice) not in help_text + ] + if missing_choices: + issues.append( + f"{field_label} missing help choices {missing_choices}" + ) + continue + discovery_command = field.get("discovery_command") + if not isinstance(discovery_command, str) or not discovery_command: + issues.append( + f"{field_label} has {len(choices)} choices without discovery" + ) + + assert issues == [] + + +def test_schema_discovery_commands_point_to_existing_help_surfaces() -> None: + data = get_schema_result().data + assert isinstance(data, dict) + commands = data["commands"] + assert isinstance(commands, list) + + known_paths = {path for path, _command in _iter_schema_command_paths(commands)} + issues: list[str] = [] + for discovery_command in sorted(_iter_discovery_commands(data)): + tokens = shlex.split(discovery_command) + if not tokens or tokens[0] != "dsctl": + issues.append(f"{discovery_command}: must start with dsctl") + continue + + command_path = _longest_known_command_path(tokens[1:], known_paths) + if command_path is None: + issues.append(f"{discovery_command}: no declared command path") + continue + + result = RUNNER.invoke(app, [*command_path, "--help"]) + if result.exit_code != 0: + issues.append( + f"{discovery_command}: {' '.join(command_path)} --help " + f"exited {result.exit_code}" + ) + continue + + help_text = normalize_cli_help(result.stdout) + issues.extend( + f"{discovery_command}: {flag} missing from {' '.join(command_path)} --help" + for flag in _option_flags_after_path(tokens[1:], command_path) + if flag not in help_text + ) + + assert issues == [] + + def test_name_first_resources_have_resolver_functions() -> None: assert set(NAME_FIRST_RESOURCE_RESOLVERS) == set(NAME_FIRST_RESOURCES) available = { @@ -287,6 +421,20 @@ def test_stable_command_docs_cover_shared_cli_surface() -> None: assert missing_by_doc == {} +def test_cli_contract_command_blocks_do_not_duplicate_rules_sections() -> None: + text = (REPO_ROOT / "docs/reference/cli-contract.md").read_text(encoding="utf-8") + violations: list[str] = [] + for block in re.split(r"(?=^## `dsctl )", text, flags=re.MULTILINE): + if not block.startswith("## `dsctl "): + continue + title = block.splitlines()[0] + rules_count = sum(1 for line in block.splitlines() if line.strip() == "Rules:") + if rules_count > 1: + violations.append(f"{title}: {rules_count} Rules sections") + + assert violations == [] + + def _iter_schema_commands(nodes: list[object]) -> list[CommandNode]: commands: list[CommandNode] = [] for node in nodes: @@ -311,7 +459,120 @@ def _iter_schema_commands(nodes: list[object]) -> list[CommandNode]: return commands +def _iter_schema_action_names(nodes: list[object]) -> set[str]: + actions: set[str] = set() + for node in nodes: + if not isinstance(node, dict): + continue + action = node.get("action") + if isinstance(action, str): + actions.add(action) + group_action = node.get("group_action") + if isinstance(group_action, dict): + group_action_name = group_action.get("action") + if isinstance(group_action_name, str): + actions.add(group_action_name) + nested = node.get("commands") + if isinstance(nested, list): + actions.update(_iter_schema_action_names(nested)) + return actions + + +def _literal_emit_result_actions() -> list[tuple[str, Path, int]]: + emitted_actions: list[tuple[str, Path, int]] = [] + for path in COMMANDS_DIR.glob("*.py"): + module = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(module): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "emit_result" + and node.args + ): + continue + first_arg = node.args[0] + if isinstance(first_arg, ast.Constant) and isinstance(first_arg.value, str): + emitted_actions.append((first_arg.value, path, node.lineno)) + return emitted_actions + + SurfacePath = tuple[str, ...] +SchemaCommandPath = tuple[SurfacePath, dict[str, object]] + + +def _iter_schema_command_paths( + nodes: list[object], + prefix: SurfacePath = (), +) -> list[SchemaCommandPath]: + commands: list[SchemaCommandPath] = [] + for node in nodes: + if not isinstance(node, dict): + continue + name = node.get("name") + if not isinstance(name, str): + continue + path = (*prefix, name) + action = node.get("action") + if isinstance(action, str): + commands.append((path, node)) + group_action = node.get("group_action") + if isinstance(group_action, dict) and isinstance( + group_action.get("action"), + str, + ): + commands.append((path, group_action)) + nested = node.get("commands") + if isinstance(nested, list): + commands.extend(_iter_schema_command_paths(nested, path)) + return commands + + +def _iter_schema_command_fields( + command: dict[str, object], +) -> list[tuple[str, dict[str, object]]]: + fields: list[tuple[str, dict[str, object]]] = [] + for field_kind in ("arguments", "options"): + value = command.get(field_kind) + if not isinstance(value, list): + continue + fields.extend((field_kind, field) for field in value if isinstance(field, dict)) + return fields + + +def _iter_discovery_commands(value: object) -> set[str]: + commands: set[str] = set() + if isinstance(value, dict): + for key, item in value.items(): + if key == "discovery_command" and isinstance(item, str): + commands.add(item) + continue + commands.update(_iter_discovery_commands(item)) + elif isinstance(value, list): + for item in value: + commands.update(_iter_discovery_commands(item)) + return commands + + +def _longest_known_command_path( + tokens: list[str], + known_paths: set[SurfacePath], +) -> SurfacePath | None: + for length in range(len(tokens), 0, -1): + candidate = tuple(tokens[:length]) + if candidate in known_paths: + return candidate + return None + + +def _option_flags_after_path( + tokens: list[str], + command_path: SurfacePath, +) -> tuple[str, ...]: + return tuple( + token.split("=", 1)[0] + for token in tokens[len(command_path) :] + if token.startswith("--") + ) def _stable_surface_leaf_paths() -> set[SurfacePath]: diff --git a/tests/services/test_task_instance.py b/tests/services/test_task_instance.py index 7eb90e4..ccd6a2f 100644 --- a/tests/services/test_task_instance.py +++ b/tests/services/test_task_instance.py @@ -17,6 +17,7 @@ ApiResultError, InvalidStateError, NotFoundError, + TaskNotDispatchedError, UserInputError, WaitTimeoutError, ) @@ -102,8 +103,12 @@ def fake_task_instance_adapter() -> FakeTaskInstanceAdapter: project_code_value=7, task_code_value=201, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("RUNNING_EXECUTION"), + start_time_value="2026-04-11 10:00:00", host="worker-1", + executor_name_value="alice", + task_execute_type_value=FakeEnumValue("BATCH"), ), FakeTaskInstance( id=3002, @@ -114,8 +119,12 @@ def fake_task_instance_adapter() -> FakeTaskInstanceAdapter: project_code_value=7, task_code_value=202, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("FAILURE"), + start_time_value="2026-04-11 10:05:00", host="worker-1", + executor_name_value="bob", + task_execute_type_value=FakeEnumValue("BATCH"), ), FakeTaskInstance( id=3003, @@ -126,7 +135,11 @@ def fake_task_instance_adapter() -> FakeTaskInstanceAdapter: project_code_value=7, task_code_value=203, task_definition_version_value=1, + process_definition_name_value="daily-sync", state_value=FakeEnumValue("SUCCESS"), + start_time_value="2026-04-11 10:10:00", + executor_name_value="alice", + task_execute_type_value=FakeEnumValue("BATCH"), ), ], log_messages_by_task_instance_id={ @@ -186,6 +199,76 @@ def test_list_task_instances_result_can_auto_exhaust_pages( assert len(items) == 2 +def test_list_task_instances_result_supports_project_scoped_filters( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, + fake_task_instance_adapter: FakeTaskInstanceAdapter, +) -> None: + _install_task_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + task_instance_adapter=fake_task_instance_adapter, + ) + + result = task_instance_service.list_task_instances_result( + project="etl-prod", + host="worker-1", + executor="bob", + start="2026-04-11 10:00:00", + end="2026-04-11 10:10:00", + execute_type="BATCH", + ) + data = _mapping(result.data) + items = _sequence(data["totalList"]) + + assert _mapping(result.resolved["project"])["code"] == 7 + assert result.resolved["host"] == "worker-1" + assert result.resolved["executor"] == "bob" + assert data["total"] == 1 + assert _mapping(items[0])["id"] == 3002 + + +def test_list_task_instances_result_requires_project_without_workflow_instance( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, + fake_task_instance_adapter: FakeTaskInstanceAdapter, +) -> None: + _install_task_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + task_instance_adapter=fake_task_instance_adapter, + ) + + with pytest.raises(UserInputError, match="Project is required"): + task_instance_service.list_task_instances_result() + + +def test_list_task_instances_result_rejects_workflow_definition_filter() -> None: + with pytest.raises(UserInputError, match="cannot reliably filter") as exc_info: + task_instance_service.list_task_instances_result( + project="etl-prod", + workflow="daily-sync", + ) + + assert exc_info.value.details["upstream_filter"] == "workflowDefinitionName" + assert "workflow-instance list" in (exc_info.value.suggestion or "") + + +def test_list_task_instances_result_rejects_redundant_workflow_with_instance() -> None: + with pytest.raises(UserInputError, match="does not accept --workflow") as exc_info: + task_instance_service.list_task_instances_result( + workflow_instance=901, + workflow="daily-sync", + ) + + assert exc_info.value.details["workflow_instance_id"] == 901 + assert "--workflow-instance already scopes" in (exc_info.value.suggestion or "") + + def test_get_task_instance_result_returns_one_payload( monkeypatch: pytest.MonkeyPatch, fake_project_adapter: FakeProjectAdapter, @@ -298,6 +381,43 @@ def test_get_task_instance_log_result_returns_tail_lines( assert data["text"] == "line-3\nline-4" +def test_get_task_instance_log_result_translates_not_dispatched( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, + fake_task_instance_adapter: FakeTaskInstanceAdapter, +) -> None: + _install_task_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + task_instance_adapter=fake_task_instance_adapter, + ) + + def not_dispatched_log( + *, + task_instance_id: int, + skip_line_num: int, + limit: int, + ) -> None: + del task_instance_id, skip_line_num, limit + raise ApiResultError( + result_code=10103, + result_message=( + "TaskInstanceLogPath is empty, maybe the taskInstance doesn't " + "be dispatched" + ), + ) + + monkeypatch.setattr(fake_task_instance_adapter, "log_chunk", not_dispatched_log) + + with pytest.raises(TaskNotDispatchedError) as exc_info: + task_instance_service.get_task_instance_log_result(3001) + + assert exc_info.value.details["result_code"] == 10103 + assert "workflow-instance digest" in (exc_info.value.suggestion or "") + + def test_get_task_instance_result_reports_missing_instance( monkeypatch: pytest.MonkeyPatch, fake_project_adapter: FakeProjectAdapter, @@ -654,11 +774,39 @@ def test_list_task_instances_result_reports_supported_state_names( ) assert exc_info.value.suggestion == ( - "Run `dsctl enum list task_execution_status` to inspect the supported DS " + "Run `dsctl enum list task-execution-status` to inspect the supported DS " "task-instance states." ) +def test_list_task_instances_result_reports_supported_execute_types( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, + fake_task_instance_adapter: FakeTaskInstanceAdapter, +) -> None: + _install_task_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + task_instance_adapter=fake_task_instance_adapter, + ) + + with pytest.raises( + UserInputError, + match="Task execute type must be one of the DS task execute-type names", + ) as exc_info: + task_instance_service.list_task_instances_result( + workflow_instance=901, + execute_type="not-real", + ) + + assert exc_info.value.suggestion == ( + "Run `dsctl enum list task-execute-type` to inspect the supported DS " + "task execute-type names." + ) + + def test_stop_task_instance_result_reports_running_state_suggestion( monkeypatch: pytest.MonkeyPatch, fake_project_adapter: FakeProjectAdapter, diff --git a/tests/services/test_template.py b/tests/services/test_template.py index cb5c2fc..2a4171f 100644 --- a/tests/services/test_template.py +++ b/tests/services/test_template.py @@ -9,9 +9,13 @@ from dsctl.services import _workflow_compile as workflow_compile_service from dsctl.services.template import ( ParameterSyntaxIndexData, + cluster_config_template_result, + datasource_template_result, + environment_config_template_result, generic_task_template_types, parameter_syntax_data, parameter_syntax_result, + supported_datasource_types, supported_task_template_types, task_template_metadata, task_template_result, @@ -30,6 +34,7 @@ def test_workflow_template_result_returns_valid_yaml_document() -> None: assert result.resolved["with_schedule"] is False yaml_text = data["yaml"] assert isinstance(yaml_text, str) + assert data["lines"][0]["line"].startswith("# Workflow YAML") document = yaml.safe_load(yaml_text) assert "# Optional task runtime controls:" in yaml_text @@ -105,6 +110,124 @@ def test_parameter_syntax_result_can_expand_specific_topics() -> None: ] +def test_datasource_template_result_returns_discovery_without_type() -> None: + result = datasource_template_result() + data = result.data + + assert isinstance(data, dict) + assert result.resolved == {"view": "list"} + assert data["default_type"] == "MYSQL" + assert data["template_command"] == "dsctl template datasource --type MYSQL" + assert data["template_command_pattern"] == "dsctl template datasource --type TYPE" + assert data["target_commands"] == [ + "dsctl datasource create --file FILE", + "dsctl datasource update DATASOURCE --file FILE", + ] + assert data["type_discovery_command"] == "dsctl enum list db-type" + assert data["supported_types"] == list(supported_datasource_types()) + assert { + "type": "MYSQL", + "template_command": "dsctl template datasource --type MYSQL", + } in data["rows"] + assert "fields" not in data + assert "rules" not in data + + +def test_environment_config_template_result_returns_shell_template() -> None: + result = environment_config_template_result() + data = result.data + + assert isinstance(data, dict) + assert result.resolved == {"template": "environment.config"} + assert data["filename"] == "env.sh" + assert "export JAVA_HOME=/opt/java" in data["config"] + assert data["target_commands"] == [ + "dsctl environment create --name NAME --config-file env.sh", + "dsctl environment update ENVIRONMENT --config-file env.sh", + ] + assert data["source_options"] == ["--config TEXT", "--config-file PATH"] + lines = data["lines"] + assert isinstance(lines, list) + assert lines[0]["line"] == "export JAVA_HOME=/opt/java" + + +def test_cluster_config_template_result_returns_json_template() -> None: + result = cluster_config_template_result() + data = result.data + + assert isinstance(data, dict) + assert result.resolved == {"template": "cluster.config"} + assert data["filename"] == "cluster-config.json" + assert data["target_commands"] == [ + "dsctl cluster create --name NAME --config-file cluster-config.json", + "dsctl cluster update CLUSTER --config-file cluster-config.json", + ] + assert data["source_options"] == ["--config TEXT", "--config-file PATH"] + payload = data["payload"] + assert isinstance(payload, dict) + assert json.loads(data["config"]) == payload + assert set(payload) == {"k8s", "yarn"} + assert "apiVersion: v1" in payload["k8s"] + assert data["rows"] == data["fields"] + + +def test_datasource_template_result_returns_json_payload_template() -> None: + result = datasource_template_result("mysql") + data = result.data + + assert isinstance(data, dict) + assert result.resolved == { + "view": "template", + "datasource_type": "MYSQL", + } + assert data["type"] == "MYSQL" + assert data["target_commands"] == [ + "dsctl datasource create --file FILE", + "dsctl datasource update DATASOURCE --file FILE", + ] + assert data["source_option"] == "--file" + payload = data["payload"] + assert isinstance(payload, dict) + assert payload == json.loads(data["json"]) + assert data["rows"] == data["fields"] + assert payload["type"] == "MYSQL" + assert payload["port"] == 3306 + assert payload["other"] == {"serverTimezone": "UTC"} + assert "payload_schema" not in data + type_fields = [ + field + for field in data["fields"] + if isinstance(field, dict) and field.get("name") == "type" + ] + assert type_fields + assert "choices" not in type_fields[0] + + +def test_datasource_template_result_handles_type_specific_payload() -> None: + result = datasource_template_result("k8s") + data = result.data + + assert isinstance(data, dict) + payload = data["payload"] + assert isinstance(payload, dict) + assert payload["type"] == "K8S" + assert payload["kubeConfig"] == "change-me" + assert payload["namespace"] == "default" + assert "host" not in payload + field_names = { + field["name"] + for field in data["fields"] + if isinstance(field, dict) and isinstance(field.get("name"), str) + } + assert "kubeConfig" in field_names + assert "namespace" in field_names + + +def test_datasource_template_result_rejects_unsupported_type() -> None: + with pytest.raises(UserInputError, match="Unsupported datasource type"): + datasource_template_result("UNKNOWN") + + @pytest.mark.parametrize( ("task_type", "expected_key", "expected_kind", "expected_category"), [ @@ -135,6 +258,7 @@ def test_task_template_result_returns_valid_yaml_for_supported_types( assert result.resolved["task_category"] == expected_category yaml_text = data["yaml"] assert isinstance(yaml_text, str) + assert data["rows"][0]["line"].startswith("# Task template") document = yaml.safe_load(yaml_text) assert "# Optional task runtime controls:" in yaml_text @@ -145,21 +269,29 @@ def test_task_template_result_returns_valid_yaml_for_supported_types( def test_task_template_result_rejects_unsupported_type() -> None: - with pytest.raises(UserInputError, match="Unsupported task template type"): + with pytest.raises(UserInputError, match="Unsupported task template type") as exc: task_template_result("SPARK_SQL") + assert exc.value.details == { + "task_type": "SPARK_SQL", + "available_task_types_count": len(supported_task_template_types()), + "discovery_command": "dsctl template task --list", + } + def test_task_template_types_result_lists_supported_types() -> None: result = task_template_types_result() data = result.data assert isinstance(data, dict) + assert result.resolved == {"mode": "list"} assert data["count"] == len(upstream_default_task_types()) assert data["task_types"] == list(upstream_default_task_types()) assert data["typed_task_types"] == list(typed_task_template_types()) assert data["generic_task_types"] == list(generic_task_template_types()) assert "Universal" in data["task_types_by_category"] assert "Logic" in data["task_types_by_category"] + assert data["rows"][0]["task_type"] == "SHELL" assert data["task_templates"]["SHELL"]["variants"] == [ "minimal", "params", @@ -224,9 +356,18 @@ def test_task_template_result_renders_discoverable_variants( def test_task_template_result_rejects_unsupported_variant() -> None: - with pytest.raises(UserInputError, match="Unsupported task template variant"): + with pytest.raises( + UserInputError, match="Unsupported task template variant" + ) as exc: task_template_result("SHELL", variant="post-json") + assert exc.value.details == { + "task_type": "SHELL", + "variant": "post-json", + "available_variants": ["minimal", "params", "resource"], + "discovery_command": "dsctl template task --list", + } + @pytest.mark.parametrize( "task_type", diff --git a/tests/services/test_workflow_instance.py b/tests/services/test_workflow_instance.py index a2848de..6092e31 100644 --- a/tests/services/test_workflow_instance.py +++ b/tests/services/test_workflow_instance.py @@ -1,5 +1,6 @@ import json from collections.abc import Mapping, Sequence +from dataclasses import replace from pathlib import Path import pytest @@ -13,6 +14,7 @@ FakeTaskInstance, FakeTaskInstanceAdapter, FakeWorkflow, + FakeWorkflowAdapter, FakeWorkflowInstance, FakeWorkflowInstanceAdapter, FakeWorkflowTaskRelation, @@ -36,6 +38,7 @@ def _install_workflow_instance_service_fakes( *, project_adapter: FakeProjectAdapter, workflow_instance_adapter: FakeWorkflowInstanceAdapter, + workflow_adapter: FakeWorkflowAdapter | None = None, task_adapter: FakeTaskAdapter | None = None, task_instance_adapter: FakeTaskInstanceAdapter | None = None, ) -> None: @@ -45,6 +48,7 @@ def _install_workflow_instance_service_fakes( lambda env_file=None: fake_service_runtime( project_adapter, profile=make_profile(), + workflow_adapter=workflow_adapter, workflow_instance_adapter=workflow_instance_adapter, task_adapter=task_adapter, task_instance_adapter=task_instance_adapter, @@ -188,6 +192,118 @@ def test_list_workflow_instances_result_returns_ds_page( assert result.resolved["state"] == "RUNNING_EXECUTION" +def test_list_workflow_instances_result_supports_project_scoped_filters( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, +) -> None: + timed_adapter = FakeWorkflowInstanceAdapter( + workflow_instances=[ + replace( + fake_workflow_instance_adapter.workflow_instances[0], + start_time_value="2026-04-11 10:00:00", + ), + replace( + fake_workflow_instance_adapter.workflow_instances[1], + start_time_value="2026-04-11 11:00:00", + ), + replace( + fake_workflow_instance_adapter.workflow_instances[2], + start_time_value="2026-04-11 12:00:00", + ), + ], + ) + workflow_adapter = FakeWorkflowAdapter( + workflows=[ + FakeWorkflow( + code=101, + name="daily-sync", + version=1, + project_code_value=7, + ) + ], + dags={}, + ) + _install_workflow_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_adapter=workflow_adapter, + workflow_instance_adapter=timed_adapter, + ) + + result = workflow_instance_service.list_workflow_instances_result( + project="etl-prod", + workflow="daily-sync", + search="daily-sync", + executor="alice", + host="master-1", + start="2026-04-11 10:30:00", + end="2026-04-11 11:30:00", + ) + data = _mapping(result.data) + items = _sequence(data["totalList"]) + + assert data["total"] == 1 + assert _mapping(items[0])["id"] == 902 + assert result.resolved["project"] == "etl-prod" + assert result.resolved["project_code"] == 7 + assert result.resolved["workflow"] == "daily-sync" + assert result.resolved["workflow_code"] == 101 + assert result.resolved["start"] == "2026-04-11 10:30:00" + assert result.resolved["end"] == "2026-04-11 11:30:00" + + +def test_list_workflow_instances_result_rejects_project_scoped_filters_without_project( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, +) -> None: + _install_workflow_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + ) + + with pytest.raises(UserInputError, match="filters require --project") as exc_info: + workflow_instance_service.list_workflow_instances_result(search="daily") + + assert exc_info.value.details == {"filters": ["search"]} + assert exc_info.value.suggestion == ( + "Pass --project PROJECT with --search or --executor, or use --workflow, " + "--state, --host, --start, and --end for global workflow-instance " + "filtering." + ) + + +def test_list_workflow_instances_result_translates_controller_fallback_error( + monkeypatch: pytest.MonkeyPatch, + fake_project_adapter: FakeProjectAdapter, + fake_workflow_instance_adapter: FakeWorkflowInstanceAdapter, +) -> None: + _install_workflow_instance_service_fakes( + monkeypatch, + project_adapter=fake_project_adapter, + workflow_instance_adapter=fake_workflow_instance_adapter, + ) + + def broken_list(**kwargs: object) -> object: + del kwargs + raise ApiResultError( + result_code=10113, + result_message="query workflow instance list paging error:null", + ) + + monkeypatch.setattr(fake_workflow_instance_adapter, "list", broken_list) + + with pytest.raises(UserInputError, match="rejected") as exc_info: + workflow_instance_service.list_workflow_instances_result( + workflow="daily-sync", + ) + + assert exc_info.value.details == {"filters": {"workflow": "daily-sync"}} + assert exc_info.value.__cause__ is not None + + def test_list_workflow_instances_result_reports_supported_state_names( monkeypatch: pytest.MonkeyPatch, fake_project_adapter: FakeProjectAdapter, @@ -205,7 +321,7 @@ def test_list_workflow_instances_result_reports_supported_state_names( ) as exc_info: workflow_instance_service.list_workflow_instances_result(state="running") assert exc_info.value.suggestion == ( - "Run `dsctl enum list workflow_execution_status` to inspect the " + "Run `dsctl enum list workflow-execution-status` to inspect the " "supported state names." ) diff --git a/tests/support.py b/tests/support.py index cfa18f9..62f8ddf 100644 --- a/tests/support.py +++ b/tests/support.py @@ -1,6 +1,9 @@ +import re + from dsctl.config import ClusterProfile TEST_API_TOKEN = "test-api-token" +ANSI_ESCAPE_PATTERN = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") def make_profile( @@ -14,3 +17,9 @@ def make_profile( api_token=api_token, ds_version=ds_version, ) + + +def normalize_cli_help(text: str) -> str: + """Return Typer/Rich help text in a stable form for substring assertions.""" + plain = ANSI_ESCAPE_PATTERN.sub("", text) + return re.sub(r"\s+", " ", plain) diff --git a/tests/test_cli_runtime.py b/tests/test_cli_runtime.py index e9eaadf..978236c 100644 --- a/tests/test_cli_runtime.py +++ b/tests/test_cli_runtime.py @@ -1,17 +1,17 @@ +import json + import pytest import typer -from dsctl.cli_runtime import emit_result +from dsctl.cli_runtime import AppState, emit_result, set_app_state from dsctl.errors import ConfigError from dsctl.output import CommandResult +from dsctl.output_formats import RenderOptions def test_emit_result_formats_dsctl_errors( - monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - printed: list[object] = [] - monkeypatch.setattr("dsctl.cli_runtime.print_json", printed.append) - def builder() -> CommandResult: message = "Missing required setting: DS_API_URL" raise ConfigError(message) @@ -20,28 +20,23 @@ def builder() -> CommandResult: emit_result("context", builder) assert exc_info.value.exit_code == 1 - assert printed == [ - { - "ok": False, - "action": "context", - "resolved": {}, - "data": {}, - "warnings": [], - "warning_details": [], - "error": { - "type": "config_error", - "message": "Missing required setting: DS_API_URL", - }, - } - ] + assert json.loads(capsys.readouterr().out) == { + "ok": False, + "action": "context", + "resolved": {}, + "data": {}, + "warnings": [], + "warning_details": [], + "error": { + "type": "config_error", + "message": "Missing required setting: DS_API_URL", + }, + } def test_emit_result_does_not_swallow_unexpected_exceptions( - monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - printed: list[object] = [] - monkeypatch.setattr("dsctl.cli_runtime.print_json", printed.append) - def builder() -> CommandResult: message = "boom" raise ValueError(message) @@ -49,4 +44,208 @@ def builder() -> CommandResult: with pytest.raises(ValueError, match="boom"): emit_result("context", builder) - assert printed == [] + assert capsys.readouterr().out == "" + + +def test_emit_result_can_render_table_rows( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions( + output_format="table", + columns=("code", "name"), + ), + ) + ) + + def builder() -> CommandResult: + return CommandResult( + data={ + "totalList": [{"code": 101, "name": "etl-prod", "description": "demo"}], + "total": 1, + } + ) + + try: + emit_result("project.list", builder) + assert capsys.readouterr().out == ( + "code | name\n-----+---------\n101 | etl-prod\n" + ) + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_uses_datasource_list_defaults_without_owner_user_name( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(output_format="table"), + ) + ) + + def builder() -> CommandResult: + return CommandResult( + data={ + "totalList": [ + { + "id": 7, + "name": "warehouse", + "type": "MYSQL", + "userName": "admin", + "createTime": "2026-04-19 10:00:00", + } + ], + "total": 1, + } + ) + + try: + emit_result("datasource.list", builder) + assert capsys.readouterr().out == ( + "id | name | type | createTime\n" + "---+-----------+-------+--------------------\n" + "7 | warehouse | MYSQL | 2026-04-19 10:00:00\n" + ) + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_can_render_empty_table_with_default_columns( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(output_format="table"), + ) + ) + + def builder() -> CommandResult: + return CommandResult(data={"totalList": [], "total": 0}) + + try: + emit_result("cluster.list", builder) + assert capsys.readouterr().out == ( + "code | name | config\n-----+------+-------\n" + ) + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_columns_wildcard_renders_all_row_fields( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(output_format="tsv", columns=("*",)), + ) + ) + + def builder() -> CommandResult: + return CommandResult( + data={ + "totalList": [ + {"id": 7, "name": "extract", "state": "SUCCESS"}, + {"id": 8, "name": "load", "host": "worker-1"}, + ], + "total": 2, + } + ) + + try: + emit_result("task-instance.list", builder) + assert capsys.readouterr().out == ( + "id\tname\tstate\thost\n7\textract\tSUCCESS\t\n8\tload\t\tworker-1\n" + ) + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_projects_json_columns_for_page_rows( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(columns=("id", "name")), + ) + ) + + def builder() -> CommandResult: + return CommandResult( + data={ + "totalList": [ + {"id": 7, "name": "extract", "state": "SUCCESS"}, + {"id": 8, "name": "load", "host": "worker-1"}, + ], + "total": 2, + } + ) + + try: + emit_result("task-instance.list", builder) + payload = json.loads(capsys.readouterr().out) + assert payload["data"] == { + "total": 2, + "totalList": [ + {"id": 7, "name": "extract"}, + {"id": 8, "name": "load"}, + ], + } + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_projects_json_columns_for_object_data( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(columns=("cli", "ds")), + ) + ) + + def builder() -> CommandResult: + return CommandResult( + data={ + "cli": "0.1.0", + "ds": "3.4.1", + "family": "workflow-3.3-plus", + } + ) + + try: + emit_result("version", builder) + payload = json.loads(capsys.readouterr().out) + assert payload["data"] == {"cli": "0.1.0", "ds": "3.4.1"} + finally: + set_app_state(AppState(env_file=None)) + + +def test_emit_result_rejects_mixed_columns_wildcard( + capsys: pytest.CaptureFixture[str], +) -> None: + set_app_state( + AppState( + env_file=None, + render_options=RenderOptions(output_format="table", columns=("*", "id")), + ) + ) + + def builder() -> CommandResult: + return CommandResult(data={"totalList": [{"id": 7}]}) + + with pytest.raises(typer.Exit) as exc_info: + emit_result("task-instance.list", builder) + + assert exc_info.value.exit_code == 1 + output = capsys.readouterr().out + assert "error.type" in output + assert "user_input_error" in output + assert "--columns '*' cannot be combined with explicit columns" in output diff --git a/tests/tools/test_check_error_translation_governance.py b/tests/tools/test_check_error_translation_governance.py index 636ee1a..babd734 100644 --- a/tests/tools/test_check_error_translation_governance.py +++ b/tests/tools/test_check_error_translation_governance.py @@ -27,7 +27,6 @@ def test_current_entries_match_reviewed_raw_findings() -> None: "page_hook|audit|_list_audit_logs_result", "page_hook|project|_list_projects_result", "page_hook|task_instance|_list_task_instances_result", - "page_hook|workflow_instance|_list_workflow_instances_result", ( "raw_matrix|alert_group|_translate_alert_group_api_error|" "CREATE_ALERT_GROUP_ERROR,LIST_PAGING_ALERT_GROUP_ERROR," diff --git a/tests/tools/test_check_quality_gate.py b/tests/tools/test_check_quality_gate.py index 9c456a5..9541eb9 100644 --- a/tests/tools/test_check_quality_gate.py +++ b/tests/tools/test_check_quality_gate.py @@ -44,9 +44,9 @@ def test_build_steps_matches_ci_shape() -> None: "python", "-m", "pytest", + "-m", + "not live", "-q", - "--ignore", - "tests/live", ) @@ -98,9 +98,9 @@ def test_build_steps_can_append_live_suite() -> None: "python", "-m", "pytest", + "-m", + "not live", "-q", - "--ignore", - "tests/live", ) assert live_tests_step.command == ( "python", diff --git a/tests/tools/test_live_command_coverage.py b/tests/tools/test_live_command_coverage.py index 89e1d66..66fb931 100644 --- a/tests/tools/test_live_command_coverage.py +++ b/tests/tools/test_live_command_coverage.py @@ -10,9 +10,13 @@ LOCAL_ONLY_COMMANDS = { "capabilities", "context", + "enum names", "enum list", "lint workflow", "schema", + "template cluster", + "template datasource", + "template environment", "template params", "template task", "template workflow", diff --git a/tests/upstream/test_ds_3_4_1.py b/tests/upstream/test_ds_3_4_1.py index da176c0..a876547 100644 --- a/tests/upstream/test_ds_3_4_1.py +++ b/tests/upstream/test_ds_3_4_1.py @@ -3484,6 +3484,79 @@ def handler(request: httpx.Request) -> httpx.Response: ] +def test_adapter_workflow_instance_list_can_use_project_scoped_endpoint() -> None: + profile = make_profile() + requests_seen: list[tuple[str, str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests_seen.append((request.method, request.url.path)) + if ( + request.method == "GET" + and request.url.path == "/dolphinscheduler/projects/7/workflow-instances" + ): + assert request.url.params["workflowDefinitionCode"] == "101" + assert request.url.params["searchVal"] == "daily" + assert request.url.params["executorName"] == "alice" + assert request.url.params["stateType"] == "SUCCESS" + assert request.url.params["host"] == "master" + assert request.url.params["startDate"] == "2026-04-11 10:00:00" + assert request.url.params["endDate"] == "2026-04-11 11:00:00" + assert request.url.params["pageNo"] == "1" + assert request.url.params["pageSize"] == "50" + return httpx.Response( + 200, + json={ + "code": 0, + "msg": "success", + "data": { + "totalList": [ + { + "id": 902, + "workflowDefinitionCode": 101, + "projectCode": 7, + "state": "SUCCESS", + "runTimes": 1, + "name": "daily-sync-902", + "executorId": 11, + } + ], + "total": 1, + "totalPage": 1, + "pageSize": 50, + "currentPage": 1, + }, + }, + ) + message = f"Unexpected request: {request.method} {request.url}" + raise AssertionError(message) + + adapter = DS341Adapter() + http_client = DolphinSchedulerClient( + profile, + transport=httpx.MockTransport(handler), + ) + + with http_client: + session = adapter.bind(profile, http_client=http_client) + page = session.workflow_instances.list( + page_no=1, + page_size=50, + project_code=7, + workflow_code=101, + search="daily", + executor="alice", + host="master", + start_time="2026-04-11 10:00:00", + end_time="2026-04-11 11:00:00", + state="SUCCESS", + ) + + assert page.total == 1 + assert page.totalList is not None + assert page.totalList[0].id == 902 + assert requests_seen == [("GET", "/dolphinscheduler/projects/7/workflow-instances")] + + def test_adapter_workflow_instance_relation_methods_use_project_scoped_endpoints() -> ( None ): @@ -3772,18 +3845,29 @@ def handler(request: httpx.Request) -> httpx.Response: requests_seen.append((request.method, request.url.path)) if ( request.method == "GET" - and request.url.path - == "/dolphinscheduler/projects/7/workflow-instances/901/tasks" + and request.url.path == "/dolphinscheduler/projects/7/task-instances" ): - assert not request.url.params + assert request.url.params["workflowInstanceId"] == "901" + assert request.url.params["workflowInstanceName"] == "daily-sync-901" + assert request.url.params["workflowDefinitionName"] == "daily-sync" + assert request.url.params["searchVal"] == "extract" + assert request.url.params["taskName"] == "extract" + assert request.url.params["taskCode"] == "201" + assert request.url.params["executorName"] == "alice" + assert request.url.params["stateType"] == "RUNNING_EXECUTION" + assert request.url.params["host"] == "worker-1" + assert request.url.params["startDate"] == "2026-04-11 10:00:00" + assert request.url.params["endDate"] == "2026-04-11 11:00:00" + assert request.url.params["taskExecuteType"] == "BATCH" + assert request.url.params["pageNo"] == "1" + assert request.url.params["pageSize"] == "20" return httpx.Response( 200, json={ "code": 0, "msg": "success", - "dataList": { - "workflowInstanceState": "RUNNING_EXECUTION", - "taskList": [ + "data": { + "totalList": [ { "id": 3001, "name": "extract", @@ -3794,17 +3878,11 @@ def handler(request: httpx.Request) -> httpx.Response: "taskDefinitionVersion": 1, "state": "RUNNING_EXECUTION", }, - { - "id": 3002, - "name": "load", - "taskType": "SHELL", - "workflowInstanceId": 901, - "projectCode": 7, - "taskCode": 202, - "taskDefinitionVersion": 1, - "state": "SUCCESS", - }, ], + "total": 1, + "totalPage": 1, + "pageSize": 20, + "currentPage": 1, }, }, ) @@ -3879,11 +3957,20 @@ def handler(request: httpx.Request) -> httpx.Response: session = adapter.bind(profile, http_client=http_client) page = session.task_instances.list( workflow_instance_id=901, + workflow_instance_name="daily-sync-901", + workflow_definition_name="daily-sync", project_code=7, page_no=1, page_size=20, search="extract", + task_name="extract", + task_code=201, + executor="alice", state="RUNNING_EXECUTION", + host="worker-1", + start_time="2026-04-11 10:00:00", + end_time="2026-04-11 11:00:00", + task_execute_type="BATCH", ) task_instance = session.task_instances.get( project_code=7, @@ -3914,7 +4001,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert log.lineNum == 2 assert log.message == "line-1\nline-2" assert requests_seen == [ - ("GET", "/dolphinscheduler/projects/7/workflow-instances/901/tasks"), + ("GET", "/dolphinscheduler/projects/7/task-instances"), ("POST", "/dolphinscheduler/v2/projects/7/task-instances/3001"), ("GET", "/dolphinscheduler/log/detail"), ("POST", "/dolphinscheduler/projects/7/task-instances/3001/force-success"), diff --git a/tests/upstream/test_enums.py b/tests/upstream/test_enums.py index ba405d9..a2fe563 100644 --- a/tests/upstream/test_enums.py +++ b/tests/upstream/test_enums.py @@ -1,4 +1,10 @@ -from dsctl.upstream import get_enum_spec, supported_enum_names +from dsctl.upstream import ( + datasource_base_payload_fields, + datasource_type_names, + get_enum_spec, + normalize_datasource_type, + supported_enum_names, +) def test_supported_enum_names_are_sorted_and_stable() -> None: @@ -24,3 +30,34 @@ def test_get_enum_spec_resolves_aliases_and_member_attributes() -> None: "descp": "mysql", "name_field": "mysql", } + + +def test_datasource_contract_exposes_db_type_and_payload_fields() -> None: + assert datasource_type_names("3.4.1")[:3] == ( + "MYSQL", + "POSTGRESQL", + "HIVE", + ) + assert normalize_datasource_type("3.4.1", "mysql") == "MYSQL" + assert ( + normalize_datasource_type("3.4.1", "aliyun-serverless-spark") + == "ALIYUN_SERVERLESS_SPARK" + ) + + fields = datasource_base_payload_fields("3.4.1") + assert [field.name for field in fields] == [ + "id", + "name", + "note", + "host", + "port", + "database", + "userName", + "password", + "other", + "type", + ] + type_field = fields[-1] + assert type_field.name == "type" + assert type_field.cli_required is True + assert "MYSQL" in type_field.choices diff --git a/tools/check_quality_gate.py b/tools/check_quality_gate.py index 6d14f0e..1bb4021 100644 --- a/tools/check_quality_gate.py +++ b/tools/check_quality_gate.py @@ -143,7 +143,7 @@ def build_steps( steps.append( Step( "Run tests", - python_cmd(python, "-m", "pytest", "-q", "--ignore", "tests/live"), + python_cmd(python, "-m", "pytest", "-m", "not live", "-q"), ) ) if include_live: diff --git a/tools/error_translation_allowlist.txt b/tools/error_translation_allowlist.txt index a06932a..af202b4 100644 --- a/tools/error_translation_allowlist.txt +++ b/tools/error_translation_allowlist.txt @@ -10,10 +10,6 @@ page_hook|project|_list_projects_result # Remaining page-hook failures are generic DS paging/controller errors rather # than stable task-instance domain semantics. page_hook|task_instance|_list_task_instances_result -# Workflow-instance list is a thin runtime paging surface over DS-native -# filters, so list-time upstream failures stay raw until upstream exposes -# stable resource-specific semantics. -page_hook|workflow_instance|_list_workflow_instances_result # These alert-group codes are generic create/list/update/delete upstream # failures, not stable domain errors such as conflict or invalid state. raw_matrix|alert_group|_translate_alert_group_api_error|CREATE_ALERT_GROUP_ERROR,LIST_PAGING_ALERT_GROUP_ERROR,UPDATE_ALERT_GROUP_ERROR,DELETE_ALERT_GROUP_ERROR diff --git a/tools/explicit_object_debt.txt b/tools/explicit_object_debt.txt index 885d544..d0c4797 100644 --- a/tools/explicit_object_debt.txt +++ b/tools/explicit_object_debt.txt @@ -144,7 +144,6 @@ src/dsctl/services/task.py|_parse_task_update_value:return|object src/dsctl/services/task_instance.py|_task_instance_action_error.details|dict[str, object] src/dsctl/services/workflow.py|_workflow_schedule_create_confirmation_payload:return|dict[str, object] src/dsctl/upstream/adapters/ds_3_4_1.py|_DS341DataSourceOperations.get:return|Mapping[str, object] -src/dsctl/upstream/adapters/ds_3_4_1.py|_enum_member_value(value)|object src/dsctl/upstream/adapters/ds_3_4_1.py|_is_http_form_scalar(value)|object src/dsctl/upstream/adapters/ds_3_4_1.py|_json_mapping(value)|object src/dsctl/upstream/adapters/ds_3_4_1.py|_json_mapping:return|Mapping[str, object]