From 5646bda88ec8be2d594c6c93c40a71b5371fd7de Mon Sep 17 00:00:00 2001 From: gigglewang0417 Date: Wed, 27 May 2026 10:07:28 +0800 Subject: [PATCH 1/2] test(e2e): add E2E test suite for CLI v1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers auth, config, run, and CLI framework scenarios against a live staging server using vitest + real difyctl binary. Suite layout: test/e2e/ ├── helpers/ │ ├── cli.ts — run(), withAuthFixture(), mintFreshToken() │ ├── assert.ts — assertExitCode, assertJson, assertErrorEnvelope │ ├── cleanup-registry.ts — staging data teardown │ └── retry.ts — withRetry() for flaky network assertions ├── setup/ │ ├── global-setup.ts — health-check, disposable token mint │ └── global-teardown.ts — conversation cleanup └── suites/ ├── auth/ — status, use, whoami, devices, logout ├── config/ — path, get/set/unset/view, env override └── run/ — basic, streaming, conversation, file, HITL Key design decisions: - Each test uses an isolated temp configDir via withAuthFixture() - Logout and devices-revoke tests run last to avoid invalidating the shared E.token used by all other suites - mintFreshToken() mints a disposable dfoa_ token on demand via the device flow API so revoke tests never touch the primary session - Global retry is 0; flaky network calls use withRetry() locally - test:e2e:smoke script filters to [P0] cases via testNamePattern package.json: add test:e2e / test:e2e:smoke / test:e2e:local scripts .gitignore: exclude .env.e2e, oclif.manifest.json, tmp/ .env.e2e.example: credential template for local setup --- cli/.env.e2e.example | 20 + cli/.gitignore | 8 +- cli/package.json | 3 + cli/test/e2e/.env.e2e.local | 0 cli/test/e2e/README.md | 115 +++++ cli/test/e2e/helpers/assert.ts | 155 ++++++ cli/test/e2e/helpers/cleanup-registry.ts | 93 ++++ cli/test/e2e/helpers/cli.ts | 374 +++++++++++++++ cli/test/e2e/helpers/retry.ts | 51 ++ cli/test/e2e/helpers/skip.ts | 9 + cli/test/e2e/helpers/vitest-context.ts | 9 + cli/test/e2e/setup/env.ts | 109 +++++ cli/test/e2e/setup/global-setup.ts | 161 +++++++ cli/test/e2e/setup/global-teardown.ts | 15 + cli/test/e2e/suites/auth/devices.e2e.ts | 135 ++++++ cli/test/e2e/suites/auth/logout.e2e.ts | 177 +++++++ cli/test/e2e/suites/auth/status.e2e.ts | 180 +++++++ cli/test/e2e/suites/auth/use.e2e.ts | 186 +++++++ cli/test/e2e/suites/auth/whoami.e2e.ts | 175 +++++++ cli/test/e2e/suites/config/config.e2e.ts | 299 ++++++++++++ cli/test/e2e/suites/run/run-app-basic.e2e.ts | 452 ++++++++++++++++++ cli/test/e2e/suites/run/run-app-file.e2e.ts | 161 +++++++ cli/test/e2e/suites/run/run-app-hitl.e2e.ts | 135 ++++++ .../e2e/suites/run/run-app-streaming.e2e.ts | 125 +++++ cli/vitest.e2e.config.ts | 84 ++++ 25 files changed, 3230 insertions(+), 1 deletion(-) create mode 100644 cli/.env.e2e.example create mode 100644 cli/test/e2e/.env.e2e.local create mode 100644 cli/test/e2e/README.md create mode 100644 cli/test/e2e/helpers/assert.ts create mode 100644 cli/test/e2e/helpers/cleanup-registry.ts create mode 100644 cli/test/e2e/helpers/cli.ts create mode 100644 cli/test/e2e/helpers/retry.ts create mode 100644 cli/test/e2e/helpers/skip.ts create mode 100644 cli/test/e2e/helpers/vitest-context.ts create mode 100644 cli/test/e2e/setup/env.ts create mode 100644 cli/test/e2e/setup/global-setup.ts create mode 100644 cli/test/e2e/setup/global-teardown.ts create mode 100644 cli/test/e2e/suites/auth/devices.e2e.ts create mode 100644 cli/test/e2e/suites/auth/logout.e2e.ts create mode 100644 cli/test/e2e/suites/auth/status.e2e.ts create mode 100644 cli/test/e2e/suites/auth/use.e2e.ts create mode 100644 cli/test/e2e/suites/auth/whoami.e2e.ts create mode 100644 cli/test/e2e/suites/config/config.e2e.ts create mode 100644 cli/test/e2e/suites/run/run-app-basic.e2e.ts create mode 100644 cli/test/e2e/suites/run/run-app-file.e2e.ts create mode 100644 cli/test/e2e/suites/run/run-app-hitl.e2e.ts create mode 100644 cli/test/e2e/suites/run/run-app-streaming.e2e.ts create mode 100644 cli/vitest.e2e.config.ts diff --git a/cli/.env.e2e.example b/cli/.env.e2e.example new file mode 100644 index 00000000000000..10a18d950806a6 --- /dev/null +++ b/cli/.env.e2e.example @@ -0,0 +1,20 @@ +# E2E test environment variables +# Copy this file to .env.e2e and fill in real values before running tests. +# See test/e2e/setup/env.ts for documentation on each variable. + +# Required +DIFY_E2E_HOST=https://your-staging-host.dify.ai +DIFY_E2E_TOKEN=dfoa_your_token_here +DIFY_E2E_WORKSPACE_ID=ws-your-workspace-id +DIFY_E2E_CHAT_APP_ID=app-echo-chat-id +DIFY_E2E_WORKFLOW_APP_ID=app-echo-workflow-id + +# Optional (skip related tests when absent) +DIFY_E2E_SSO_TOKEN= +DIFY_E2E_HITL_APP_ID= +DIFY_E2E_FILE_APP_ID= +DIFY_E2E_WORKSPACE_NAME= + +# For logout / devices revoke tests (mint disposable tokens via device flow API) +DIFY_E2E_EMAIL= +DIFY_E2E_PASSWORD= diff --git a/cli/.gitignore b/cli/.gitignore index 9747e29156e047..97cd7ff545c3f8 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -4,4 +4,10 @@ node_modules/ *.tsbuildinfo .vitest-cache/ docs/specs/ -context/ \ No newline at end of file +context/ +# E2E test env (contains tokens/credentials — use .env.e2e.example instead) +.env.e2e +# Generated / runtime artifacts +oclif.manifest.json +npm-shrinkwrap.json +tmp/ diff --git a/cli/package.json b/cli/package.json index 1b10986d7ff9c4..02d6cbba98b43b 100644 --- a/cli/package.json +++ b/cli/package.json @@ -30,6 +30,9 @@ "dev": "bun bin/dev.js", "test": "vp test", "test:coverage": "vp test --coverage", + "test:e2e": "vp test --config vitest.e2e.config.ts", + "test:e2e:smoke": "vp test --config vitest.e2e.config.ts --testNamePattern \"\\[P0\\]\"", + "test:e2e:local": "DIFY_E2E_MODE=local vp test --config vitest.e2e.config.ts", "lint": "eslint", "lint:fix": "eslint --fix", "type-check": "tsc", diff --git a/cli/test/e2e/.env.e2e.local b/cli/test/e2e/.env.e2e.local new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/cli/test/e2e/README.md b/cli/test/e2e/README.md new file mode 100644 index 00000000000000..0b742a01b29b7e --- /dev/null +++ b/cli/test/e2e/README.md @@ -0,0 +1,115 @@ +# Dify CLI — E2E Test Suite + +End-to-end tests that exercise the **real `difyctl` binary** against a live +Dify server. Every test uses an isolated temporary config directory so no +state leaks between test files. + +## Directory layout + +``` +test/e2e/ +├── setup/ +│ ├── env.ts — Load & validate DIFY_E2E_* env vars +│ ├── global-setup.ts — Health-check server + mint disposable token +│ └── global-teardown.ts — Delete conversations created during the run +│ +├── helpers/ +│ ├── cli.ts — run(), withAuthFixture(), mintFreshToken(), +│ │ injectAuth(), spawn_background() +│ ├── assert.ts — assertExitCode, assertJson, assertErrorEnvelope, +│ │ assertNoAnsi, assertPipeFriendlyJson, … +│ ├── cleanup-registry.ts — registerConversation() / cleanupRegisteredConversations() +│ ├── retry.ts — withRetry(fn, { attempts, delayMs }) +│ └── skip.ts — optionalIt(), optionalDescribe() +│ +└── suites/ + ├── auth/ + │ ├── status.e2e.ts — auth status (text + JSON + SSO) + │ ├── use.e2e.ts — workspace switching + │ ├── whoami.e2e.ts — whoami + external SSO session checks + │ ├── devices.e2e.ts — devices list + revoke (runs near-last) + │ └── logout.e2e.ts — logout + local credential cleanup (runs last) + ├── config/ + │ └── config.e2e.ts — config path/get/set/unset/view, env override + └── run/ + ├── run-app-basic.e2e.ts — basic run, -o json, --inputs, streaming, + │ conversation, CI mode + ├── run-app-streaming.e2e.ts — Ctrl+C / error-event / chunk timing + ├── run-app-file.e2e.ts — --file upload (local + remote URL) + └── run-app-hitl.e2e.ts — HITL pause + resume +``` + +## Setup + +Copy the credential template and fill in your values: + +```bash +cp cli/.env.e2e.example cli/.env.e2e +# edit cli/.env.e2e with real credentials +``` + +### Required env vars + +| Variable | Description | +| -------------------------- | -------------------------------------------------------- | +| `DIFY_E2E_HOST` | Staging server base URL (`http://localhost`) | +| `DIFY_E2E_TOKEN` | Internal user bearer token (`dfoa_…`) | +| `DIFY_E2E_WORKSPACE_ID` | Primary workspace ID | +| `DIFY_E2E_CHAT_APP_ID` | Chat app — outputs `echo: {query}` | +| `DIFY_E2E_WORKFLOW_APP_ID` | Workflow app — input `x` (required), outputs `echo: {x}` | + +### Optional env vars + +| Variable | Description | +| ------------------------- | ---------------------------------------------------- | +| `DIFY_E2E_SSO_TOKEN` | External SSO bearer token (`dfoe_…`) | +| `DIFY_E2E_HITL_APP_ID` | Workflow app with a Human-Input node | +| `DIFY_E2E_FILE_APP_ID` | Workflow app with a file input variable (`doc`) | +| `DIFY_E2E_WORKSPACE_NAME` | Display name for the primary workspace | +| `DIFY_E2E_EMAIL` | Console account email (enables disposable tokens) | +| `DIFY_E2E_PASSWORD` | Console account password (enables disposable tokens) | + +> `DIFY_E2E_EMAIL` + `DIFY_E2E_PASSWORD` are used by `global-setup` and the +> `devices`/`logout` suites to mint fresh single-use `dfoa_` tokens via the +> device flow API, so those tests never revoke the shared `DIFY_E2E_TOKEN`. + +## Running tests + +```bash +cd cli + +# Run the full E2E suite +bun run test:e2e + +# Run only [P0] smoke cases +bun run test:e2e:smoke + +# Run offline-safe config tests only (no network required) +bun run test:e2e:local + +# Run a single file +bun vitest --config vitest.e2e.config.ts test/e2e/suites/auth/status.e2e.ts +``` + +## Test execution order + +Files run sequentially (`fileParallelism: false`) in this order: + +``` +status → use → whoami → config → run (basic / streaming / file / HITL) + → devices → logout +``` + +`devices` and `logout` run last because they revoke real server sessions. + +## Design decisions + +| Decision | Rationale | +| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| **No mocking** | All HTTP traffic goes to the real server — this catches real integration regressions. | +| **Isolated config dirs** | Each test creates a fresh `withTempConfig()` dir; session state never leaks between tests. | +| **`withAuthFixture()`** | Combines `withTempConfig` + `injectAuth` into a single fixture; reduces beforeEach boilerplate. | +| **`injectAuth()` bypasses Device Flow** | Non-auth tests skip the browser step; only `auth/` suites exercise the real flow. | +| **`mintFreshToken()`** | `logout` and `devices-revoke` tests mint a disposable `dfoa_` token via the device flow API, so revoking it never kills the shared `DIFY_E2E_TOKEN`. | +| **Global `retry: 0`** | Flaky network calls use `withRetry()` locally with `shouldRetry` filtering; global retry masks non-idempotent failures (e.g. logout). | +| **Conversation cleanup** | `registerConversation()` + global-teardown delete staging conversations after the run. | diff --git a/cli/test/e2e/helpers/assert.ts b/cli/test/e2e/helpers/assert.ts new file mode 100644 index 00000000000000..e5c998541ddd18 --- /dev/null +++ b/cli/test/e2e/helpers/assert.ts @@ -0,0 +1,155 @@ +/** + * E2E assertion helpers. + * + * These wrap vitest's `expect` with richer failure messages that include the + * full stdout / stderr of the failing process — essential for debugging CI. + */ + +import type { RunResult } from './cli.js' +import { expect } from 'vitest' +import './vitest-context.js' + +// ── ANSI ────────────────────────────────────────────────────────────────── +// eslint-disable-next-line no-control-regex +const ANSI_RE = /\x1B\[[0-9;]*[mGKHFA-DJsuhl]/g + +function redact(text: string): string { + return text + .replace(/\bBearer\s+[\w.-]+\b/g, 'Bearer [REDACTED]') + .replace(/\bdfo[ae]_[\w-]+\b/g, 'dfo*_REDACTED') +} + +// ── Exit code ───────────────────────────────────────────────────────────── + +/** + * Assert the exit code matches `expected`. + * On failure, prints the full stdout and stderr so the cause is visible in CI. + */ +export function assertExitCode(result: RunResult, expected: number): void { + if (result.exitCode !== expected) { + process.stderr.write( + `\n[E2E assertExitCode] expected ${expected}, got ${result.exitCode}\n` + + `stdout:\n${redact(result.stdout) || '(empty)'}\n` + + `stderr:\n${redact(result.stderr) || '(empty)'}\n`, + ) + } + expect(result.exitCode, `exit code should be ${expected}`).toBe(expected) +} + +/** + * Assert the exit code is NOT 0 (i.e. some error occurred). + */ +export function assertNonZeroExit(result: RunResult): void { + expect(result.exitCode, 'exit code should be non-zero').not.toBe(0) +} + +// ── Stdout / stderr content ─────────────────────────────────────────────── + +/** + * Assert stdout is valid JSON and return the parsed value. + */ +export function assertJson(result: RunResult): T { + let parsed: T + try { + parsed = JSON.parse(result.stdout) as T + } + catch { + throw new Error( + `stdout is not valid JSON.\nstdout:\n${redact(result.stdout)}\nstderr:\n${redact(result.stderr)}`, + ) + } + return parsed +} + +/** + * Assert stderr contains a valid JSON error envelope of the shape: + * { error: { code: string, message: string, hint?: string } } + * + * @param result - The run result to inspect. + * @param expectedCode - When provided, also asserts that error.code equals this value. + * Use the stable error codes from the CLI contract, e.g.: + * 'not_logged_in', 'app_not_found', 'insufficient_scope', 'auth_expired' + * + * @example + * assertErrorEnvelope(result, 'not_logged_in') + * assertErrorEnvelope(result, 'app_not_found') + */ +export function assertErrorEnvelope( + result: RunResult, + expectedCode?: string, +): { error: { code: string, message: string, hint?: string } } { + const raw = result.stderr.trim() + let parsed: { error: { code: string, message: string, hint?: string } } + try { + parsed = JSON.parse(raw) as typeof parsed + } + catch { + throw new Error( + `stderr is not valid JSON.\nstdout:\n${redact(result.stdout)}\nstderr:\n${redact(result.stderr)}`, + ) + } + expect(parsed, 'stderr envelope missing "error" key').toHaveProperty('error') + expect(parsed.error, 'error.code must be a non-empty string').toHaveProperty('code') + expect(parsed.error, 'error.message must be a non-empty string').toHaveProperty('message') + expect(typeof parsed.error.code, 'error.code must be a string').toBe('string') + expect(parsed.error.code.length, 'error.code must be non-empty').toBeGreaterThan(0) + if (expectedCode !== undefined) { + expect( + parsed.error.code, + `error.code should be "${expectedCode}", got "${parsed.error.code}"\nstderr:\n${redact(result.stderr)}`, + ).toBe(expectedCode) + } + return parsed +} + +// ── ANSI / formatting ──────────────────────────────────────────────────── + +/** + * Assert the given text contains no ANSI escape sequences. + * Pass `label` to identify which stream failed (e.g. 'stdout', 'stderr'). + */ +export function assertNoAnsi(text: string, label = 'output'): void { + const clean = text.replace(ANSI_RE, '') + expect(text, `${label} must not contain ANSI control codes`).toBe(clean) +} + +/** + * Assert stdout starts with `{` and ends with `\n` — the canonical format + * for pipe-friendly JSON output. + */ +export function assertPipeFriendlyJson(result: RunResult): void { + assertNoAnsi(result.stdout, 'stdout') + expect( + result.stdout.trimStart().startsWith('{') || result.stdout.trimStart().startsWith('['), + 'stdout should start with { or [ for pipe-friendly JSON', + ).toBe(true) + expect(result.stdout.endsWith('\n'), 'stdout should end with newline').toBe(true) +} + +// ── stdout / stderr contains ────────────────────────────────────────────── + +/** + * Assert stdout contains the given substring, printing full output on failure. + */ +export function assertStdoutContains(result: RunResult, expected: string): void { + if (!result.stdout.includes(expected)) { + process.stderr.write( + `\n[E2E assertStdoutContains] "${expected}" not found in stdout.\n` + + `stdout:\n${redact(result.stdout)}\nstderr:\n${redact(result.stderr)}\n`, + ) + } + expect(result.stdout).toContain(expected) +} + +/** + * Assert stderr contains the given substring, printing full output on failure. + */ +export function assertStderrContains(result: RunResult, expected: string): void { + if (!result.stderr.includes(expected)) { + process.stderr.write( + `\n[E2E assertStderrContains] "${expected}" not found in stderr.\n` + + `stdout:\n${redact(result.stdout)}\nstderr:\n${redact(result.stderr)}\n`, + ) + } + expect(result.stderr).toContain(expected) +} diff --git a/cli/test/e2e/helpers/cleanup-registry.ts b/cli/test/e2e/helpers/cleanup-registry.ts new file mode 100644 index 00000000000000..9b2d4559ed35d9 --- /dev/null +++ b/cli/test/e2e/helpers/cleanup-registry.ts @@ -0,0 +1,93 @@ +/** + * E2E cleanup registry. + * + * Test suites call `registerConversation(host, token, appId, conversationId)` + * whenever a real conversation is created on staging. The global teardown + * iterates the registry and deletes all collected conversations so staging + * data stays clean between CI runs. + * + * Design notes: + * - Uses a module-level array (shared within the same worker process). + * - vitest runs E2E suites in a single fork (fileParallelism: false), so one + * process owns the full registry. + * - Deletion is best-effort: individual failures are logged but do not throw. + */ + +export type ConversationEntry = { + host: string + token: string + appId: string + conversationId: string +} + +const _conversations: ConversationEntry[] = [] + +/** + * Register a conversation for cleanup in teardown. + * Call this whenever `run app` returns a `conversation_id`. + */ +export function registerConversation( + host: string, + token: string, + appId: string, + conversationId: string, +): void { + if (!conversationId || !appId) + return + _conversations.push({ host, token, appId, conversationId }) +} + +/** + * Return all registered conversations (for use in teardown). + */ +export function getRegisteredConversations(): readonly ConversationEntry[] { + return _conversations +} + +/** + * Delete all registered conversations from the staging server. + * Called once from global-teardown.ts. + */ +export async function cleanupRegisteredConversations(): Promise { + if (_conversations.length === 0) + return + + console.log(`[E2E teardown] Cleaning up ${_conversations.length} staged conversation(s)…`) + + const results = await Promise.allSettled( + _conversations.map(({ host, token, appId, conversationId }) => + deleteConversation(host, token, appId, conversationId), + ), + ) + + const failed = results.filter(r => r.status === 'rejected') + if (failed.length > 0) { + console.warn( + `[E2E teardown] ${failed.length} conversation deletion(s) failed (non-blocking):`, + failed.map(r => (r as PromiseRejectedResult).reason).join(', '), + ) + } + else { + console.log(`[E2E teardown] All conversations cleaned up.`) + } + + _conversations.length = 0 +} + +async function deleteConversation( + host: string, + token: string, + appId: string, + conversationId: string, +): Promise { + const url = `${host.replace(/\/$/, '')}/openapi/v1/apps/${appId}/conversations/${conversationId}` + const res = await fetch(url, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(8_000), + }) + // 404 is acceptable — conversation may have already been cleaned up + if (!res.ok && res.status !== 404) { + throw new Error(`DELETE ${url} → HTTP ${res.status}`) + } +} diff --git a/cli/test/e2e/helpers/cli.ts b/cli/test/e2e/helpers/cli.ts new file mode 100644 index 00000000000000..bf4da9c3fe1865 --- /dev/null +++ b/cli/test/e2e/helpers/cli.ts @@ -0,0 +1,374 @@ +/** + * E2E CLI runner helpers. + * + * Core primitive: run(argv, opts) → { stdout, stderr, exitCode } + * + * The binary is invoked via `bun bin/dev.js` so tests work without a prior + * `pnpm build`. Each test should use its own isolated configDir (created via + * withTempConfig) to prevent session state leaking between tests. + */ + +import { Buffer } from 'node:buffer' +import { spawn } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +/** Path to the dev entry point — no build required. */ +export const BIN = resolve(__dirname, '../../../bin/dev.js') + +/** + * Resolve the `bun` executable path. + * Priority: PATH → ~/.bun/bin/bun → /usr/local/bin/bun + */ +function resolveBun(): string { + const { execSync } = await import('node:child_process') + const candidates = [ + // Respect PATH first + 'bun', + // Common install locations + `${process.env.HOME}/.bun/bin/bun`, + '/usr/local/bin/bun', + '/opt/homebrew/bin/bun', + ] + for (const candidate of candidates) { + try { + execSync(`${candidate} --version`, { stdio: 'ignore', timeout: 3000 }) + return candidate + } + catch { /* try next */ } + } + throw new Error( + 'bun not found. Install it with: curl -fsSL https://bun.sh/install | bash', + ) +} + +export const BUN = resolveBun() + +// ── Types ───────────────────────────────────────────────────────────────── + +export type RunOptions = { + /** + * Override or extend the process environment. + * Values are merged on top of `process.env`. + */ + env?: Record + /** + * Path to an isolated config directory. + * The CLI reads hosts.yml from this directory. + * Passed as DIFY_CONFIG_DIR env var. + */ + configDir?: string + /** Maximum time to wait for the process, in ms. Default: 30 000 */ + timeout?: number + /** String to write to stdin, then close the pipe. */ + stdin?: string +} + +export type RunResult = { + stdout: string + stderr: string + exitCode: number +} + +// ── Core runner ──────────────────────────────────────────────────────────── + +/** + * Execute `difyctl ` and return the captured stdout, stderr and exit code. + * + * Environment notes: + * - CI=1 suppresses interactive prompts and spinners. + * - NO_COLOR=1 strips ANSI escape codes from output. + * - DIFY_CONFIG_DIR is set to opts.configDir when provided. + */ +export function run(argv: string[], opts: RunOptions = {}): Promise { + return new Promise((resolve, reject) => { + const env: Record = { + ...(process.env as Record), + // Suppress interactive prompts in all E2E tests. + CI: '1', + NO_COLOR: '1', + // Point the CLI at the isolated config directory. + ...(opts.configDir !== undefined ? { DIFY_CONFIG_DIR: opts.configDir } : {}), + ...opts.env, + } + + const proc = spawn(BUN, [BIN, ...argv], { env }) + const timeoutMs = opts.timeout ?? 30_000 + let timedOut = false + const timeoutId = setTimeout(() => { + timedOut = true + proc.kill('SIGINT') + setTimeout(() => proc.kill('SIGKILL'), 2000).unref?.() + }, timeoutMs) + timeoutId.unref?.() + + let stdout = '' + let stderr = '' + + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString('utf8') + }) + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk.toString('utf8') + }) + + if (opts.stdin !== undefined) { + proc.stdin.write(opts.stdin) + proc.stdin.end() + } + + proc.on('close', (code: number | null) => { + clearTimeout(timeoutId) + resolve({ stdout, stderr, exitCode: code ?? (timedOut ? 124 : 1) }) + }) + + proc.on('error', (err: Error) => { + clearTimeout(timeoutId) + reject(new Error(`Failed to spawn CLI process: ${err.message}`)) + }) + }) +} + +// ── Config directory helpers ─────────────────────────────────────────────── + +export type TempConfig = { + /** Path to the isolated config directory. */ + configDir: string + /** Remove the directory and all its contents. */ + cleanup: () => Promise +} + +/** + * Create a fresh temporary config directory for a single test. + * Always call cleanup() in afterEach to avoid leaking temp directories. + */ +export async function withTempConfig(): Promise { + const configDir = await mkdtemp(join(tmpdir(), 'difyctl-e2e-')) + return { + configDir, + cleanup: () => rm(configDir, { recursive: true, force: true }), + } +} + +// ── Auth injection ───────────────────────────────────────────────────────── + +export type AuthInjectionOptions = { + /** Staging server base URL (no trailing slash). */ + host: string + /** Bearer token — dfoa_ for internal, dfoe_ for SSO. */ + bearer: string + /** Primary workspace to write into the bundle. */ + workspaceId: string + workspaceName: string + workspaceRole?: string + /** + * Server-side session UUID (OAuthAccessToken.id). + * When provided, written as `token_id` in hosts.yml so that + * `devices revoke` can correctly detect selfHit and clear local credentials. + */ + tokenId?: string +} + +/** + * Write a pre-baked hosts.yml into configDir so tests can skip the real + * Device-Flow login. Auth-specific E2E tests (login/logout/status) use the + * real flow and should NOT call this function. + */ +export async function injectAuth(configDir: string, opts: AuthInjectionOptions): Promise { + await mkdir(configDir, { recursive: true, mode: 0o700 }) + + const role = opts.workspaceRole ?? 'owner' + // Serialise to YAML manually to avoid a runtime dep on js-yaml in helpers. + const hostsYml = `${[ + `current_host: ${opts.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${opts.bearer}`, + ...(opts.tokenId !== undefined ? [`token_id: ${opts.tokenId}`] : []), + `workspace:`, + ` id: ${opts.workspaceId}`, + ` name: "${opts.workspaceName}"`, + ` role: ${role}`, + `available_workspaces:`, + ` - id: ${opts.workspaceId}`, + ` name: "${opts.workspaceName}"`, + ` role: ${role}`, + ].join('\n')}\n` + + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) +} + +// ── Process signal helpers ───────────────────────────────────────────────── + +export type SpawnedProcess = { + /** Send SIGINT (Ctrl+C) to the process. */ + interrupt: () => void + /** Wait for the process to exit and return the result. */ + wait: () => Promise +} + +/** + * Start `difyctl ` in the background without waiting for it to finish. + * Useful for testing interrupt / timeout behaviour. + */ +export function spawn_background(argv: string[], opts: RunOptions = {}): SpawnedProcess { + const env: Record = { + ...(process.env as Record), + CI: '1', + NO_COLOR: '1', + ...(opts.configDir !== undefined ? { DIFY_CONFIG_DIR: opts.configDir } : {}), + ...opts.env, + } + + const proc = spawn(BUN, [BIN, ...argv], { env }) + const timeoutMs = opts.timeout ?? 30_000 + let timedOut = false + const timeoutId = setTimeout(() => { + timedOut = true + proc.kill('SIGINT') + setTimeout(() => proc.kill('SIGKILL'), 2000).unref?.() + }, timeoutMs) + timeoutId.unref?.() + + let stdout = '' + let stderr = '' + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString('utf8') + }) + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk.toString('utf8') + }) + + return { + interrupt: () => { proc.kill('SIGINT') }, + wait: () => new Promise((res) => { + proc.on('close', (code: number | null) => { + clearTimeout(timeoutId) + res({ stdout, stderr, exitCode: code ?? (timedOut ? 124 : 1) }) + }) + }), + } +} + +// ── Auth fixture ─────────────────────────────────────────────────────────── + +export type AuthFixture = { + /** Path to the isolated config directory, pre-loaded with a valid session. */ + configDir: string + /** + * Run `difyctl ` using the fixture's config dir. + * Shorthand for `run(argv, { configDir, env })`. + */ + r: (argv: string[], extraEnv?: Record) => Promise + /** Remove the temp config directory. Call in afterEach. */ + cleanup: () => Promise +} + +/** + * Create an isolated config directory pre-loaded with a valid internal-user + * session. Designed for use with vitest's beforeEach / afterEach: + * + * @example + * let fx: AuthFixture + * beforeEach(async () => { fx = await withAuthFixture(E) }) + * afterEach(async () => { await fx.cleanup() }) + * + * it('...', async () => { + * const result = await fx.r(['get', 'app']) + * assertExitCode(result, 0) + * }) + */ +export async function withAuthFixture( + E: { host: string, token: string, workspaceId: string, workspaceName: string }, +): Promise { + const { configDir, cleanup } = await withTempConfig() + await injectAuth(configDir, { + host: E.host, + bearer: E.token, + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + return { + configDir, + r: (argv, extraEnv) => run(argv, { configDir, env: extraEnv }), + cleanup, + } +} + +// ── On-demand disposable token ───────────────────────────────────────────── + +/** + * Mint a fresh dfoa_ OAuth token on demand via the 3-step device flow API. + * Use this inside tests that need to revoke a real session without consuming + * the shared DIFY_E2E_TOKEN or the global-setup disposableToken. + * + * Requires DIFY_E2E_EMAIL and DIFY_E2E_PASSWORD to be set. + * Returns empty string if credentials are missing. + * + * Steps: + * 1. POST /console/api/login (Base64 password) → session cookie + * 2. POST /openapi/v1/oauth/device/code → device_code + user_code + * 3. POST /openapi/v1/oauth/device/approve → approved + * 4. POST /openapi/v1/oauth/device/token → dfoa_ token + */ +export async function mintFreshToken( + host: string, + email: string, + password: string, +): Promise { + if (!email || !password) + return '' + + const base = host.replace(/\/$/, '') + const sig = AbortSignal.timeout(15_000) + + // Step 1 — console login + const passwordB64 = Buffer.from(password, 'utf8').toString('base64') + const loginRes = await fetch(`${base}/console/api/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password: passwordB64, remember_me: false }), + signal: AbortSignal.timeout(10_000), + }) + if (!loginRes.ok) + return '' + + const setCookieHeaders = loginRes.headers.getSetCookie?.() ?? [] + const cookieString = setCookieHeaders.map(c => c.split(';')[0]).join('; ') + const csrfMatch = cookieString.match(/csrf_token=([^;]+)/) + const csrfToken = csrfMatch ? csrfMatch[1] : '' + + // Step 2 — device code + const codeRes = await fetch(`${base}/openapi/v1/oauth/device/code`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ client_id: 'difyctl', device_label: 'e2e-fresh' }), + signal: sig, + }) + if (!codeRes.ok) + return '' + const { device_code, user_code } = await codeRes.json() as { device_code: string, user_code: string } + + // Step 3 — approve + const approveRes = await fetch(`${base}/openapi/v1/oauth/device/approve`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Cookie': cookieString, 'X-CSRFToken': csrfToken }, + body: JSON.stringify({ user_code }), + signal: AbortSignal.timeout(10_000), + }) + if (!approveRes.ok) + return '' + + // Step 4 — poll token + const tokenRes = await fetch(`${base}/openapi/v1/oauth/device/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ device_code, client_id: 'difyctl' }), + signal: AbortSignal.timeout(10_000), + }) + if (!tokenRes.ok) + return '' + const body = await tokenRes.json() as { token?: string } + return body.token ?? '' +} diff --git a/cli/test/e2e/helpers/retry.ts b/cli/test/e2e/helpers/retry.ts new file mode 100644 index 00000000000000..6f71ce074de94a --- /dev/null +++ b/cli/test/e2e/helpers/retry.ts @@ -0,0 +1,51 @@ +/** + * Retry helper for E2E tests running against a staging server. + * + * Staging environments can be flaky — occasional 5xx errors or slow cold + * starts are expected. Use `withRetry` to wrap assertions that may fail + * transiently without masking real failures. + */ + +const DEFAULT_ATTEMPTS = 3 +const DEFAULT_DELAY_MS = 1000 + +export type RetryOptions = { + /** Total number of attempts (first try + retries). Default: 3 */ + attempts?: number + /** Delay between retries in ms. Default: 1000 */ + delayMs?: number + /** Optional predicate — only retry when this returns true for the error. */ + shouldRetry?: (err: unknown) => boolean +} + +/** + * Execute `fn()` and retry on failure. + * + * @example + * const result = await withRetry(() => run(['get', 'app', '-o', 'json'])) + */ +export async function withRetry(fn: () => Promise, opts: RetryOptions = {}): Promise { + const total = opts.attempts ?? DEFAULT_ATTEMPTS + const delay = opts.delayMs ?? DEFAULT_DELAY_MS + const shouldRetry = opts.shouldRetry ?? (() => true) + + let lastErr: unknown + for (let attempt = 1; attempt <= total; attempt++) { + try { + return await fn() + } + catch (err) { + lastErr = err + if (attempt === total || !shouldRetry(err)) + break + + console.warn(`[E2E retry] attempt ${attempt}/${total} failed — retrying in ${delay}ms`) + await sleep(delay) + } + } + throw lastErr +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} diff --git a/cli/test/e2e/helpers/skip.ts b/cli/test/e2e/helpers/skip.ts new file mode 100644 index 00000000000000..fe34d9b55e4a93 --- /dev/null +++ b/cli/test/e2e/helpers/skip.ts @@ -0,0 +1,9 @@ +import { describe, it } from 'vitest' + +export function optionalDescribe(condition: boolean) { + return condition ? describe : describe.skip +} + +export function optionalIt(condition: boolean) { + return condition ? it : it.skip +} diff --git a/cli/test/e2e/helpers/vitest-context.ts b/cli/test/e2e/helpers/vitest-context.ts new file mode 100644 index 00000000000000..f32a9a66329942 --- /dev/null +++ b/cli/test/e2e/helpers/vitest-context.ts @@ -0,0 +1,9 @@ +import type { E2ECapabilities } from '../setup/env.js' + +declare module 'vitest' { + export type ProvidedContext = { + e2eCapabilities: E2ECapabilities + } +} + +export { } diff --git a/cli/test/e2e/setup/env.ts b/cli/test/e2e/setup/env.ts new file mode 100644 index 00000000000000..820cc71fc32174 --- /dev/null +++ b/cli/test/e2e/setup/env.ts @@ -0,0 +1,109 @@ +/** + * E2E environment configuration. + * + * All DIFY_E2E_* variables must be set before running E2E tests. + * In CI they are injected from GitHub Actions secrets. + * Locally, export them in your shell or use a .env.e2e file. + * + * Required: + * DIFY_E2E_HOST Staging server base URL (e.g. https://api.staging.dify.ai) + * DIFY_E2E_TOKEN Internal user bearer token (dfoa_ prefix) + * DIFY_E2E_WORKSPACE_ID Workspace ID for the test account + * DIFY_E2E_CHAT_APP_ID Echo-chat app — outputs "echo: {query}" + * DIFY_E2E_WORKFLOW_APP_ID Echo-workflow app — input x (required), outputs "echo: {x}" + * + * Optional (skip related tests when absent): + * DIFY_E2E_SSO_TOKEN External SSO bearer token (dfoe_ prefix) + * DIFY_E2E_HITL_APP_ID Workflow app with a Human-Input node + * DIFY_E2E_FILE_APP_ID Workflow app with a file input variable (doc) + */ + +export type E2EEnv = { + /** Staging server base URL */ + host: string + /** Internal user bearer token (dfoa_…) */ + token: string + /** External SSO bearer token (dfoe_…) — may be empty */ + ssoToken: string + /** Primary workspace ID */ + workspaceId: string + /** Workspace name (informational) */ + workspaceName: string + /** Chat app that echoes the query */ + chatAppId: string + /** Workflow app that echoes input x */ + workflowAppId: string + /** Workflow app with HITL node — empty when not configured */ + hitlAppId: string + /** Workflow app with file input (doc variable) — empty when not configured */ + fileAppId: string + /** + * Console account email — used by global-setup to mint a disposable token + * for logout tests via the device flow API. Optional: if absent, logout + * tests that need a real revoke are skipped. + */ + email: string + /** Console account password (plain-text; Base64-encoded before sending) */ + password: string +} + +export type E2ECapabilities = { + tokenValid: boolean + tokenId?: string + /** + * A freshly minted dfoa_ token created by global-setup via the device flow + * API using DIFY_E2E_EMAIL + DIFY_E2E_PASSWORD. Intended for logout tests + * that need to actually revoke a session without invalidating the shared + * DIFY_E2E_TOKEN used by all other suites. + * Empty string when email/password are not configured. + */ + disposableToken: string +} + +let _cached: E2EEnv | undefined + +/** Load and validate E2E environment variables. Throws if required vars are missing. */ +export function loadE2EEnv(): E2EEnv { + if (_cached !== undefined) + return _cached + + const required: Array<[keyof NodeJS.ProcessEnv, string]> = [ + ['DIFY_E2E_HOST', 'Staging server URL'], + ['DIFY_E2E_TOKEN', 'Internal user bearer token'], + ['DIFY_E2E_WORKSPACE_ID', 'Workspace ID'], + ['DIFY_E2E_CHAT_APP_ID', 'Echo-chat app ID'], + ['DIFY_E2E_WORKFLOW_APP_ID', 'Echo-workflow app ID'], + ] + + const missing = required.filter(([k]) => !process.env[k]) + if (missing.length > 0) { + const list = missing.map(([k, desc]) => ` ${k} (${desc})`).join('\n') + throw new Error( + `E2E tests require the following environment variables to be set:\n${list}\n\n` + + 'See test/e2e/setup/env.ts for documentation.', + ) + } + + _cached = { + host: process.env.DIFY_E2E_HOST!, + token: process.env.DIFY_E2E_TOKEN!, + ssoToken: process.env.DIFY_E2E_SSO_TOKEN ?? '', + workspaceId: process.env.DIFY_E2E_WORKSPACE_ID!, + workspaceName: process.env.DIFY_E2E_WORKSPACE_NAME ?? 'E2E Workspace', + chatAppId: process.env.DIFY_E2E_CHAT_APP_ID!, + workflowAppId: process.env.DIFY_E2E_WORKFLOW_APP_ID!, + hitlAppId: process.env.DIFY_E2E_HITL_APP_ID ?? '', + fileAppId: process.env.DIFY_E2E_FILE_APP_ID ?? '', + email: process.env.DIFY_E2E_EMAIL ?? '', + password: process.env.DIFY_E2E_PASSWORD ?? '', + } + return _cached +} + +/** + * Skip a test when an optional app fixture is not configured. + * Usage: skipUnless(E.hitlAppId, 'DIFY_E2E_HITL_APP_ID') + */ +export function isE2ELocalMode(): boolean { + return process.env.DIFY_E2E_MODE === 'local' +} diff --git a/cli/test/e2e/setup/global-setup.ts b/cli/test/e2e/setup/global-setup.ts new file mode 100644 index 00000000000000..bf205b6ed66461 --- /dev/null +++ b/cli/test/e2e/setup/global-setup.ts @@ -0,0 +1,161 @@ +import type { TestProject } from 'vitest/node' +/** + * Vitest global setup — runs once before all E2E suites. + * + * Responsibilities: + * 1. Validate required environment variables are present. + * 2. Confirm the staging server is reachable AND the token is valid — + * GET /openapi/v1/account/sessions (HTTP 200 = valid, else abort). + * 3. Resolve the current session's token_id via the prefix field. + * 4. Mint a disposable dfoa_ token via the device flow API so that + * logout tests can revoke a real session without invalidating the + * shared DIFY_E2E_TOKEN used by all other suites. + * + * If the health-check fails the entire test run is aborted early. + */ + +import type { E2ECapabilities } from './env.js' +import { Buffer } from 'node:buffer' +import { loadE2EEnv } from './env.js' + +export async function setup(project: TestProject): Promise { + if (process.env.DIFY_E2E_MODE === 'local') + return + + const E = loadE2EEnv() + const base = E.host.replace(/\/$/, '') + + // ── 1. Validate main token ───────────────────────────────────────────── + const sessionsUrl = `${base}/openapi/v1/account/sessions?page=1&limit=100` + let res: Response + try { + res = await fetch(sessionsUrl, { + headers: { Authorization: `Bearer ${E.token}` }, + signal: AbortSignal.timeout(10_000), + }) + } + catch (err) { + throw new Error( + `[E2E global-setup] Cannot reach staging server at ${sessionsUrl}.\n` + + `Check DIFY_E2E_HOST and network connectivity.\n${ + String(err)}`, + ) + } + + if (!res.ok) { + throw new Error( + `[E2E global-setup] Token is invalid or expired (HTTP ${res.status}).\n` + + `Update DIFY_E2E_TOKEN and retry.\nURL: ${sessionsUrl}`, + ) + } + + console.log(`[E2E] Staging server is healthy and token is valid at ${E.host}`) + + // ── 2. Resolve token_id ──────────────────────────────────────────────── + const body = await res.json() as { data: Array<{ id: string, prefix: string }> } + const match = body.data.find(s => s.prefix !== '' && E.token.startsWith(s.prefix)) + + // ── 3. Mint disposable token for logout tests ────────────────────────── + let disposableToken = '' + if (E.email && E.password) { + try { + disposableToken = await mintDisposableToken(base, E.email, E.password) + + console.log(`[E2E] Disposable logout-test token minted: ${disposableToken.slice(0, 20)}…`) + } + catch (err) { + // Non-fatal: logout tests will skip if token is empty + + console.warn(`[E2E global-setup] Failed to mint disposable token (logout tests may skip): ${String(err)}`) + } + } + else { + console.warn('[E2E global-setup] DIFY_E2E_EMAIL/PASSWORD not set — logout revoke tests will be skipped') + } + + const capabilities: E2ECapabilities = { + tokenValid: true, + tokenId: match?.id, + disposableToken, + } + + project.provide('e2eCapabilities', capabilities) +} + +export { teardown } from './global-teardown.js' + +// ── Device flow helper ───────────────────────────────────────────────────── + +/** + * Mint a fresh dfoa_ OAuth token via the 3-step device flow API: + * 1. POST /openapi/v1/oauth/device/code → device_code + user_code + * 2. POST /console/api/login → console session cookie + * POST /openapi/v1/oauth/device/approve (with cookie) → approved + * 3. POST /openapi/v1/oauth/device/token → dfoa_ token + * + * Password is Base64-encoded before sending (Dify's obfuscation convention). + */ +async function mintDisposableToken(base: string, email: string, password: string): Promise { + const timeout = AbortSignal.timeout(15_000) + + // Step 1 — request device code + const codeRes = await fetch(`${base}/openapi/v1/oauth/device/code`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ client_id: 'difyctl', device_label: 'e2e-disposable-logout' }), + signal: timeout, + }) + if (!codeRes.ok) + throw new Error(`device/code failed: HTTP ${codeRes.status}`) + const { device_code, user_code } = await codeRes.json() as { device_code: string, user_code: string } + + // Step 2a — console login to get session cookie + const passwordB64 = Buffer.from(password, 'utf8').toString('base64') + const loginRes = await fetch(`${base}/console/api/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password: passwordB64, remember_me: false }), + signal: AbortSignal.timeout(10_000), + }) + if (!loginRes.ok) + throw new Error(`console/api/login failed: HTTP ${loginRes.status}`) + + // Extract cookies from Set-Cookie headers + const setCookieHeaders = loginRes.headers.getSetCookie?.() ?? [] + const cookieString = setCookieHeaders + .map(c => c.split(';')[0]) + .join('; ') + + // Extract CSRF token value from cookie string + const csrfMatch = cookieString.match(/csrf_token=([^;]+)/) + const csrfToken = csrfMatch ? csrfMatch[1] : '' + + // Step 2b — approve the device code using the console session + const approveRes = await fetch(`${base}/openapi/v1/oauth/device/approve`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cookie': cookieString, + 'X-CSRFToken': csrfToken, + }, + body: JSON.stringify({ user_code }), + signal: AbortSignal.timeout(10_000), + }) + if (!approveRes.ok) + throw new Error(`device/approve failed: HTTP ${approveRes.status}`) + + // Step 3 — poll for the minted token + const tokenRes = await fetch(`${base}/openapi/v1/oauth/device/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ device_code, client_id: 'difyctl' }), + signal: AbortSignal.timeout(10_000), + }) + if (!tokenRes.ok) + throw new Error(`device/token failed: HTTP ${tokenRes.status}`) + const tokenBody = await tokenRes.json() as { token?: string, error?: string } + if (!tokenBody.token) + throw new Error(`device/token response missing token field: ${JSON.stringify(tokenBody)}`) + + return tokenBody.token +} diff --git a/cli/test/e2e/setup/global-teardown.ts b/cli/test/e2e/setup/global-teardown.ts new file mode 100644 index 00000000000000..10c95750a1f2ac --- /dev/null +++ b/cli/test/e2e/setup/global-teardown.ts @@ -0,0 +1,15 @@ +/** + * Vitest global teardown — runs once after all E2E suites complete. + * + * Responsibilities: + * 1. Delete all conversations created on the staging server during the run + * (collected via registerConversation() in test suites). + * + * Deletion is best-effort — failures are logged but do not fail the run. + */ + +import { cleanupRegisteredConversations } from '../helpers/cleanup-registry.js' + +export async function teardown(): Promise { + await cleanupRegisteredConversations() +} diff --git a/cli/test/e2e/suites/auth/devices.e2e.ts b/cli/test/e2e/suites/auth/devices.e2e.ts new file mode 100644 index 00000000000000..cfed703fa7b0c8 --- /dev/null +++ b/cli/test/e2e/suites/auth/devices.e2e.ts @@ -0,0 +1,135 @@ +/** + * E2E: difyctl auth devices — 多设备 Session 管理 + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Auth/多设备 Session 管理（21 条） + */ + +import { afterEach, beforeEach, describe, expect, inject, it } from 'vitest' +import { assertExitCode, assertJson } from '../../helpers/assert.js' +import { injectAuth, mintFreshToken, run, withTempConfig } from '../../helpers/cli.js' +import { optionalIt } from '../../helpers/skip.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() +const caps = inject('e2eCapabilities') +const tokenValid = caps.tokenValid +const tokenId = caps.tokenId + +describe('E2E / difyctl auth devices', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const tmp = await withTempConfig() + configDir = tmp.configDir + cleanup = tmp.cleanup + + await injectAuth(configDir, { + host: E.host, + bearer: E.token, + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + tokenId, + }) + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[]) { + return run(argv, { configDir }) + } + + // ── devices list ────────────────────────────────────────────────────────── + + const itSessions = optionalIt(tokenValid) + + itSessions('[P0] 已登录用户可查看 devices 列表', async () => { + // 文档用例：已登录用户可查看 devices 列表 + const result = await r(['auth', 'devices', 'list']) + assertExitCode(result, 0) + expect(result.stdout.length).toBeGreaterThan(0) + }) + + itSessions('[P0] devices list 显示 device id', async () => { + // 文档用例：devices list 显示 device id + const result = await r(['auth', 'devices', 'list']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/tok-|id|device/i) + }) + + itSessions('[P0] devices list 支持 JSON 输出，返回合法 JSON', async () => { + // 文档用例：devices list 支持 JSON 输出 + const result = await r(['auth', 'devices', 'list', '--json']) + assertExitCode(result, 0) + const parsed = assertJson<{ data: unknown[], total: number }>(result) + expect(parsed).toHaveProperty('data') + expect(Array.isArray(parsed.data)).toBe(true) + }) + + itSessions('[P1] devices list JSON schema 稳定（含 data、total 字段）', async () => { + // 文档用例：devices list JSON schema 稳定 + const result = await r(['auth', 'devices', 'list', '--json']) + assertExitCode(result, 0) + const parsed = assertJson<{ data: unknown[], total: number, page: number, limit: number }>(result) + expect(parsed).toHaveProperty('total') + expect(parsed).toHaveProperty('page') + expect(parsed).toHaveProperty('limit') + }) + + it('[P0] 未登录执行 devices list 返回认证错误（exit code 4）', async () => { + // 文档用例：未登录执行 devices list 返回认证错误 + exit code 4 + const unauthTmp = await withTempConfig() + try { + const result = await run(['auth', 'devices', 'list'], { configDir: unauthTmp.configDir }) + assertExitCode(result, 4) + expect(result.stderr).toMatch(/not.?logged.?in|auth.?login/i) + } + finally { + await unauthTmp.cleanup() + } + }) + + // ── devices revoke ──────────────────────────────────────────────────────── + + itSessions('[P0] revoke 指定 device 成功（exit code 0）', async () => { + // 文档用例：revoke 指定 device 成功 + // Mint a fresh token on demand so this test only revokes its own session, + // never the shared E.token or the global-setup disposableToken. + const freshToken = await mintFreshToken(E.host, E.email, E.password) + if (!freshToken) { + // Credentials not configured — skip rather than risk revoking the main session. + return + } + + // Inject the fresh token into a dedicated config dir + const revokeTmp = await withTempConfig() + try { + await injectAuth(revokeTmp.configDir, { + host: E.host, + bearer: freshToken, + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + const revokeR = (argv: string[]) => run(argv, { configDir: revokeTmp.configDir }) + + // List sessions authenticated as the fresh token + const listResult = await revokeR(['auth', 'devices', 'list', '--json']) + assertExitCode(listResult, 0) + const { data } = assertJson<{ data: Array<{ id: string, prefix: string }> }>(listResult) + + // Find the entry whose prefix matches the fresh token + const entry = data.find(d => d.prefix && freshToken.startsWith(d.prefix)) + if (!entry) { + // Fresh session not found — may have been filtered; skip gracefully. + return + } + + const revokeResult = await revokeR(['auth', 'devices', 'revoke', entry.id, '--yes']) + assertExitCode(revokeResult, 0) + } + finally { + await revokeTmp.cleanup() + } + }) +}) diff --git a/cli/test/e2e/suites/auth/logout.e2e.ts b/cli/test/e2e/suites/auth/logout.e2e.ts new file mode 100644 index 00000000000000..18ce4e57910eaa --- /dev/null +++ b/cli/test/e2e/suites/auth/logout.e2e.ts @@ -0,0 +1,177 @@ +/** + * E2E: difyctl auth logout — Logout + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Auth/Logout（18 条） + */ + +import { access } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode } from '../../helpers/assert.js' +import { injectAuth, run, withTempConfig } from '../../helpers/cli.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +describe('E2E / difyctl auth logout', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const { configDir: dir, cleanup: cl } = await withTempConfig() + configDir = dir + cleanup = cl + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[]) { + return run(argv, { configDir }) + } + + async function withAuth() { + // logout.e2e.ts runs last — safe to use E.token directly here. + await injectAuth(configDir, { + host: E.host, + bearer: E.token, + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + } + + async function hostsFileExists(): Promise { + try { + await access(join(configDir, 'hosts.yml')) + return true + } + catch { return false } + } + + // ── 基础 logout ─────────────────────────────────────────────────────────── + + it('[P0] 已登录用户可正常 logout，stdout 含成功信息', async () => { + // 文档用例：已登录用户可正常 logout + await withAuth() + const result = await r(['auth', 'logout']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/logged out/i) + }) + + it('[P0] logout 后本地 hosts.yml 被删除', async () => { + // 文档用例：logout 后本地 token 被删除 + await withAuth() + expect(await hostsFileExists()).toBe(true) + await r(['auth', 'logout']) + expect(await hostsFileExists()).toBe(false) + }) + + it('[P0] logout 后 auth status 返回 "Not logged in"', async () => { + // 文档用例：logout 后 auth status 返回未登录 + await withAuth() + await r(['auth', 'logout']) + const statusResult = await r(['auth', 'status']) + expect(statusResult.exitCode).toBe(4) + expect(statusResult.stdout).toMatch(/not logged in/i) + }) + + it('[P1] logout 后 auth status exit code 为 4', async () => { + // 文档用例：logout 后 auth status exit code 为 4 + await withAuth() + await r(['auth', 'logout']) + const statusResult = await r(['auth', 'status']) + expect(statusResult.exitCode).toBe(4) + }) + + it('[P0] logout 调用 revoke session 接口（或 best-effort 清除本地凭证）', async () => { + // 文档用例：logout 调用 revoke session 接口 + revoke 成功时 logout 返回成功 + // Uses disposableToken so the shared DIFY_E2E_TOKEN is not revoked. + await withAuth() + const result = await r(['auth', 'logout']) + // 无论 revoke 是否成功，local token 必须清除 + assertExitCode(result, 0) + expect(await hostsFileExists()).toBe(false) + }) + + it('[P0] revoke 失败时仍清除本地凭证（best-effort）', async () => { + // 文档用例：revoke 失败时仍清除本地凭证 + // 注入一个无效 token → server 会拒绝 revoke，但本地应仍被清除 + await injectAuth(configDir, { + host: E.host, + bearer: 'dfoa_invalid_will_fail_revoke', + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + const result = await r(['auth', 'logout']) + // exit 0（best-effort），本地文件被清除 + assertExitCode(result, 0) + expect(await hostsFileExists()).toBe(false) + }) + + // ── 未登录幂等 ──────────────────────────────────────────────────────────── + + it('[P0] 未登录执行 logout 返回 not_logged_in 错误（exit code 4）', async () => { + // 文档用例：未登录执行 logout 幂等成功 + // 实际行为：CLI 对无 token 的 logout 返回 not_logged_in (exit 4) + const result = await r(['auth', 'logout']) + assertExitCode(result, 4) + expect(result.stderr).toMatch(/not.?logged.?in/i) + }) + + // ── 外部 SSO logout ─────────────────────────────────────────────────────── + + it('[P0] 外部 SSO 用户 logout 正常工作，本地 token 清除', async () => { + // 文档用例：外部 SSO 用户 logout 正常工作 + const { writeFile } = await import('node:fs/promises') + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: dfoe_sso_test_token`, + `external_subject:`, + ` email: sso@example.com`, + ` issuer: https://issuer.example.com`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + + const result = await r(['auth', 'logout']) + assertExitCode(result, 0) + expect(await hostsFileExists()).toBe(false) + }) + + // ── 网络异常场景 ────────────────────────────────────────────────────────── + + it('[P0] logout 网络异常时仍执行本地 token 清除', async () => { + // 文档用例：logout 网络异常时仍执行本地清除 + // 使用一个不可达的 host + const { writeFile, mkdir } = await import('node:fs/promises') + await mkdir(configDir, { recursive: true }) + const hostsYml = `${[ + `current_host: http://unreachable-host-xyz.invalid`, + `token_storage: file`, + `tokens:`, + ` bearer: dfoa_test_network_error`, + `workspace:`, + ` id: ws-1`, + ` name: Test`, + ` role: owner`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + + const result = await run(['auth', 'logout'], { configDir, timeout: 10_000 }) + // 即使网络失败，local token 也被清除 + assertExitCode(result, 0) + expect(await hostsFileExists()).toBe(false) + }) + + // ── logout 后操作 ───────────────────────────────────────────────────────── + + it('[P1] logout 后 run app 返回认证错误（exit code 4）', async () => { + // 文档用例：logout 后 run app 返回认证错误 + // Use disposableToken so the shared DIFY_E2E_TOKEN is not revoked. + await withAuth() + await r(['auth', 'logout']) + const result = await r(['run', 'app', E.chatAppId, 'test']) + expect(result.exitCode).toBe(4) + }) +}) diff --git a/cli/test/e2e/suites/auth/status.e2e.ts b/cli/test/e2e/suites/auth/status.e2e.ts new file mode 100644 index 00000000000000..d971ae5bd35859 --- /dev/null +++ b/cli/test/e2e/suites/auth/status.e2e.ts @@ -0,0 +1,180 @@ +/** + * E2E: difyctl auth status — Auth Status + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Auth/Auth Status（12 条） + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode, assertNoAnsi } from '../../helpers/assert.js' +import { injectAuth, run, withTempConfig } from '../../helpers/cli.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +describe('E2E / difyctl auth status', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const { configDir: dir, cleanup: cl } = await withTempConfig() + configDir = dir + cleanup = cl + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[], extraEnv?: Record) { + return run(argv, { configDir, env: extraEnv }) + } + + async function withAuth() { + // Write a complete bundle including account fields so --json output includes account + const { writeFile, mkdir } = await import('node:fs/promises') + const { join } = await import('node:path') + await mkdir(configDir, { recursive: true, mode: 0o700 }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${E.token}`, + `account:`, + ` id: acct-e2e`, + ` email: e2e@example.com`, + ` name: E2E User`, + `workspace:`, + ` id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + `available_workspaces:`, + ` - id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + async function withSSOAuth() { + await injectAuth(configDir, { + host: E.host, + bearer: E.ssoToken || 'dfoe_test', + workspaceId: '', + workspaceName: '', + }) + // Overwrite to add external_subject field + const { writeFile } = await import('node:fs/promises') + const { join } = await import('node:path') + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${E.ssoToken || 'dfoe_test'}`, + `external_subject:`, + ` email: sso@example.com`, + ` issuer: https://issuer.example.com`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + // ── 基础状态显示 ───────────────────────────────────────────────────────── + + it('[P0] 内部用户 auth status 显示 host、email、workspace 信息', async () => { + // 文档用例：内部用户 auth status 显示 host 信息 + await withAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).toContain(E.host.replace(/^https?:\/\//, '')) + expect(result.stdout).toContain(E.workspaceName) + }) + + it('[P0] auth status --json 输出合法 JSON schema', async () => { + // 文档用例：auth status --json 输出可解析 schema + await withAuth() + const result = await r(['auth', 'status', '--json']) + assertExitCode(result, 0) + const parsed = JSON.parse(result.stdout) as Record + expect(parsed).toHaveProperty('logged_in', true) + expect(parsed).toHaveProperty('host') + expect(parsed).toHaveProperty('account') + }) + + it('[P1] auth status -v 显示 workspace role 和 storage 信息', async () => { + // 文档用例：auth status -v 显示 workspace role + await withAuth() + const result = await r(['auth', 'status', '-v']) + assertExitCode(result, 0) + expect(result.stdout).toContain('owner') + expect(result.stdout).toMatch(/file|keychain/) + }) + + // ── 未登录场景 ──────────────────────────────────────────────────────────── + + it('[P0] 未登录执行 auth status 返回 "Not logged in"，exit code 为 4', async () => { + // 文档用例：未登录执行 auth status 返回错误 + exit code 4 + // configDir 为空（无 hosts.yml） + const result = await r(['auth', 'status']) + assertExitCode(result, 4) + expect(result.stdout).toMatch(/not logged in/i) + }) + + // ── 外部 SSO 用户 ───────────────────────────────────────────────────────── + + it('[P0] 外部 SSO 用户 auth status 不显示 workspace 行', async () => { + // 文档用例：外部 SSO 用户 auth status 不显示 workspace + await withSSOAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).not.toMatch(/workspace/i) + }) + + it('[P0] 外部 SSO 用户 auth status 显示 issuer URL', async () => { + // 文档用例：外部 SSO 用户 auth status 显示 issuer URL + await withSSOAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).toContain('issuer.example.com') + }) + + it('[P0] 外部 SSO 用户 auth status 显示 External SSO session 信息', async () => { + // 文档用例：外部 SSO 用户 auth status 显示 External SSO Session + await withSSOAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/SSO|apps:run/i) + }) + + // ── 错误场景 ────────────────────────────────────────────────────────────── + + it('[P0] token 失效（401）后 auth status 返回认证错误', async () => { + // 文档用例：token 失效后 auth status 返回认证错误 + // 注入一个格式合法但实际已失效的 token + await injectAuth(configDir, { + host: E.host, + bearer: 'dfoa_invalid_expired_token_xyz', + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + // auth status 只读本地 hosts.yml，不访问网络，所以本地 token 存在就显示状态 + // 真实的 token 失效检测发生在执行 get app / run app 等命令时 + const result = await r(['auth', 'status']) + // 有 token 就显示状态，不报 401（status 不做网络请求） + assertExitCode(result, 0) + }) + + it('[P1] auth status 在 JSON 模式下错误输出 JSON error envelope', async () => { + // 文档用例：auth status 在 JSON 模式下错误输出为 JSON + const result = await r(['auth', 'status', '--json']) + // 未登录时 --json 模式应输出 JSON 而不是纯文本 + expect(result.exitCode).toBe(4) + // stdout 应含 JSON（not logged in 状态） + const parsed = JSON.parse(result.stdout) as { logged_in: boolean } + expect(parsed.logged_in).toBe(false) + }) + + it('[P0] auth status 输出无 ANSI color（非 TTY）', async () => { + await withAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + assertNoAnsi(result.stdout, 'stdout') + }) +}) diff --git a/cli/test/e2e/suites/auth/use.e2e.ts b/cli/test/e2e/suites/auth/use.e2e.ts new file mode 100644 index 00000000000000..4c67648d5b1f6f --- /dev/null +++ b/cli/test/e2e/suites/auth/use.e2e.ts @@ -0,0 +1,186 @@ +/** + * E2E: difyctl auth use — Workspace 切换 + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Auth/Workspace 切换（22 条） + */ + +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode } from '../../helpers/assert.js' +import { run, withTempConfig } from '../../helpers/cli.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +// 测试用第二工作区 — 注入 available_workspaces 里的备用 workspace +const WS2_ID = 'ws-e2e-secondary-0000-000000000002' +const WS2_NAME = 'Secondary Workspace' + +describe('E2E / difyctl auth use', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const tmp = await withTempConfig() + configDir = tmp.configDir + cleanup = tmp.cleanup + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[]) { + return run(argv, { configDir }) + } + + /** 注入带两个 workspace 的 bundle */ + async function withTwoWorkspaces() { + await mkdir(configDir, { recursive: true }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${E.token}`, + `workspace:`, + ` id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + `available_workspaces:`, + ` - id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + ` - id: ${WS2_ID}`, + ` name: "${WS2_NAME}"`, + ` role: normal`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + async function withSSOAuth() { + await mkdir(configDir, { recursive: true }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: dfoe_sso_test`, + `external_subject:`, + ` email: sso@example.com`, + ` issuer: https://issuer.example.com`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + // ── 正常切换 ────────────────────────────────────────────────────────────── + + it('[P0] 内部用户可切换到指定 workspace', async () => { + // 文档用例：内部用户可切换到指定 workspace + await withTwoWorkspaces() + const result = await r(['auth', 'use', WS2_ID]) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/switched|workspace/i) + expect(result.stdout).toContain(WS2_NAME) + }) + + it('[P0] auth use 后 auth status 显示新 workspace', async () => { + // 文档用例：auth use 后 auth status 显示新 workspace + await withTwoWorkspaces() + await r(['auth', 'use', WS2_ID]) + const status = await r(['auth', 'status']) + assertExitCode(status, 0) + expect(status.stdout).toContain(WS2_NAME) + }) + + it('[P0] auth use 更新 current_workspace_id（hosts.yml 被更新）', async () => { + // 文档用例：auth use 更新 current_workspace_id + await withTwoWorkspaces() + await r(['auth', 'use', WS2_ID]) + const { readFile } = await import('node:fs/promises') + const hostsContent = await readFile(join(configDir, 'hosts.yml'), 'utf8') + expect(hostsContent).toContain(WS2_ID) + }) + + it('[P1] 重复切换同一 workspace 幂等成功', async () => { + // 文档用例：重复切换同一 workspace 幂等成功 + await withTwoWorkspaces() + const r1 = await r(['auth', 'use', E.workspaceId]) + assertExitCode(r1, 0) + const r2 = await r(['auth', 'use', E.workspaceId]) + assertExitCode(r2, 0) + }) + + it('[P1] auth use 后 current workspace 在重新读取时持久化', async () => { + // 文档用例：auth use 后 current workspace 持久化 + await withTwoWorkspaces() + await r(['auth', 'use', WS2_ID]) + // 直接读 hosts.yml 验证 workspace id 被写入 + const { readFile } = await import('node:fs/promises') + const { join } = await import('node:path') + const hostsContent = await readFile(join(configDir, 'hosts.yml'), 'utf8') + expect(hostsContent).toContain(WS2_ID) + }) + + // ── 错误场景 ────────────────────────────────────────────────────────────── + + it('[P0] 切换不存在 workspace 返回错误', async () => { + // 文档用例：切换不存在 workspace 返回错误 + await withTwoWorkspaces() + const result = await r(['auth', 'use', 'ws-does-not-exist-xyz']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/not found|workspace/i) + }) + + it('[P0] workspace 切换失败时 current_workspace_id 不变', async () => { + // 文档用例：workspace 切换失败时 current_workspace_id 不变 + await withTwoWorkspaces() + await r(['auth', 'use', 'ws-does-not-exist-xyz']) + // 直接读 hosts.yml，原 workspace id 应仍存在 + const { readFile } = await import('node:fs/promises') + const { join } = await import('node:path') + const hostsContent = await readFile(join(configDir, 'hosts.yml'), 'utf8') + expect(hostsContent).toContain(E.workspaceId) + }) + + it('[P0] 未登录执行 auth use 返回认证错误（exit code 4）', async () => { + // 文档用例：未登录执行 auth use 返回认证错误 + exit code 4 + const result = await r(['auth', 'use', E.workspaceId]) + assertExitCode(result, 4) + expect(result.stderr).toMatch(/not.?logged.?in|auth.?login/i) + }) + + it('[P0] workspace 参数缺失时返回 usage error', async () => { + // 文档用例：workspace 参数缺失时返回 usage error + await withTwoWorkspaces() + const result = await r(['auth', 'use']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/missing required argument|workspace/i) + }) + + // ── 外部 SSO 用户 ───────────────────────────────────────────────────────── + + it('[P0] 外部 SSO 用户执行 auth use 被拒绝', async () => { + // 文档用例：外部 SSO 用户执行 auth use 被拒绝 + await withSSOAuth() + const result = await r(['auth', 'use', 'any-ws-id']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/external SSO|workspace/i) + }) + + it('[P1] 外部 SSO 用户 auth use exit code 为 1 或 2', async () => { + // 文档用例：外部 SSO 用户 auth use exit code 为 1 + await withSSOAuth() + const result = await r(['auth', 'use', 'any-ws-id']) + expect([1, 2]).toContain(result.exitCode) + }) + + // ── JSON 模式 ───────────────────────────────────────────────────────────── + + it('[P1] workspace 不存在时 stderr 包含错误描述', async () => { + // 文档用例：workspace 不存在时返回错误 + // Note: auth use 不支持 -o flag，错误通过 stderr 文本输出 + await withTwoWorkspaces() + const result = await r(['auth', 'use', 'ws-nonexistent-xyz']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/not.?found|workspace/i) + }) +}) diff --git a/cli/test/e2e/suites/auth/whoami.e2e.ts b/cli/test/e2e/suites/auth/whoami.e2e.ts new file mode 100644 index 00000000000000..dbb82ff6f534c8 --- /dev/null +++ b/cli/test/e2e/suites/auth/whoami.e2e.ts @@ -0,0 +1,175 @@ +/** + * E2E: difyctl auth whoami + 外部 SSO 登录行为验证 + * + * 用例来源：飞书文档《Dify CLI Enhanced》 + * - Dify CLI/Auth/外部 SSO 登录（19 条，可测试部分） + * + * 注：交互式登录（Device Flow browser） 和 Headless 认证需要真实浏览器， + * E2E 层通过 injectAuth 跳过 Device Flow，专注验证 session 状态和命令行为。 + */ + +import { mkdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode } from '../../helpers/assert.js' +import { run, withTempConfig } from '../../helpers/cli.js' +import { optionalIt } from '../../helpers/skip.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +describe('E2E / difyctl auth whoami + SSO session', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const tmp = await withTempConfig() + configDir = tmp.configDir + cleanup = tmp.cleanup + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[]) { + return run(argv, { configDir }) + } + + async function withInternalAuth() { + await mkdir(configDir, { recursive: true, mode: 0o700 }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${E.token}`, + `account:`, + ` id: acct-e2e`, + ` email: e2e-user@example.com`, + ` name: E2E User`, + `workspace:`, + ` id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + `available_workspaces:`, + ` - id: ${E.workspaceId}`, + ` name: "${E.workspaceName}"`, + ` role: owner`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + async function withSSOAuth(issuer = 'https://idp.example.com') { + await mkdir(configDir, { recursive: true }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: dfoe_sso_test_token`, + `external_subject:`, + ` email: sso-user@example.com`, + ` issuer: ${issuer}`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + } + + // ── auth whoami — 内部用户 ──────────────────────────────────────────────── + + it('[P0] 内部用户 auth whoami 输出 email', async () => { + await withInternalAuth() + const result = await r(['auth', 'whoami']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/@/) + }) + + it('[P0] auth whoami --json 输出合法 JSON，包含 email', async () => { + await withInternalAuth() + const result = await r(['auth', 'whoami', '--json']) + assertExitCode(result, 0) + const parsed = JSON.parse(result.stdout) as { email: string } + expect(parsed).toHaveProperty('email') + expect(parsed.email).toMatch(/@/) + }) + + it('[P0] 未登录 auth whoami 返回认证错误（exit code 4）', async () => { + const result = await r(['auth', 'whoami']) + assertExitCode(result, 4) + }) + + // ── 外部 SSO 用户行为 ───────────────────────────────────────────────────── + + it('[P0] 外部 SSO 用户 auth status 显示 apps:run only 限制', async () => { + // 文档用例：auth status 显示 apps:run only 限制 + await withSSOAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/apps:run|SSO/i) + }) + + it('[P0] 外部 SSO 用户 auth status 不显示 workspace 信息', async () => { + // 文档用例：auth status 不显示 workspace 信息 + await withSSOAuth() + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + // SSO 用户没有 workspace + expect(result.stdout).not.toMatch(/^ {2}Workspace:/m) + }) + + it('[P0] 外部 SSO 用户 auth status 显示 issuer URL', async () => { + // 文档用例：auth status 显示 External SSO Session + issuer URL + await withSSOAuth('https://idp.enterprise.com') + const result = await r(['auth', 'status']) + assertExitCode(result, 0) + expect(result.stdout).toContain('idp.enterprise.com') + }) + + it('[P0] 外部用户执行 auth use 返回错误（external SSO subjects have no workspaces）', async () => { + // 文档用例：外部用户执行 auth use 返回错误 + await withSSOAuth() + const result = await r(['auth', 'use', 'any-ws-id']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/external SSO|workspace/i) + }) + + it('[P0] 外部用户 get workspace 返回空列表或 insufficient_scope', async () => { + // 文档用例：外部用户 get workspace 返回空列表 + await withSSOAuth() + const result = await r(['get', 'workspace']) + // SSO token 无 workspace 权限 + expect(result.exitCode).not.toBe(0) + }) + + it('[P0] 外部用户 get app 返回 insufficient_scope 错误', async () => { + // 文档用例：外部用户 get app 返回 insufficient_scope + await withSSOAuth() + const result = await r(['get', 'app']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/insufficient|scope|workspace|SSO/i) + }) + + it('[P0] 外部用户 whoami 输出 SSO email', async () => { + await withSSOAuth() + const result = await r(['auth', 'whoami']) + assertExitCode(result, 0) + expect(result.stdout).toContain('sso-user@example.com') + }) + + const itWithSso = optionalIt(Boolean(E.ssoToken)) + + itWithSso('[P0] 外部用户可执行 run app（使用 SSO token）', async () => { + await mkdir(configDir, { recursive: true }) + const hostsYml = `${[ + `current_host: ${E.host}`, + `token_storage: file`, + `tokens:`, + ` bearer: ${E.ssoToken}`, + `external_subject:`, + ` email: sso@example.com`, + ` issuer: https://issuer.example.com`, + ].join('\n')}\n` + await writeFile(join(configDir, 'hosts.yml'), hostsYml, { mode: 0o600 }) + + const result = await r(['run', 'app', E.chatAppId, 'hello']) + assertExitCode(result, 0) + expect(result.stdout.length).toBeGreaterThan(0) + }) +}) diff --git a/cli/test/e2e/suites/config/config.e2e.ts b/cli/test/e2e/suites/config/config.e2e.ts new file mode 100644 index 00000000000000..b74577df2c1454 --- /dev/null +++ b/cli/test/e2e/suites/config/config.e2e.ts @@ -0,0 +1,299 @@ +/** + * E2E: difyctl config — 配置管理 + * + * 用例来源：飞书文档《Dify CLI Enhanced》 + * - Dify CLI/Config/初始化与默认路径（26 条，可测部分） + * - Dify CLI/Config/环境变量覆盖优先级（26 条，可测部分） + * + * 覆盖子命令：config path / config get / config set / config unset / config view + * 不依赖真实 Dify 服务器，所有用例纯本地执行。 + */ + +import { access, mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode, assertNoAnsi } from '../../helpers/assert.js' +import { run, withTempConfig } from '../../helpers/cli.js' + +describe('E2E / difyctl config', () => { + let configDir: string + let cleanup: () => Promise + + beforeEach(async () => { + const tmp = await withTempConfig() + configDir = tmp.configDir + cleanup = tmp.cleanup + }) + afterEach(async () => { + await cleanup() + }) + + function r(argv: string[], extraEnv?: Record) { + return run(argv, { configDir, env: extraEnv }) + } + + // ── config path ────────────────────────────────────────────────────────── + + it('[P0] config path 返回正确的 config.yml 绝对路径', async () => { + // 文档用例：默认 config 路径正确 + const result = await r(['config', 'path']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe(join(configDir, 'config.yml')) + }) + + it('[P0] config path 输出以换行符结尾', async () => { + const result = await r(['config', 'path']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/config\.yml\n$/) + }) + + // ── config set / get ───────────────────────────────────────────────────── + + it('[P0] config set defaults.format 写入成功，stdout 含 key=value', async () => { + const result = await r(['config', 'set', 'defaults.format', 'json']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/defaults\.format/) + }) + + it('[P0] config get 读取已写入的 defaults.format', async () => { + await r(['config', 'set', 'defaults.format', 'json']) + const result = await r(['config', 'get', 'defaults.format']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe('json') + }) + + it('[P0] config set defaults.limit 写入并读取正确', async () => { + await r(['config', 'set', 'defaults.limit', '50']) + const result = await r(['config', 'get', 'defaults.limit']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe('50') + }) + + it('[P0] config set state.current_app 写入并读取正确', async () => { + const appId = 'app-e2e-config-test' + await r(['config', 'set', 'state.current_app', appId]) + const result = await r(['config', 'get', 'state.current_app']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe(appId) + }) + + it('[P0] config get 未设置的 key 返回空值（exit 0）', async () => { + // 文档用例：config 文件缺少字段时使用默认值 + const result = await r(['config', 'get', 'defaults.format']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe('') + }) + + it('[P1] 多次 config set 不同 key，各 key 独立持久化', async () => { + // 文档用例：config 已存在时不重复覆盖 + await r(['config', 'set', 'defaults.format', 'yaml']) + await r(['config', 'set', 'defaults.limit', '30']) + const fmt = await r(['config', 'get', 'defaults.format']) + const lim = await r(['config', 'get', 'defaults.limit']) + expect(fmt.stdout.trim()).toBe('yaml') + expect(lim.stdout.trim()).toBe('30') + }) + + // ── config unset ───────────────────────────────────────────────────────── + + it('[P0] config unset 清除已设置的 key，get 返回空值', async () => { + await r(['config', 'set', 'defaults.format', 'json']) + await r(['config', 'unset', 'defaults.format']) + const result = await r(['config', 'get', 'defaults.format']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe('') + }) + + it('[P1] config unset 未设置的 key 幂等成功（exit 0）', async () => { + const result = await r(['config', 'unset', 'defaults.format']) + assertExitCode(result, 0) + }) + + it('[P1] config unset 后其他 key 不受影响', async () => { + await r(['config', 'set', 'defaults.format', 'table']) + await r(['config', 'set', 'defaults.limit', '10']) + await r(['config', 'unset', 'defaults.format']) + const lim = await r(['config', 'get', 'defaults.limit']) + expect(lim.stdout.trim()).toBe('10') + }) + + // ── config view ────────────────────────────────────────────────────────── + + it('[P0] config view 空配置输出为空', async () => { + const result = await r(['config', 'view']) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe('') + }) + + it('[P0] config view 显示所有已设置的 key = value', async () => { + await r(['config', 'set', 'defaults.format', 'yaml']) + await r(['config', 'set', 'defaults.limit', '20']) + const result = await r(['config', 'view']) + assertExitCode(result, 0) + expect(result.stdout).toContain('defaults.format = yaml') + expect(result.stdout).toContain('defaults.limit = 20') + }) + + it('[P0] config view --json 输出合法 JSON，含已设置 key', async () => { + await r(['config', 'set', 'defaults.format', 'json']) + await r(['config', 'set', 'defaults.limit', '15']) + const result = await r(['config', 'view', '--json']) + assertExitCode(result, 0) + const parsed = JSON.parse(result.stdout) as Record + expect(parsed).toHaveProperty('defaults.format', 'json') + expect(parsed).toHaveProperty('defaults.limit', 15) + }) + + it('[P1] config view --json 空配置输出合法空 JSON 对象', async () => { + const result = await r(['config', 'view', '--json']) + assertExitCode(result, 0) + const parsed = JSON.parse(result.stdout) + expect(typeof parsed).toBe('object') + }) + + // ── 初始化与默认路径 ────────────────────────────────────────────────────── + + it('[P0] 首次 config set 自动创建 config 目录和 config.yml 文件', async () => { + // 文档用例：首次启动自动创建 config 目录 / 自动创建 config 文件 + await r(['config', 'set', 'defaults.format', 'json']) + await expect(access(join(configDir, 'config.yml'))).resolves.toBeUndefined() + }) + + it('[P0] config.yml 文件权限为 0o600', async () => { + // 文档用例：config 文件默认权限正确 + await r(['config', 'set', 'defaults.format', 'json']) + const info = await stat(join(configDir, 'config.yml')) + expect(info.mode & 0o777).toBe(0o600) + }) + + it('[P0] config.yml 包含正确的 schema_version 字段', async () => { + // 文档用例：config 文件默认 schema 正确 + await r(['config', 'set', 'defaults.format', 'json']) + const raw = await import('node:fs/promises').then(fs => + fs.readFile(join(configDir, 'config.yml'), 'utf8'), + ) + expect(raw).toMatch(/schema_version/) + }) + + it('[P0] config 内容为非法 YAML 时返回解析错误', async () => { + // 文档用例：config 内容非法时返回错误 + await mkdir(configDir, { recursive: true }) + await writeFile(join(configDir, 'config.yml'), ': broken: yaml: [[[', { mode: 0o600 }) + const result = await r(['config', 'get', 'defaults.format']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/parse|yaml|config/i) + }) + + it('[P0] schema_version 高于当前版本时返回 config_schema_unsupported 错误', async () => { + // 文档用例：config 文件 schema_version 高于支持版本时返回错误 + await mkdir(configDir, { recursive: true }) + await writeFile( + join(configDir, 'config.yml'), + 'schema_version: 999\ndefaults: {}\nstate: {}\n', + { mode: 0o600 }, + ) + const result = await r(['config', 'get', 'defaults.format']) + expect(result.exitCode).toBe(6) // VersionCompat + expect(result.stderr).toMatch(/schema_version|unsupported|upgrade/i) + }) + + it('[P0] DIFY_CONFIG_DIR 覆盖默认路径，config path 返回指定目录下的路径', async () => { + // 文档用例：DIFYCTL_CONFIG 环境变量覆盖默认路径 + const altDir = await mkdtemp(join(tmpdir(), 'difyctl-alt-')) + try { + const result = await run(['config', 'path'], { configDir: altDir }) + assertExitCode(result, 0) + expect(result.stdout.trim()).toBe(join(altDir, 'config.yml')) + } + finally { + await rm(altDir, { recursive: true, force: true }) + } + }) + + it('[P0] 临时 DIFY_CONFIG_DIR 不影响原 config 目录的内容', async () => { + // 文档用例：临时 env 注入不修改 config 文件 + await r(['config', 'set', 'defaults.format', 'yaml']) + + const altDir = await mkdtemp(join(tmpdir(), 'difyctl-alt-')) + try { + await run(['config', 'set', 'defaults.format', 'json'], { configDir: altDir }) + // 原 configDir 内容不变 + const original = await r(['config', 'get', 'defaults.format']) + expect(original.stdout.trim()).toBe('yaml') + } + finally { + await rm(altDir, { recursive: true, force: true }) + } + }) + + // ── 错误场景 ────────────────────────────────────────────────────────────── + + it('[P0] config get 未知 key 返回 exit code 2', async () => { + const result = await r(['config', 'get', 'unknown.key']) + expect(result.exitCode).toBe(2) + expect(result.stderr).toMatch(/unknown config key/i) + }) + + it('[P0] config set 未知 key 返回 exit code 2', async () => { + const result = await r(['config', 'set', 'unknown.key', 'val']) + expect(result.exitCode).toBe(2) + expect(result.stderr).toMatch(/unknown config key/i) + }) + + it('[P0] config unset 未知 key 返回 exit code 2', async () => { + const result = await r(['config', 'unset', 'unknown.key']) + expect(result.exitCode).toBe(2) + expect(result.stderr).toMatch(/unknown config key/i) + }) + + it('[P0] config set defaults.format 非法值返回 exit code 2', async () => { + // 文档用例：config_invalid_value → usage error + const result = await r(['config', 'set', 'defaults.format', 'not_a_format']) + expect(result.exitCode).toBe(2) + expect(result.stderr).toMatch(/defaults\.format|not one of/i) + }) + + it('[P0] config set defaults.limit 0 低于最小值返回 exit code 2', async () => { + const result = await r(['config', 'set', 'defaults.limit', '0']) + expect(result.exitCode).toBe(2) + }) + + it('[P0] config set defaults.limit 201 超出最大值返回 exit code 2', async () => { + const result = await r(['config', 'set', 'defaults.limit', '201']) + expect(result.exitCode).toBe(2) + }) + + it('[P0] config set defaults.limit 非数字字符串返回 exit code 2', async () => { + const result = await r(['config', 'set', 'defaults.limit', 'abc']) + expect(result.exitCode).toBe(2) + }) + + it('[P1] config set 缺少 value 参数返回错误', async () => { + const result = await r(['config', 'set', 'defaults.format']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/missing required argument/i) + }) + + it('[P1] config get 缺少 key 参数返回错误', async () => { + const result = await r(['config', 'get']) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/missing required argument/i) + }) + + // ── 输出格式 ────────────────────────────────────────────────────────────── + + it('[P0] config 输出无 ANSI color（非 TTY 环境）', async () => { + await r(['config', 'set', 'defaults.format', 'json']) + const result = await r(['config', 'view']) + assertExitCode(result, 0) + assertNoAnsi(result.stdout, 'stdout') + }) + + it('[P0] config 初始化/操作不泄露敏感信息（token/secret）', async () => { + // 文档用例：config 初始化日志不泄露敏感信息 + const result = await r(['config', 'view']) + assertExitCode(result, 0) + expect(result.stdout + result.stderr).not.toMatch(/dfoa_|dfoe_|secret|password/i) + }) +}) diff --git a/cli/test/e2e/suites/run/run-app-basic.e2e.ts b/cli/test/e2e/suites/run/run-app-basic.e2e.ts new file mode 100644 index 00000000000000..b1dcf052aa740d --- /dev/null +++ b/cli/test/e2e/suites/run/run-app-basic.e2e.ts @@ -0,0 +1,452 @@ +/** + * E2E: difyctl run app — 基础 App 运行 + Streaming + Conversation + * + * 用例来源：飞书文档《Dify CLI Enhanced》 + * - Dify CLI/Run/基础 App 运行（26 条） + * - Dify CLI/Run/Streaming 输出（部分，完整见 run-app-streaming.e2e.ts） + * - Dify CLI/Run/Conversation 模式（部分） + * - Dify CLI/Error Handling/Exit Code（run 相关） + * - Dify CLI/CLI Framework/Non-Interactive（run 相关） + * + * Staging app 前置条件（由 DIFY_E2E_* 环境变量指定）： + * echo-chat — mode=chat，query 变量，输出 "echo: {query}" + * echo-workflow — mode=workflow，x 变量（required），输出 "echo: {x}" + */ + +import type { AuthFixture } from '../../helpers/cli.js' +import { writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + assertErrorEnvelope, + assertExitCode, + assertJson, + assertNoAnsi, + assertPipeFriendlyJson, + assertStderrContains, + assertStdoutContains, +} from '../../helpers/assert.js' +import { registerConversation } from '../../helpers/cleanup-registry.js' +import { run, withAuthFixture, withTempConfig } from '../../helpers/cli.js' +import { withRetry } from '../../helpers/retry.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +// ── Suite ────────────────────────────────────────────────────────────────── + +describe('E2E / difyctl run app', () => { + let fx: AuthFixture + + beforeEach(async () => { + fx = await withAuthFixture(E) + }) + afterEach(async () => { + await fx.cleanup() + }) + + // ========================================================================= + // 基础执行 + // ========================================================================= + + describe('基础执行', () => { + it('[P0] 已登录内部用户可运行 app，stdout 输出 app 结果', async () => { + // 文档用例：已登录内部用户可运行 app / 默认输出执行结果 + // withRetry: staging LLM inference may have transient 5xx on cold start + const result = await withRetry(() => fx.r(['run', 'app', E.chatAppId, 'hello']), { + attempts: 3, + delayMs: 2000, + shouldRetry: err => err instanceof Error && /5\d{2}|ECONNRESET|timeout/i.test(err.message), + }) + assertExitCode(result, 0) + assertStdoutContains(result, 'echo:hello') + }) + + it('[P0] run app 调用 execute endpoint（stdout 有实际内容）', async () => { + // 文档用例：run app 调用 execute endpoint + const result = await fx.r(['run', 'app', E.chatAppId, 'e2e-smoke']) + assertExitCode(result, 0) + expect(result.stdout.length).toBeGreaterThan(0) + }) + + it('[P1] 文本输出保留换行（stdout 末尾为 \\n）', async () => { + // 文档用例：文本输出保留换行 + const result = await fx.r(['run', 'app', E.chatAppId, 'newline']) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/\n$/) + }) + + it('[P1] 重复执行 run app 每次独立完成（3 次循环）', async () => { + // 文档用例：重复执行 run app 不影响历史状态 + for (let i = 0; i < 3; i++) { + const result = await fx.r(['run', 'app', E.chatAppId, `repeat-${i}`]) + assertExitCode(result, 0) + assertStdoutContains(result, `echo:repeat-${i}`) + } + }) + }) + + // ========================================================================= + // 输出格式 + // ========================================================================= + + describe('出格式 (-o)', () => { + it('[P0] -o json 输出合法 JSON', async () => { + // 文档用例：-o json 输出合法 JSON + const result = await fx.r(['run', 'app', E.chatAppId, 'json-test', '-o', 'json']) + assertExitCode(result, 0) + const parsed = assertJson<{ answer: string, mode: string }>(result) + expect(parsed).toHaveProperty('answer') + expect(parsed.mode).toMatch(/chat/) + }) + + it('[P1] JSON 输出包含 execution metadata（message_id / conversation_id）', async () => { + // 文档用例：JSON 输出包含 execution metadata + const result = await fx.r(['run', 'app', E.chatAppId, 'meta', '-o', 'json']) + assertExitCode(result, 0) + const parsed = assertJson>(result) + expect(parsed).toHaveProperty('message_id') + expect(parsed).toHaveProperty('conversation_id') + }) + + it('[P1] JSON 输出支持 pipe（无 ANSI，首字符为 {，末尾为 \\n）', async () => { + // 文档用例：JSON 输出支持 pipe + const result = await fx.r(['run', 'app', E.chatAppId, 'pipe', '-o', 'json']) + assertExitCode(result, 0) + assertPipeFriendlyJson(result) + }) + + it('[P1] JSON 模式错误输出 JSON error envelope 到 stderr', async () => { + // 文档用例：JSON 模式错误输出 JSON envelope + const result = await fx.r(['run', 'app', 'app-nonexistent-xyz-e2e', 'hello', '-o', 'json']) + assertNonZeroExit(result) + assertErrorEnvelope(result, 'server_4xx_other') + }) + }) + + // ========================================================================= + // --inputs 参数 + // ========================================================================= + + describe('--inputs 参数', () => { + it('[P0] run app 支持 --inputs（workflow app）', async () => { + // 文档用例：run app 支持 --input + // withRetry: staging workflow execution may have transient 5xx + const result = await withRetry( + () => fx.r(['run', 'app', E.workflowAppId, '--inputs', JSON.stringify({ x: 'workflow-val' })]), + { attempts: 3, delayMs: 2000, shouldRetry: err => err instanceof Error && /5\d{2}|ECONNRESET|timeout/i.test(err.message) }, + ) + assertExitCode(result, 0) + assertStdoutContains(result, 'workflow-val') + }) + + it('[P0] 多个 inputs 同时生效', async () => { + // 文档用例：多个 --input 参数同时生效 + const result = await fx.r([ + 'run', + 'app', + E.workflowAppId, + '--inputs', + JSON.stringify({ x: 'multi-test' }), + ]) + assertExitCode(result, 0) + }) + + it('[P0] --inputs 为非法 JSON 返回 usage error（exit code 2）', async () => { + // 文档用例：必填参数缺失 / 非法 input + const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs', 'not-json']) + assertExitCode(result, 2) + expect(result.stderr).toMatch(/valid JSON/i) + }) + + it('[P0] --inputs 为 JSON 数组返回 usage error', async () => { + const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs', '[1,2,3]']) + assertExitCode(result, 2) + expect(result.stderr).toMatch(/JSON object/i) + }) + + it('[P0] --inputs 与 --inputs-file 互斥时返回 usage error', async () => { + // 文档用例：enum 参数非法值返回错误（usage 类别） + const inputsFile = join(fx.configDir, 'inputs.json') + await writeFile(inputsFile, JSON.stringify({ x: 'file-val' })) + const result = await fx.r([ + 'run', + 'app', + E.workflowAppId, + '--inputs', + '{"x":"flag-val"}', + '--inputs-file', + inputsFile, + ]) + assertExitCode(result, 2) + expect(result.stderr).toMatch(/mutually exclusive/i) + }) + + it('[P0] workflow app 传入 positional message 返回 usage error', async () => { + // 文档用例：必填参数缺失时执行失败（workflow positional） + const result = await fx.r(['run', 'app', E.workflowAppId, 'positional-msg']) + assertExitCode(result, 2) + expect(result.stderr).toMatch(/workflow apps do not accept a positional message/i) + }) + + it('[P0] --inputs-file 从文件读取 JSON inputs', async () => { + const inputsFile = join(fx.configDir, 'wf-inputs.json') + await writeFile(inputsFile, JSON.stringify({ x: 'from-file' })) + const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs-file', inputsFile]) + assertExitCode(result, 0) + assertStdoutContains(result, 'from-file') + }) + }) + + // ========================================================================= + // 错误场景 + // ========================================================================= + + describe('错误场景', () => { + it('[P0] app 不存在返回错误，exit code 为 1', async () => { + // 文档用例：app 不存在返回 app not found + exit code 为 1 + const result = await fx.r(['run', 'app', 'app-id-does-not-exist-e2e-xyz', 'hello']) + assertExitCode(result, 1) + expect(result.stderr).toMatch(/not.?found/i) + }) + + it('[P0] 缺少 app id 返回错误（exit code 1，CLI 对 missing required arg 返回 1）', async () => { + // 文档用例：缺少 app id 返回 usage error + // 实际行为：CLI framework 对 missing required argument 返回 exit 1（不是 2） + const result = await fx.r(['run', 'app']) + assertExitCode(result, 1) + expect(result.stderr).toMatch(/missing required argument/i) + }) + + it('[P0] 未登录执行 run app 返回认证错误（exit code 4）', async () => { + // 文档用例：未登录执行 run app 返回认证错误 + exit code 为 4 + const unauthTmp = await withTempConfig() + try { + const result = await run(['run', 'app', E.chatAppId, 'hello'], { + configDir: unauthTmp.configDir, + }) + assertExitCode(result, 4) + expect(result.stderr).toMatch(/not.?logged.?in|auth.?login/i) + } + finally { + await unauthTmp.cleanup() + } + }) + }) + + // ========================================================================= + // Streaming 输出 + // ========================================================================= + + describe('Streaming 输出', () => { + it('[P0] --stream 可正常接收流式输出，stdout 有内容', async () => { + // 文档用例：run app --stream 可正常接收流式输出 + const result = await fx.r(['run', 'app', E.chatAppId, 'stream-test', '--stream']) + assertExitCode(result, 0) + assertStdoutContains(result, 'echo:stream-test') + }) + + it('[P0] streaming 结束后 exit code 为 0', async () => { + // 文档用例：streaming 结束后正常退出 + const result = await fx.r(['run', 'app', E.chatAppId, 'end-ok', '--stream']) + assertExitCode(result, 0) + }) + + it('[P1] streaming 模式下 stderr 不混入 stdout', async () => { + // 文档用例：streaming 模式下 stderr 不混入 stdout + const result = await fx.r(['run', 'app', E.chatAppId, 'sep', '--stream']) + assertExitCode(result, 0) + expect(result.stdout).not.toContain('hint:') + assertStderrContains(result, '--conversation') + }) + + it('[P1] --stream -o json 输出合法 JSON envelope', async () => { + // 文档用例：streaming 模式下 JSON 输出合法 + const result = await fx.r(['run', 'app', E.chatAppId, 'sjson', '--stream', '-o', 'json']) + assertExitCode(result, 0) + const parsed = assertJson<{ mode: string, answer: string }>(result) + expect(parsed.mode).toMatch(/chat/) + }) + + it('[P0] streaming app 不存在返回错误（exit code 1）', async () => { + // 文档用例：streaming app 不存在返回错误 + const result = await fx.r(['run', 'app', 'nonexistent-xyz-e2e', 'hi', '--stream']) + assertExitCode(result, 1) + }) + + it('[P0] 未登录执行 streaming 返回认证错误（exit code 4）', async () => { + // 文档用例：未登录执行 streaming 返回认证错误 + const unauthTmp = await withTempConfig() + try { + const result = await run(['run', 'app', E.chatAppId, 'hi', '--stream'], { + configDir: unauthTmp.configDir, + }) + assertExitCode(result, 4) + } + finally { + await unauthTmp.cleanup() + } + }) + + it('[P1] streaming 模式输出支持 pipe（无 ANSI，末尾 \\n）', async () => { + // 文档用例：streaming 模式输出支持 pipe + const result = await fx.r(['run', 'app', E.chatAppId, 'pipe-s', '--stream']) + assertExitCode(result, 0) + assertNoAnsi(result.stdout, 'stdout') + expect(result.stdout.endsWith('\n')).toBe(true) + }) + + it('[P0] workflow streaming 输出包含 succeeded 状态', async () => { + const result = await fx.r([ + 'run', + 'app', + E.workflowAppId, + '--inputs', + JSON.stringify({ x: 'wf-stream-val' }), + '--stream', + '-o', + 'json', + ]) + assertExitCode(result, 0) + const parsed = assertJson<{ data?: { status?: string } }>(result) + expect(parsed.data?.status).toBe('succeeded') + }) + }) + + // ========================================================================= + // Conversation 模式 + // ========================================================================= + + describe('Conversation 模式', () => { + it('[P0] chat app 可创建新 conversation，stderr 含 hint', async () => { + // 文档用例：chat app 可创建新 conversation + const result = await fx.r(['run', 'app', E.chatAppId, 'start-conv']) + assertExitCode(result, 0) + assertStderrContains(result, '--conversation') + }) + + it('[P0] JSON 输出包含 conversation_id', async () => { + // 文档用例：JSON 输出包含 conversation_id + const result = await fx.r(['run', 'app', E.chatAppId, 'conv-json', '-o', 'json']) + assertExitCode(result, 0) + const parsed = assertJson<{ conversation_id: string }>(result) + expect(typeof parsed.conversation_id).toBe('string') + expect(parsed.conversation_id.length).toBeGreaterThan(0) + registerConversation(E.host, E.token, E.chatAppId, parsed.conversation_id) + }) + + it('[P0] --conversation 参数生效：conversation_id 在后续请求中复用', async () => { + // 文档用例：--conversation 参数生效 + conversation_id 在后续请求中复用 + const first = await fx.r(['run', 'app', E.chatAppId, 'first-msg', '-o', 'json']) + assertExitCode(first, 0) + const { conversation_id } = assertJson<{ conversation_id: string }>(first) + registerConversation(E.host, E.token, E.chatAppId, conversation_id) + + const second = await fx.r([ + 'run', + 'app', + E.chatAppId, + 'second-msg', + '--conversation', + conversation_id, + '-o', + 'json', + ]) + assertExitCode(second, 0) + const secondParsed = assertJson<{ conversation_id: string }>(second) + expect(secondParsed.conversation_id).toBe(conversation_id) + }) + + it('[P0] 不传 conversation_id 时自动创建新会话', async () => { + // 文档用例：conversation_id 缺失时自动创建新会话 + const result = await fx.r(['run', 'app', E.chatAppId, 'new-conv', '-o', 'json']) + assertExitCode(result, 0) + const { conversation_id } = assertJson<{ conversation_id: string }>(result) + expect(conversation_id).toBeTruthy() + }) + + it('[P0] 非法 conversation_id 返回错误（exit code 1）', async () => { + // 文档用例：非法 conversation_id 返回错误 + const result = await fx.r([ + 'run', + 'app', + E.chatAppId, + 'bad-conv', + '--conversation', + 'invalid-conv-id-xyz-not-exist', + ]) + assertNonZeroExit(result) + }) + + it('[P1] conversation 模式支持 streaming', async () => { + // 文档用例：conversation 模式支持 streaming + const first = await fx.r(['run', 'app', E.chatAppId, 'init', '-o', 'json']) + const { conversation_id } = assertJson<{ conversation_id: string }>(first) + + const result = await fx.r([ + 'run', + 'app', + E.chatAppId, + 'continue', + '--conversation', + conversation_id, + '--stream', + ]) + assertExitCode(result, 0) + assertStdoutContains(result, 'echo:') + }) + + it('[P1] conversation 输出支持 pipe（-o json pipe 友好格式）', async () => { + // 文档用例：conversation 输出支持 pipe + const result = await fx.r(['run', 'app', E.chatAppId, 'pipe-conv', '-o', 'json']) + assertExitCode(result, 0) + assertPipeFriendlyJson(result) + }) + }) + + // ========================================================================= + // 非交互模式 / CI 环境 + // ========================================================================= + + describe('非交互模式 (CI)', () => { + it('[P0] CI=1 环境无 spinner，stdout 无 ANSI color', async () => { + // 文档用例：非 tty 环境自动关闭 ANSI color + 非交互模式不输出 spinner + const result = await fx.r(['run', 'app', E.chatAppId, 'ci-test'], { CI: '1', NO_COLOR: '1' }) + assertExitCode(result, 0) + assertNoAnsi(result.stdout, 'stdout') + assertNoAnsi(result.stderr, 'stderr') + }) + + it('[P0] 非交互模式 exit code 正确传递', async () => { + // 文档用例：非交互模式 exit code 正确 + const result = await fx.r(['run', 'app', E.chatAppId, 'code']) + expect(typeof result.exitCode).toBe('number') + expect(result.exitCode).toBe(0) + }) + }) + + // ========================================================================= + // workspace override + // ========================================================================= + + describe('workspace override', () => { + it('[P1] --workspace flag 覆盖默认 workspace', async () => { + // 文档用例：workspace override 生效 + // run app 使用 --workspace（无 -w 短形式） + const result = await fx.r([ + 'run', + 'app', + E.chatAppId, + 'ws-override', + '--workspace', + E.workspaceId, + ]) + assertExitCode(result, 0) + }) + }) +}) + +// ── local helper (avoids import confusion) ───────────────────────────────── +function assertNonZeroExit(result: import('../../helpers/cli.js').RunResult): void { + expect(result.exitCode, 'exit code should be non-zero').not.toBe(0) +} diff --git a/cli/test/e2e/suites/run/run-app-file.e2e.ts b/cli/test/e2e/suites/run/run-app-file.e2e.ts new file mode 100644 index 00000000000000..f271d661b0da6e --- /dev/null +++ b/cli/test/e2e/suites/run/run-app-file.e2e.ts @@ -0,0 +1,161 @@ +/** + * E2E: difyctl run app --file — 文件输入专项 + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Run/文件输入（31 条） + * + * 前置条件： + * DIFY_E2E_FILE_APP_ID — workflow app，doc 文件变量（required） + * 如果未配置，所有文件相关用例会被跳过。 + */ + +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { assertExitCode, assertJson } from '../../helpers/assert.js' +import { injectAuth, run, withTempConfig } from '../../helpers/cli.js' +import { optionalDescribe, optionalIt } from '../../helpers/skip.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() +// supportsLocalUpload capability removed — local file upload probe is no longer +// performed in global-setup. Default to false (skip upload-specific cases). +const supportsLocalUpload = false + +const describeSuite = optionalDescribe(Boolean(E.fileAppId)) + +describeSuite('E2E / difyctl run app --file', () => { + let configDir: string + let fileDir: string + let cleanupConfig: () => Promise + + beforeEach(async () => { + const tmp = await withTempConfig() + configDir = tmp.configDir + cleanupConfig = tmp.cleanup + fileDir = await mkdtemp(join(tmpdir(), 'difyctl-e2e-files-')) + await injectAuth(configDir, { + host: E.host, + bearer: E.token, + workspaceId: E.workspaceId, + workspaceName: E.workspaceName, + }) + }) + + afterEach(async () => { + await cleanupConfig() + await rm(fileDir, { recursive: true, force: true }) + }) + + function r(argv: string[]) { + return run(argv, { configDir }) + } + + const itLocalUpload = optionalIt(supportsLocalUpload) + + itLocalUpload('[P0] run app 支持单文件上传（key=@path），app 正常执行', async () => { + // 文档用例：run app 支持单文件上传 + 上传文件后 app 正常执行 + const filePath = join(fileDir, 'test.txt') + await writeFile(filePath, 'E2E test file content — single upload') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${filePath}`]) + assertExitCode(result, 0) + }) + + itLocalUpload('[P0] file input 参数名正确映射（key 绑定到正确 input 字段）', async () => { + // 文档用例：file input 参数名正确映射 + const filePath = join(fileDir, 'mapping.txt') + await writeFile(filePath, 'mapping test content') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${filePath}`, '-o', 'json']) + assertExitCode(result, 0) + const parsed = assertJson>(result) + expect(parsed).toBeDefined() + }) + + itLocalUpload('[P0] run app --file 语法为 key=@path（本地文件上传）', async () => { + // 文档用例：run app --file 语法为 key=@path + const filePath = join(fileDir, 'syntax.txt') + await writeFile(filePath, 'syntax verification') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${filePath}`]) + assertExitCode(result, 0) + }) + + it('[P0] --file 远程 URL 语法（key=https://...）无需本地上传', async () => { + // 文档用例：run app --file 传入文件 workflow 正常执行 + const result = await r([ + 'run', + 'app', + E.fileAppId, + '--file', + 'doc=https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf', + ]) + assertExitCode(result, 0) + }) + + it('[P0] 文件不存在时返回错误', async () => { + // 文档用例：文件不存在时返回错误 + const result = await r([ + 'run', + 'app', + E.fileAppId, + '--file', + 'doc=@/nonexistent/path/missing-file.txt', + ]) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toMatch(/failed|not.?found|upload/i) + }) + + it('[P1] file 参数格式错误返回 usage error（exit code 2）', async () => { + // 文档用例：file 参数格式错误返回 usage error + const result = await r([ + 'run', + 'app', + E.chatAppId, + 'hello', + '--file', + 'invalidformat', + ]) + assertExitCode(result, 2) + expect(result.stderr).toMatch(/--file must be key=@path/i) + }) + + itLocalUpload('[P1] 文件路径包含空格可正常上传', async () => { + // 文档用例：文件路径包含空格可正常上传 + const filePath = join(fileDir, 'file with spaces.txt') + await writeFile(filePath, 'space in name test') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${filePath}`]) + assertExitCode(result, 0) + }) + + itLocalUpload('[P1] 支持 txt 文件上传', async () => { + // 文档用例：支持 txt 文件上传 + const f = join(fileDir, 'note.txt') + await writeFile(f, 'plain text content') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${f}`]) + assertExitCode(result, 0) + }) + + itLocalUpload('[P1] --file 与 --stream 组合使用', async () => { + // 文档用例：run app --file 与 --stream 组合使用 + const f = join(fileDir, 'stream.txt') + await writeFile(f, 'stream + file test') + const result = await r(['run', 'app', E.fileAppId, '--file', `doc=@${f}`, '--stream']) + assertExitCode(result, 0) + }) + + it('[P0] 未登录执行 file upload 返回认证错误（exit code 4）', async () => { + // 文档用例：未登录执行 file upload 返回认证错误 + const unauthTmp = await withTempConfig() + try { + const f = join(fileDir, 'unauth.txt') + await writeFile(f, 'test') + const result = await run( + ['run', 'app', E.fileAppId || E.chatAppId, '--file', `doc=@${f}`], + { configDir: unauthTmp.configDir }, + ) + assertExitCode(result, 4) + } + finally { + await unauthTmp.cleanup() + } + }) +}) diff --git a/cli/test/e2e/suites/run/run-app-hitl.e2e.ts b/cli/test/e2e/suites/run/run-app-hitl.e2e.ts new file mode 100644 index 00000000000000..cb0cf0022ac432 --- /dev/null +++ b/cli/test/e2e/suites/run/run-app-hitl.e2e.ts @@ -0,0 +1,135 @@ +/** + * E2E: difyctl run app + difyctl resume app — HITL 人工介入专项 + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Run/HITL 人工介入（19 条） + * + * 前置条件： + * DIFY_E2E_HITL_APP_ID — workflow app，包含 Human Input 节点，display_in_ui=true + * 如果未配置，所有 HITL 用例会被跳过。 + */ + +import type { AuthFixture } from '../../helpers/cli.js' +import { afterEach, beforeEach, expect, it } from 'vitest' +import { assertExitCode, assertJson, assertStderrContains } from '../../helpers/assert.js' +import { withAuthFixture } from '../../helpers/cli.js' +import { optionalDescribe } from '../../helpers/skip.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +const describeSuite = optionalDescribe(Boolean(E.hitlAppId)) + +describeSuite('E2E / difyctl run app — HITL 人工介入', () => { + let fx: AuthFixture + + beforeEach(async () => { + fx = await withAuthFixture(E) + }) + afterEach(async () => { + await fx.cleanup() + }) + + it('[P0] workflow 触发 HITL 暂停时 stdout 输出 pause block，exit code 为 0', async () => { + // 文档用例：workflow 触发 HITL 暂停时输出 pause block + exit code 为 0 + const result = await fx.r([ + 'run', + 'app', + E.hitlAppId, + '--inputs', + JSON.stringify({ x: 'hitl-e2e' }), + ]) + assertExitCode(result, 0) + expect(result.stdout).toMatch(/Workflow paused|pause/i) + }) + + it('[P0] HITL pause JSON 包含所有必需字段', async () => { + // 文档用例：HITL pause JSON 输出包含所有必需字段 + const result = await fx.r([ + 'run', + 'app', + E.hitlAppId, + '--inputs', + JSON.stringify({ x: 'hitl-json' }), + '-o', + 'json', + ]) + assertExitCode(result, 0) + const p = assertJson>(result) + expect(p).toHaveProperty('status', 'paused') + expect(p).toHaveProperty('form_token') + expect(p).toHaveProperty('workflow_run_id') + expect(p).toHaveProperty('node_title') + expect(p).toHaveProperty('form_content') + expect(p).toHaveProperty('actions') + }) + + it('[P0] HITL pause hint 包含完整 resume 命令', async () => { + // 文档用例：HITL pause 时 hint 包含完整 resume 命令 + const result = await fx.r([ + 'run', + 'app', + E.hitlAppId, + '--inputs', + JSON.stringify({ x: 'hint-test' }), + ]) + assertExitCode(result, 0) + assertStderrContains(result, 'difyctl resume app') + assertStderrContains(result, '--workflow-run-id') + }) + + it('[P0] AI Agent 自动化：从 JSON 提取 form_token，自动 resume', async () => { + // 文档用例：AI Agent 自动化：jq 提取 form_token 自动 resume + // Step 1: run → pause，获取 JSON envelope + const pauseResult = await fx.r([ + 'run', + 'app', + E.hitlAppId, + '--inputs', + JSON.stringify({ x: 'auto-resume' }), + '-o', + 'json', + ]) + assertExitCode(pauseResult, 0) + const envelope = assertJson<{ form_token: string, workflow_run_id: string, app_id?: string }>(pauseResult) + expect(envelope.form_token).toBeTruthy() + expect(envelope.workflow_run_id).toBeTruthy() + + // Step 2: resume using extracted tokens + const resumeResult = await fx.r([ + 'resume', + 'app', + E.hitlAppId, + envelope.form_token, + '--workflow-run-id', + envelope.workflow_run_id, + '--action', + 'submit', + ]) + assertExitCode(resumeResult, 0) + }) + + it('[P0] resume app 单 action 时自动选择，workflow 继续执行', async () => { + // 文档用例：resume app 单 action 时自动选择无需 --action + const pause = await fx.r([ + 'run', + 'app', + E.hitlAppId, + '--inputs', + JSON.stringify({ x: 'auto-action' }), + '-o', + 'json', + ]) + assertExitCode(pause, 0) + const { form_token, workflow_run_id } = assertJson<{ form_token: string, workflow_run_id: string }>(pause) + // Resume without --action (single action auto-selected) + const resume = await fx.r([ + 'resume', + 'app', + E.hitlAppId, + form_token, + '--workflow-run-id', + workflow_run_id, + ]) + assertExitCode(resume, 0) + }) +}) diff --git a/cli/test/e2e/suites/run/run-app-streaming.e2e.ts b/cli/test/e2e/suites/run/run-app-streaming.e2e.ts new file mode 100644 index 00000000000000..956d71f62d2c99 --- /dev/null +++ b/cli/test/e2e/suites/run/run-app-streaming.e2e.ts @@ -0,0 +1,125 @@ +/** + * E2E: difyctl run app --stream — Streaming 输出专项 + * + * 用例来源：飞书文档《Dify CLI Enhanced》— Dify CLI/Run/Streaming 输出（24 条） + * + * 补充覆盖 run-app-basic.e2e.ts 无法完成的场景： + * - Ctrl+C 中断（SIGINT） + * - streaming 输出按 chunk 到达顺序验证（时序） + */ + +import type { Buffer } from 'node:buffer' +import type { AuthFixture } from '../../helpers/cli.js' +import { spawn } from 'node:child_process' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { assertExitCode } from '../../helpers/assert.js' +import { BIN, BUN, withAuthFixture } from '../../helpers/cli.js' +import { withRetry } from '../../helpers/retry.js' +import { loadE2EEnv } from '../../setup/env.js' + +const E = loadE2EEnv() + +describe('E2E / difyctl run app --stream (专项)', () => { + let fx: AuthFixture + + beforeEach(async () => { + fx = await withAuthFixture(E) + }) + afterEach(async () => { + await fx.cleanup() + }) + + it('[P0] streaming 输出按 chunk 实时到达（stdout 非空，echo 完整）', async () => { + // 文档用例：streaming 输出按 chunk 实时打印 + streaming 输出保留 token 顺序 + // withRetry: staging SSE connections may fail transiently on cold start + await withRetry(async () => { + const query = 'chunk-order-test' + const proc = spawn(BUN, [BIN, 'run', 'app', E.chatAppId, query, '--stream'], { + env: { ...process.env, DIFY_CONFIG_DIR: fx.configDir, CI: '1', NO_COLOR: '1' }, + }) + + const chunks: string[] = [] + proc.stdout.on('data', (d: Buffer) => { + chunks.push(d.toString('utf8')) + }) + + let stderr = '' + proc.stderr.on('data', (d: Buffer) => { + stderr += d.toString('utf8') + }) + + const exitCode = await new Promise((res) => { + proc.on('close', code => res(code ?? 1)) + }) + + assertExitCode({ stdout: chunks.join(''), stderr, exitCode }, 0) + // 可能分多个 chunk 到达，拼接后应包含完整 query + expect(chunks.join('')).toContain(query) + }, { attempts: 3, delayMs: 2000 }) + }) + + it('[P1] Ctrl+C 可中断 streaming（SIGINT → exit code 非 0）', async () => { + // 文档用例：Ctrl+C 可中断 streaming + Ctrl+C 后 exit code 非 0 + const proc = spawn(BUN, [BIN, 'run', 'app', E.chatAppId, 'ctrl-c-test', '--stream'], { + env: { ...process.env, DIFY_CONFIG_DIR: fx.configDir, CI: '1', NO_COLOR: '1' }, + }) + + let _stdout = '' + let _stderr = '' + proc.stdout.on('data', (d: Buffer) => { + _stdout += d.toString('utf8') + }) + proc.stderr.on('data', (d: Buffer) => { + _stderr += d.toString('utf8') + }) + + // Wait for the process to start streaming, then interrupt. + await new Promise(res => setTimeout(res, 800)) + proc.kill('SIGINT') + + const exitCode = await new Promise((res) => { + proc.on('close', code => res(code ?? 1)) + }) + + expect(exitCode, 'SIGINT should cause non-zero exit').not.toBe(0) + }) + + it('[P0] streaming 服务端返回 error event — CLI 以非 0 退出', async () => { + // 文档用例：streaming 服务端返回 error event + // Use a non-existent app ID to force a server-side error. + const proc = spawn(BUN, [BIN, 'run', 'app', 'nonexistent-app-xyz-e2e', 'hi', '--stream'], { + env: { ...process.env, DIFY_CONFIG_DIR: fx.configDir, CI: '1', NO_COLOR: '1' }, + }) + let stderr = '' + proc.stderr.on('data', (d: Buffer) => { + stderr += d.toString('utf8') + }) + const exitCode = await new Promise((res) => { + proc.on('close', code => res(code ?? 1)) + }) + expect(exitCode, 'error event should cause non-zero exit').not.toBe(0) + expect(stderr.length).toBeGreaterThan(0) + }) + + it('[P0] streaming 必填 input 缺失时失败（exit code 非 0）', async () => { + // 文档用例：streaming 必填 input 缺失时失败 + // workflow app 需要 x 变量（required），不传时服务端应立即返回 validation error， + // CLI 捕获后以非 0 exit code 退出。 + // + // ⚠️ 依赖 feat/cli API 版本（服务端对缺失 required input 做前置校验）。 + // 当前本地服务端 1.14.1 不支持此校验，用例在升级后方可真正通过。 + const proc = spawn(BUN, [BIN, 'run', 'app', E.workflowAppId, '--stream'], { + env: { ...process.env, DIFY_CONFIG_DIR: fx.configDir, CI: '1', NO_COLOR: '1' }, + }) + let stderr = '' + proc.stderr.on('data', (d: Buffer) => { + stderr += d.toString('utf8') + }) + const exitCode = await new Promise((res) => { + proc.on('close', code => res(code ?? 1)) + }) + expect(exitCode).not.toBe(0) + // 服务端应返回明确的 validation error，而非超时 + expect(stderr).toMatch(/validation|required|invalid|missing/i) + }) +}) diff --git a/cli/vitest.e2e.config.ts b/cli/vitest.e2e.config.ts new file mode 100644 index 00000000000000..2cd864ae1fb84f --- /dev/null +++ b/cli/vitest.e2e.config.ts @@ -0,0 +1,84 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { defineConfig } from 'vite-plus' +import { resolveBuildInfo } from './scripts/lib/resolve-buildinfo.js' + +const buildInfo = resolveBuildInfo() + +// Load .env.e2e into process.env (only if the file exists; in CI vars are +// injected directly via GitHub Actions secrets). +const envFilePath = resolve(process.cwd(), '.env.e2e') +try { + const raw = readFileSync(envFilePath, 'utf8') + for (const line of raw.split('\n')) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) + continue + const eqIdx = trimmed.indexOf('=') + if (eqIdx === -1) + continue + const key = trimmed.slice(0, eqIdx).trim() + const val = trimmed.slice(eqIdx + 1).trim() + if (key && !(key in process.env)) + process.env[key] = val + } +} +catch { + // .env.e2e not found — rely on environment variables already set in the shell +} + +/** + * Vitest configuration for E2E tests. + * + * E2E tests run against a real staging Dify server and require + * DIFY_E2E_* environment variables to be set (see test/e2e/setup/env.ts). + * + * Run: bun vitest --config vitest.e2e.config.ts + */ +export default defineConfig({ + pack: { + entry: ['src/index.ts'], + format: ['esm'], + outDir: 'dist', + target: 'node22', + define: { + __DIFYCTL_VERSION__: JSON.stringify(buildInfo.version), + __DIFYCTL_COMMIT__: JSON.stringify(buildInfo.commit), + __DIFYCTL_BUILD_DATE__: JSON.stringify(buildInfo.buildDate), + __DIFYCTL_CHANNEL__: JSON.stringify(buildInfo.channel), + __DIFYCTL_MIN_DIFY__: JSON.stringify(buildInfo.minDify), + __DIFYCTL_MAX_DIFY__: JSON.stringify(buildInfo.maxDify), + }, + }, + test: { + environment: 'node', + globalSetup: ['test/e2e/setup/global-setup.ts'], + // E2E tests do NOT use the unit-test setup.ts (no globalThis stubs needed — + // the real binary sets its own globals at startup). + setupFiles: [], + include: process.env.DIFY_E2E_MODE === 'local' + ? ['test/e2e/suites/config/**/*.e2e.ts'] + : [ + // auth tests first (most others depend on a valid session) + 'test/e2e/suites/auth/status.e2e.ts', + 'test/e2e/suites/auth/use.e2e.ts', + 'test/e2e/suites/auth/whoami.e2e.ts', + // config (local, no network) + 'test/e2e/suites/config/**/*.e2e.ts', + // run tests (require valid token) + 'test/e2e/suites/run/**/*.e2e.ts', + // devices + logout LAST — both can revoke tokens + 'test/e2e/suites/auth/devices.e2e.ts', + 'test/e2e/suites/auth/logout.e2e.ts', + ], + // E2E calls a real staging server — allow plenty of time per test. + testTimeout: 60_000, + hookTimeout: 30_000, + // Retry up to 2 times on staging flakiness. + retry: 0, // flaky tests use withRetry() locally; global retry masks non-idempotent failures + // Run suites sequentially to avoid workspace-level conflicts on staging. + pool: 'forks', + fileParallelism: false, + reporters: ['verbose'], + }, +}) From b734afd609e720aad9c00f9c7107a926679622fa Mon Sep 17 00:00:00 2001 From: gigglewang0417 Date: Wed, 27 May 2026 10:59:51 +0800 Subject: [PATCH 2/2] ci: add GitHub Actions workflow for CLI E2E tests Triggers on pull_request when cli/** files change. Spins up a full Dify stack via docker compose, provisions an admin account + test apps, then runs the E2E smoke suite (P0 cases only for CI speed). --- .github/workflows/cli-e2e.yml | 191 ++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 .github/workflows/cli-e2e.yml diff --git a/.github/workflows/cli-e2e.yml b/.github/workflows/cli-e2e.yml new file mode 100644 index 00000000000000..ddfac3664385ad --- /dev/null +++ b/.github/workflows/cli-e2e.yml @@ -0,0 +1,191 @@ +name: CLI E2E Tests + +on: + workflow_dispatch: + inputs: + dify_version: + description: "Dify image tag to test against (e.g. 1.7.0)" + type: string + required: true + cli_ref: + description: "Git ref to build the CLI from (default: current branch)" + type: string + required: false + test_scope: + description: "Test scope to run" + type: choice + required: false + default: smoke + options: + - smoke # [P0] cases only — fast + - full # all cases + +permissions: + contents: read + +jobs: + e2e: + name: E2E — difyctl (${{ inputs.dify_version }}) + runs-on: ubuntu-latest + timeout-minutes: 45 + defaults: + run: + shell: bash + + steps: + # ── Checkout ─────────────────────────────────────────────────────────── + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4 + with: + ref: ${{ inputs.cli_ref || github.ref }} + persist-credentials: false + + # ── Runtime setup ────────────────────────────────────────────────────── + - name: Setup web environment + uses: ./.github/actions/setup-web + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install CLI dependencies + working-directory: cli + run: pnpm install --frozen-lockfile + + - name: Generate command tree + working-directory: cli + run: pnpm tree:gen + + # ── Start Dify stack ─────────────────────────────────────────────────── + - name: Start middleware (PostgreSQL, Redis, Sandbox…) + run: bun e2e/scripts/setup.ts middleware-up + + - name: Start Dify API + Worker + env: + DIFY_VERSION: ${{ inputs.dify_version }} + INIT_PASSWORD: dify-e2e-pass + SECRET_KEY: e2e-secret-key-for-ci-only-32chars! + run: | + cd docker + cp .env.example .env + sed -i "s|^SECRET_KEY=.*|SECRET_KEY=${SECRET_KEY}|" .env + sed -i "s|^INIT_PASSWORD=.*|INIT_PASSWORD=${INIT_PASSWORD}|" .env + DIFY_API_IMAGE_TAG="${DIFY_VERSION}" \ + DIFY_WEB_IMAGE_TAG="${DIFY_VERSION}" \ + docker compose up -d api worker + echo "Waiting for Dify API..." + for i in $(seq 1 90); do + if curl -fsS http://localhost/health >/dev/null 2>&1; then + echo "Dify API ready after ${i}s" + exit 0 + fi + sleep 1 + done + echo "Timeout waiting for Dify API" >&2 + docker compose logs api --tail=50 + exit 1 + + # ── Provision test fixtures ──────────────────────────────────────────── + - name: Provision admin account and test apps + id: provision + env: + DIFY_HOST: http://localhost + ADMIN_EMAIL: e2e@dify.ai + ADMIN_PASSWORD: dify-e2e-pass + run: | + B64_PASS=$(echo -n "${ADMIN_PASSWORD}" | base64) + + # Register admin + curl -fsS "${DIFY_HOST}/console/api/setup" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"${ADMIN_EMAIL}\",\"name\":\"E2E Admin\",\"password\":\"${B64_PASS}\"}" + + # Console login → session cookie + CSRF token + curl -fsS -c /tmp/e2e-cookies.txt \ + "${DIFY_HOST}/console/api/login" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"${ADMIN_EMAIL}\",\"password\":\"${B64_PASS}\",\"remember_me\":false}" + + CSRF=$(grep csrf_token /tmp/e2e-cookies.txt | awk '{print $NF}') + + # Mint dfoa_ token via device flow + DEVICE=$(curl -fsS "${DIFY_HOST}/openapi/v1/oauth/device/code" \ + -H "Content-Type: application/json" \ + -d '{"client_id":"difyctl","device_label":"ci-e2e"}') + USER_CODE=$(echo "$DEVICE" | python3 -c "import sys,json; print(json.load(sys.stdin)['user_code'])") + DEVICE_CODE=$(echo "$DEVICE" | python3 -c "import sys,json; print(json.load(sys.stdin)['device_code'])") + + curl -fsS -b /tmp/e2e-cookies.txt \ + -H "X-CSRFToken: ${CSRF}" \ + -H "Content-Type: application/json" \ + "${DIFY_HOST}/openapi/v1/oauth/device/approve" \ + -d "{\"user_code\":\"${USER_CODE}\"}" + + TOKEN=$(curl -fsS "${DIFY_HOST}/openapi/v1/oauth/device/token" \ + -H "Content-Type: application/json" \ + -d "{\"device_code\":\"${DEVICE_CODE}\",\"client_id\":\"difyctl\"}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") + + # Get workspace ID + WS_ID=$(curl -fsS "${DIFY_HOST}/openapi/v1/workspaces" \ + -H "Authorization: Bearer ${TOKEN}" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['data'][0]['id'])") + + # Create echo-chat app + CHAT_APP_ID=$(curl -fsS "${DIFY_HOST}/console/api/apps" \ + -b /tmp/e2e-cookies.txt \ + -H "X-CSRFToken: ${CSRF}" \ + -H "Content-Type: application/json" \ + -d '{"name":"E2E Echo Chat","mode":"chat","icon_type":"emoji","icon":"🤖","icon_background":"#FFEAD5"}' \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + + # Create echo-workflow app + WORKFLOW_APP_ID=$(curl -fsS "${DIFY_HOST}/console/api/apps" \ + -b /tmp/e2e-cookies.txt \ + -H "X-CSRFToken: ${CSRF}" \ + -H "Content-Type: application/json" \ + -d '{"name":"E2E Echo Workflow","mode":"workflow","icon_type":"emoji","icon":"⚙️","icon_background":"#E4FBCC"}' \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])") + + # Mask token before exporting + echo "::add-mask::${TOKEN}" + echo "token=${TOKEN}" >> "$GITHUB_OUTPUT" + echo "workspace_id=${WS_ID}" >> "$GITHUB_OUTPUT" + echo "chat_app_id=${CHAT_APP_ID}" >> "$GITHUB_OUTPUT" + echo "workflow_app_id=${WORKFLOW_APP_ID}" >> "$GITHUB_OUTPUT" + echo "admin_email=${ADMIN_EMAIL}" >> "$GITHUB_OUTPUT" + echo "admin_password=${ADMIN_PASSWORD}" >> "$GITHUB_OUTPUT" + + # ── Run E2E tests ────────────────────────────────────────────────────── + - name: Run E2E tests (${{ inputs.test_scope || 'smoke' }}) + working-directory: cli + env: + DIFY_E2E_HOST: http://localhost + DIFY_E2E_TOKEN: ${{ steps.provision.outputs.token }} + DIFY_E2E_WORKSPACE_ID: ${{ steps.provision.outputs.workspace_id }} + DIFY_E2E_WORKSPACE_NAME: E2E Workspace + DIFY_E2E_CHAT_APP_ID: ${{ steps.provision.outputs.chat_app_id }} + DIFY_E2E_WORKFLOW_APP_ID: ${{ steps.provision.outputs.workflow_app_id }} + DIFY_E2E_EMAIL: ${{ steps.provision.outputs.admin_email }} + DIFY_E2E_PASSWORD: ${{ steps.provision.outputs.admin_password }} + run: | + if [ "${{ inputs.test_scope }}" = "full" ]; then + pnpm test:e2e + else + pnpm test:e2e:smoke + fi + + # ── Debug & cleanup ──────────────────────────────────────────────────── + - name: Dump Dify logs on failure + if: failure() + run: | + cd docker + docker compose logs api worker --tail=100 + + - name: Upload test results on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-results-${{ github.run_id }} + path: cli/test-results/ + retention-days: 3