diff --git a/docs/docs/api/appkit/Interface.AgentDefinition.md b/docs/docs/api/appkit/Interface.AgentDefinition.md index 87138d35d..8996e759c 100644 --- a/docs/docs/api/appkit/Interface.AgentDefinition.md +++ b/docs/docs/api/appkit/Interface.AgentDefinition.md @@ -37,6 +37,20 @@ future calls and accumulate unbounded state in the default *** +### generationParams? + +```ts +optional generationParams: GenerationParams; +``` + +Optional generation parameters (`temperature`, `top_p`, `stop`, +`frequency_penalty`, `presence_penalty`) forwarded to the OpenAI-compatible +serving request body. Only set keys are sent. Applied only when AppKit +builds the adapter itself (string or omitted `model`); when you pass a +pre-built `AgentAdapter`, configure generation params on it directly. + +*** + ### instructions ```ts diff --git a/docs/docs/api/appkit/Interface.GenerationParams.md b/docs/docs/api/appkit/Interface.GenerationParams.md new file mode 100644 index 000000000..d4f226fe3 --- /dev/null +++ b/docs/docs/api/appkit/Interface.GenerationParams.md @@ -0,0 +1,56 @@ +# Interface: GenerationParams + +Optional generation parameters forwarded to the OpenAI-compatible serving +request body. Names match the serving API wire keys. Only keys that are set +are sent — undefined values are omitted so the endpoint applies its own +defaults. Ranges are not validated here; the serving endpoint validates. + +## Properties + +### frequency\_penalty? + +```ts +optional frequency_penalty: number; +``` + +Penalize tokens by frequency. + +*** + +### presence\_penalty? + +```ts +optional presence_penalty: number; +``` + +Penalize tokens by prior presence. + +*** + +### stop? + +```ts +optional stop: string | string[]; +``` + +Stop sequence(s) that end generation. + +*** + +### temperature? + +```ts +optional temperature: number; +``` + +Sampling temperature. + +*** + +### top\_p? + +```ts +optional top_p: number; +``` + +Nucleus sampling probability mass (`top_p`). diff --git a/docs/docs/api/appkit/Interface.RegisteredAgent.md b/docs/docs/api/appkit/Interface.RegisteredAgent.md index ead127e6e..4cbe04682 100644 --- a/docs/docs/api/appkit/Interface.RegisteredAgent.md +++ b/docs/docs/api/appkit/Interface.RegisteredAgent.md @@ -28,6 +28,16 @@ Mirrors `AgentDefinition.ephemeral` — skip thread persistence. *** +### generationParams? + +```ts +optional generationParams: GenerationParams; +``` + +Mirrors `AgentDefinition.generationParams`. + +*** + ### instructions ```ts diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 4d9950512..24caa020e 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -48,6 +48,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. | | [FunctionTool](Interface.FunctionTool.md) | - | | [GenerateDatabaseCredentialRequest](Interface.GenerateDatabaseCredentialRequest.md) | Request parameters for generating database OAuth credentials | +| [GenerationParams](Interface.GenerationParams.md) | Optional generation parameters forwarded to the OpenAI-compatible serving request body. Names match the serving API wire keys. Only keys that are set are sent — undefined values are omitted so the endpoint applies its own defaults. Ranges are not validated here; the serving endpoint validates. | | [IJobsConfig](Interface.IJobsConfig.md) | Configuration for the Jobs plugin. | | [ITelemetry](Interface.ITelemetry.md) | Plugin-facing interface for OpenTelemetry instrumentation. Provides a thin abstraction over OpenTelemetry APIs for plugins. | | [JobAPI](Interface.JobAPI.md) | User-facing API for a single configured job. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index c74d62326..4bd5fa9fb 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -172,6 +172,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.GenerateDatabaseCredentialRequest", label: "GenerateDatabaseCredentialRequest" }, + { + type: "doc", + id: "api/appkit/Interface.GenerationParams", + label: "GenerationParams" + }, { type: "doc", id: "api/appkit/Interface.IJobsConfig", diff --git a/packages/appkit/src/agents/databricks.ts b/packages/appkit/src/agents/databricks.ts index 83f89a59b..71a788160 100644 --- a/packages/appkit/src/agents/databricks.ts +++ b/packages/appkit/src/agents/databricks.ts @@ -27,6 +27,44 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +/** + * Optional generation parameters forwarded to the OpenAI-compatible serving + * request body. Names match the serving API wire keys. Only keys that are set + * are sent — undefined values are omitted so the endpoint applies its own + * defaults. Ranges are not validated here; the serving endpoint validates. + */ +export interface GenerationParams { + /** Sampling temperature. */ + temperature?: number; + /** Nucleus sampling probability mass (`top_p`). */ + top_p?: number; + /** Stop sequence(s) that end generation. */ + stop?: string | string[]; + /** Penalize tokens by frequency. */ + frequency_penalty?: number; + /** Penalize tokens by prior presence. */ + presence_penalty?: number; +} + +const GENERATION_PARAM_KEYS = [ + "temperature", + "top_p", + "stop", + "frequency_penalty", + "presence_penalty", +] as const satisfies readonly (keyof GenerationParams)[]; + +/** Copy only the set generation params onto the request body. */ +function applyGenerationParams( + body: Record, + params: GenerationParams, +): void { + for (const key of GENERATION_PARAM_KEYS) { + const value = params[key]; + if (value !== undefined) body[key] = value; + } +} + function extractLlamaToolJsonSlice(text: string): string | undefined { const start = text.indexOf("[{"); if (start < 0) return undefined; @@ -84,6 +122,8 @@ interface RawFetchAdapterOptions { authenticate: () => Promise>; maxSteps?: number; maxTokens?: number; + /** Optional generation params forwarded to the serving request body. */ + generationParams?: GenerationParams; /** Max length of one SSE line (including an incomplete tail in the buffer). */ maxSseLineChars?: number; /** Max total length of assistant `delta.content` across the stream. */ @@ -102,6 +142,7 @@ interface StreamBodyAdapterOptions { streamBody: StreamBody; maxSteps?: number; maxTokens?: number; + generationParams?: GenerationParams; maxSseLineChars?: number; maxStreamTextChars?: number; maxToolArgumentsChars?: number; @@ -135,6 +176,7 @@ interface ServingEndpointOptions { endpointName: string; maxSteps?: number; maxTokens?: number; + generationParams?: GenerationParams; maxSseLineChars?: number; maxStreamTextChars?: number; maxToolArgumentsChars?: number; @@ -143,6 +185,7 @@ interface ServingEndpointOptions { interface ModelServingOptions { maxSteps?: number; maxTokens?: number; + generationParams?: GenerationParams; workspaceClient?: WorkspaceClientLike; maxSseLineChars?: number; maxStreamTextChars?: number; @@ -238,6 +281,7 @@ export class DatabricksAdapter implements AgentAdapter { private streamBody: StreamBody; private maxSteps: number; private maxTokens: number; + private generationParams: GenerationParams; private maxSseLineChars: number; private maxStreamTextChars: number; private maxToolArgumentsChars: number; @@ -245,6 +289,7 @@ export class DatabricksAdapter implements AgentAdapter { constructor(options: DatabricksAdapterOptions) { this.maxSteps = options.maxSteps ?? 10; this.maxTokens = options.maxTokens ?? 4096; + this.generationParams = options.generationParams ?? {}; this.maxSseLineChars = options.maxSseLineChars ?? DEFAULT_MAX_SSE_LINE_CHARS; this.maxStreamTextChars = @@ -298,6 +343,7 @@ export class DatabricksAdapter implements AgentAdapter { endpointName, maxSteps, maxTokens, + generationParams, maxSseLineChars, maxStreamTextChars, maxToolArgumentsChars, @@ -315,6 +361,7 @@ export class DatabricksAdapter implements AgentAdapter { ), maxSteps, maxTokens, + generationParams, maxSseLineChars, maxStreamTextChars, maxToolArgumentsChars, @@ -370,6 +417,7 @@ export class DatabricksAdapter implements AgentAdapter { endpointName: resolvedEndpoint, maxSteps: options?.maxSteps, maxTokens: options?.maxTokens, + generationParams: options?.generationParams, maxSseLineChars: options?.maxSseLineChars, maxStreamTextChars: options?.maxStreamTextChars, maxToolArgumentsChars: options?.maxToolArgumentsChars, @@ -495,6 +543,8 @@ export class DatabricksAdapter implements AgentAdapter { max_tokens: this.maxTokens, }; + applyGenerationParams(body, this.generationParams); + if (tools.length > 0) { body.tools = tools; } diff --git a/packages/appkit/src/agents/tests/databricks.test.ts b/packages/appkit/src/agents/tests/databricks.test.ts index 84f0c6717..665f60051 100644 --- a/packages/appkit/src/agents/tests/databricks.test.ts +++ b/packages/appkit/src/agents/tests/databricks.test.ts @@ -1,6 +1,10 @@ import type { AgentEvent, AgentToolDefinition, Message } from "shared"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -import { DatabricksAdapter, parseTextToolCalls } from "../databricks"; +import { + DatabricksAdapter, + type GenerationParams, + parseTextToolCalls, +} from "../databricks"; const mockAuthenticate = vi .fn() @@ -93,6 +97,7 @@ function createAdapter(overrides?: { authenticate?: () => Promise>; maxSteps?: number; maxTokens?: number; + generationParams?: GenerationParams; maxSseLineChars?: number; maxStreamTextChars?: number; maxToolArgumentsChars?: number; @@ -566,6 +571,56 @@ describe("DatabricksAdapter", () => { }); }); + test("forwards set generation params to the request body", async () => { + globalThis.fetch = mockFetch([textDelta("Hi"), sseChunk("[DONE]")]); + + const adapter = createAdapter({ + generationParams: { + temperature: 0.2, + top_p: 0.9, + stop: ["END"], + frequency_penalty: 0.5, + presence_penalty: 0.1, + }, + }); + + for await (const _ of adapter.run( + { messages: createTestMessages(), tools: [], threadId: "t1" }, + { executeTool: vi.fn() }, + )) { + // drain + } + + const [, init] = (globalThis.fetch as any).mock.calls[0]; + const body = JSON.parse(init.body); + expect(body.temperature).toBe(0.2); + expect(body.top_p).toBe(0.9); + expect(body.stop).toEqual(["END"]); + expect(body.frequency_penalty).toBe(0.5); + expect(body.presence_penalty).toBe(0.1); + }); + + test("omits generation param keys that are not set", async () => { + globalThis.fetch = mockFetch([textDelta("Hi"), sseChunk("[DONE]")]); + + const adapter = createAdapter({ generationParams: { temperature: 0.7 } }); + + for await (const _ of adapter.run( + { messages: createTestMessages(), tools: [], threadId: "t1" }, + { executeTool: vi.fn() }, + )) { + // drain + } + + const [, init] = (globalThis.fetch as any).mock.calls[0]; + const body = JSON.parse(init.body); + expect(body.temperature).toBe(0.7); + expect(body).not.toHaveProperty("top_p"); + expect(body).not.toHaveProperty("stop"); + expect(body).not.toHaveProperty("frequency_penalty"); + expect(body).not.toHaveProperty("presence_penalty"); + }); + test("forwards tool thread fields from input messages to the request body", async () => { globalThis.fetch = mockFetch([textDelta("Done"), sseChunk("[DONE]")]); diff --git a/packages/appkit/src/beta.ts b/packages/appkit/src/beta.ts index 3f5bba80c..cffaf1adc 100644 --- a/packages/appkit/src/beta.ts +++ b/packages/appkit/src/beta.ts @@ -18,7 +18,11 @@ export type { ToolAnnotations, ToolProvider, } from "shared"; -export { DatabricksAdapter, parseTextToolCalls } from "./agents/databricks"; +export { + DatabricksAdapter, + type GenerationParams, + parseTextToolCalls, +} from "./agents/databricks"; // Agent runtime export { createAgent } from "./core/agent/create-agent"; diff --git a/packages/appkit/src/core/agent/load-agents.ts b/packages/appkit/src/core/agent/load-agents.ts index 13b2ff70d..5f535cafc 100644 --- a/packages/appkit/src/core/agent/load-agents.ts +++ b/packages/appkit/src/core/agent/load-agents.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import yaml from "js-yaml"; import type { AgentAdapter } from "shared"; +import type { GenerationParams } from "../../agents/databricks"; import type { AgentDefinition, AgentTool, @@ -75,6 +76,12 @@ interface Frontmatter { agents?: string[]; maxSteps?: number; maxTokens?: number; + /** + * Optional OpenAI-compatible generation params forwarded to the serving + * request body (`temperature`, `top_p`, `stop`, `frequency_penalty`, + * `presence_penalty`). Parsed defensively in {@link buildDefinition}. + */ + generationParams?: Record; default?: boolean; baseSystemPrompt?: false | string; ephemeral?: boolean; @@ -120,6 +127,7 @@ const ALLOWED_KEYS = new Set([ "agents", "maxSteps", "maxTokens", + "generationParams", "default", "baseSystemPrompt", "ephemeral", @@ -340,6 +348,73 @@ export function parseFrontmatter( return { data: data as Frontmatter, content: match[2].trim() }; } +const isNumber = (v: unknown): v is number => typeof v === "number"; + +/** + * Per-key validators for {@link GenerationParams} frontmatter. Keyed by wire + * name and typed as `Record`, so a new param on + * the interface is a compile error until it gets an entry here — parsing, + * unknown-key detection, and warnings all derive from this one table. + */ +const GENERATION_PARAM_SPECS: Record< + keyof GenerationParams, + { label: string; valid: (v: unknown) => boolean } +> = { + temperature: { label: "number", valid: isNumber }, + top_p: { label: "number", valid: isNumber }, + frequency_penalty: { label: "number", valid: isNumber }, + presence_penalty: { label: "number", valid: isNumber }, + stop: { + label: "string or string[]", + valid: (v) => + typeof v === "string" || + (Array.isArray(v) && v.every((s) => typeof s === "string")), + }, +}; + +/** + * Defensively maps a frontmatter `generationParams` map to {@link GenerationParams}. + * Picks only known keys with the expected wire types. A key present with the + * wrong type (e.g. `temperature: "0.5"`) or an unknown key (e.g. `top-p`) is + * dropped and logged at warn level, so a silently-ignored param is visible + * rather than mistaken for "applied". Returns `undefined` when no valid key is + * present. + */ +function parseGenerationParams( + value: unknown, + sourcePath?: string, +): GenerationParams | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return undefined; + } + const raw = value as Record; + const out: Record = {}; + const where = sourcePath ?? ""; + + for (const [key, v] of Object.entries(raw)) { + const spec = GENERATION_PARAM_SPECS[key as keyof GenerationParams]; + if (!spec) { + logger.warn( + "Ignoring unknown generationParams key '%s' in %s", + key, + where, + ); + } else if (spec.valid(v)) { + out[key] = v; + } else { + logger.warn( + "Ignoring generationParams.%s in %s: expected %s, got %s", + key, + where, + spec.label, + typeof v, + ); + } + } + + return Object.keys(out).length > 0 ? (out as GenerationParams) : undefined; +} + function buildDefinition( name: string, raw: string, @@ -364,6 +439,7 @@ function buildDefinition( tools: Object.keys(tools).length > 0 ? tools : undefined, maxSteps: typeof fm.maxSteps === "number" ? fm.maxSteps : undefined, maxTokens: typeof fm.maxTokens === "number" ? fm.maxTokens : undefined, + generationParams: parseGenerationParams(fm.generationParams, filePath), baseSystemPrompt, ephemeral: typeof fm.ephemeral === "boolean" ? fm.ephemeral : undefined, }; diff --git a/packages/appkit/src/core/agent/tests/load-agents.test.ts b/packages/appkit/src/core/agent/tests/load-agents.test.ts index 195a31796..de07e5be9 100644 --- a/packages/appkit/src/core/agent/tests/load-agents.test.ts +++ b/packages/appkit/src/core/agent/tests/load-agents.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { z } from "zod"; import { buildToolkitEntries } from "../../../core/agent/build-toolkit"; import { @@ -103,6 +103,62 @@ describe("loadAgentFromFile", () => { expect(def.name).toBe("router"); expect(def.instructions).toBe("Route traffic."); }); + + describe("generationParams", () => { + test("picks known keys with the expected wire types", async () => { + const p = writeRoot( + "gen.md", + "---\nendpoint: e\ngenerationParams:\n temperature: 0.5\n top_p: 0.9\n stop: STOP\n frequency_penalty: 0.1\n presence_penalty: 0.2\n---\nBody.", + ); + const def = await loadAgentFromFile(p, {}); + expect(def.generationParams).toEqual({ + temperature: 0.5, + top_p: 0.9, + stop: "STOP", + frequency_penalty: 0.1, + presence_penalty: 0.2, + }); + }); + + test("accepts stop as a string array", async () => { + const p = writeRoot( + "gen.md", + "---\nendpoint: e\ngenerationParams:\n stop:\n - END\n - STOP\n---\nBody.", + ); + const def = await loadAgentFromFile(p, {}); + expect(def.generationParams).toEqual({ stop: ["END", "STOP"] }); + }); + + test("drops a wrong-typed key and warns instead of silently ignoring", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const p = writeRoot( + "gen.md", + '---\nendpoint: e\ngenerationParams:\n temperature: "0.5"\n---\nBody.', + ); + const def = await loadAgentFromFile(p, {}); + expect(def.generationParams).toBeUndefined(); + const msg = warn.mock.calls.map((c) => c.join(" ")).join("\n"); + expect(msg).toContain("generationParams.temperature"); + expect(msg).toContain("expected number"); + expect(msg).toContain("got string"); + expect(msg).toContain("gen.md"); + warn.mockRestore(); + }); + + test("warns on an unknown key (e.g. hyphenated top-p) and keeps valid ones", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const p = writeRoot( + "gen.md", + "---\nendpoint: e\ngenerationParams:\n top-p: 0.9\n temperature: 0.3\n---\nBody.", + ); + const def = await loadAgentFromFile(p, {}); + expect(def.generationParams).toEqual({ temperature: 0.3 }); + const msg = warn.mock.calls.map((c) => c.join(" ")).join("\n"); + expect(msg).toContain("unknown generationParams key 'top-p'"); + expect(msg).toContain("gen.md"); + warn.mockRestore(); + }); + }); }); describe("loadAgentsFromDir", () => { diff --git a/packages/appkit/src/core/agent/types.ts b/packages/appkit/src/core/agent/types.ts index cf47845f7..0dbbcfac2 100644 --- a/packages/appkit/src/core/agent/types.ts +++ b/packages/appkit/src/core/agent/types.ts @@ -5,6 +5,7 @@ import type { ThreadStore, ToolAnnotations, } from "shared"; +import type { GenerationParams } from "../../agents/databricks"; import type { McpHostPolicyConfig } from "../../connectors/mcp"; import type { FunctionTool } from "./tools/function-tool"; import type { HostedTool } from "./tools/hosted-tools"; @@ -162,6 +163,14 @@ export interface AgentDefinition { baseSystemPrompt?: BaseSystemPromptOption; maxSteps?: number; maxTokens?: number; + /** + * Optional generation parameters (`temperature`, `top_p`, `stop`, + * `frequency_penalty`, `presence_penalty`) forwarded to the OpenAI-compatible + * serving request body. Only set keys are sent. Applied only when AppKit + * builds the adapter itself (string or omitted `model`); when you pass a + * pre-built `AgentAdapter`, configure generation params on it directly. + */ + generationParams?: GenerationParams; /** * When true, the thread used for a chat request against this agent is * deleted from `ThreadStore` after the stream completes (success or @@ -309,6 +318,8 @@ export interface RegisteredAgent { baseSystemPrompt?: BaseSystemPromptOption; maxSteps?: number; maxTokens?: number; + /** Mirrors `AgentDefinition.generationParams`. */ + generationParams?: GenerationParams; /** Mirrors `AgentDefinition.ephemeral` — skip thread persistence. */ ephemeral?: boolean; } diff --git a/packages/appkit/src/plugins/agents/agents.ts b/packages/appkit/src/plugins/agents/agents.ts index c63ec094f..ce40c514e 100644 --- a/packages/appkit/src/plugins/agents/agents.ts +++ b/packages/appkit/src/plugins/agents/agents.ts @@ -446,6 +446,7 @@ export class AgentsPlugin extends Plugin implements ToolProvider { baseSystemPrompt: def.baseSystemPrompt, maxSteps: def.maxSteps, maxTokens: def.maxTokens, + generationParams: def.generationParams, ephemeral: def.ephemeral, }; } @@ -458,9 +459,15 @@ export class AgentsPlugin extends Plugin implements ToolProvider { // Per-agent adapter knobs from `AgentDefinition` / markdown frontmatter. // Only applied when AppKit builds the adapter itself (string or omitted // model). Users who pass a pre-built `AgentAdapter` own these settings. - const adapterOptions: { maxSteps?: number; maxTokens?: number } = {}; + const adapterOptions: { + maxSteps?: number; + maxTokens?: number; + generationParams?: AgentDefinition["generationParams"]; + } = {}; if (def.maxSteps !== undefined) adapterOptions.maxSteps = def.maxSteps; if (def.maxTokens !== undefined) adapterOptions.maxTokens = def.maxTokens; + if (def.generationParams !== undefined) + adapterOptions.generationParams = def.generationParams; if (!source) { const { DatabricksAdapter } = await import("../../agents/databricks");