Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ export interface ApiHandler {

getModel(): { id: string; info: ModelInfo }

/**
* Ensures model metadata has been fetched from the remote API so that getModel()
* returns accurate info (context window, pricing, etc.) instead of hardcoded defaults.
* Only router providers that discover models over the network implement this.
*/
ensureModelFetched?(): Promise<void>

/**
* Optional context window for context-management / auto-condense when it must differ from
* getModel().info.contextWindow. Only VS Code LM overrides it (static `maxInputTokens` vs its
Expand Down
88 changes: 88 additions & 0 deletions src/api/providers/__tests__/zoo-gateway.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -635,4 +635,92 @@ describe("ZooGatewayHandler", () => {
)
})
})

describe("ensureModelFetched", () => {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the rejection path covered anywhere? If getModels rejects once and the finally cleanup in router-provider.ts (L71-74) ever regresses, the stored rejected promise would poison every subsequent call. A reject-then-resolve test here would lock that in.

it("fetches models when instance models are empty", async () => {
const handler = new ZooGatewayHandler(mockOptions)
const { getModels } = await import("../fetchers/modelCache")

expect(handler.getModel().info.contextWindow).toBe(200000)

await handler.ensureModelFetched()

expect(getModels).toHaveBeenCalled()
})

it("skips the fetch when models are already populated", async () => {
const handler = new ZooGatewayHandler(mockOptions)
const { getModels } = await import("../fetchers/modelCache")

await handler.ensureModelFetched()
vitest.mocked(getModels).mockClear()

await handler.ensureModelFetched()
expect(getModels).not.toHaveBeenCalled()
})

it("short-circuits a subsequent fetchModel call after models are populated", async () => {
const handler = new ZooGatewayHandler(mockOptions)
const { getModels } = await import("../fetchers/modelCache")

await handler.ensureModelFetched()
vitest.mocked(getModels).mockClear()

await handler.fetchModel()
expect(getModels).not.toHaveBeenCalled()
})

it("deduplicates concurrent calls into a single fetch", async () => {
const handler = new ZooGatewayHandler(mockOptions)
const { getModels } = await import("../fetchers/modelCache")
vitest.mocked(getModels).mockClear()

await Promise.all([handler.ensureModelFetched(), handler.ensureModelFetched()])

expect(getModels).toHaveBeenCalledTimes(1)
})

it("recovers after a rejected fetch so later calls are not poisoned", async () => {
const handler = new ZooGatewayHandler(mockOptions)
const { getModels } = await import("../fetchers/modelCache")

vitest.mocked(getModels).mockRejectedValueOnce(new Error("network down"))
await expect(handler.ensureModelFetched()).rejects.toThrow("network down")

vitest.mocked(getModels).mockResolvedValueOnce({
"anthropic/claude-sonnet-4": {
maxTokens: 64000,
contextWindow: 1000000,
supportsImages: true,
supportsPromptCache: true,
},
})
await handler.ensureModelFetched()

expect(handler.getModel().info.contextWindow).toBe(1000000)
})

it("makes getModel return the fetched context window instead of the default", async () => {
const { getModels } = await import("../fetchers/modelCache")
vitest.mocked(getModels).mockResolvedValueOnce({
"google/gemini-2.5-pro": {
maxTokens: 65536,
contextWindow: 1048576,
supportsImages: true,
supportsPromptCache: false,
},
})

const handler = new ZooGatewayHandler({
...mockOptions,
zooGatewayModelId: "google/gemini-2.5-pro",
})

expect(handler.getModel().info.contextWindow).toBe(200000)

await handler.ensureModelFetched()

expect(handler.getModel().info.contextWindow).toBe(1048576)
})
})
})
28 changes: 26 additions & 2 deletions src/api/providers/router-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,33 @@ export abstract class RouterProvider extends BaseProvider {
})
}

private modelFetchPromise?: Promise<{ id: string; info: ModelInfo }>

public async fetchModel() {
this.models = await getModels({ provider: this.name, apiKey: this.client.apiKey, baseUrl: this.client.baseURL })
return this.getModel()
if (Object.keys(this.models).length > 0) {
return this.getModel()
}

if (!this.modelFetchPromise) {
this.modelFetchPromise = getModels({
provider: this.name,
apiKey: this.client.apiKey,
baseUrl: this.client.baseURL,
})
.then((models) => {
this.models = models
return this.getModel()
})
.finally(() => {
this.modelFetchPromise = undefined
})
}

return this.modelFetchPromise
}

async ensureModelFetched(): Promise<void> {
await this.fetchModel()
}

override getModel(): { id: string; info: ModelInfo } {
Expand Down
20 changes: 20 additions & 0 deletions src/core/task/Task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2762,6 +2762,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {

await this.diffViewProvider.reset()

await this.safeEnsureModelFetched()

// Cache model info once per API request to avoid repeated calls during streaming
// This is especially important for tools and background usage collection
this.cachedStreamingModel = this.api.getModel()
Expand Down Expand Up @@ -3837,11 +3839,28 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
)
}

/**
* Ensures router-provider model metadata is loaded before getModel() is used for
* context management or streaming. Failures fall back to hardcoded defaults rather
* than aborting the task.
*/
private async safeEnsureModelFetched(): Promise<void> {
try {
await this.api.ensureModelFetched?.()
} catch (error) {
console.error(
`[Task#${this.taskId}] Failed to fetch model metadata:`,
error instanceof Error ? error.message : error,
)
}
}

private async handleContextWindowExceededError(): Promise<void> {
const state = await this.providerRef.deref()?.getState()
const { profileThresholds = {}, mode, apiConfiguration } = state ?? {}

const { contextTokens } = this.getTokenUsage()
await this.safeEnsureModelFetched()
const modelInfo = this.api.getModel().info

const maxTokens = getModelMaxOutputTokens({
Expand Down Expand Up @@ -4042,6 +4061,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
const { contextTokens } = this.getTokenUsage()

if (contextTokens) {
await this.safeEnsureModelFetched()
const modelInfo = this.api.getModel().info

const maxTokens = getModelMaxOutputTokens({
Expand Down
Loading
Loading