feat(openai): add computer use tool

Author: christian-bromann

.changeset/wet-drinks-remain.md

@@ -0,0 +1,5 @@
+---
+"@langchain/openai": minor
+---
+
+add support for computer use tool

libs/providers/langchain-openai/README.md

@@ -408,6 +408,62 @@ Supported models: `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-n
 
 For more information, see [OpenAI's Image Generation Documentation](https://platform.openai.com/docs/guides/tools-image-generation).
 
+### Computer Use Tool
+
+The Computer Use tool allows models to control computer interfaces by simulating mouse clicks, keyboard input, scrolling, and more. It uses OpenAI's Computer-Using Agent (CUA) model to understand screenshots and suggest actions.
+
+> **Beta**: Computer use is in beta. Use in sandboxed environments only and do not use for high-stakes or authenticated tasks. Always implement human-in-the-loop for important decisions.
+
+**How it works**: The tool operates in a continuous loop:
+
+1. Model sends computer actions (click, type, scroll, etc.)
+2. Your code executes these actions in a controlled environment
+3. You capture a screenshot of the result
+4. Send the screenshot back to the model
+5. Repeat until the task is complete
+
+```typescript
+import { ChatOpenAI, tools } from "@langchain/openai";
+
+const model = new ChatOpenAI({ model: "computer-use-preview" });
+
+// With execute callback for automatic action handling
+const computer = tools.computerUse({
+  displayWidth: 1024,
+  displayHeight: 768,
+  environment: "browser",
+  execute: async (action) => {
+    if (action.type === "screenshot") {
+      return captureScreenshot();
+    }
+    if (action.type === "click") {
+      await page.mouse.click(action.x, action.y, { button: action.button });
+      return captureScreenshot();
+    }
+    if (action.type === "type") {
+      await page.keyboard.type(action.text);
+      return captureScreenshot();
+    }
+    if (action.type === "scroll") {
+      await page.mouse.move(action.x, action.y);
+      await page.evaluate(
+        `window.scrollBy(${action.scroll_x}, ${action.scroll_y})`
+      );
+      return captureScreenshot();
+    }
+    // Handle other actions...
+    return captureScreenshot();
+  },
+});
+
+const llmWithComputer = model.bindTools([computer]);
+const response = await llmWithComputer.invoke(
+  "Check the latest news on bing.com"
+);
+```
+
+For more information, see [OpenAI's Computer Use Documentation](https://platform.openai.com/docs/guides/tools-computer-use).
+
 ## Embeddings
 
 This package also adds support for OpenAI's embeddings model.

libs/providers/langchain-openai/package.json

@@ -48,6 +48,7 @@
     "@langchain/standard-tests": "workspace:*",
     "@langchain/tsconfig": "workspace:*",
     "@tsconfig/recommended": "^1.0.10",
+    "@types/node": "^24.10.1",
     "@vitest/coverage-v8": "^3.2.4",
     "dotenv": "^17.2.1",
     "dpdm": "^3.14.0",
@@ -99,4 +100,4 @@
     "LICENSE"
   ],
   "module": "./dist/index.js"
-}
+}

libs/providers/langchain-openai/src/azure/chat_models/tests/index.int.test.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "vitest";
+import { test, expect, vi, describe } from "vitest";
 import { z } from "zod/v3";
 import {
   BaseMessage,
@@ -509,8 +509,8 @@ test("Test Azure ChatOpenAI Function calling with streaming", async () => {
 
 test("Test Azure ChatOpenAI can cache generations", async () => {
   const memoryCache = new InMemoryCache();
-  const lookupSpy = jest.spyOn(memoryCache, "lookup");
-  const updateSpy = jest.spyOn(memoryCache, "update");
+  const lookupSpy = vi.spyOn(memoryCache, "lookup");
+  const updateSpy = vi.spyOn(memoryCache, "update");
   const chat = new AzureChatOpenAI({
     model: "gpt-3.5-turbo",
     maxTokens: 10,
@@ -530,16 +530,16 @@ test("Test Azure ChatOpenAI can cache generations", async () => {
 
 test("Test Azure ChatOpenAI can write and read cached generations", async () => {
   const memoryCache = new InMemoryCache();
-  const lookupSpy = jest.spyOn(memoryCache, "lookup");
-  const updateSpy = jest.spyOn(memoryCache, "update");
+  const lookupSpy = vi.spyOn(memoryCache, "lookup");
+  const updateSpy = vi.spyOn(memoryCache, "update");
 
   const chat = new AzureChatOpenAI({
     model: "gpt-3.5-turbo",
     maxTokens: 100,
     n: 1,
     cache: memoryCache,
   });
-  const generateUncachedSpy = jest.spyOn(chat, "_generateUncached");
+  const generateUncachedSpy = vi.spyOn(chat, "_generateUncached");
 
   const messages = [
     [
@@ -568,8 +568,8 @@ test("Test Azure ChatOpenAI can write and read cached generations", async () =>
 
 test("Test Azure ChatOpenAI should not reuse cache if function call args have changed", async () => {
   const memoryCache = new InMemoryCache();
-  const lookupSpy = jest.spyOn(memoryCache, "lookup");
-  const updateSpy = jest.spyOn(memoryCache, "update");
+  const lookupSpy = vi.spyOn(memoryCache, "lookup");
+  const updateSpy = vi.spyOn(memoryCache, "update");
 
   const chat = new AzureChatOpenAI({
     model: "gpt-3.5-turbo",
@@ -578,7 +578,7 @@ test("Test Azure ChatOpenAI should not reuse cache if function call args have ch
     cache: memoryCache,
   });
 
-  const generateUncachedSpy = jest.spyOn(chat, "_generateUncached");
+  const generateUncachedSpy = vi.spyOn(chat, "_generateUncached");
 
   const messages = [
     [

libs/providers/langchain-openai/src/chat_models/tests/index-extended.int.test.ts

@@ -114,8 +114,9 @@ test("Test ChatOpenAI streaming logprobs", async () => {
     logprobs: true,
   });
   const res = await model.invoke("Print hello world.");
-  // console.log(res.response_metadata.logprobs.content);
-  expect(res.response_metadata.logprobs.content.length).toBeGreaterThan(0);
+  expect(
+    (res.response_metadata.logprobs as any).content.length
+  ).toBeGreaterThan(0);
 });
 
 test("Test ChatOpenAI with search preview model", async () => {
@@ -172,7 +173,6 @@ test("Test ChatOpenAI tool calling with ToolMessages", async () => {
   const res = await chat.invoke([
     ["human", "What's the weather like in San Francisco, Tokyo, and Paris?"],
   ]);
-  // console.log(JSON.stringify(res));
   expect(res.additional_kwargs.tool_calls?.length).toBeGreaterThan(1);
   const toolMessages = res.additional_kwargs.tool_calls!.map(
     (toolCall) =>
@@ -194,7 +194,6 @@ test("Test ChatOpenAI tool calling with ToolMessages", async () => {
     toolError = e;
   }
   expect(toolError).toBeDefined();
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   expect((toolError as any)?.lc_error_code).toEqual("INVALID_TOOL_RESULTS");
   // @eslint-disable-next-line/@typescript-eslint/ban-ts-comment
   // @ts-expect-error unused var
@@ -642,22 +641,23 @@ test.skip("system prompt caching", async () => {
     },
   ];
   const res = await model.invoke(messages);
-  expect(res.response_metadata?.usage.prompt_tokens_details.cached_tokens).toBe(
-    0
-  );
+  expect(
+    (res.response_metadata?.usage as any).prompt_tokens_details.cached_tokens
+  ).toBe(0);
   await new Promise((resolve) => {
     setTimeout(resolve, 5000);
   });
   const res2 = await model.invoke(messages);
   expect(
-    res2.response_metadata?.usage.prompt_tokens_details.cached_tokens
+    (res2.response_metadata?.usage as any).prompt_tokens_details.cached_tokens
   ).toBeGreaterThan(0);
   let aggregate;
   for await (const chunk of await model.stream(messages)) {
     aggregate = aggregate ? concat(aggregate, chunk) : chunk;
   }
   expect(
-    aggregate?.response_metadata?.usage.prompt_tokens_details.cached_tokens
+    (aggregate?.response_metadata?.usage as any).prompt_tokens_details
+      .cached_tokens
   ).toBeGreaterThan(0);
 });
 
@@ -707,11 +707,11 @@ public class User
     }
   );
   expect(
-    typeof res.response_metadata?.usage?.completion_tokens_details
+    typeof (res.response_metadata?.usage as any).completion_tokens_details
       .accepted_prediction_tokens
   ).toBe("number");
   expect(
-    typeof res.response_metadata?.usage?.completion_tokens_details
+    typeof (res.response_metadata?.usage as any).completion_tokens_details
       .rejected_prediction_tokens
   ).toBe("number");
 });