Merge pull request #158 from SentienceAPI/permissions

solution for handle Chrome permission bubble

Author: rcholic

src/agent-runtime.ts

@@ -492,6 +492,17 @@ export class AgentRuntime {
     const hasEval = typeof (this as any).evaluateJs === 'function';
     const hasKeyboard = Boolean((this.page as any)?.keyboard);
     const hasDownloads = this.downloads.length >= 0;
+    let hasPermissions = false;
+    try {
+      const context =
+        typeof (this.page as any)?.context === 'function' ? (this.page as any).context() : null;
+      hasPermissions =
+        Boolean(context) &&
+        typeof context.clearPermissions === 'function' &&
+        typeof context.grantPermissions === 'function';
+    } catch {
+      hasPermissions = false;
+    }
     let hasFiles = false;
     if (this.toolRegistry) {
       hasFiles = Boolean(this.toolRegistry.get('read_file'));
@@ -502,6 +513,7 @@ export class AgentRuntime {
       downloads: hasDownloads,
       filesystem_tools: hasFiles,
       keyboard: hasKeyboard,
+      permissions: hasPermissions,
     };
   }
 

src/browser.ts

@@ -12,6 +12,15 @@ import { SnapshotOptions } from './snapshot';
 import { IBrowser } from './protocols/browser-protocol';
 import { snapshot as snapshotFunction } from './snapshot';
 
+export type PermissionDefault = 'clear' | 'deny' | 'grant';
+
+export type PermissionPolicy = {
+  default?: PermissionDefault;
+  autoGrant?: string[];
+  geolocation?: { latitude: number; longitude: number; accuracy?: number };
+  origin?: string;
+};
+
 export function normalizeDomain(domain: string): string {
   const raw = domain.trim();
   let host = raw;
@@ -88,6 +97,7 @@ export class SentienceBrowser implements IBrowser {
   private _allowedDomains?: string[];
   private _prohibitedDomains?: string[];
   private _keepAlive: boolean;
+  private _permissionPolicy?: PermissionPolicy;
 
   /**
    * Create a new SentienceBrowser instance
@@ -121,7 +131,8 @@ export class SentienceBrowser implements IBrowser {
     deviceScaleFactor?: number,
     allowedDomains?: string[],
     prohibitedDomains?: string[],
-    keepAlive: boolean = false
+    keepAlive: boolean = false,
+    permissionPolicy?: PermissionPolicy
   ) {
     this._apiKey = apiKey;
 
@@ -162,6 +173,24 @@ export class SentienceBrowser implements IBrowser {
     this._allowedDomains = allowedDomains;
     this._prohibitedDomains = prohibitedDomains;
     this._keepAlive = keepAlive;
+    this._permissionPolicy = permissionPolicy;
+  }
+
+  private async applyPermissionPolicy(
+    context: BrowserContext,
+    policy: PermissionPolicy
+  ): Promise<void> {
+    const defaultPolicy = policy.default ?? 'clear';
+    if (defaultPolicy === 'clear' || defaultPolicy === 'deny') {
+      await context.clearPermissions();
+    }
+    if (policy.geolocation) {
+      await context.setGeolocation(policy.geolocation);
+    }
+    if (policy.autoGrant && policy.autoGrant.length > 0) {
+      const options = policy.origin ? { origin: policy.origin } : undefined;
+      await context.grantPermissions(policy.autoGrant, options);
+    }
   }
 
   async start(): Promise<void> {
@@ -276,6 +305,10 @@ export class SentienceBrowser implements IBrowser {
 
     this.context = await chromium.launchPersistentContext(this.userDataDir, launchOptions);
 
+    if (this._permissionPolicy) {
+      await this.applyPermissionPolicy(this.context, this._permissionPolicy);
+    }
+
     this.page = this.context.pages()[0] || (await this.context.newPage());
 
     // Inject storage state if provided (must be after context creation)

src/index.ts

@@ -2,7 +2,7 @@
  * Sentience TypeScript SDK - AI Agent Browser Automation
  */
 
-export { SentienceBrowser } from './browser';
+export { SentienceBrowser, PermissionPolicy } from './browser';
 export { snapshot, SnapshotOptions } from './snapshot';
 export { query, find, parseSelector } from './query';
 export {

src/tools/defaults.ts

@@ -1,7 +1,7 @@
 import { z } from 'zod';
 import type { AgentRuntime } from '../agent-runtime';
 import type { ActionResult, Snapshot, EvaluateJsResult } from '../types';
-import { ToolContext } from './context';
+import { ToolContext, UnsupportedCapabilityError } from './context';
 import { defineTool, ToolRegistry } from './registry';
 
 const snapshotSchema = z
@@ -83,6 +83,19 @@ const evaluateJsInput = z.object({
   truncate: z.boolean().default(true),
 });
 
+const grantPermissionsInput = z.object({
+  permissions: z.array(z.string()).min(1),
+  origin: z.string().optional(),
+});
+
+const clearPermissionsInput = z.object({});
+
+const setGeolocationInput = z.object({
+  latitude: z.number(),
+  longitude: z.number(),
+  accuracy: z.number().optional(),
+});
+
 function getRuntime(ctx: ToolContext | null, runtime?: ToolContext | AgentRuntime): AgentRuntime {
   if (ctx) return ctx.runtime;
   if (runtime instanceof ToolContext) return runtime.runtime;
@@ -342,5 +355,87 @@ export function registerDefaultTools(
     })
   );
 
+  registry.register(
+    defineTool<z.infer<typeof grantPermissionsInput>, ActionResult, ToolContext | null>({
+      name: 'grant_permissions',
+      description: 'Grant browser permissions for the current context.',
+      input: grantPermissionsInput,
+      output: actionResultSchema,
+      handler: async (ctx, params): Promise<ActionResult> => {
+        const runtimeRef = getRuntime(ctx, runtime);
+        if (ctx) {
+          ctx.require('permissions');
+        } else if (!runtimeRef.can('permissions')) {
+          throw new UnsupportedCapabilityError('permissions');
+        }
+        const context =
+          typeof (runtimeRef.page as any)?.context === 'function'
+            ? (runtimeRef.page as any).context()
+            : null;
+        if (!context) {
+          throw new Error('Permission context unavailable');
+        }
+        await context.grantPermissions(params.permissions, params.origin);
+        return { success: true, duration_ms: 0, outcome: 'dom_updated' };
+      },
+    })
+  );
+
+  registry.register(
+    defineTool<z.infer<typeof clearPermissionsInput>, ActionResult, ToolContext | null>({
+      name: 'clear_permissions',
+      description: 'Clear browser permissions for the current context.',
+      input: clearPermissionsInput,
+      output: actionResultSchema,
+      handler: async (ctx): Promise<ActionResult> => {
+        const runtimeRef = getRuntime(ctx, runtime);
+        if (ctx) {
+          ctx.require('permissions');
+        } else if (!runtimeRef.can('permissions')) {
+          throw new UnsupportedCapabilityError('permissions');
+        }
+        const context =
+          typeof (runtimeRef.page as any)?.context === 'function'
+            ? (runtimeRef.page as any).context()
+            : null;
+        if (!context) {
+          throw new Error('Permission context unavailable');
+        }
+        await context.clearPermissions();
+        return { success: true, duration_ms: 0, outcome: 'dom_updated' };
+      },
+    })
+  );
+
+  registry.register(
+    defineTool<z.infer<typeof setGeolocationInput>, ActionResult, ToolContext | null>({
+      name: 'set_geolocation',
+      description: 'Set geolocation for the current browser context.',
+      input: setGeolocationInput,
+      output: actionResultSchema,
+      handler: async (ctx, params): Promise<ActionResult> => {
+        const runtimeRef = getRuntime(ctx, runtime);
+        if (ctx) {
+          ctx.require('permissions');
+        } else if (!runtimeRef.can('permissions')) {
+          throw new UnsupportedCapabilityError('permissions');
+        }
+        const context =
+          typeof (runtimeRef.page as any)?.context === 'function'
+            ? (runtimeRef.page as any).context()
+            : null;
+        if (!context) {
+          throw new Error('Permission context unavailable');
+        }
+        await context.setGeolocation({
+          latitude: params.latitude,
+          longitude: params.longitude,
+          accuracy: params.accuracy,
+        });
+        return { success: true, duration_ms: 0, outcome: 'dom_updated' };
+      },
+    })
+  );
+
   return registry;
 }

src/tools/registry.ts

@@ -84,7 +84,11 @@ export class ToolRegistry {
     name: string,
     payload: unknown,
     ctx: {
-      runtime?: { tracer?: { emit: (...args: any[]) => void }; stepId?: string; step_id?: string };
+      runtime?: {
+        tracer?: { emit: (...args: any[]) => void };
+        stepId?: string | null;
+        step_id?: string | null;
+      };
     } | null = null
   ): Promise<TOutput> {
     const start = Date.now();

src/types.ts

@@ -259,6 +259,7 @@ export interface BackendCapabilities {
   downloads: boolean;
   filesystem_tools: boolean;
   keyboard: boolean;
+  permissions: boolean;
 }
 
 export interface EvaluateJsRequest {

tests/actions.test.ts

@@ -23,7 +23,7 @@ import {
   uncheck,
   uploadFile,
 } from '../src';
-import { createTestBrowser, getPageOrThrow } from './test-utils';
+import { createTestBrowser, getPageOrThrow, patchExampleDotCom } from './test-utils';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
@@ -220,8 +220,8 @@ describe('Actions', () => {
       try {
         await browser.start();
         const page = getPageOrThrow(browser);
+        patchExampleDotCom(page);
         await page.goto('https://example.com');
-        await page.waitForLoadState('networkidle', { timeout: 10000 });
 
         await expect(search(browser, 'sentience sdk', 'duckduckgo')).rejects.toThrow(
           'domain not allowed'

tests/browser.test.ts

@@ -7,6 +7,7 @@ import { chromium, BrowserContext, Page } from 'playwright';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
+import { patchExampleDotCom } from './test-utils';
 
 describe('Browser Proxy Support', () => {
   describe('Proxy Parsing', () => {
@@ -228,6 +229,7 @@ describe('Browser Proxy Support', () => {
         if (!page) {
           throw new Error('Browser page is not available');
         }
+        patchExampleDotCom(page);
         await page.goto('https://example.com', { waitUntil: 'domcontentloaded', timeout: 20000 });
 
         const viewportSize = await page.evaluate(() => ({
@@ -293,6 +295,7 @@ describe('Browser Proxy Support', () => {
         expect(sentienceBrowser.getContext()).toBe(context);
 
         // Test that we can use it
+        patchExampleDotCom(page);
         await page.goto('https://example.com');
         await page.waitForLoadState('networkidle', { timeout: 10000 });
 

tests/test-utils.ts

@@ -13,6 +13,10 @@ export async function createTestBrowser(headless?: boolean): Promise<SentienceBr
   const browser = new SentienceBrowser(undefined, undefined, headless);
   try {
     await browser.start();
+    const page = browser.getPage();
+    if (page) {
+      patchExampleDotCom(page);
+    }
     return browser;
   } catch (e: any) {
     // Clean up browser on failure to prevent resource leaks
@@ -44,3 +48,28 @@ export function getPageOrThrow(browser: SentienceBrowser): Page {
   }
   return page;
 }
+
+const DEFAULT_TEST_HTML = `<!doctype html>
+<html>
+  <head><meta charset="utf-8" /></head>
+  <body>
+    <a id="link" href="#ok">Example Link</a>
+    <input id="text" type="text" value="hello" />
+    <button id="btn" type="button">Click me</button>
+    <div style="height: 2000px;"></div>
+  </body>
+</html>`;
+
+export async function setTestPageContent(page: Page, html?: string): Promise<void> {
+  await page.setContent(html ?? DEFAULT_TEST_HTML, { waitUntil: 'domcontentloaded' });
+}
+
+export function patchExampleDotCom(page: Page): void {
+  void page.route(/https?:\/\/example\.com\/?.*/, async route => {
+    await route.fulfill({
+      status: 200,
+      contentType: 'text/html',
+      body: DEFAULT_TEST_HTML,
+    });
+  });
+}