+ +
+
+

1. Let's make a test request

+
The gateway supports 250+ models across 36 AI providers. Choose your provider and API + key below.
+
+
🐍 Python
+
📦 Node.js
+
🌀 cURL
+
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+

2. Create a routing config

+
Gateway configs allow you to route requests to different providers and models. You can load balance, set fallbacks, and configure automatic retries & timeouts. Learn more
+
+
Simple Config
+
Load Balancing
+
Fallbacks
+
Retries & Timeouts
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+

3. More Features to Explore

+
Discover advanced capabilities of the Portkey AI Gateway.
+
+ + + + + + + +

Agents

+

Seamlessly integrate with popular agent frameworks like Autogen, CrewAI, and LangChain.

+ + + + + + + +

Multi-modal AI

+

Handle vision, audio, and image generation requests across multiple providers.

+ + + + + +

Guardrails

+

Verify LLM inputs and outputs with 20+ pre-built checks or build your own.

+ + + + + + + + +

Supported Providers

+

Access 250+ models from 35+ providers including OpenAI, Anthropic, and Google.

+ +
+
+ +
+ + +
+
+ + + + +

Setup a Call

+

Get personalized support and learn how Portkey can be tailored to your needs.

+ Schedule Consultation +
+
+ + + + + +

Enterprise Features

+

Explore advanced features and see how Portkey can scale with your business.

+ View Enterprise Plan +
+
+ + + + +

Join Our Community

+

Connect with other developers, share ideas, and get help from the Portkey team.

+ Join Discord +
+
+
+
+ +
+
+

Real-time Logs

+
+ + +
+
+ + + + + + + + + + + + + + + + + + +
TimeMethodEndpointStatusDurationActions
+
+ Listening for logs... +
+
+
+
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/services/transformToProviderRequest.ts b/src/services/transformToProviderRequest.ts index 6abb36096..af4d56876 100644 --- a/src/services/transformToProviderRequest.ts +++ b/src/services/transformToProviderRequest.ts @@ -183,10 +183,11 @@ const transformToProviderRequestFormData = ( export const transformToProviderRequest = ( provider: string, params: Params, - inputParams: Params | FormData, + inputParams: Params | FormData | ArrayBuffer, fn: endpointStrings ) => { - if (inputParams instanceof FormData) return inputParams; + if (inputParams instanceof FormData || inputParams instanceof ArrayBuffer) + return inputParams; if (fn === 'proxy') { return params; diff --git a/src/start-server.ts b/src/start-server.ts index 2a0369c70..f58da4231 100644 --- a/src/start-server.ts +++ b/src/start-server.ts @@ -1,8 +1,11 @@ #!/usr/bin/env node import { serve } from '@hono/node-server'; -import { createNodeWebSocket } from '@hono/node-ws'; + import app from './index'; +import { streamSSE } from 'hono/streaming'; +import { Context } from 'hono'; +import { createNodeWebSocket } from '@hono/node-ws'; import { realTimeHandlerNode } from './handlers/realtimeHandlerNode'; import { requestValidator } from './middlewares/requestValidator'; @@ -12,6 +15,126 @@ const args = process.argv.slice(2); const portArg = args.find((arg) => arg.startsWith('--port=')); const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; +const isHeadless = args.includes('--headless'); + +// Setup static file serving only if not in headless mode +if ( + !isHeadless && + !( + process.env.NODE_ENV === 'production' || + process.env.ENVIRONMENT === 'production' + ) +) { + const setupStaticServing = async () => { + const { join, dirname } = await import('path'); + const { fileURLToPath } = await import('url'); + const { readFileSync } = await import('fs'); + + const scriptDir = dirname(fileURLToPath(import.meta.url)); + + // Serve the index.html content directly for both routes + const indexPath = join(scriptDir, 'public/index.html'); + const indexContent = readFileSync(indexPath, 'utf-8'); + + const serveIndex = (c: Context) => { + return c.html(indexContent); + }; + + // Set up routes + app.get('/public/logs', serveIndex); + app.get('/public/', serveIndex); + + // Redirect `/public` to `/public/` + app.get('/public', (c: Context) => { + return c.redirect('/public/'); + }); + }; + + // Initialize static file serving + await setupStaticServing(); + + /** + * A helper function to enforce a timeout on SSE sends. + * @param fn A function that returns a Promise (e.g. stream.writeSSE()) + * @param timeoutMs The timeout in milliseconds (default: 2000) + */ + async function sendWithTimeout(fn: () => Promise, timeoutMs = 200) { + const timeoutPromise = new Promise((_, reject) => { + const id = setTimeout(() => { + clearTimeout(id); + reject(new Error('Write timeout')); + }, timeoutMs); + }); + + return Promise.race([fn(), timeoutPromise]); + } + + app.get('/log/stream', (c: Context) => { + const clientId = Date.now().toString(); + + // Set headers to prevent caching + c.header('Cache-Control', 'no-cache'); + c.header('X-Accel-Buffering', 'no'); + + return streamSSE(c, async (stream) => { + const addLogClient: any = c.get('addLogClient'); + const removeLogClient: any = c.get('removeLogClient'); + + const client = { + sendLog: (message: any) => + sendWithTimeout(() => stream.writeSSE(message)), + }; + // Add this client to the set of log clients + addLogClient(clientId, client); + + // If the client disconnects (closes the tab, etc.), this signal will be aborted + const onAbort = () => { + removeLogClient(clientId); + }; + c.req.raw.signal.addEventListener('abort', onAbort); + + try { + // Send an initial connection event + await sendWithTimeout(() => + stream.writeSSE({ event: 'connected', data: clientId }) + ); + + // Use an interval instead of a while loop + const heartbeatInterval = setInterval(async () => { + if (c.req.raw.signal.aborted) { + clearInterval(heartbeatInterval); + return; + } + + try { + await sendWithTimeout(() => + stream.writeSSE({ event: 'heartbeat', data: 'pulse' }) + ); + } catch (error) { + // console.error(`Heartbeat failed for client ${clientId}:`, error); + clearInterval(heartbeatInterval); + removeLogClient(clientId); + } + }, 10000); + + // Wait for abort signal + await new Promise((resolve) => { + c.req.raw.signal.addEventListener('abort', () => { + clearInterval(heartbeatInterval); + resolve(undefined); + }); + }); + } catch (error) { + // console.error(`Error in log stream for client ${clientId}:`, error); + } finally { + // Remove this client when the connection is closed + removeLogClient(clientId); + c.req.raw.signal.removeEventListener('abort', onAbort); + } + }); + }); +} + const { injectWebSocket, upgradeWebSocket } = createNodeWebSocket({ app }); app.get( @@ -25,5 +148,43 @@ const server = serve({ port: port, }); +const url = `http://localhost:${port}`; + injectWebSocket(server); -console.log(`Your AI Gateway is now running on http://localhost:${port} 🚀`); + +// Loading animation function +async function showLoadingAnimation() { + const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + let i = 0; + + return new Promise((resolve) => { + const interval = setInterval(() => { + process.stdout.write(`\r${frames[i]} Starting AI Gateway...`); + i = (i + 1) % frames.length; + }, 80); + + // Stop after 1 second + setTimeout(() => { + clearInterval(interval); + process.stdout.write('\r'); + resolve(undefined); + }, 1000); + }); +} + +// Clear the console and show animation before main output +console.clear(); +await showLoadingAnimation(); + +// Main server information with minimal spacing +console.log('\x1b[1m%s\x1b[0m', '🚀 Your AI Gateway is running at:'); +console.log(' ' + '\x1b[1;4;32m%s\x1b[0m', `${url}`); + +// Secondary information on single lines +if (!isHeadless) { + console.log('\n\x1b[90m📱 UI:\x1b[0m \x1b[36m%s\x1b[0m', `${url}/public/`); +} +// console.log('\x1b[90m📚 Docs:\x1b[0m \x1b[36m%s\x1b[0m', 'https://portkey.ai/docs'); + +// Single-line ready message +console.log('\n\x1b[32m✨ Ready for connections!\x1b[0m'); diff --git a/src/types/requestBody.ts b/src/types/requestBody.ts index 4787cc7d0..4f7833dd1 100644 --- a/src/types/requestBody.ts +++ b/src/types/requestBody.ts @@ -217,15 +217,18 @@ export enum MESSAGE_ROLES { ASSISTANT = 'assistant', FUNCTION = 'function', TOOL = 'tool', + DEVELOPER = 'developer', } +export const SYSTEM_MESSAGE_ROLES = ['system', 'developer']; + export type OpenAIMessageRole = | 'system' | 'user' | 'assistant' | 'function' - | 'tool'; - + | 'tool' + | 'developer'; /** * A message in the conversation. * @interface @@ -301,7 +304,7 @@ export interface Tool extends AnthropicPromptCache { /** The name of the function. */ type: string; /** A description of the function. */ - function?: Function; + function: Function; } /** diff --git a/start-test.js b/start-test.js index 369be1668..b4ba7beb9 100644 --- a/start-test.js +++ b/start-test.js @@ -2,7 +2,7 @@ import { spawn } from 'node:child_process'; console.log('Starting the application...'); -const app = spawn('node', ['build/start-server.js'], { +const app = spawn('node', ['build/start-server.js', '--headless'], { stdio: 'inherit', });