+ +
+
+

1. Let's make a test request

+
The gateway supports 250+ models across 36 AI providers. Choose your provider and API + key below.
+
+
🐍 Python
+
📦 Node.js
+
🌀 cURL
+
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+

2. Create a routing config

+
Gateway configs allow you to route requests to different providers and models. You can load balance, set fallbacks, and configure automatic retries & timeouts. Learn more
+
+
Simple Config
+
Load Balancing
+
Fallbacks
+
Retries & Timeouts
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+

3. More Features to Explore

+
Discover advanced capabilities of the Portkey AI Gateway.
+
+ + + + + + + +

Agents

+

Seamlessly integrate with popular agent frameworks like Autogen, CrewAI, and LangChain.

+ + + + + + + +

Multi-modal AI

+

Handle vision, audio, and image generation requests across multiple providers.

+ + + + + +

Guardrails

+

Verify LLM inputs and outputs with 20+ pre-built checks or build your own.

+ + + + + + + + +

Supported Providers

+

Access 250+ models from 35+ providers including OpenAI, Anthropic, and Google.

+ +
+
+ +
+ + +
+
+ + + + +

Setup a Call

+

Get personalized support and learn how Portkey can be tailored to your needs.

+ Schedule Consultation +
+
+ + + + + +

Enterprise Features

+

Explore advanced features and see how Portkey can scale with your business.

+ View Enterprise Plan +
+
+ + + + +

Join Our Community

+

Connect with other developers, share ideas, and get help from the Portkey team.

+ Join Discord +
+
+
+
+ +
+
+

Real-time Logs

+
+ + +
+
+ + + + + + + + + + + + + + + + + + +
TimeMethodEndpointStatusDurationActions
+
+ Listening for logs... +
+
+
+
+ + + + + + + + + + + + + + \ No newline at end of file diff --git a/public/main.js b/public/main.js new file mode 100644 index 000000000..b90a0df0e --- /dev/null +++ b/public/main.js @@ -0,0 +1,536 @@ +function getTestRequestCodeBlock(language, vars) { + switch (language) { + case 'nodejs': + case 'nodejs': + return ` +import Portkey from 'portkey-ai' + +const portkey = new Portkey({ + provider: "${vars.provider || '[Click to edit]'}"${vars.provider != 'bedrock' ? `, + Authorization: "${vars.providerDetails?.apiKey || '[Click to edit]'}"`: ''}${vars.provider === 'azure-openai' ? `, + azureResourceName: "${vars.providerDetails?.azureResourceName || '[Click to edit]'}", + azureDeploymentId: "${vars.providerDetails?.azureDeploymentId || '[Click to edit]'}", + azureApiVersion: "${vars.providerDetails?.azureApiVersion || '[Click to edit]'}", + azureModelName: "${vars.providerDetails?.azureModelName || '[Click to edit]'}"` : ''}${vars.provider === 'bedrock' ? `, + awsAccessKeyId: "${vars.providerDetails?.awsAccessKeyId || '[Click to edit]'}", + awsSecretAccessKey: "${vars.providerDetails?.awsSecretAccessKey || '[Click to edit]'}", + awsRegion: "${vars.providerDetails?.awsRegion || '[Click to edit]'}"${vars.providerDetails?.awsSessionToken ? `, + awsSessionToken: "${vars.providerDetails.awsSessionToken}"` : ''}` : ''} +}) + +// Example: Send a chat completion request +const response = await portkey.chat.completion.create({ + messages: [{ role: 'user', content: 'Hello, how are you?' }], + model: "${modelMap[vars.provider] || ''}"${vars.provider=="anthropic"?`, + max_tokens: 40`:''} +}) +console.log(response.choices[0].message.content)`.trim(); + + case 'python': + return ` +from portkey_ai import Portkey + +client = Portkey( + provider="${vars.provider || '[Click to edit]'}"${vars.provider != 'bedrock' ? `, + Authorization="${vars.providerDetails?.apiKey || '[Click to edit]'}"`: ''}${vars.provider === 'azure-openai' ? `, + azure_resource_name="${vars.providerDetails?.azureResourceName || '[Click to edit]'}", + azure_deployment_id="${vars.providerDetails?.azureDeploymentId || '[Click to edit]'}", + azure_api_version="${vars.providerDetails?.azureApiVersion || '[Click to edit]'}", + azure_model_name="${vars.providerDetails?.azureModelName || '[Click to edit]'}"` : ''}${vars.provider === 'bedrock' ? `, + aws_access_key_id="${vars.providerDetails?.awsAccessKeyId || '[Click to edit]'}", + aws_secret_access_key="${vars.providerDetails?.awsSecretAccessKey || '[Click to edit]'}", + aws_region="${vars.providerDetails?.awsRegion || '[Click to edit]'}"${vars.providerDetails?.awsSessionToken ? `, + aws_session_token="${vars.providerDetails.awsSessionToken}"` : ''}` : ''} +) + +# Example: Send a chat completion request +response = client.chat.completion.create( + messages=[{"role": "user", "content": "Hello, how are you?"}], + model="${modelMap[vars.provider] || ''}" +) +print(response.choices[0].message.content)`.trim(); + + case 'curl': + return `curl -X POST \\ +https://api.portkey.ai/v1/chat/completions \\ +-H "Content-Type: application/json" \\ +-H "x-portkey-provider: ${vars.provider || '[Click to edit]'}" \\${vars.provider != 'bedrock' ? ` +-H "Authorization: ${vars.providerDetails?.apiKey || '[Click to edit]'}" \\`: '' }${vars.provider === 'azure-openai' ? `\n-H "x-portkey-azure-resource-name: ${vars.providerDetails?.azureResourceName || '[Click to edit]'}" \\ +-H "x-portkey-azure-deployment-id: ${vars.providerDetails?.azureDeploymentId || '[Click to edit]'}" \\ +-H "x-portkey-azure-api-version: ${vars.providerDetails?.azureApiVersion || '[Click to edit]'}" \\ +-H "x-portkey-azure-model-name: ${vars.providerDetails?.azureModelName || '[Click to edit]'}" \\` : ''}${vars.provider === 'bedrock' ? `\n-H "x-portkey-aws-access-key-id: ${vars.providerDetails?.awsAccessKeyId || '[Click to edit]'}" \\ +-H "x-portkey-aws-secret-access-key: ${vars.providerDetails?.awsSecretAccessKey || '[Click to edit]'}" \\ +-H "x-portkey-aws-region: ${vars.providerDetails?.awsRegion || '[Click to edit]'}" \\${vars.providerDetails?.awsSessionToken ? `\n-H "x-portkey-aws-session-token: ${vars.providerDetails.awsSessionToken}" \\` : ''}` : ''} +-d '{ + "messages": [ + { "role": "user", "content": "Hello, how are you?" }, + ], + "model": ""${modelMap[vars.provider] || ''}"" +}'`.trim(); + } +} + + +function getRoutingConfigCodeBlock(language, type) { + return configs[language][type]; +} + +// Needed for highlight.js +const lngMap = {"nodejs": "js", "python": "py", "curl": "sh"} + +const modelMap = { + "openai": "gpt-4o-mini", + "anthropic": "claude-3-5-sonnet-20240620", + "groq": "llama3-70b-8192", + "bedrock": "anthropic.claude-3-sonnet-20240229-v1:0", + "azure-openai": "gpt-4o-mini" +} + +// Initialize Lucide icons +lucide.createIcons(); + +// Variables +let provider = ''; +let apiKey = ''; +let providerDetails = {}; +let logCounter = 0; + +// DOM Elements +const providerValue = document.getElementById('providerValue'); +const apiKeyValue = document.getElementById('apiKeyValue'); +const copyBtn = document.getElementById('copyBtn'); +const testRequestBtn = document.getElementById('testRequestBtn'); +const logsContent = document.getElementById('logsContent'); +const providerDialog = document.getElementById('providerDialog'); +const apiKeyDialog = document.getElementById('apiKeyDialog'); +const providerSelect = document.getElementById('providerSelect'); +const apiKeyInput = document.getElementById('apiKeyInput'); +const saveApiKeyBtn = document.getElementById('saveApiKeyBtn'); +const saveApiDetailsBtn = document.getElementById('saveApiDetailsBtn'); +const languageSelect = document.getElementById('languageSelect'); +const copyConfigBtn = document.getElementById('copyConfigBtn'); + +const camelToSnakeCase = str => str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); +const camelToKebabCase = str => str.replace(/[A-Z]/g, letter => `-${letter.toLowerCase()}`); + +// Dummy function for test request +function dummyTestRequest() { + // Make an API request to the Portkey API + // Use the provider and providerDetails to make the request + const myHeaders = new Headers(); + Object.keys(providerDetails).forEach(key => { + if (key === 'apiKey') { + myHeaders.append("Authorization", providerDetails[key]); + } else { + myHeaders.append("x-portkey-" + camelToKebabCase(key), providerDetails[key]); + } + }) + myHeaders.append("Content-Type", "application/json"); + myHeaders.append("x-portkey-provider", provider); + + const raw = JSON.stringify({ + "messages": [{"role": "user","content": "How are you?"}], + "model": modelMap[provider], + "max_tokens": 40 + }); + + const requestOptions = {method: "POST", headers: myHeaders, body: raw}; + + // Add loading class to testRequestBtn + testRequestBtn.classList.add('loading'); + + fetch("/v1/chat/completions", requestOptions) + .then((response) => { + if (!response.ok) { + return response.json().then(error => { + const responseDiv = document.getElementById('testRequestResponse'); + responseDiv.innerHTML = `[${response.status} ${response.statusText}]: ${error.message || error.error.message}`; + responseDiv.style.display = 'block'; + throw new Error(error); + }); + } + return response.json(); + }) + .then((result) => { + const responseDiv = document.getElementById('testRequestResponse'); + responseDiv.innerHTML = `${result.choices[0].message.content}`; + responseDiv.style.display = 'block'; + responseDiv.classList.remove('error'); + }) + .catch((error) => { + console.error('Error:', error); + }) + .finally(() => { + // Remove loading class from testRequestBtn + testRequestBtn.classList.remove('loading'); + }); +} + +// Functions + +function switchTab(tabsContainer, tabName, updateRoutingConfigFlag = true) { + const tabs = tabsContainer.querySelectorAll('.tab'); + const tabContents = tabsContainer.closest('.card').querySelectorAll('.tab-content'); + + tabs.forEach(tab => tab.classList.remove('active')); + tabContents.forEach(content => content.classList.remove('active')); + + tabsContainer.querySelector(`.tab[data-tab="${tabName}"]`).classList.add('active'); + tabsContainer.closest('.card').querySelector(`#${tabName}Content`).classList.add('active'); + + if (tabsContainer.classList.contains('test-request-tabs')) { + updateAllCommands(); + // Update the language select with the active tab + languageSelect.value = tabName; + updateRoutingConfigFlag ? updateRoutingConfig() : null; + } else if (tabsContainer.classList.contains('routing-config-tabs')) { + updateRoutingConfig(); + } +} + +function updateAllCommands() { + ["nodejs", "python", "curl"].forEach(language => { + const command = document.getElementById(`${language}Command`); + const code = getTestRequestCodeBlock(language, {provider, providerDetails}); + command.innerHTML = code; + }); + addClickListeners(); +} + +function highlightElement(element) { + element.classList.add('animate-highlight'); + setTimeout(() => element.classList.remove('animate-highlight'), 1000); +} + +function showProviderDialog() { + providerDialog.style.display = 'flex'; +} + +function getProviderFields(provider) { + switch(provider) { + case 'openai': + case 'anthropic': + case 'groq': + return [{ id: 'apiKey', placeholder: 'Enter your API key' }]; + case 'azure-openai': + return [ + { id: 'apiKey', placeholder: 'Enter your API key' }, + { id: 'azureResourceName', placeholder: 'Azure Resource Name' }, + { id: 'azureDeploymentId', placeholder: 'Azure Deployment ID' }, + { id: 'azureApiVersion', placeholder: 'Azure API Version' }, + { id: 'azureModelName', placeholder: 'Azure Model Name' } + ]; + case 'bedrock': + return [ + { id: 'awsAccessKeyId', placeholder: 'AWS Access Key ID' }, + { id: 'awsSecretAccessKey', placeholder: 'AWS Secret Access Key' }, + { id: 'awsRegion', placeholder: 'AWS Region' }, + { id: 'awsSessionToken', placeholder: 'AWS Session Token (optional)' } + ]; + default: + return [{ id: 'apiKey', placeholder: 'Enter your API key' }]; + } +} + +function showApiKeyDialog() { + // apiKeyDialog.style.display = 'flex'; + const form = document.getElementById('apiDetailsForm'); + form.innerHTML = ''; // Clear existing fields + + const fields = getProviderFields(provider); + fields.forEach(field => { + const label = document.createElement('label'); + label.textContent = field.placeholder; + label.for = field.id; + form.appendChild(label); + const input = document.createElement('input'); + // input.type = 'password'; + input.id = field.id; + input.className = 'input'; + // input.placeholder = field.placeholder; + input.value = providerDetails[field.id] || ""; + form.appendChild(input); + }); + + apiKeyDialog.style.display = 'flex'; +} + +function updateRoutingConfig() { + const language = languageSelect.value; + const activeTab = document.querySelector('.routing-config-tabs .tab.active').dataset.tab; + const codeElement = document.getElementById(`${activeTab}Code`); + + // Also change the tabs for test request + switchTab(document.querySelector('.test-request-tabs'), language, false); + + const code = getRoutingConfigCodeBlock(language, activeTab); + codeElement.innerHTML = hljs.highlight(code, {language: lngMap[language]}).value; +} + +function addClickListeners() { + const providerValueSpans = document.querySelectorAll('.highlighted-value:not(#providerValue)'); + const providerValues = document.querySelectorAll('[id^="providerValue"]'); + // const apiKeyValues = document.querySelectorAll('[id^="apiKeyValue"]'); + + providerValues.forEach(el => el.addEventListener('click', showProviderDialog)); + // apiKeyValues.forEach(el => el.addEventListener('click', showApiKeyDialog)); + providerValueSpans.forEach(el => el.addEventListener('click', showApiKeyDialog)); +} + + +// Event Listeners +testRequestBtn.addEventListener('click', dummyTestRequest); + +document.querySelectorAll('.tabs').forEach(tabsContainer => { + tabsContainer.querySelectorAll('.tab').forEach(tab => { + tab.addEventListener('click', () => switchTab(tabsContainer, tab.dataset.tab)); + }); +}); + +copyBtn.addEventListener('click', () => { + const activeContent = document.querySelector('.curl-command .tab-content.active code'); + navigator.clipboard.writeText(activeContent.innerText); + copyBtn.innerHTML = ''; + lucide.createIcons(); + setTimeout(() => { + copyBtn.innerHTML = ''; + lucide.createIcons(); + }, 2000); + // addLog('Code example copied to clipboard'); +}); + +copyConfigBtn.addEventListener('click', () => { + const activeContent = document.querySelector('.routing-config .tab-content.active code'); + navigator.clipboard.writeText(activeContent.textContent); + copyConfigBtn.innerHTML = ''; + lucide.createIcons(); + setTimeout(() => { + copyConfigBtn.innerHTML = ''; + lucide.createIcons(); + }, 2000); + // addLog('Routing config copied to clipboard'); +}); + +// Modify existing event listeners +providerSelect.addEventListener('change', (e) => { + provider = e.target.value; + updateAllCommands(); + providerDialog.style.display = 'none'; + highlightElement(document.getElementById('providerValue')); + // Find if there are any provider details in localStorage for this provider + let localDetails = localStorage.getItem(`providerDetails-${provider}`); + if(localDetails) { + console.log('Provider details found in localStorage', localDetails); + providerDetails = JSON.parse(localDetails); + updateAllCommands(); + highlightElement(document.getElementById('apiKeyValue')); + } + // addLog(`Provider set to ${provider}`); +}); + +saveApiDetailsBtn.addEventListener('click', () => { + const fields = getProviderFields(provider); + providerDetails = {}; + fields.forEach(field => { + const input = document.getElementById(field.id); + providerDetails[field.id] = input.value; + }); + // Save all provider details in localStorage for this provider + localStorage.setItem(`providerDetails-${provider}`, JSON.stringify(providerDetails)); + updateAllCommands(); + apiKeyDialog.style.display = 'none'; + highlightElement(document.getElementById('apiKeyValue')); +}); + +languageSelect.addEventListener('change', updateRoutingConfig); + +// Initialize +updateAllCommands(); +updateRoutingConfig(); + +// Close dialogs when clicking outside +window.addEventListener('click', (e) => { + if (e.target.classList.contains('dialog-overlay')) { + e.target.style.display = 'none'; + } +}); + +// Close dialogs when hitting escape +window.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { + providerDialog.style.display = 'none'; + apiKeyDialog.style.display = 'none'; + logDetailsModal.style.display = 'none'; + } +}); + +// Tab functionality +const tabButtons = document.querySelectorAll('.tab-button'); +const tabContents = document.querySelectorAll('.main-tab-content'); + +function mainTabFocus(tabName) { + if(tabName === 'logs') { + resetLogCounter(); + } + tabButtons.forEach(btn => btn.classList.remove('active')); + tabContents.forEach(content => content.classList.remove('active')); + + document.getElementById(`${tabName}-tab-button`).classList.add('active'); + document.getElementById(`${tabName}-tab`).classList.add('active'); +} + +tabButtons.forEach(button => { + button.addEventListener('click', (e) => { + e.preventDefault(); + let tabName = button.getAttribute('data-tab'); + const href = tabName === 'logs' ? '/public/logs' : '/public/'; + history.pushState(null, '', href); + mainTabFocus(tabName); + }); +}); + +function managePage() { + if(window.location.pathname === '/public/logs') { + mainTabFocus('logs'); + } else { + mainTabFocus('main'); + } +} + +window.addEventListener('popstate', () => { + managePage() +}); + +managePage() + +// Logs functionality +const logsTableBody = document.getElementById('logsTableBody'); +const logDetailsModal = document.getElementById('logDetailsModal'); +const logDetailsContent = document.getElementById('logDetailsContent'); +const closeModal = document.querySelector('.close'); +const clearLogsBtn = document.querySelector('.btn-clear-logs'); + +// SSE for the logs +const logSource = new EventSource('/log/stream'); + +function setupLogSource() { + logSource.addEventListener('connected', (event) => { + console.log('Connected to log stream', event.data); + }); + + logSource.addEventListener('log', (event) => { + const entry = JSON.parse(event.data); + console.log('Received log entry', entry); + addLogEntry(entry.time, entry.method, entry.endpoint, entry.status, entry.duration, entry.requestOptions); + }); + + // Handle heartbeat to keep connection alive + logSource.addEventListener('heartbeat', (event) => { + console.log('Received heartbeat'); + }); + + logSource.onerror = (error) => { + console.error('SSE error (logs):', error); + reconnectLogSource(); + }; +} + +function reconnectLogSource() { + if (logSource) { + logSource.close(); + } + console.log('Attempting to reconnect to log stream...'); + setTimeout(() => { + logSource = new EventSource('/log/stream'); + setupLogSource(); + }, 5000); // Wait 5 seconds before attempting to reconnect +} + +setupLogSource(); + +function addLogEntry(time, method, endpoint, status, duration, requestOptions) { + const tr = document.createElement('tr'); + tr.classList.add('new-row'); + tr.innerHTML = ` + ${time} + ${method} + ${endpoint} + ${status} + ${duration}ms + + `; + + const viewDetailsBtn = tr.querySelector('.btn-view-details'); + viewDetailsBtn.addEventListener('click', () => showLogDetails(time, method, endpoint, status, duration, requestOptions)); + + if (logsTableBody.children.length > 1) { + logsTableBody.insertBefore(tr, logsTableBody.children[1]); + } else { + logsTableBody.appendChild(tr); + } + + incrementLogCounter(); + + setTimeout(() => { + tr.className = ''; + }, 500); +} + +function showLogDetails(time, method, endpoint, status, duration, requestOptions) { + logDetailsContent.innerHTML = ` +

Request Details

+

Time: ${time}

+

Method: ${method}

+

Endpoint: ${endpoint}

+

Status: ${status}

+

Duration: ${duration}ms

+

Request: 

${JSON.stringify(requestOptions[0].requestParams, null, 2)}

+

Response: 

${JSON.stringify(requestOptions[0].response, null, 2)}

+ `; + logDetailsModal.style.display = 'block'; +} + +function incrementLogCounter() { + if(window.location.pathname != '/public/logs') { + logCounter++; + const badge = document.querySelector('header .badge'); + badge.textContent = logCounter; + badge.style.display = 'inline-block'; + } +} + +function resetLogCounter() { + logCounter = 0; + const badge = document.querySelector('header .badge'); + badge.textContent = logCounter; + badge.style.display = 'none'; +} + +closeModal.addEventListener('click', () => { + logDetailsModal.style.display = 'none'; +}); + +window.addEventListener('click', (event) => { + if (event.target === logDetailsModal) { + logDetailsModal.style.display = 'none'; + } +}); + +window.addEventListener('beforeunload', () => { + console.log('Page is being unloaded'); + logSource.close(); +}); + + +window.onload = function() { + // Run the confetti function only once by storing the state in localStorage + if(!localStorage.getItem('confettiRun')) { + confetti(); + localStorage.setItem('confettiRun', 'true'); + } + // confetti({ + // particleCount: 100, + // spread: 70, + // origin: { y: 0.6 } + // }); +}; \ No newline at end of file diff --git a/public/snippets.js b/public/snippets.js new file mode 100644 index 000000000..ee0316e69 --- /dev/null +++ b/public/snippets.js @@ -0,0 +1,230 @@ +const configs = {"nodejs": {}, "python": {}, "curl": {}} + +// Node.js - Simple +configs["nodejs"]["simple"] = ` +// 1. Create config with provider and API key +const config = { + "provider": 'openai', + "api_key": 'Your OpenAI API key', +}; + +// 2. Add this config to the client +const client = new Portkey({config}); + +// 3. Use the client in completion requests +await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello, world!' }], +});` + +// Node.js - Load Balancing +configs["nodejs"]["loadBalancing"] = ` +// 1. Create the load-balanced config +const lbConfig = { + "strategy": { "mode": "loadbalance" }, + "targets": [{ + "provider": 'openai', + "api_key": 'Your OpenAI API key', + "weight": 0.7 + },{ + "provider": 'anthropic', + "api_key": 'Your Anthropic API key', + "weight": 0.3, + "override_params": { + "model": 'claude-3-opus-20240229' // Any params you want to override + }, + }], +}; + +// 2. Use the config in completion requests +await client.chat.completions.create({ + model: 'gpt-4o', // The model will be replaced with the one specified in the config + messages: [{ role: 'user', content: 'Hello, world!' }], +}, {config: lbConfig});` + +// Node.js - Fallbacks +configs["nodejs"]["fallbacks"] = ` +// 1. Create the fallback config +const fallbackConfig = { + "strategy": { "mode": "fallback" }, + "targets": [{ // The primary target + "provider": 'openai', + "api_key": 'Your OpenAI API key', + },{ // The fallback target + "provider": 'anthropic', + "api_key": 'Your Anthropic API key', + }], +}; + +// 2. Use the config in completion requests +await client.chat.completions.create({ + model: 'gpt-4o', // The model will be replaced with the one specified in the config + messages: [{ role: 'user', content: 'Hello, world!' }], +}, {config: fallbackConfig});` + +// Node.js - Retries & Timeouts +configs["nodejs"]["autoRetries"] = ` +// 1. Create the retry and timeout config +const retryTimeoutConfig = { + "retry": { + "attempts": 3, + "on_status_codes": [429, 502, 503, 504] // Optional + }, + "request_timeout": 10000, + "provider": 'openai', + "api_key": 'Your OpenAI API key' +}; + +// 2. Use the config in completion requests +await client.chat.completions.create({ + model: 'gpt-4o', // The model will be replaced with the one specified in the config + messages: [{ role: 'user', content: 'Hello, world!' }], +}, {config: retryTimeoutConfig});` + +// Python - Simple +configs["python"]["simple"] = ` +# 1. Create config with provider and API key +config = { + "provider": 'openai', + "api_key": 'Your OpenAI API key', +} + +# 2. Add this config to the client +client = Portkey(config=config) + +# 3. Use the client in completion requests +client.chat.completions.create( + model = 'gpt-4o', + messages = [{ role: 'user', content: 'Hello, world!' }], +)` + +// Python - Load Balancing +configs["python"]["loadBalancing"] = ` +# 1. Create the load-balanced config +lb_config = { + "strategy": { "mode": "loadbalance" }, + "targets": [{ + "provider": 'openai', + "api_key": 'Your OpenAI API key', + "weight": 0.7 + },{ + "provider": 'anthropic', + "api_key": 'Your Anthropic API key', + "weight": 0.3, + "override_params": { + "model": 'claude-3-opus-20240229' # Any params you want to override + }, + }], +} + +# 2. Use the config in completion requests +client.with_options(config=lb_config).chat.completions.create( + model = 'gpt-4o', + messages = [{ role: 'user', content: 'Hello, world!' }], +)` + +// Python - Fallbacks +configs["python"]["fallbacks"] = ` +# 1. Create the fallback config +fallback_config = { + "strategy": { "mode": "fallback" }, + "targets": [{ # The primary target + "provider": 'openai', + "api_key": 'Your OpenAI API key', + },{ # The fallback target + "provider": 'anthropic', + "api_key": 'Your Anthropic API key', + "override_params": { + "model": 'claude-3-opus-20240229' # Any params you want to override + }, + }], +} + +# 2. Use the config in completion requests +client.with_options(config=fallback_config).chat.completions.create( + model = 'gpt-4o', + messages = [{ role: 'user', content: 'Hello, world!' }], +)` + +// Python - Retries & Timeouts +configs["python"]["autoRetries"] = ` +# 1. Create the retry and timeout config +retry_timeout_config = { + "retry": { + "attempts": 3, + "on_status_codes": [429, 502, 503, 504] # Optional + }, + "request_timeout": 10000, + "provider": 'openai', + "api_key": 'Your OpenAI API key' +} + +# 2. Use the config in completion requests +client.with_options(config=retry_timeout_config).chat.completions.create( + model = 'gpt-4o', + messages = [{ role: 'user', content: 'Hello, world!' }], +)` + +// Curl - Simple +configs["curl"]["simple"] = ` +# Store the config in a variable +simple_config='{"provider":"openai","api_key":"Your OpenAI API Key"}' + +# Use the config in completion requests +curl http://localhost:8787/v1/chat/completions \ +\n-H "Content-Type: application/json" \ +\n-H "x-portkey-config: $simple_config" \ +\n-d '{ + "model": "gpt-4o", + "messages": [ + { "role": "user", "content": "Hello!" } + ] +}'` + +// Curl - Load Balancing +configs["curl"]["loadBalancing"] = ` +# Store the config in a variable +lb_config='{"strategy":{"mode":"loadbalance"},"targets":[{"provider":"openai","api_key":"Your OpenAI API key","weight": 0.7 },{"provider":"anthropic","api_key":"Your Anthropic API key","weight": 0.3,"override_params":{"model":"claude-3-opus-20240229"}}]}' + +# Use the config in completion requests +curl http://localhost:8787/v1/chat/completions \ +\n-H "Content-Type: application/json" \ +\n-H "x-portkey-config: $lb_config" \ +\n-d '{ + "model": "gpt-4o", + "messages": [ + { "role": "user", "content": "Hello!" } + ] +}'` + +// Curl - Fallbacks +configs["curl"]["fallbacks"] = ` +# Store the config in a variable +fb_config='{"strategy":{"mode":"fallback"},"targets":[{"provider":"openai","api_key":"Your OpenAI API key"},{"provider":"anthropic","api_key":"Your Anthropic API key","override_params":{"model":"claude-3-opus-20240229"}}]}' + +# Use the config in completion requests +curl http://localhost:8787/v1/chat/completions \ +\n-H "Content-Type: application/json" \ +\n-H "x-portkey-config: $fb_config" \ +\n-d '{ + "model": "gpt-4o", + "messages": [ + { "role": "user", "content": "Hello!" } + ] +}'` + +// Curl - Retries & Timeouts +configs["curl"]["autoRetries"] = ` +# Store the config in a variable +rt_config='{"retry":{"attempts": 3,"on_status_codes": [429, 502, 503, 504]},"request_timeout": 10000, "provider": "openai", "api_key": "Your OpenAI API key"}' + +# Use the config in completion requests +curl http://localhost:8787/v1/chat/completions \ +\n-H "Content-Type: application/json" \ +\n-H "x-portkey-config: $rt_config" \ +\n-d '{ + "model": "gpt-4o", + "messages": [ + { "role": "user", "content": "Hello!" } + ] +}'` \ No newline at end of file diff --git a/public/styles/buttons.css b/public/styles/buttons.css new file mode 100644 index 000000000..954aa297e --- /dev/null +++ b/public/styles/buttons.css @@ -0,0 +1,63 @@ +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0.5rem 1rem; + border-radius: 0.375rem; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: background-color 0.2s; + background-color: rgb(24, 24, 27); + color: white; + border: 0px; + position: relative; + overflow: hidden; +} + +.btn:hover { + background-color: rgba(24, 24, 27,0.9) +} + +.btn-outline { + border: 1px solid #b8bcc2; + background-color: white; + color: rgb(24, 24, 27); +} + +.btn-outline:hover { + background-color: #f3f4f6; +} + +/* Loading state */ +.btn.loading { + cursor: not-allowed; + opacity: 0.7; +} + +.btn.loading::after { + content: ''; + position: absolute; + width: 1rem; + height: 1rem; + border: 2px solid rgba(255, 255, 255, 0.3); + border-radius: 50%; + border-top-color: white; + animation: spin 0.8s linear infinite; +} + +.btn.loading .btn-text { + visibility: hidden; +} + +.btn-outline.loading::after { + border-color: rgba(24, 24, 27, 0.3); + border-top-color: rgb(24, 24, 27); +} + +@keyframes spin { + to { + transform: rotate(360deg); + } +} \ No newline at end of file diff --git a/public/styles/header.css b/public/styles/header.css new file mode 100644 index 000000000..9a49b7b44 --- /dev/null +++ b/public/styles/header.css @@ -0,0 +1,103 @@ +/* Header styles */ +header { + background-color: white; + box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.1); + padding: 0.75rem 0; + position: fixed; + top: 0; + left: 0; + right: 0; + z-index: 100; +} + +.container { + max-width: 1200px; + margin: 0 auto; + display: flex; + justify-content: space-between; + align-items: center; + padding: 0 1rem; +} + +.logo { + display: flex; + align-items: center; +} + +.logo img { + margin-right: 0.5rem; + max-height: 2rem; +} + +.logo span { + font-size: 0.875rem; + font-weight: normal; + display: flex; + align-items: center; +} + +.status-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background-color: #22c55e; + margin-left: 8px; + animation: blink 1s infinite; +} + +@keyframes blink { + 0% { opacity: 0; } + 50% { opacity: 1; } + 100% { opacity: 0; } +} + +.header-links { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.header-links a { + color: #2563eb; + text-decoration: none; + font-size: 0.875rem; +} + +.header-links a:hover { + color: #1d4ed8; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .container { + flex-direction: column; + align-items: flex-start; + } + + .logo { + margin-bottom: 0.5rem; + } + + .tabs-container { + margin-bottom: 0.5rem; + } + + .header-links { + width: 100%; + justify-content: space-between; + } +} + +header .badge { + background-color: rgb(239, 68, 68); + color: white; + padding: 0.25rem 0.25rem; + border-radius: 100px; + font-size: 0.65rem; + font-weight: normal; + margin-left: 5px; + min-width: 13px; + /* display: inline-block; */ + text-align: center; + display: none; +} \ No newline at end of file diff --git a/public/styles/interative-code.css b/public/styles/interative-code.css new file mode 100644 index 000000000..9c0b30b13 --- /dev/null +++ b/public/styles/interative-code.css @@ -0,0 +1,178 @@ +pre { + background-color: #f3f4f6; + padding: 0.75rem; + border-radius: 0.375rem; + overflow-x: auto; + font-size: 0.875rem; + position: relative; +} + +.copy-btn { + position: absolute; + top: 0.5rem; + right: 0.5rem; + padding: 0.25rem; + background-color: white; + border: 1px solid #d1d5db; + border-radius: 0.25rem; + cursor: pointer; + z-index: 10; + height: 28px; +} + +.copy-btn svg { + width: 20px; + height: 18px; + color: #393d45; +} + +/* Highlighted values */ +.highlighted-value { + display: inline-block; + position: relative; + cursor: pointer; + transition: transform 0.2s; + padding: 0 0.25rem; + margin: 2px 0; +} + +.highlighted-value.filled { + font-weight: bold; +} + +.highlighted-value:hover { + transform: scale(1.05); +} + +.highlighted-value::before { + content: ''; + position: absolute; + inset: 0; + border-radius: 0.25rem; + transition: all 0.2s; +} + +.highlighted-value.empty::before { + background-color: rgba(252, 165, 165, 0.3); + border: 1px solid rgba(248, 113, 113, 0.5); +} + +.highlighted-value.filled::before { + background-color: rgba(134, 239, 172, 0.2); + border: 1px solid rgba(74, 222, 128, 0.5); +} + +.highlighted-value:hover::before { + opacity: 0.4; +} + +.highlighted-value span { + position: relative; + z-index: 10; +} + +.highlighted-value.empty span { + color: #dc2626; +} + +.highlighted-value.filled span { + color: #16a34a; +} + +@keyframes highlight { + 0% { + background-color: rgba(253, 224, 71, 0.2); + transform: scale(1); + } + 20% { + background-color: rgba(253, 224, 71, 1); + transform: scale(1.05); + } + 100% { + background-color: rgba(253, 224, 71, 0.2); + transform: scale(1); + } +} + +/* Dialog styles */ +.dialog-overlay { + position: fixed; + inset: 0; + background-color: rgba(0, 0, 0, 0.5); + display: flex; + justify-content: center; + align-items: center; + z-index: 50; +} + +.dialog { + background-color: white; + border-radius: 0.5rem; + padding: 1.5rem; + width: 90%; + max-width: 500px; +} + +.dialog h3 { + font-size: 1.25rem; + font-weight: bold; + margin-bottom: 0; + margin-top: 0; +} + +.dialog p { + font-size: 0.875rem; + color: #6b7280; + margin-bottom: 1rem; + margin-top: 0; +} + +.select-wrapper { + position: relative; +} + +.select { + width: 100%; + padding: 0.5rem; + border: 1px solid #d1d5db; + border-radius: 0.375rem; + appearance: none; + background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%236b7280' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e"); + background-position: right 0.5rem center; + background-repeat: no-repeat; + background-size: 1.5em 1.5em; + font-size: 0.75rem; +} + +.input { + width: 90%; + padding: 0.5rem; + border: 1px solid #d1d5db; + border-radius: 0.375rem; + margin-bottom: 0.5rem; +} + +.dialog label { + font-size: 0.75rem; + font-weight: bold; + display: inline-block; + padding: 0.25rem; +} + +.dialog .btn { + margin-top: 0.5rem; +} + +.animate-highlight { + animation: highlight 1s ease-out; +} + +.language-select-wrapper { + width: 100px; + display: inline-block; + position: absolute; + z-index: 2; + right: 45px; + top: 0.5rem; + font-size: 12px; +} \ No newline at end of file diff --git a/public/styles/logs.css b/public/styles/logs.css new file mode 100644 index 000000000..0f7420438 --- /dev/null +++ b/public/styles/logs.css @@ -0,0 +1,150 @@ +/* Logs styles */ +.card.logs-card { + margin-top: 1rem; + max-width: 800px; +} + +.logs-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +.logs-table-container { + background-color: white; + border-radius: 8px; + box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.1), 0 1px 2px 0 rgba(0, 0, 0, 0.06); + overflow: hidden; + width: 100%; + max-width: 800px; +} + +.logs-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; +} + +.logs-table th, +.logs-table td { + padding: 0.75rem; + text-align: left; + border-bottom: 1px solid #e5e7eb; + cursor: default; +} + +.logs-table th { + background-color: #f3f4f6; + font-weight: 600; + color: #374151; + text-transform: uppercase; + font-size: 12px; + letter-spacing: 0.05em; +} + +.logs-table tr:hover { + background-color: #f3f4f6; +} + +/* .logs-table td:last-child { + text-align: right; +} */ + +.loading-row { + background-color: #f3f4f6; + color: #6b7280; + font-style: italic; +} + +.loading-row td { + padding: 0.25rem 0.75rem; +} + +.loading-animation { + display: inline-block; + width: 12px; + height: 12px; + border: 2px solid #6b7280; + border-radius: 50%; + border-top-color: transparent; + animation: spin 1s linear infinite; + margin-right: 8px; + vertical-align: middle; +} +@keyframes spin { + to { + transform: rotate(360deg); + } +} + +.new-row { + animation: fadeInSlideDown 0.2s ease-out; +} +@keyframes fadeInSlideDown { + from { + opacity: 0; + transform: translateY(-20px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.log-time { + font-family: monospace; + font-size: 0.875rem; +} + +.log-method span { + padding: 0.3rem; + background-color: rgba(0, 0, 0, 0.1); + border-radius: 4px; + font-weight: 600; +} + +.log-status span.success { + padding: 0.3rem; + background-color: green; + border-radius: 4px; + color: white; + font-weight: 700; +} + +.log-status span.error { + padding: 0.3rem; + background-color: red; + border-radius: 4px; + color: white; + font-weight: 700; +} + + + +.btn-view-details { + padding: 0.25rem 0.5rem; + background-color: #3b82f6; + color: white; + border: none; + border-radius: 0.25rem; + cursor: pointer; + transition: background-color 0.3s ease; +} + +.btn-view-details:hover { + background-color: #2563eb; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .logs-header { + flex-direction: column; + align-items: stretch; + } + + .logs-search { + width: 100%; + margin-bottom: 1rem; + } +} \ No newline at end of file diff --git a/public/styles/modal.css b/public/styles/modal.css new file mode 100644 index 000000000..439a4f597 --- /dev/null +++ b/public/styles/modal.css @@ -0,0 +1,35 @@ +/* Modal styles */ +.modal { + display: none; + position: fixed; + z-index: 1000; + left: 0; + top: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.5); +} + +.modal-content { + background-color: white; + margin: 0 0 0 auto; + padding: 2rem; + border-radius: 0rem; + width: 80%; + max-width: 500px; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); + height: 100vh; + overflow-y: auto; +} + +.close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; + cursor: pointer; +} + +.close:hover { + color: #000; +} \ No newline at end of file diff --git a/public/styles/style.css b/public/styles/style.css new file mode 100644 index 000000000..d15659346 --- /dev/null +++ b/public/styles/style.css @@ -0,0 +1,220 @@ +/* Base styles */ +body { + font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; + line-height: 1.5; + font-weight: 400; + font-size: 14px; + margin: 0; + padding: 0; + min-height: 100vh; + background-color: #f3f4f6; + color: #213547; + margin-top: 4rem; +} + +a { + color: rgb(24, 24, 27); + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +.relative { + position: relative; +} + +/* Main content styles */ +.main-content { + max-width: 1200px; + margin: 1rem auto; + padding: 0 1rem; +} + +/* Main content styles */ +.main-content { + max-width: 1200px; + margin: 0 auto; + padding: 1rem; + transition: margin-bottom 0.3s ease; +} + +.left-column { + width: 65%; + display: flex; + flex-direction: column; + gap: 1rem; +} + +.right-column { + width: 35%; + display: flex; + flex-direction: column; +} + +.card { + background-color: white; + border-radius: 0.75rem; + box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1), 0 2px 4px -1px rgba(0, 0, 0, 0.06); + padding: 1.5rem; + max-width: 600px; + margin: 0rem auto 2rem auto; +} + +.left-column .card { + width: 100%; + max-width: 500px; + margin: 0 auto; +} + +/* Responsive adjustments */ +@media (max-width: 1024px) { + .main-content { + flex-direction: column; + } + + .left-column, + .right-column { + width: 100%; + } +} + +.card-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +h2 { + font-size: 1.125rem; + font-weight: bold; + margin: 0; +} + +.card-subtitle { + font-size: 0.875rem; + color: #6b7280; + margin-bottom: 1rem; +} + +/* Features to Explore Card Styles */ +.features-grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 1rem; + margin-top: 1rem; +} + +.feature-item { + background-color: #f9fafb; + border-radius: 0.5rem; + padding: 1rem; + display: flex; + flex-direction: column; + align-items: center; + text-align: center; +} + +.feature-item:hover { + box-shadow: 0px 0px 3px 1px rgba(0, 0, 0, 0.1); + cursor: pointer; + transition: all 0.2s; + text-decoration: none; +} + +.feature-item .icon { + width: 2rem; + height: 2rem; + color: #3b82f6; + margin-bottom: 0.5rem; +} + +.feature-item h3 { + font-size: 1rem; + font-weight: 600; + margin-bottom: 0.5rem; +} + +.feature-item p { + font-size: 0.875rem; + color: #6b7280; +} + +/* Next Steps Card Styles */ +.card.next-steps { + margin-top: 1rem; + background-color: transparent; + /* border-top: 1px solid #ccc; */ + padding-top: 2rem; + box-shadow: none; + border-radius: 0; + width: 90%; + max-width: 700px; +} + +.next-steps-grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 1rem; + margin-top: 1rem; +} + +.next-step-item { + border: 1px solid #babcc0; + border-radius: 0.5rem; + padding: 1rem; + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + box-shadow: 0px 5px 3px 2px rgba(0, 0, 0, 0.1); +} + +.next-step-item .icon { + width: 2rem; + height: 2rem; + color: #3b82f6; + margin-bottom: 0.5rem; +} + +.next-step-item h3 { + font-size: 1rem; + font-weight: 600; + margin-bottom: 0.5rem; +} + +.next-step-item p { + font-size: 0.875rem; + color: #6b7280; + margin-bottom: 1rem; +} + +.next-step-item .btn { + margin-top: auto; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .features-grid, + .next-steps-grid { + grid-template-columns: 1fr; + } +} + +#testRequestResponse { + margin-top: 0.5rem; + border-radius: 4px; + font-family: monospace; + /* dark background color */ + background-color: #213547; + color: #f9fafb; + padding: 0.5rem; + display: none; +} + +#testRequestResponse .error { + /* red color that looks good on dark background */ + color: #ff7f7f; +} \ No newline at end of file diff --git a/public/styles/tabs.css b/public/styles/tabs.css new file mode 100644 index 000000000..951765321 --- /dev/null +++ b/public/styles/tabs.css @@ -0,0 +1,73 @@ +/* Tabs styles */ +.tabs-container { + display: flex; + background-color: #f4f4f5; + padding: 0.25rem; + border-radius: 6px; + color: rgb(113, 113, 122); +} + +.tab-button:hover { + color: rgb(9, 9, 11); +} + +.tab-button.active { + color: rgb(9, 9, 11); + /* border-bottom-color: #3b82f6; */ + background-color: white; + font-weight: 500; + box-shadow: rgba(0, 0, 0, 0) 0px 0px 0px 0px, rgba(0, 0, 0, 0) 0px 0px 0px 0px, rgba(0, 0, 0, 0.05) 0px 1px 2px 0px; +} + +.tabs-container .tab-button { + min-width: 100px; + padding: 0.3rem 0.875rem; +} + +/* Tab content styles */ +.tab-content { + display: none; +} + +.tab-content.active { + display: block; +} + +.main-tab-content { + display: none; +} + +.main-tab-content.active { + display: block; +} + +.tab-button { + padding: 0.5rem 1rem; + background: none; + border: none; + border-bottom: 2px solid transparent; + cursor: pointer; + font-size: 0.875rem; + font-weight: 500; + color: #6b7280; + transition: all 0.3s ease; + border-radius: 0.375rem; + /* margin-right: 0.5rem; */ +} + +.tabs { + display: flex; + border-bottom: 1px solid #d1d5db; + margin-bottom: 1rem; +} + +.tab { + padding: 0.5rem 1rem; + cursor: pointer; + border-bottom: 2px solid transparent; +} + +.tab.active { + border-bottom-color: #3b82f6; + font-weight: bold; +} \ No newline at end of file diff --git a/src/index.html b/src/index.html deleted file mode 100644 index 66e880e47..000000000 --- a/src/index.html +++ /dev/null @@ -1,479 +0,0 @@ - - - - - - AI Gateway - Up and Running - - - - - - -
-
-
Gateway is Live
-

🚀 AI Gateway is Up and Running!

-

- Your AI Gateway is now running on http://localhost:8787. -

- AI Gateway Demo - - -
- -
-
-
1
-

Get Started

-
-

- Use the Gateway to route requests to 200+ LLMs with a unified API. -

-
- - - - - -
-
- 

-from openai import OpenAI
-
-gateway = OpenAI(
-    api_key="ANTHROPIC_API_KEY",
-    base_url="http://localhost:8787/v1",
-    default_headers={
-        "x-portkey-provider": "anthropic"
-    }
-)
-
-chat_complete = gateway.chat.completions.create(
-    model="claude-3-sonnet-20240229",
-    messages=[{"role": "user", "content": "What's a fractal?"}],
-    max_tokens=512
-)
-
-
-
- 

-import OpenAI from 'openai';
-
-const gateway = new OpenAI({
-    apiKey: 'ANTHROPIC_API_KEY',
-    baseURL: 'http://localhost:8787/v1',
-    defaultHeaders: {
-        'x-portkey-provider': 'anthropic'
-    }
-});
-
-async function main() {
-    const chatCompletion = await gateway.chat.completions.create({
-        messages: [{ role: 'user', content: "What's a fractal?" }],
-        model: 'claude-3-sonnet-20240229',
-        max_tokens: 512
-    });
-    console.log(chatCompletion.choices[0].message.content);
-}
-
-main();
-
-
-
- 

-            // Go code example will be added here
-
-
- -
- 

-            // Java code example will be added here
-
-
- -
- 

-            // C# code example will be added here
-
-
-
- -
-
-
2
-

Explore Features

-
-
-
- -

Fallbacks

- Learn more -
-
- -

Automatic Retries

- Learn more -
-
- -

Load Balancing

- Learn more -
-
- -

Request Timeouts

- Learn more -
-
-
- -
-
-
4
-

Choose Your Gateway Option

-
-
-
-

Self-Hosted

-

Deploy and manage the Gateway yourself:

- -
-
-

Hosted by Portkey

-

Quick setup without infrastructure concerns.

-

- Powers billions of tokens daily for companies like Postman, - Haptik, Turing, and more. -

- Sign up for free developer plan -
-
-
- -
-

Enterprise Version

-

For enhanced security, privacy, and support:

-
    -
  • Secure Key Management
    • -
  • Simple & Semantic Caching
    • -
  • Access Control & Inbound Rules
    • -
  • PII Redaction
    • -
  • SOC2, ISO, HIPAA, GDPR Compliances
    • -
  • Professional Support
    • -
- Schedule a call for enterprise deployments -
- -
-

Need Help?

- -
-
- - - - - - diff --git a/src/index.ts b/src/index.ts index 4ad184a56..3bd32bcf3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { Hono } from 'hono'; import { prettyJSON } from 'hono/pretty-json'; import { HTTPException } from 'hono/http-exception'; +import { streamSSE } from 'hono/streaming' // import { env } from 'hono/adapter' // Have to set this up for multi-environment deployment import { completeHandler } from './handlers/completeHandler'; @@ -19,6 +20,7 @@ import { completionsHandler } from './handlers/completionsHandler'; import { embeddingsHandler } from './handlers/embeddingsHandler'; import { requestValidator } from './middlewares/requestValidator'; import { hooks } from './middlewares/hooks'; +import { logger } from './middlewares/log' import { compress } from 'hono/compress'; import { getRuntimeKey } from 'hono/adapter'; import { imageGenerationsHandler } from './handlers/imageGenerationsHandler'; @@ -50,6 +52,9 @@ app.get('/', (c) => c.text('AI Gateway says hey!')); // Use prettyJSON middleware for all routes app.use('*', prettyJSON()); +// Use logger middleware for all routes +app.use(logger()) + // Use hooks middleware for all routes app.use('*', hooks); @@ -152,5 +157,41 @@ app.get('/v1/*', requestValidator, proxyGetHandler); app.delete('/v1/*', requestValidator, proxyGetHandler); +app.get('/log/stream', (c) => { + const clientId = Date.now().toString() + + // Set headers to prevent caching + c.header('Cache-Control', 'no-cache') + c.header('X-Accel-Buffering', 'no') + + return streamSSE(c, async (stream) => { + const client = { + sendLog: (message:any) => stream.writeSSE(message) + } + // Add this client to the set of log clients + const addLogClient:any = c.get('addLogClient') + addLogClient(clientId, client) + + + + try { + // Send an initial connection event + await stream.writeSSE({ event: 'connected', data: clientId }) + + // Keep the connection open + while (true) { + await stream.sleep(10000) // Heartbeat every 10 seconds + await stream.writeSSE({ event: 'heartbeat', data: 'pulse' }) + } + } catch (error) { + console.error(`Error in log stream for client ${clientId}:`, error) + } finally { + // Remove this client when the connection is closed + const removeLogClient:any = c.get('removeLogClient') + removeLogClient(clientId) + } + }) +}) + // Export the app export default app; diff --git a/src/middlewares/log/index.ts b/src/middlewares/log/index.ts new file mode 100644 index 000000000..588278d9c --- /dev/null +++ b/src/middlewares/log/index.ts @@ -0,0 +1,75 @@ +import { Context } from 'hono'; + +let logId = 0 + +// Map to store all connected log clients +const logClients:any = new Map() + +const addLogClient = (clientId:any, client:any) => { + logClients.set(clientId, client) + console.log(`New client ${clientId} connected. Total clients: ${logClients.size}`) +} + +const removeLogClient = (clientId:any) => { + logClients.delete(clientId) + console.log(`Client ${clientId} disconnected. Total clients: ${logClients.size}`) +} + +const broadcastLog = async (log:any) => { + const message = { + data: log, + event: 'log', + id: String(logId++) + } + + const deadClients:any = [] + + for (const [id, client] of logClients) { + try { + await Promise.race([ + client.sendLog(message), + new Promise((_, reject) => setTimeout(() => reject(new Error('Send timeout')), 1000)) + ]) + } catch (error:any) { + console.error(`Failed to send log to client ${id}:`, error.message) + deadClients.push(id) + } + } + + // Remove dead clients after iteration + deadClients.forEach((id:any) => { + removeLogClient(id) + }) +} + +export const logger = () => { + return async (c: Context, next: any) => { + + c.set('addLogClient', addLogClient) + c.set('removeLogClient', removeLogClient) + + const start = Date.now() + + await next(); + + const ms = Date.now() - start + if(!c.req.url.includes('/v1/')) return + + const requestOptionsArray = c.get('requestOptions'); + if (requestOptionsArray[0].requestParams.stream) { + requestOptionsArray[0].response = {"message": "The response was a stream."} + } else { + const response = await c.res.clone().json(); + requestOptionsArray[0].response = response; + } + + await broadcastLog(JSON.stringify({ + time: new Date().toLocaleString(), + method: c.req.method, + endpoint: c.req.url.split(":8787")[1], + status: c.res.status, + duration: ms, + requestOptions: requestOptionsArray + })) + }; +}; diff --git a/src/start-server.ts b/src/start-server.ts index 8b0934765..91a8ee2a5 100644 --- a/src/start-server.ts +++ b/src/start-server.ts @@ -1,6 +1,8 @@ #!/usr/bin/env node import { serve } from '@hono/node-server'; +import { serveStatic } from '@hono/node-server/serve-static' +import { exec } from 'child_process'; import app from './index'; @@ -10,9 +12,37 @@ const args = process.argv.slice(2); const portArg = args.find((arg) => arg.startsWith('--port=')); const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; +app.get('/public/*', serveStatic({ root: './' })); +app.get('/public/logs', serveStatic({ path: './public/index.html' })); + serve({ fetch: app.fetch, port: port, }); -console.log(`Your AI Gateway is now running on http://localhost:${port} 🚀`); +const url = `http://localhost:${port}`; +console.log(`Your AI Gateway is now running on ${url} 🚀`); + +// Function to open URL in the default browser +function openBrowser(url: string) { + let command: string; + switch (process.platform) { + case 'darwin': + command = `open ${url}`; + break; + case 'win32': + command = `start ${url}`; + break; + default: + command = `xdg-open ${url}`; + } + + exec(command, (error) => { + if (error) { + console.error('Failed to open browser:', error); + } + }); +} + +// Open the browser +openBrowser(`${url}/public/`); \ No newline at end of file From b783072aa8cc7d912d96a99e8ca3a80086c34bdc Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Mon, 21 Oct 2024 20:25:18 +0530 Subject: [PATCH 006/119] Add headless, format --- src/index.ts | 41 +---------------- src/middlewares/log/index.ts | 89 ++++++++++++++++++++---------------- src/start-server.ts | 52 +++++++++++++++++++-- 3 files changed, 98 insertions(+), 84 deletions(-) diff --git a/src/index.ts b/src/index.ts index 3bd32bcf3..5d593bb24 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,6 @@ import { Hono } from 'hono'; import { prettyJSON } from 'hono/pretty-json'; import { HTTPException } from 'hono/http-exception'; -import { streamSSE } from 'hono/streaming' // import { env } from 'hono/adapter' // Have to set this up for multi-environment deployment import { completeHandler } from './handlers/completeHandler'; @@ -20,7 +19,7 @@ import { completionsHandler } from './handlers/completionsHandler'; import { embeddingsHandler } from './handlers/embeddingsHandler'; import { requestValidator } from './middlewares/requestValidator'; import { hooks } from './middlewares/hooks'; -import { logger } from './middlewares/log' +import { logger } from './middlewares/log'; import { compress } from 'hono/compress'; import { getRuntimeKey } from 'hono/adapter'; import { imageGenerationsHandler } from './handlers/imageGenerationsHandler'; @@ -53,7 +52,7 @@ app.get('/', (c) => c.text('AI Gateway says hey!')); app.use('*', prettyJSON()); // Use logger middleware for all routes -app.use(logger()) +app.use(logger()); // Use hooks middleware for all routes app.use('*', hooks); @@ -157,41 +156,5 @@ app.get('/v1/*', requestValidator, proxyGetHandler); app.delete('/v1/*', requestValidator, proxyGetHandler); -app.get('/log/stream', (c) => { - const clientId = Date.now().toString() - - // Set headers to prevent caching - c.header('Cache-Control', 'no-cache') - c.header('X-Accel-Buffering', 'no') - - return streamSSE(c, async (stream) => { - const client = { - sendLog: (message:any) => stream.writeSSE(message) - } - // Add this client to the set of log clients - const addLogClient:any = c.get('addLogClient') - addLogClient(clientId, client) - - - - try { - // Send an initial connection event - await stream.writeSSE({ event: 'connected', data: clientId }) - - // Keep the connection open - while (true) { - await stream.sleep(10000) // Heartbeat every 10 seconds - await stream.writeSSE({ event: 'heartbeat', data: 'pulse' }) - } - } catch (error) { - console.error(`Error in log stream for client ${clientId}:`, error) - } finally { - // Remove this client when the connection is closed - const removeLogClient:any = c.get('removeLogClient') - removeLogClient(clientId) - } - }) -}) - // Export the app export default app; diff --git a/src/middlewares/log/index.ts b/src/middlewares/log/index.ts index 588278d9c..1acbe02cb 100644 --- a/src/middlewares/log/index.ts +++ b/src/middlewares/log/index.ts @@ -1,75 +1,84 @@ import { Context } from 'hono'; -let logId = 0 +let logId = 0; // Map to store all connected log clients -const logClients:any = new Map() +const logClients: any = new Map(); -const addLogClient = (clientId:any, client:any) => { - logClients.set(clientId, client) - console.log(`New client ${clientId} connected. Total clients: ${logClients.size}`) -} +const addLogClient = (clientId: any, client: any) => { + logClients.set(clientId, client); + console.log( + `New client ${clientId} connected. Total clients: ${logClients.size}` + ); +}; -const removeLogClient = (clientId:any) => { - logClients.delete(clientId) - console.log(`Client ${clientId} disconnected. Total clients: ${logClients.size}`) -} +const removeLogClient = (clientId: any) => { + logClients.delete(clientId); + console.log( + `Client ${clientId} disconnected. Total clients: ${logClients.size}` + ); +}; -const broadcastLog = async (log:any) => { +const broadcastLog = async (log: any) => { const message = { data: log, event: 'log', - id: String(logId++) - } + id: String(logId++), + }; - const deadClients:any = [] + const deadClients: any = []; for (const [id, client] of logClients) { try { await Promise.race([ client.sendLog(message), - new Promise((_, reject) => setTimeout(() => reject(new Error('Send timeout')), 1000)) - ]) - } catch (error:any) { - console.error(`Failed to send log to client ${id}:`, error.message) - deadClients.push(id) + new Promise((_, reject) => + setTimeout(() => reject(new Error('Send timeout')), 1000) + ), + ]); + } catch (error: any) { + console.error(`Failed to send log to client ${id}:`, error.message); + deadClients.push(id); } } // Remove dead clients after iteration - deadClients.forEach((id:any) => { - removeLogClient(id) - }) -} + deadClients.forEach((id: any) => { + removeLogClient(id); + }); +}; export const logger = () => { return async (c: Context, next: any) => { - - c.set('addLogClient', addLogClient) - c.set('removeLogClient', removeLogClient) + c.set('addLogClient', addLogClient); + c.set('removeLogClient', removeLogClient); - const start = Date.now() + const start = Date.now(); await next(); - const ms = Date.now() - start - if(!c.req.url.includes('/v1/')) return - + const ms = Date.now() - start; + if (!c.req.url.includes('/v1/')) return; + const requestOptionsArray = c.get('requestOptions'); if (requestOptionsArray[0].requestParams.stream) { - requestOptionsArray[0].response = {"message": "The response was a stream."} + requestOptionsArray[0].response = { + message: 'The response was a stream.', + }; } else { const response = await c.res.clone().json(); requestOptionsArray[0].response = response; } - - await broadcastLog(JSON.stringify({ - time: new Date().toLocaleString(), - method: c.req.method, - endpoint: c.req.url.split(":8787")[1], - status: c.res.status, - duration: ms, - requestOptions: requestOptionsArray - })) + + await broadcastLog( + JSON.stringify({ + time: new Date().toLocaleString(), + method: c.req.method, + endpoint: c.req.url.split(':8787')[1], + status: c.res.status, + duration: ms, + requestOptions: requestOptionsArray, + }) + ); }; }; diff --git a/src/start-server.ts b/src/start-server.ts index 91a8ee2a5..b82cdd3fd 100644 --- a/src/start-server.ts +++ b/src/start-server.ts @@ -1,19 +1,59 @@ #!/usr/bin/env node import { serve } from '@hono/node-server'; -import { serveStatic } from '@hono/node-server/serve-static' +import { serveStatic } from '@hono/node-server/serve-static'; import { exec } from 'child_process'; import app from './index'; +import { streamSSE } from 'hono/streaming'; // Extract the port number from the command line arguments const defaultPort = 8787; const args = process.argv.slice(2); +console.log(args, process.argv); const portArg = args.find((arg) => arg.startsWith('--port=')); const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; -app.get('/public/*', serveStatic({ root: './' })); -app.get('/public/logs', serveStatic({ path: './public/index.html' })); +const isHeadless = args.includes('--headless'); + +if (!isHeadless) { + app.get('/public/*', serveStatic({ root: './' })); + app.get('/public/logs', serveStatic({ path: './public/index.html' })); + + app.get('/log/stream', (c) => { + const clientId = Date.now().toString(); + + // Set headers to prevent caching + c.header('Cache-Control', 'no-cache'); + c.header('X-Accel-Buffering', 'no'); + + return streamSSE(c, async (stream) => { + const client = { + sendLog: (message: any) => stream.writeSSE(message), + }; + // Add this client to the set of log clients + const addLogClient: any = c.get('addLogClient'); + addLogClient(clientId, client); + + try { + // Send an initial connection event + await stream.writeSSE({ event: 'connected', data: clientId }); + + // Keep the connection open + while (true) { + await stream.sleep(10000); // Heartbeat every 10 seconds + await stream.writeSSE({ event: 'heartbeat', data: 'pulse' }); + } + } catch (error) { + console.error(`Error in log stream for client ${clientId}:`, error); + } finally { + // Remove this client when the connection is closed + const removeLogClient: any = c.get('removeLogClient'); + removeLogClient(clientId); + } + }); + }); +} serve({ fetch: app.fetch, @@ -44,5 +84,7 @@ function openBrowser(url: string) { }); } -// Open the browser -openBrowser(`${url}/public/`); \ No newline at end of file +// Open the browser only when --headless is not provided +if (!isHeadless) { + openBrowser(`${url}/public/`); +} From f6b2b9fcb72d54721fbb44f0fcfe28bbbe581894 Mon Sep 17 00:00:00 2001 From: Keshav Krishna Date: Thu, 24 Oct 2024 13:38:08 +0530 Subject: [PATCH 007/119] making options parameter optional --- plugins/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/types.ts b/plugins/types.ts index c4f37a7f5..898366dee 100644 --- a/plugins/types.ts +++ b/plugins/types.ts @@ -20,7 +20,7 @@ export type PluginHandler = ( context: PluginContext, parameters: PluginParameters, eventType: HookEventType, - options: { + options?: { env: Record; } ) => Promise; From eb5094b05625f9ca8f6a7c3778a67592ad69bfb5 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Fri, 8 Nov 2024 20:23:48 +0530 Subject: [PATCH 008/119] Minor changes to speed up streaming & support docker --- Dockerfile | 13 ++++------ docs/installation-deployments.md | 22 ++++++++-------- public/main.js | 20 ++++++++++----- src/middlewares/log/index.ts | 31 ++++++++++++----------- src/start-server.ts | 43 +++++++++++++++++++------------- 5 files changed, 73 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4014004b0..20cec8ec1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Use an official Node.js runtime as a parent image +# Use the official Node.js runtime as a parent image FROM node:20-alpine # Set the working directory in the container @@ -10,19 +10,16 @@ COPY package*.json ./ # Install app dependencies RUN npm install -COPY ./ ./ +# Copy the rest of the application code +COPY . . +# Build the application and clean up RUN npm run build \ && rm -rf node_modules \ && npm install --production -# Bundle app source -COPY . . - -# Expose the port your app runs on +# Expose port 8787 EXPOSE 8787 ENTRYPOINT ["npm"] - -# Define the command to run your app CMD ["run", "start:node"] \ No newline at end of file diff --git a/docs/installation-deployments.md b/docs/installation-deployments.md index b88271b54..8f02471b5 100644 --- a/docs/installation-deployments.md +++ b/docs/installation-deployments.md @@ -8,20 +8,20 @@ Portkey runs this same Gateway on our API and processes **billions of tokens** daily. Portkey's API is in production with companies like Postman, Haptik, Turing, MultiOn, SiteGPT, and more. -Sign up for the free developer plan (10K request/month) [here](https://app.portkey.ai/) or [discuss here](https://calendly.com/rohit-portkey/noam) for enterprise deployments. +Sign up for the free developer plan [here](https://app.portkey.ai/) or [discuss here](https://calendly.com/portkey-ai/quick-meeting?utm_source=github&utm_campaign=install_page) for enterprise deployments. Check out the [API docs](https://portkey.ai/docs/welcome/make-your-first-request) here. ## Local Deployment -1. Do [NPM](#node) or [Bun](#bun) Install -2. Run a [Node.js Server](https://github.com/Portkey-AI/gateway/blob/main/docs/installation-deployments.md#run-a-nodejs-server) -3. Deploy on [App Stack](#deploy-to-app-stack) -4. Deploy on [Cloudflare Workers](https://github.com/Portkey-AI/gateway/blob/main/docs/installation-deployments.md#deploy-to-cloudflare-workers) -5. Deploy using [Docker](https://github.com/Portkey-AI/gateway/blob/main/docs/installation-deployments.md#deploy-using-docker) -6. Deploy using [Docker Compose](https://github.com/Portkey-AI/gateway/blob/main/docs/installation-deployments.md#deploy-using-docker-compose) +1. Run through [NPX](#node) or [BunX](#bun) Install +2. Run a [Node.js Server](#nodejs-server) +3. Deploy using [Docker](#docker) +4. Deploy using [Docker Compose](#docker-compose) +5. Deploy on [Cloudflare Workers](#cloudflare-workers) +6. Deploy on [App Stack](#deploy-to-app-stack) 7. Deploy on [Replit](#replit) -8. Deploy on [Zeabur](https://github.com/Portkey-AI/gateway/blob/main/docs/installation-deployments.md#deploy-to-zeabur) +8. Deploy on [Zeabur](#zeabur) ### Node @@ -39,7 +39,7 @@ $ bunx @portkey-ai/gateway
-# Deploy to App Stack +### Deploy to App Stack F5 Distributed Cloud 1. [Create an App Stack Site](https://docs.cloud.f5.com/docs/how-to/site-management/create-voltstack-site) @@ -189,7 +189,7 @@ node build/start-server.js ### Docker -**Run using Docker directly:** +**Run through the latest Docker Hub image:** ```sh docker run -d -p 8787:8787 portkeyai/gateway:latest @@ -268,6 +268,6 @@ Make your AI app more reliable and forward compatible, whi ✅  SOC2, ISO, HIPAA, GDPR Compliances - for best security practices
✅  Professional Support - along with feature prioritization
-[Schedule a call to discuss enterprise deployments](https://calendly.com/rohit-portkey/noam) +[Schedule a call to discuss enterprise deployments](https://calendly.com/portkey-ai/quick-meeting?utm_source=github&utm_campaign=install_page)
diff --git a/public/main.js b/public/main.js index b90a0df0e..48a98c1cd 100644 --- a/public/main.js +++ b/public/main.js @@ -411,9 +411,11 @@ const closeModal = document.querySelector('.close'); const clearLogsBtn = document.querySelector('.btn-clear-logs'); // SSE for the logs -const logSource = new EventSource('/log/stream'); +let logSource; function setupLogSource() { + logSource = new EventSource('/log/stream'); + logSource.addEventListener('connected', (event) => { console.log('Connected to log stream', event.data); }); @@ -435,13 +437,20 @@ function setupLogSource() { }; } +function cleanupLogSource() { + if (logSource) { + console.log('Closing log stream connection'); + logSource.close(); + logSource = null; + } +} + function reconnectLogSource() { if (logSource) { logSource.close(); } console.log('Attempting to reconnect to log stream...'); setTimeout(() => { - logSource = new EventSource('/log/stream'); setupLogSource(); }, 5000); // Wait 5 seconds before attempting to reconnect } @@ -516,10 +525,9 @@ window.addEventListener('click', (event) => { } }); -window.addEventListener('beforeunload', () => { - console.log('Page is being unloaded'); - logSource.close(); -}); +// Update event listeners for page unload +window.addEventListener('beforeunload', cleanupLogSource); +window.addEventListener('unload', cleanupLogSource); window.onload = function() { diff --git a/src/middlewares/log/index.ts b/src/middlewares/log/index.ts index 1acbe02cb..d0515c720 100644 --- a/src/middlewares/log/index.ts +++ b/src/middlewares/log/index.ts @@ -3,7 +3,7 @@ import { Context } from 'hono'; let logId = 0; // Map to store all connected log clients -const logClients: any = new Map(); +const logClients: Map = new Map(); const addLogClient = (clientId: any, client: any) => { logClients.set(clientId, client); @@ -28,19 +28,22 @@ const broadcastLog = async (log: any) => { const deadClients: any = []; - for (const [id, client] of logClients) { - try { - await Promise.race([ - client.sendLog(message), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Send timeout')), 1000) - ), - ]); - } catch (error: any) { - console.error(`Failed to send log to client ${id}:`, error.message); - deadClients.push(id); - } - } + // Run all sends in parallel + await Promise.all( + Array.from(logClients.entries()).map(async ([id, client]) => { + try { + await Promise.race([ + client.sendLog(message), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Send timeout')), 1000) + ), + ]); + } catch (error: any) { + console.error(`Failed to send log to client ${id}:`, error.message); + deadClients.push(id); + } + }) + ); // Remove dead clients after iteration deadClients.forEach((id: any) => { diff --git a/src/start-server.ts b/src/start-server.ts index b82cdd3fd..c4cba33b6 100644 --- a/src/start-server.ts +++ b/src/start-server.ts @@ -6,11 +6,11 @@ import { exec } from 'child_process'; import app from './index'; import { streamSSE } from 'hono/streaming'; +import { Context } from 'hono'; // Extract the port number from the command line arguments const defaultPort = 8787; const args = process.argv.slice(2); -console.log(args, process.argv); const portArg = args.find((arg) => arg.startsWith('--port=')); const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; @@ -20,7 +20,7 @@ if (!isHeadless) { app.get('/public/*', serveStatic({ root: './' })); app.get('/public/logs', serveStatic({ path: './public/index.html' })); - app.get('/log/stream', (c) => { + app.get('/log/stream', (c: Context) => { const clientId = Date.now().toString(); // Set headers to prevent caching @@ -33,6 +33,7 @@ if (!isHeadless) { }; // Add this client to the set of log clients const addLogClient: any = c.get('addLogClient'); + const removeLogClient: any = c.get('removeLogClient'); addLogClient(clientId, client); try { @@ -46,9 +47,9 @@ if (!isHeadless) { } } catch (error) { console.error(`Error in log stream for client ${clientId}:`, error); + removeLogClient(clientId); } finally { // Remove this client when the connection is closed - const removeLogClient: any = c.get('removeLogClient'); removeLogClient(clientId); } }); @@ -66,22 +67,30 @@ console.log(`Your AI Gateway is now running on ${url} 🚀`); // Function to open URL in the default browser function openBrowser(url: string) { let command: string; - switch (process.platform) { - case 'darwin': - command = `open ${url}`; - break; - case 'win32': - command = `start ${url}`; - break; - default: - command = `xdg-open ${url}`; + // In Docker container, just log the URL in a clickable format + if (process.env.DOCKER || process.env.CONTAINER) { + console.log('\n🔗 Access your AI Gateway at: \x1b[36m%s\x1b[0m\n', url); + command = ''; // No-op for Docker/containers + } else { + switch (process.platform) { + case 'darwin': + command = `open ${url}`; + break; + case 'win32': + command = `start ${url}`; + break; + default: + command = `xdg-open ${url}`; + } } - exec(command, (error) => { - if (error) { - console.error('Failed to open browser:', error); - } - }); + if (command) { + exec(command, (error) => { + if (error) { + console.log('\n🔗 Access your AI Gateway at: \x1b[36m%s\x1b[0m\n', url); + } + }); + } } // Open the browser only when --headless is not provided From b607b319f9ffaf73f1a9023d12f1eca2d5388568 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Tue, 12 Nov 2024 19:19:55 +0530 Subject: [PATCH 009/119] Support shorthand format for guardrails in config --- .gitignore | 1 + src/handlers/handlerUtils.ts | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.gitignore b/.gitignore index a5530aba1..1d40a1d0c 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,4 @@ build .idea plugins/**/.creds.json plugins/**/creds.json +src/handlers/test.ts diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index afdb5f033..dc2877290 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -222,6 +222,32 @@ export const fetchProviderOptionsFromConfig = ( return providerOptions; }; +export function convertGuardrailsShorthand(guardrailsArr: any, type:string) { + + return guardrailsArr.map((guardrails:any) => { + let hooksObject: any = { + "type": "guardrail", + "id": `${type}_guardrail_${Math.random().toString(36).substring(2, 5)}`, + }; + + // if the deny key is present (true or false), add it to hooksObject and remove it from guardrails + ['deny', 'on_fail', 'on_success', 'async', 'onFail', 'onSuccess'].forEach(key => { + if (guardrails.hasOwnProperty(key)) { + hooksObject[key] = guardrails[key]; + delete guardrails[key]; + } + }); + + // Now, add all the checks to the checks array + hooksObject.checks = Object.keys(guardrails).map((key) => ({ + id: key, + parameters: guardrails[key], + })); + + return hooksObject; + }); +} + /** * @deprecated * Makes a request (GET or POST) to a provider and returns the response. @@ -784,6 +810,16 @@ export async function tryTargetsRecursively( currentTarget.requestTimeout = inheritedConfig.requestTimeout; } + if (currentTarget.inputGuardrails) { + currentTarget.beforeRequestHooks = + convertGuardrailsShorthand(currentTarget.inputGuardrails, "input"); + } + + if (currentTarget.outputGuardrails) { + currentTarget.afterRequestHooks = + convertGuardrailsShorthand(currentTarget.outputGuardrails, "output"); + } + if (currentTarget.afterRequestHooks) { currentInheritedConfig.afterRequestHooks = [ ...currentTarget.afterRequestHooks, From 6791237d4a43296ffd71761ae8f320f05c064cc2 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Tue, 12 Nov 2024 20:29:19 +0530 Subject: [PATCH 010/119] Appending to existing hooks if they exist. --- src/handlers/handlerUtils.ts | 39 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index dc2877290..746693a18 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -222,28 +222,29 @@ export const fetchProviderOptionsFromConfig = ( return providerOptions; }; -export function convertGuardrailsShorthand(guardrailsArr: any, type:string) { - - return guardrailsArr.map((guardrails:any) => { +export function convertGuardrailsShorthand(guardrailsArr: any, type: string) { + return guardrailsArr.map((guardrails: any) => { let hooksObject: any = { - "type": "guardrail", - "id": `${type}_guardrail_${Math.random().toString(36).substring(2, 5)}`, + type: 'guardrail', + id: `${type}_guardrail_${Math.random().toString(36).substring(2, 5)}`, }; - + // if the deny key is present (true or false), add it to hooksObject and remove it from guardrails - ['deny', 'on_fail', 'on_success', 'async', 'onFail', 'onSuccess'].forEach(key => { - if (guardrails.hasOwnProperty(key)) { - hooksObject[key] = guardrails[key]; - delete guardrails[key]; + ['deny', 'on_fail', 'on_success', 'async', 'onFail', 'onSuccess'].forEach( + (key) => { + if (guardrails.hasOwnProperty(key)) { + hooksObject[key] = guardrails[key]; + delete guardrails[key]; + } } - }); - + ); + // Now, add all the checks to the checks array hooksObject.checks = Object.keys(guardrails).map((key) => ({ id: key, parameters: guardrails[key], })); - + return hooksObject; }); } @@ -811,13 +812,17 @@ export async function tryTargetsRecursively( } if (currentTarget.inputGuardrails) { - currentTarget.beforeRequestHooks = - convertGuardrailsShorthand(currentTarget.inputGuardrails, "input"); + currentTarget.beforeRequestHooks = [ + ...(currentTarget.beforeRequestHooks || []), + ...convertGuardrailsShorthand(currentTarget.inputGuardrails, 'input'), + ]; } if (currentTarget.outputGuardrails) { - currentTarget.afterRequestHooks = - convertGuardrailsShorthand(currentTarget.outputGuardrails, "output"); + currentTarget.afterRequestHooks = [ + ...(currentTarget.afterRequestHooks || []), + ...convertGuardrailsShorthand(currentTarget.outputGuardrails, 'output'), + ]; } if (currentTarget.afterRequestHooks) { From e1919bf12c269106859eabeec25b42952d5f8e61 Mon Sep 17 00:00:00 2001 From: Mahesh Date: Wed, 20 Nov 2024 20:34:53 +0530 Subject: [PATCH 011/119] feat: add mistral as a new guardrail provider --- plugins/index.ts | 21 +++++ plugins/mistral/index.ts | 112 +++++++++++++++++++++++++++ plugins/mistral/manifest.json | 135 +++++++++++++++++++++++++++++++++ plugins/types.ts | 6 +- src/middlewares/hooks/index.ts | 3 +- 5 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 plugins/mistral/index.ts create mode 100644 plugins/mistral/manifest.json diff --git a/plugins/index.ts b/plugins/index.ts index 03e7c6785..3bccb8fbf 100644 --- a/plugins/index.ts +++ b/plugins/index.ts @@ -32,6 +32,20 @@ import { handler as patronusnoRacialBias } from './patronus/noRacialBias'; import { handler as patronusretrievalAnswerRelevance } from './patronus/retrievalAnswerRelevance'; import { handler as patronustoxicity } from './patronus/toxicity'; import { handler as patronuscustom } from './patronus/custom'; +import { mistralGuardrailHandler } from './mistral'; +import { PluginHandler } from './types'; + +const mistralGuardCategories = [ + 'sexual', + 'hate_and_discrimination', + 'violence_and_threats', + 'dangerous_and_criminal_content', + 'selfharm', + 'health', + 'financial', + 'law', + 'pii', +]; export const plugins = { default: { @@ -80,4 +94,11 @@ export const plugins = { toxicity: patronustoxicity, custom: patronuscustom, }, + mistral: mistralGuardCategories.reduce( + (config, category) => { + config[category] = mistralGuardrailHandler; + return config; + }, + {} as Record + ), }; diff --git a/plugins/mistral/index.ts b/plugins/mistral/index.ts new file mode 100644 index 000000000..19bcf336c --- /dev/null +++ b/plugins/mistral/index.ts @@ -0,0 +1,112 @@ +import { + HookEventType, + PluginContext, + PluginHandler, + PluginParameters, +} from '../types'; +import { getText, post } from '../utils'; + +interface MistralResponse { + id: string; + model: string; + results: [ + { + categories: { + sexual: boolean; + hate_and_discrimination: boolean; + violence_and_threats: boolean; + dangerous_and_criminal_content: boolean; + selfharm: boolean; + health: boolean; + financial: boolean; + law: boolean; + pii: boolean; + }; + category_score: { + sexual: number; + hate_and_discrimination: number; + violence_and_threats: number; + dangerous_and_criminal_content: number; + selfharm: number; + health: number; + financial: number; + law: number; + pii: number; + }; + }, + ]; +} + +type GuardrailFunction = keyof MistralResponse['results'][0]['categories']; + +export const mistralGuardrailHandler: PluginHandler = async ( + context: PluginContext, + parameters: PluginParameters, + eventType: HookEventType, + _options, + fn: string +) => { + let error = null; + let verdict = true; + let data = null; + + const creds = parameters.credentials as Record; + if (!creds.apiKey) { + return { + error: 'Mistral API key not provided.', + verdict: false, + data: null, + }; + } + + let model = 'mistral-moderation-latest'; + + if (parameters.model) { + // Model can be passed dynamically + model = parameters.model; + } + + const guardrailFunction = fn as GuardrailFunction; + + const text = getText(context, eventType); + const messages = context.request?.json?.messages ?? []; + + if (!text || !Array.isArray(messages) || !messages.length) { + return { + error: 'Mistral: Invalid Request body', + verdict: false, + data: null, + }; + } + + // Use conversation guardrail if it's a chatcomplete and before hook + const shouldUseConversation = + eventType === 'beforeRequestHook' && context.requestType === 'chatComplete'; + const url = shouldUseConversation + ? 'https://api.mistral.ai/v1/chat/moderations' + : 'https://api.mistral.ai/v1/moderations'; + + try { + const request = await post( + url, + { + model: model, + ...(!shouldUseConversation && { input: [text] }), + ...(shouldUseConversation && { input: [messages] }), + }, + { + headers: { + Authorization: `Bearer ${creds.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ); + + verdict = request.results?.[0]?.categories[guardrailFunction]; + } catch (error) { + error = error; + verdict = false; + } + + return { error, verdict, data }; +}; diff --git a/plugins/mistral/manifest.json b/plugins/mistral/manifest.json new file mode 100644 index 000000000..4a0fb148c --- /dev/null +++ b/plugins/mistral/manifest.json @@ -0,0 +1,135 @@ +{ + "id": "mistral", + "description": "Mistral Content Moderation classifier leverages the most relevant policy categories for effective guardrails and introduces a pragmatic approach to LLM safety by addressing model-generated harms such as unqualified advice and PII", + "credentials": { + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "label": "API Key", + "description": "Find your API key in the Mistral la-plateforme", + "encrypted": true + } + }, + "required": ["apiKey"] + }, + "functions": [ + { + "name": "Detect PII", + "id": "pii", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that requests, shares, or attempts to elicit personal identifying information such as full names, addresses, phone numbers, social security numbers, or financial account details." + } + ], + "parameters": {} + }, + { + "name": "Detect Sexual Content", + "id": "sexual", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Material that explicitly depicts, describes, or promotes sexual activities, nudity, or sexual services. This includes pornographic content, graphic descriptions of sexual acts, and solicitation for sexual purposes. Educational or medical content about sexual health presented in a non-explicit, informational context is generally exempted." + } + ], + "parameters": {} + }, + { + "name": "Detect Hate & Discrimination", + "id": "hate_and_discrimination", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that expresses prejudice, hostility, or advocates discrimination against individuals or groups based on protected characteristics such as race, ethnicity, religion, gender, sexual orientation, or disability. This includes slurs, dehumanizing language, calls for exclusion or harm targeted at specific groups, and persistent harassment or bullying of individuals based on these characteristics." + } + ], + "parameters": {} + }, + { + "name": "Detect Violent & Thereat", + "id": "violence_and_threats", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that describes, glorifies, incites, or threatens physical violence against individuals or groups. This includes graphic depictions of injury or death, explicit threats of harm, and instructions for carrying out violent acts. This category covers both targeted threats and general promotion or glorification of violence." + } + ], + "parameters": {} + }, + { + "name": "Detect Dangerous & Criminal Content", + "id": "dangerous_and_criminal_content", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that promotes or provides instructions for illegal activities or extremely hazardous behaviors that pose a significant risk of physical harm, death, or legal consequences. This includes guidance on creating weapons or explosives, encouragement of extreme risk-taking behaviors, and promotion of non-violent crimes such as fraud, theft, or drug trafficking." + } + ], + "parameters": {} + }, + { + "name": "Detect Selfharm", + "id": "selfharm", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that promotes, instructs, plans, or encourages deliberate self-injury, suicide, eating disorders, or other self-destructive behaviors. This includes detailed methods, glorification, statements of intent, dangerous challenges, and related slang terms" + } + ], + "parameters": {} + }, + { + "name": "Detect Health", + "id": "health", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that contains or tries to elicit detailed or tailored medical advice." + } + ], + "parameters": {} + }, + { + "name": "Detect Finance", + "id": "financial", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that contains or tries to elicit detailed or tailored financial advice." + } + ], + "parameters": {} + }, + { + "name": "Detect Law", + "id": "law", + "supportedHooks": ["beforeRequestHook", "afterRequestHook"], + "type": "guardrail", + "description": [ + { + "type": "subHeading", + "text": "Content that contains or tries to elicit detailed or tailored legal advice." + } + ], + "parameters": {} + } + ] +} diff --git a/plugins/types.ts b/plugins/types.ts index c4f37a7f5..0c1f02f1b 100644 --- a/plugins/types.ts +++ b/plugins/types.ts @@ -1,5 +1,7 @@ export interface PluginContext { [key: string]: any; + requestType: 'complete' | 'chatComplete'; + provider: string; } export interface PluginParameters { @@ -22,5 +24,7 @@ export type PluginHandler = ( eventType: HookEventType, options: { env: Record; - } + }, + // Handler function, useful in cases for a provider with multiple guardrails ex: mistral + fn: string ) => Promise; diff --git a/src/middlewares/hooks/index.ts b/src/middlewares/hooks/index.ts index bfdca842a..6c6246e97 100644 --- a/src/middlewares/hooks/index.ts +++ b/src/middlewares/hooks/index.ts @@ -261,7 +261,8 @@ export class HooksManager { context, check.parameters, eventType, - options + options, + fn ); return { ...result, From a8cc903b927b86970458266c5f045960b939a080 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Mon, 25 Nov 2024 16:57:42 -0800 Subject: [PATCH 012/119] Logging middleware async --- src/middlewares/log/index.ts | 56 ++++++++++++++++++++++-------------- src/start-server.ts | 2 +- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/middlewares/log/index.ts b/src/middlewares/log/index.ts index d0515c720..919c26920 100644 --- a/src/middlewares/log/index.ts +++ b/src/middlewares/log/index.ts @@ -1,4 +1,5 @@ import { Context } from 'hono'; +import { getRuntimeKey } from 'hono/adapter'; let logId = 0; @@ -51,6 +52,33 @@ const broadcastLog = async (log: any) => { }); }; +async function processLog(c: Context, start: number) { + const ms = Date.now() - start; + if (!c.req.url.includes('/v1/')) return; + + const requestOptionsArray = c.get('requestOptions'); + + if (requestOptionsArray[0].requestParams.stream) { + requestOptionsArray[0].response = { + message: 'The response was a stream.', + }; + } else { + const response = await c.res.clone().json(); + requestOptionsArray[0].response = response; + } + + await broadcastLog( + JSON.stringify({ + time: new Date().toLocaleString(), + method: c.req.method, + endpoint: c.req.url.split(':8787')[1], + status: c.res.status, + duration: ms, + requestOptions: requestOptionsArray, + }) + ); +} + export const logger = () => { return async (c: Context, next: any) => { c.set('addLogClient', addLogClient); @@ -60,28 +88,12 @@ export const logger = () => { await next(); - const ms = Date.now() - start; - if (!c.req.url.includes('/v1/')) return; - - const requestOptionsArray = c.get('requestOptions'); - if (requestOptionsArray[0].requestParams.stream) { - requestOptionsArray[0].response = { - message: 'The response was a stream.', - }; - } else { - const response = await c.res.clone().json(); - requestOptionsArray[0].response = response; - } + const runtime = getRuntimeKey(); - await broadcastLog( - JSON.stringify({ - time: new Date().toLocaleString(), - method: c.req.method, - endpoint: c.req.url.split(':8787')[1], - status: c.res.status, - duration: ms, - requestOptions: requestOptionsArray, - }) - ); + if (runtime == 'workerd') { + c.executionCtx.waitUntil(processLog(c, start)); + } else if (['node', 'bun', 'deno'].includes(runtime)) { + processLog(c, start).then().catch(console.error); + } }; }; diff --git a/src/start-server.ts b/src/start-server.ts index c4cba33b6..ee58636a6 100644 --- a/src/start-server.ts +++ b/src/start-server.ts @@ -16,7 +16,7 @@ const port = portArg ? parseInt(portArg.split('=')[1]) : defaultPort; const isHeadless = args.includes('--headless'); -if (!isHeadless) { +if (!isHeadless && process.env.NODE_ENV !== 'production') { app.get('/public/*', serveStatic({ root: './' })); app.get('/public/logs', serveStatic({ path: './public/index.html' })); From cf25af712dff25a260c6426fc9617445f2f42d5c Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Tue, 26 Nov 2024 13:13:56 -0800 Subject: [PATCH 013/119] shorter logs, max 100 logs viewed --- public/main.js | 19 +++++++++++++++++++ src/middlewares/log/index.ts | 6 +++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/public/main.js b/public/main.js index 48a98c1cd..8c4e96fdc 100644 --- a/public/main.js +++ b/public/main.js @@ -478,6 +478,25 @@ function addLogEntry(time, method, endpoint, status, duration, requestOptions) { logsTableBody.appendChild(tr); } + // Ensure the log table does not exceed 100 rows + while (logsTableBody.children.length > 100) { + logsTableBody.removeChild(logsTableBody.lastChild); + } + + // Add a message to the last line of the table + if (logsTableBody.children.length === 100) { + let messageRow = logsTableBody.querySelector('.log-message-row'); + if (!messageRow) { + messageRow = document.createElement('tr'); + messageRow.classList.add('log-message-row'); + const messageCell = document.createElement('td'); + messageCell.colSpan = 6; // Assuming there are 6 columns in the table + messageCell.textContent = 'Only the latest 100 logs are being shown.'; + messageRow.appendChild(messageCell); + logsTableBody.appendChild(messageRow); + } + } + incrementLogCounter(); setTimeout(() => { diff --git a/src/middlewares/log/index.ts b/src/middlewares/log/index.ts index 919c26920..50bec2a26 100644 --- a/src/middlewares/log/index.ts +++ b/src/middlewares/log/index.ts @@ -64,7 +64,11 @@ async function processLog(c: Context, start: number) { }; } else { const response = await c.res.clone().json(); - requestOptionsArray[0].response = response; + const maxLength = 1000; // Set a reasonable limit for the response length + const responseString = JSON.stringify(response); + requestOptionsArray[0].response = responseString.length > maxLength + ? JSON.parse(responseString.substring(0, maxLength) + '...') + : response; } await broadcastLog( From a9f6b112b9ed4c85bcd7851ab60a0b9e789b8cb7 Mon Sep 17 00:00:00 2001 From: Mahesh Date: Thu, 28 Nov 2024 00:35:01 +0530 Subject: [PATCH 014/119] fix: add test cases for mistral --- plugins/mistral/index.ts | 9 ++- plugins/mistral/mistra.test.ts | 109 +++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 plugins/mistral/mistra.test.ts diff --git a/plugins/mistral/index.ts b/plugins/mistral/index.ts index 19bcf336c..1a7643a9b 100644 --- a/plugins/mistral/index.ts +++ b/plugins/mistral/index.ts @@ -69,9 +69,14 @@ export const mistralGuardrailHandler: PluginHandler = async ( const guardrailFunction = fn as GuardrailFunction; const text = getText(context, eventType); - const messages = context.request?.json?.messages ?? []; + const messages = context.request?.json?.messages; - if (!text || !Array.isArray(messages) || !messages.length) { + // should contain text or should contain messages array + if ( + (!text && !Array.isArray(messages)) || + (Array.isArray(messages) && messages.length === 0) + ) { + console.log(!text, messages); return { error: 'Mistral: Invalid Request body', verdict: false, diff --git a/plugins/mistral/mistra.test.ts b/plugins/mistral/mistra.test.ts new file mode 100644 index 000000000..69143449d --- /dev/null +++ b/plugins/mistral/mistra.test.ts @@ -0,0 +1,109 @@ +import { PluginContext } from '../types'; +import testCreds from './.creds.json'; +import { mistralGuardrailHandler } from './index'; + +function getParameters() { + return { + credentials: testCreds, + }; +} + +describe('validateProject handler', () => { + it('should fail if the apiKey is invalid', async () => { + const eventType = 'beforeRequestHook'; + const context = { + request: { text: 'this is a test string for moderations' }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + parameters.credentials.apiKey = 'invalid-api-key'; + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(false); + expect(result.error).toBeDefined(); + expect(result.data).toBeNull(); + }); + + it('should return pii true for pii function', async () => { + const eventType = 'beforeRequestHook'; + const context = { + request: { + text: 'Say Hi. My name is Jhon Doe and my email is user@example.com', + }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(true); + expect(result.error).toBeNull(); + expect(result.data).toBeNull(); + }); + + it('should be false when pii is not present', async () => { + const eventType = 'beforeRequestHook'; + const context = { + request: { text: 'this text is safe text' }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(false); + expect(result.error).toBeNull(); + expect(result.data).toBeNull(); + }); + + it('should work pii for chatComplete messages', async () => { + const eventType = 'beforeRequestHook'; + const context = { + requestType: 'chatComplete', + request: { + json: { + messages: [ + { + role: 'user', + content: + 'Say Hi. My name is Jhon Doe and my email is user@example.com', + }, + ], + }, + }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(true); + expect(result.error).toBeNull(); + expect(result.data).toBeNull(); + }); +}); From 3881aaa571ce2ff6d9333fb2e253e9dd29268554 Mon Sep 17 00:00:00 2001 From: Mahesh Date: Thu, 28 Nov 2024 00:51:06 +0530 Subject: [PATCH 015/119] chore: add extra test cases --- plugins/mistral/index.ts | 6 ++- plugins/mistral/mistra.test.ts | 76 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/plugins/mistral/index.ts b/plugins/mistral/index.ts index 1a7643a9b..b73d4f130 100644 --- a/plugins/mistral/index.ts +++ b/plugins/mistral/index.ts @@ -69,14 +69,16 @@ export const mistralGuardrailHandler: PluginHandler = async ( const guardrailFunction = fn as GuardrailFunction; const text = getText(context, eventType); - const messages = context.request?.json?.messages; + const messages = + eventType === 'beforeRequestHook' + ? context.request?.json?.messages + : context.response?.json?.messages; // should contain text or should contain messages array if ( (!text && !Array.isArray(messages)) || (Array.isArray(messages) && messages.length === 0) ) { - console.log(!text, messages); return { error: 'Mistral: Invalid Request body', verdict: false, diff --git a/plugins/mistral/mistra.test.ts b/plugins/mistral/mistra.test.ts index 69143449d..2cd5509cb 100644 --- a/plugins/mistral/mistra.test.ts +++ b/plugins/mistral/mistra.test.ts @@ -8,7 +8,7 @@ function getParameters() { }; } -describe('validateProject handler', () => { +describe('mistral guardrail handler', () => { it('should fail if the apiKey is invalid', async () => { const eventType = 'beforeRequestHook'; const context = { @@ -106,4 +106,78 @@ describe('validateProject handler', () => { expect(result.error).toBeNull(); expect(result.data).toBeNull(); }); + + it('should give error on invalid request body', async () => { + const eventType = 'beforeRequestHook'; + const context = { + request: {}, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(false); + expect(result.error).toBe('Mistral: Invalid Request body'); + expect(result.data).toBeNull(); + }); + + it('should work for afterRequestHook', async () => { + const eventType = 'afterRequestHook'; + const context = { + response: { text: 'this text is safe text' }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(false); + expect(result.error).toBeNull(); + expect(result.data).toBeNull(); + }); + + it('should work for afterRequestHook with chatCompletion messages', async () => { + const eventType = 'afterRequestHook'; + const context = { + requestType: 'chatComplete', + response: { + json: { + messages: [ + { + role: 'user', + content: + 'Say Hi. My name is Jhon Doe and my email is user@example.com', + }, + ], + }, + }, + }; + const parameters = JSON.parse(JSON.stringify(getParameters())); + + const result = await mistralGuardrailHandler( + context as unknown as PluginContext, + parameters, + eventType, + { env: {} }, + 'pii' + ); + + expect(result).toBeDefined(); + expect(result.verdict).toBe(false); + expect(result.error).toBeNull(); + expect(result.data).toBeNull(); + }); }); From 3362553354173e9b38b19a8c6c438666d87758f1 Mon Sep 17 00:00:00 2001 From: vrushankportkey Date: Fri, 29 Nov 2024 17:27:45 +0530 Subject: [PATCH 016/119] Change to all dub.co links --- README.md | 170 +++++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 2191719bf..6321a67c2 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,19 @@ Portkey AI Gateway Demo showing LLM routing capabilities -[Docs](https://portkey.ai/docs) | [Enterprise](https://portkey.ai/docs/product/enterprise-offering) | [Hosted Gateway](https://app.portkey.ai/) | [Changelog](https://new.portkey.ai) | [API Reference](https://portkey.ai/docs/api-reference/inference-api/introduction) +[Docs](https://portkey.wiki/gh-1) | [Enterprise](https://portkey.wiki/gh-2) | [Hosted Gateway](https://portkey.wiki/gh-3) | [Changelog](https://portkey.wiki/gh-4) | [API Reference](https://portkey.wiki/gh-5) [![License](https://img.shields.io/github/license/Ileriayo/markdown-badges)](./LICENSE) -[![Discord](https://img.shields.io/discord/1143393887742861333)](https://portkey.ai/community) -[![Twitter](https://img.shields.io/twitter/url/https/twitter/follow/portkeyai?style=social&label=Follow%20%40PortkeyAI)](https://x.com/portkeyai) -[![npm version](https://badge.fury.io/js/%40portkey-ai%2Fgateway.svg)](https://www.npmjs.com/package/@portkey-ai/gateway) -[![Better Stack Badge](https://uptime.betterstack.com/status-badges/v1/monitor/q94g.svg)](https://status.portkey.ai/?utm_source=status_badge) +[![Discord](https://img.shields.io/discord/1143393887742861333)](https://portkey.wiki/gh-6) +[![Twitter](https://img.shields.io/twitter/url/https/twitter/follow/portkeyai?style=social&label=Follow%20%40PortkeyAI)](https://portkey.wiki/gh-7) +[![npm version](https://badge.fury.io/js/%40portkey-ai%2Fgateway.svg)](https://portkey.wiki/gh-8) +[![Better Stack Badge](https://uptime.betterstack.com/status-badges/v1/monitor/q94g.svg)](https://portkey.wiki/gh-9)