diff --git a/docs/WORKLOG.md b/docs/WORKLOG.md index 5f33143..cc939f2 100644 --- a/docs/WORKLOG.md +++ b/docs/WORKLOG.md @@ -1,3 +1,9 @@ +2025-12-26 20:31 UTC - Provider contract VCR replay tests +- Files: tests/contracts/provider.contract.test.ts; tests/contracts/vcr/loadCassette.ts; tests/contracts/vcr/types.ts; tests/contracts/cassettes/*.json; docs/WORKLOG.md +- Why: Provider contract tests were a skipped scaffold; we need deterministic replay tests to validate real adapter behavior without network calls and without placeholder assertions. +- Details: Adds replay-only VCR cassettes that drive real `OpenAIAdapter.translate()` and `GeminiAdapter.translate()` while mocking only provider SDK boundaries; asserts JSON parsing, token accounting, cost wiring, and OpenAI metrics recording. +- Tests: `npm test` + 2025-12-24 11:23 UTC - Migration recovery UI gate - Files: App.tsx; components/MigrationRecovery.tsx; tests/components/MigrationRecovery.test.tsx; docs/WORKLOG.md - Why: When the DB is newer/corrupted/blocked or a migration failed, users need a clear recovery path (restore from backup, upload backup, or start fresh) instead of a silent failure. diff --git a/tests/contracts/cassettes/gemini-happy-path.json b/tests/contracts/cassettes/gemini-happy-path.json new file mode 100644 index 0000000..7237475 --- /dev/null +++ b/tests/contracts/cassettes/gemini-happy-path.json @@ -0,0 +1,24 @@ +{ + "name": "gemini-happy-path", + "provider": "Gemini", + "model": "gemini-2.5-flash", + "request": { + "title": "Test Title", + "content": "今天天气很好。", + "systemPrompt": "Translate this Chinese text to English.", + "temperature": 0.3, + "maxOutputTokens": 512 + }, + "mock": { + "responseText": "{\"translatedTitle\":\"Test Title\",\"translation\":\"The weather is very good today.\",\"footnotes\":[],\"suggestedIllustrations\":[],\"proposal\":null}", + "usageMetadata": { "promptTokenCount": 14, "candidatesTokenCount": 9 } + }, + "expected": { + "translatedTitle": "Test Title", + "translation": "The weather is very good today.", + "promptTokens": 14, + "completionTokens": 9, + "estimatedCost": 0.000008 + } +} + diff --git a/tests/contracts/cassettes/openai-happy-path.json b/tests/contracts/cassettes/openai-happy-path.json new file mode 100644 index 0000000..3198b34 --- /dev/null +++ b/tests/contracts/cassettes/openai-happy-path.json @@ -0,0 +1,33 @@ +{ + "name": "openai-happy-path", + "provider": "OpenAI", + "model": "gpt-4o-mini", + "request": { + "title": "Test Title", + "content": "今天天气很好。", + "systemPrompt": "Translate this Chinese text to English.", + "temperature": 0.3, + "maxOutputTokens": 512 + }, + "mock": { + "sdkResponse": { + "choices": [ + { + "finish_reason": "stop", + "message": { + "content": "{\"translatedTitle\":\"Test Title\",\"translation\":\"The weather is very nice today.\",\"footnotes\":[],\"suggestedIllustrations\":[]}" + } + } + ], + "usage": { "prompt_tokens": 15, "completion_tokens": 8 } + } + }, + "expected": { + "translatedTitle": "Test Title", + "translation": "The weather is very nice today.", + "promptTokens": 15, + "completionTokens": 8, + "estimatedCost": 0.00003 + } +} + diff --git a/tests/contracts/cassettes/openai-medium-chapter.json b/tests/contracts/cassettes/openai-medium-chapter.json new file mode 100644 index 0000000..8d6e52f --- /dev/null +++ b/tests/contracts/cassettes/openai-medium-chapter.json @@ -0,0 +1,33 @@ +{ + "name": "openai-medium-chapter", + "provider": "OpenAI", + "model": "gpt-4o-mini", + "request": { + "title": "Chapter 1", + "content": "그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다. 그날 하늘은 맑았다.", + "systemPrompt": "Translate this Korean web novel chapter to English.", + "temperature": 0.7, + "maxOutputTokens": 2048 + }, + "mock": { + "sdkResponse": { + "choices": [ + { + "finish_reason": "stop", + "message": { + "content": "{\"translatedTitle\":\"Chapter 1\",\"translation\":\"That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear.\",\"footnotes\":[],\"suggestedIllustrations\":[]}" + } + } + ], + "usage": { "prompt_tokens": 1000, "completion_tokens": 850 } + } + }, + "expected": { + "translatedTitle": "Chapter 1", + "translation": "That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear. That day, the sky was clear.", + "promptTokens": 1000, + "completionTokens": 850, + "estimatedCost": 0.00092 + } +} + diff --git a/tests/contracts/provider.contract.test.ts b/tests/contracts/provider.contract.test.ts index 58929f0..bd83f53 100644 --- a/tests/contracts/provider.contract.test.ts +++ b/tests/contracts/provider.contract.test.ts @@ -1,320 +1,252 @@ /** - * Provider Contract Tests + * Provider Contract Tests (VCR replay-only) * - * STATUS: SCAFFOLD - All tests skipped pending VCR infrastructure - * TEST-QUALITY: 2/10 (current) → 8.5/10 (target when implemented) + * Goal: Exercise real adapter logic deterministically (no network) using + * small cassette fixtures that represent provider SDK responses. * - * This file defines the STRUCTURE for provider contract tests but the VCR - * (Video Cassette Recording) infrastructure for deterministic replay is not built. - * All tests are currently skipped. + * This avoids placeholder tests while still providing integration-like coverage across: + * - prompt building + * - SDK request/response handling + * - JSON extraction/parsing + * - token accounting + cost call wiring + * - metrics recording (where applicable) * - * Target construct: "Given a prompt + text, provider returns well-formed TranslationResult - * with correct token accounting and typed errors within timeout." - * - * NOTE: Adversarial tests (rate limits, timeouts, malformed responses) are now - * implemented in the individual adapter test files where they can be properly tested: - * - tests/adapters/providers/OpenAIAdapter.test.ts (adversarial scenarios section) + * Adversarial scenarios (rate limits, timeouts, malformed responses) live in: + * - tests/adapters/providers/OpenAIAdapter.test.ts * - tests/adapters/providers/GeminiAdapter.test.ts - * - * To make these contract tests real: - * 1. Implement VCR cassette recording/replay (see TODO at bottom) - * 2. Hook up actual adapters instead of inline mock responses - * 3. Remove .skip from test cases */ import { describe, it, expect, beforeEach, vi } from 'vitest'; -import type { TranslationResult } from '../../types'; - -// VCR-style recording types -interface RecordedRequest { - provider: string; - model: string; - prompt: string; - timestamp: number; -} - -interface RecordedResponse { - translatedText: string; - tokenCount?: { prompt: number; completion: number; total: number }; - cost?: number; - latencyMs: number; - error?: string; -} - -interface Cassette { - request: RecordedRequest; - response: RecordedResponse; -} - -/** - * VCR Replay Mechanism - * - * In real implementation, this would: - * 1. Check if cassette file exists - * 2. If yes, replay from disk - * 3. If no + LIVE_API_TEST=1, make real call and record - * 4. If no + not live, fail with helpful message - * - * For now, we use inline cassettes for demonstration. - */ -class CassettePlayer { - private cassettes: Map = new Map(); - - record(key: string, cassette: Cassette) { - this.cassettes.set(key, cassette); - } - - replay(key: string): Cassette | null { - return this.cassettes.get(key) || null; - } - - clear() { - this.cassettes.clear(); +import type { AppSettings } from '../../types'; +import type { TranslationRequest } from '../../services/translate/Translator'; +import { OpenAIAdapter } from '../../adapters/providers/OpenAIAdapter'; +import { GeminiAdapter } from '../../adapters/providers/GeminiAdapter'; +import { createMockAppSettings } from '../utils/test-data'; +import { loadCassette } from './vcr/loadCassette'; +import type { GeminiCassette, OpenAICassette } from './vcr/types'; + +const openAiMocks = vi.hoisted(() => { + const create = vi.fn(); + const ctor = vi.fn(); + class OpenAI { + chat = { + completions: { + create: (...args: any[]) => create(...args), + }, + }; + constructor(...args: any[]) { + ctor(...args); + } } -} - -const vcr = new CassettePlayer(); - -// Contract test cases - shared across all providers -interface ContractTestCase { - name: string; - input: { - systemPrompt: string; - text: string; - temperature?: number; - }; - assertions: { - hasTranslation: boolean; - minTokens?: number; - maxCost?: number; - maxLatency?: number; - }; -} + return { OpenAI, create, ctor }; +}); -const SHARED_CONTRACT_CASES: ContractTestCase[] = [ - { - name: 'happy path: small translation', - input: { - systemPrompt: 'Translate this Chinese text to English.', - text: '今天天气很好。', - temperature: 0.3, - }, - assertions: { - hasTranslation: true, - minTokens: 10, - maxCost: 0.001, // Should be cheap for small text - maxLatency: 5000, // 5 seconds +vi.mock('openai', () => ({ __esModule: true, default: openAiMocks.OpenAI })); + +const geminiMocks = vi.hoisted(() => { + const generateContent = vi.fn(); + const getGenerativeModel = vi.fn(() => ({ + generateContent: (...args: any[]) => generateContent(...args), + })); + const ctor = vi.fn(); + class GoogleGenerativeAI { + constructor(...args: any[]) { + ctor(...args); } - }, - { - name: 'medium chapter: ~1000 tokens', - input: { - systemPrompt: 'Translate this Korean web novel chapter to English.', - text: '그날 하늘은 맑았다. '.repeat(100), // ~1000 tokens worth - temperature: 0.7, - }, - assertions: { - hasTranslation: true, - minTokens: 500, - maxCost: 0.05, // Should be reasonable for medium text - maxLatency: 15000, // 15 seconds + getGenerativeModel(...args: any[]) { + return getGenerativeModel(...args); } } -]; + return { GoogleGenerativeAI, generateContent, getGenerativeModel, ctor }; +}); -describe('Provider Contract: OpenAI', () => { - const PROVIDER = 'OpenAI'; - const MODEL = 'gpt-4o-mini'; +vi.mock('@google/generative-ai', () => ({ + GoogleGenerativeAI: geminiMocks.GoogleGenerativeAI, + GenerateContentResult: Object, + SchemaType: { + OBJECT: 'OBJECT', + ARRAY: 'ARRAY', + STRING: 'STRING', + NUMBER: 'NUMBER', + BOOLEAN: 'BOOLEAN', + }, +})); - beforeEach(() => { - vcr.clear(); - - // Record cassettes (in real implementation, these come from files) - vcr.record('openai-happy-path', { - request: { - provider: PROVIDER, - model: MODEL, - prompt: SHARED_CONTRACT_CASES[0].input.text, - timestamp: Date.now(), - }, - response: { - translatedText: 'The weather is very nice today.', - tokenCount: { prompt: 15, completion: 8, total: 23 }, - cost: 0.00003, - latencyMs: 1200, - } - }); +const supportsStructuredOutputsMock = vi.fn().mockResolvedValue(false); +const supportsParametersMock = vi.fn().mockResolvedValue(false); - vcr.record('openai-medium-chapter', { - request: { - provider: PROVIDER, - model: MODEL, - prompt: SHARED_CONTRACT_CASES[1].input.text, - timestamp: Date.now(), - }, - response: { - translatedText: 'That day, the sky was clear. '.repeat(100), - tokenCount: { prompt: 1000, completion: 850, total: 1850 }, - cost: 0.00092, - latencyMs: 3500, - } - }); - }); +vi.mock('../../services/capabilityService', () => ({ + supportsStructuredOutputs: (...args: any[]) => supportsStructuredOutputsMock(...args), + supportsParameters: (...args: any[]) => supportsParametersMock(...args), +})); - it.skip('[Contract] happy path: returns well-formed result with correct tokens', async () => { - const testCase = SHARED_CONTRACT_CASES[0]; - const cassette = vcr.replay('openai-happy-path')!; - - // Simulate adapter response based on cassette - const result: TranslationResult = { - translatedTitle: 'Test', - translation: cassette.response.translatedText, - proposal: null, - footnotes: [], - suggestedIllustrations: [], - usageMetrics: { - totalTokens: cassette.response.tokenCount!.total, - promptTokens: cassette.response.tokenCount!.prompt, - completionTokens: cassette.response.tokenCount!.completion, - estimatedCost: cassette.response.cost!, - requestTime: cassette.response.latencyMs / 1000, - provider: PROVIDER as any, - model: MODEL, - } - }; +const rateLimitMock = vi.fn().mockResolvedValue(undefined); - // Contract assertions - expect(result.translation).toBeTruthy(); - expect(result.translation.length).toBeGreaterThan(0); +vi.mock('../../services/rateLimitService', () => ({ + rateLimitService: { + canMakeRequest: (...args: any[]) => rateLimitMock(...args), + }, +})); - // Token accounting - expect(result.usageMetrics.totalTokens).toBeGreaterThan(testCase.assertions.minTokens!); - expect(result.usageMetrics.totalTokens).toBe( - result.usageMetrics.promptTokens + result.usageMetrics.completionTokens - ); +const calculateCostMock = vi.fn(); - // Cost calculation - expect(result.usageMetrics.estimatedCost).toBeLessThan(testCase.assertions.maxCost!); - expect(result.usageMetrics.estimatedCost).toBeGreaterThan(0); +vi.mock('../../services/aiService', () => ({ + calculateCost: (...args: any[]) => calculateCostMock(...args), +})); - // Latency - expect(result.usageMetrics.requestTime * 1000).toBeLessThan(testCase.assertions.maxLatency!); - }); +const recordMetricMock = vi.fn().mockResolvedValue(undefined); - it.skip('[Contract] medium chapter: scales correctly', async () => { - const testCase = SHARED_CONTRACT_CASES[1]; - const cassette = vcr.replay('openai-medium-chapter')!; - - const result: TranslationResult = { - translatedTitle: 'Chapter 1', - translation: cassette.response.translatedText, - proposal: null, - footnotes: [], - suggestedIllustrations: [], - usageMetrics: { - totalTokens: cassette.response.tokenCount!.total, - promptTokens: cassette.response.tokenCount!.prompt, - completionTokens: cassette.response.tokenCount!.completion, - estimatedCost: cassette.response.cost!, - requestTime: cassette.response.latencyMs / 1000, - provider: PROVIDER as any, - model: MODEL, - } - }; +vi.mock('../../services/apiMetricsService', () => ({ + apiMetricsService: { + recordMetric: (...args: any[]) => recordMetricMock(...args), + }, +})); + +vi.mock('../../services/env', () => ({ + getEnvVar: (_key: string) => undefined, +})); + +vi.mock('../../services/defaultApiKeyService', () => ({ + getDefaultApiKey: () => undefined, +})); + +function buildSettings(provider: 'OpenAI' | 'Gemini', cassette: OpenAICassette | GeminiCassette): AppSettings { + const base: Partial = { + provider, + model: cassette.model, + systemPrompt: cassette.request.systemPrompt, + temperature: cassette.request.temperature ?? 0.3, + maxOutputTokens: cassette.request.maxOutputTokens, + includeFanTranslationInPrompt: cassette.request.includeFanTranslationInPrompt ?? true, + enableAmendments: cassette.request.enableAmendments ?? false, + }; - // Should handle larger input - expect(result.translation.length).toBeGreaterThan(100); - expect(result.usageMetrics.totalTokens).toBeGreaterThan(testCase.assertions.minTokens!); - expect(result.usageMetrics.estimatedCost).toBeLessThan(testCase.assertions.maxCost!); + return createMockAppSettings({ + ...base, + apiKeyOpenAI: provider === 'OpenAI' ? 'test-openai-key' : '', + apiKeyGemini: provider === 'Gemini' ? 'test-gemini-key' : '', + apiKeyDeepSeek: '', }); -}); +} -describe('Provider Contract: Gemini', () => { - const PROVIDER = 'Gemini'; - const MODEL = 'gemini-2.5-flash'; +function buildRequest(cassette: OpenAICassette | GeminiCassette, settings: AppSettings): TranslationRequest { + return { + title: cassette.request.title, + content: cassette.request.content, + settings, + history: [], + chapterId: `contract:${cassette.name}`, + }; +} +describe('Provider Contract (VCR replay-only)', () => { beforeEach(() => { - vcr.clear(); - - vcr.record('gemini-happy-path', { - request: { - provider: PROVIDER, - model: MODEL, - prompt: SHARED_CONTRACT_CASES[0].input.text, - timestamp: Date.now(), - }, - response: { - translatedText: 'The weather is very good today.', - tokenCount: { prompt: 14, completion: 9, total: 23 }, - cost: 0.000008, // Gemini Flash is cheaper - latencyMs: 900, - } - }); + openAiMocks.create.mockReset(); + openAiMocks.ctor.mockClear(); + geminiMocks.generateContent.mockReset(); + geminiMocks.getGenerativeModel.mockClear(); + geminiMocks.ctor.mockClear(); + + rateLimitMock.mockClear(); + calculateCostMock.mockReset(); + recordMetricMock.mockClear(); + + supportsStructuredOutputsMock.mockResolvedValue(false); + supportsParametersMock.mockResolvedValue(false); }); - it.skip('[Contract] happy path: Gemini-specific token counting', async () => { - const testCase = SHARED_CONTRACT_CASES[0]; - const cassette = vcr.replay('gemini-happy-path')!; - - const result: TranslationResult = { - translatedTitle: 'Test', - translation: cassette.response.translatedText, - proposal: null, - footnotes: [], - suggestedIllustrations: [], - usageMetrics: { - totalTokens: cassette.response.tokenCount!.total, - promptTokens: cassette.response.tokenCount!.prompt, - completionTokens: cassette.response.tokenCount!.completion, - estimatedCost: cassette.response.cost!, - requestTime: cassette.response.latencyMs / 1000, - provider: PROVIDER as any, - model: MODEL, - } - }; + describe('OpenAIAdapter', () => { + it('replays happy-path cassette via adapter.translate()', async () => { + const cassette = loadCassette('openai-happy-path'); + const settings = buildSettings('OpenAI', cassette); + + calculateCostMock.mockResolvedValueOnce(cassette.expected.estimatedCost); + openAiMocks.create.mockResolvedValueOnce(cassette.mock.sdkResponse); + + const adapter = new OpenAIAdapter(); + const result = await adapter.translate(buildRequest(cassette, settings)); + + expect(result.translatedTitle).toBe(cassette.expected.translatedTitle); + expect(result.translation).toBe(cassette.expected.translation); + expect(result.usageMetrics.promptTokens).toBe(cassette.expected.promptTokens); + expect(result.usageMetrics.completionTokens).toBe(cassette.expected.completionTokens); + expect(result.usageMetrics.totalTokens).toBe(cassette.expected.promptTokens + cassette.expected.completionTokens); + expect(result.usageMetrics.estimatedCost).toBe(cassette.expected.estimatedCost); + + expect(rateLimitMock).toHaveBeenCalledWith(cassette.model); + expect(calculateCostMock).toHaveBeenCalledWith(cassette.model, cassette.expected.promptTokens, cassette.expected.completionTokens); + expect(recordMetricMock).toHaveBeenCalledWith(expect.objectContaining({ + apiType: 'translation', + provider: 'OpenAI', + model: cassette.model, + success: true, + chapterId: `contract:${cassette.name}`, + })); + + expect(openAiMocks.ctor).toHaveBeenCalledWith(expect.objectContaining({ + apiKey: 'test-openai-key', + baseURL: 'https://api.openai.com/v1', + })); + + const requestOptions = openAiMocks.create.mock.calls[0]?.[0]; + expect(requestOptions).toEqual(expect.objectContaining({ model: cassette.model })); + expect(JSON.stringify(requestOptions)).toContain(cassette.request.title); + expect(JSON.stringify(requestOptions)).toContain(cassette.request.content); + }); + + it('replays a larger cassette and preserves token accounting', async () => { + const cassette = loadCassette('openai-medium-chapter'); + const settings = buildSettings('OpenAI', cassette); + + calculateCostMock.mockResolvedValueOnce(cassette.expected.estimatedCost); + openAiMocks.create.mockResolvedValueOnce(cassette.mock.sdkResponse); - // Gemini-specific assertions - expect(result.translation).toBeTruthy(); - expect(result.usageMetrics.estimatedCost).toBeLessThan(testCase.assertions.maxCost!); + const adapter = new OpenAIAdapter(); + const result = await adapter.translate(buildRequest(cassette, settings)); - // Gemini Flash should be faster and cheaper than GPT-4o-mini - expect(result.usageMetrics.requestTime).toBeLessThan(2); - expect(result.usageMetrics.estimatedCost).toBeLessThan(0.0001); + expect(result.translatedTitle).toBe(cassette.expected.translatedTitle); + expect(result.translation).toBe(cassette.expected.translation); + expect(result.translation.length).toBeGreaterThan(100); + expect(result.usageMetrics.totalTokens).toBe(cassette.expected.promptTokens + cassette.expected.completionTokens); + }); }); -}); -// NOTE: Adversarial contract tests (rate limits, timeouts, malformed responses) -// have been moved to individual adapter test files where they can be properly tested: -// - tests/adapters/providers/OpenAIAdapter.test.ts → "adversarial scenarios" describe block -// - tests/adapters/providers/GeminiAdapter.test.ts -// This avoids placeholder stubs that inflate test counts without testing behavior. + describe('GeminiAdapter', () => { + it('replays happy-path cassette via adapter.translate()', async () => { + const cassette = loadCassette('gemini-happy-path'); + const settings = buildSettings('Gemini', cassette); + + calculateCostMock.mockResolvedValueOnce(cassette.expected.estimatedCost); + geminiMocks.generateContent.mockResolvedValueOnce({ + response: { + text: () => cassette.mock.responseText, + usageMetadata: cassette.mock.usageMetadata, + }, + }); + + const adapter = new GeminiAdapter(); + const result = await adapter.translate(buildRequest(cassette, settings)); + + expect(result.translatedTitle).toBe(cassette.expected.translatedTitle); + expect(result.translation).toBe(cassette.expected.translation); + expect(result.usageMetrics.promptTokens).toBe(cassette.expected.promptTokens); + expect(result.usageMetrics.completionTokens).toBe(cassette.expected.completionTokens); + expect(result.usageMetrics.totalTokens).toBe(cassette.expected.promptTokens + cassette.expected.completionTokens); + expect(result.usageMetrics.estimatedCost).toBe(cassette.expected.estimatedCost); + + expect(rateLimitMock).toHaveBeenCalledWith(cassette.model); + expect(calculateCostMock).toHaveBeenCalledWith(cassette.model, cassette.expected.promptTokens, cassette.expected.completionTokens); + + expect(geminiMocks.ctor).toHaveBeenCalledWith('test-gemini-key'); + expect(geminiMocks.getGenerativeModel).toHaveBeenCalledWith(expect.objectContaining({ model: cassette.model })); + + const callArg = geminiMocks.generateContent.mock.calls[0]?.[0]; + const promptText = callArg?.contents?.[0]?.parts?.[0]?.text; + expect(typeof promptText).toBe('string'); + expect(promptText).toContain(cassette.request.title); + expect(promptText).toContain(cassette.request.content); + expect(callArg?.generationConfig?.responseMimeType).toBe('application/json'); + }); + }); +}); -/** - * Implementation TODO (for full 8.5/10 score): - * - * 1. Real VCR implementation: - * - Save cassettes to tests/contracts/cassettes/*.json - * - Load from disk in replay mode - * - Record to disk in LIVE_API_TEST mode - * - * 2. Hook up actual adapters: - * - Import OpenAIAdapter, GeminiAdapter, ClaudeAdapter - * - Call real adapter.translate() methods - * - Intercept HTTP at network layer (using nock or MSW) - * - * 3. Add more adversarial cases: - * - Concurrent requests (check for race conditions) - * - Very large inputs (>100K tokens) - * - Unicode edge cases - * - Network failures (ECONNRESET, etc.) - * - * 4. Add calibration tests: - * - Compare token counts to manual verification - * - Compare costs to actual provider billing - * - Validate latency buckets (p50, p95, p99) - * - * 5. CI integration: - * - Fast lane: replay only (no network) - * - Nightly: optional live test (rate-limited) - * - Fail on cassette drift (warn if recording changes) - */ diff --git a/tests/contracts/vcr/loadCassette.ts b/tests/contracts/vcr/loadCassette.ts new file mode 100644 index 0000000..fb57f8f --- /dev/null +++ b/tests/contracts/vcr/loadCassette.ts @@ -0,0 +1,30 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +/** + * Load a VCR cassette fixture from `tests/contracts/cassettes/.json`. + * + * These cassettes are "replay-only": no network calls, deterministic outputs. + */ +export function loadCassette(name: string): T { + if (!name || typeof name !== 'string') { + throw new Error(`[VCR] loadCassette(name): expected non-empty string, got: ${String(name)}`); + } + + const normalized = name.endsWith('.json') ? name.slice(0, -'.json'.length) : name; + const cassettePath = path.resolve( + process.cwd(), + 'tests', + 'contracts', + 'cassettes', + `${normalized}.json`, + ); + + try { + const raw = readFileSync(cassettePath, 'utf-8'); + return JSON.parse(raw) as T; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + throw new Error(`[VCR] Failed to load cassette "${normalized}" (${cassettePath}): ${msg}`); + } +} diff --git a/tests/contracts/vcr/types.ts b/tests/contracts/vcr/types.ts new file mode 100644 index 0000000..b699b23 --- /dev/null +++ b/tests/contracts/vcr/types.ts @@ -0,0 +1,46 @@ +export type ProviderName = 'OpenAI' | 'Gemini'; + +export interface CassetteRequest { + title: string; + content: string; + systemPrompt: string; + temperature?: number; + maxOutputTokens?: number; + includeFanTranslationInPrompt?: boolean; + enableAmendments?: boolean; +} + +export interface CassetteExpected { + translatedTitle: string; + translation: string; + promptTokens: number; + completionTokens: number; + estimatedCost: number; +} + +export interface BaseCassette { + name: string; + provider: ProviderName; + model: string; + request: CassetteRequest; + expected: CassetteExpected; +} + +export interface OpenAICassette extends BaseCassette { + provider: 'OpenAI'; + mock: { + sdkResponse: unknown; + }; +} + +export interface GeminiCassette extends BaseCassette { + provider: 'Gemini'; + mock: { + responseText: string; + usageMetadata: { + promptTokenCount: number; + candidatesTokenCount: number; + }; + }; +} +