diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionRequest.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionRequest.java deleted file mode 100644 index 4d39c3d..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionRequest.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -import com.unispeaking.domain.vo.ai.AiCallContext; -import com.unispeaking.domain.vo.evaluation.AudioInput; - -public record AudioTranscriptionRequest(AiCallContext context, AudioInput audio) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionResponse.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionResponse.java deleted file mode 100644 index 94c39c6..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/AudioTranscriptionResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -public record AudioTranscriptionResponse(String text) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskRequest.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskRequest.java deleted file mode 100644 index cfb403c..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskRequest.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -import com.unispeaking.domain.vo.ai.AiCallContext; - -public record LlmTaskRequest(AiCallContext context, String prompt) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskResponse.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskResponse.java deleted file mode 100644 index defe918..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/LlmTaskResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -public record LlmTaskResponse(Object data) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationRequest.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationRequest.java deleted file mode 100644 index 35236cb..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationRequest.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -import com.unispeaking.domain.vo.ai.AiCallContext; -import com.unispeaking.domain.vo.evaluation.AudioInput; - -public record PronunciationEvaluationRequest( - AiCallContext context, - AudioInput audio, - String referenceText) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationResponse.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationResponse.java deleted file mode 100644 index 01c91b3..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/PronunciationEvaluationResponse.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -public record PronunciationEvaluationResponse( - Integer totalScore, - Integer fluency, - Integer pronunciation, - Integer rhythm, - Integer tone) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeRequest.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeRequest.java deleted file mode 100644 index c0a40e9..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeRequest.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -import com.unispeaking.domain.vo.ai.AiCallContext; - -public record RealtimeSdpExchangeRequest( - AiCallContext context, - String model, - String offerSdp, - String apiKey) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeResponse.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeResponse.java deleted file mode 100644 index 632de2a..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/RealtimeSdpExchangeResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -public record RealtimeSdpExchangeResponse(String answerSdp, String aiCallId) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioRequest.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioRequest.java deleted file mode 100644 index c7ad1f7..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioRequest.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -import com.unispeaking.domain.vo.ai.AiCallContext; - -public record SpeechAudioRequest(AiCallContext context, String text) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioResponse.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioResponse.java deleted file mode 100644 index 9ac90ea..0000000 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/dto/ai/SpeechAudioResponse.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.unispeaking.domain.dto.ai; - -public record SpeechAudioResponse(byte[] audioData, String audioFormat, String contentType) { -} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/ai/AiModelDefinition.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/ai/AiModelDefinition.java index cb8c0fe..13898d9 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/ai/AiModelDefinition.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/ai/AiModelDefinition.java @@ -1,10 +1,8 @@ package com.unispeaking.domain.vo.ai; -import com.unispeaking.domain.vo.realtime.ProviderType; - public record AiModelDefinition( String modelId, - ProviderType providerType, + String providerId, AiCapability capability, boolean defaultModel) { } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/realtime/ProviderType.java b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/realtime/ProviderType.java index 3798308..d0634f9 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/realtime/ProviderType.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/domain/vo/realtime/ProviderType.java @@ -6,5 +6,6 @@ public enum ProviderType { DEEPSEEK, IFLYTEK, ALIYUN, - MINIMAX + MINIMAX, + DOUBAO } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/aliyun/AliyunTtsProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/aliyun/AliyunTtsProvider.java index 9cf8889..060655e 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/aliyun/AliyunTtsProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/aliyun/AliyunTtsProvider.java @@ -1,22 +1,428 @@ package com.unispeaking.infrastructure.ai.aliyun; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; +import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.TtsProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; @Component public class AliyunTtsProvider extends TtsProvider { - public AliyunTtsProvider() { - super(ProviderType.ALIYUN, Set.of(AiProviderRegistry.ALIYUN_TTS)); + private static final int MAX_TEXT_LENGTH = 5_000; + private static final int MAX_JSON_RESPONSE_BYTES = 1024 * 1024; + private static final int DEFAULT_MAX_AUDIO_BYTES = 10 * 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final String voice; + private final String format; + private final int sampleRate; + private final Duration readTimeout; + private final int maxAudioBytes; + + @Autowired + public AliyunTtsProvider( + ObjectMapper objectMapper, + @Value("${DASHSCOPE_API_KEY:}") String apiKey, + @Value("${BAILIAN_WORKSPACE_ID:}") String workspaceId, + @Value("${BAILIAN_REGION:cn-beijing}") String region, + @Value("${ALIYUN_TTS_MODEL:cosyvoice-v3-flash}") String model, + @Value("${ALIYUN_TTS_VOICE:loongemily_v3}") String voice, + @Value("${ALIYUN_TTS_FORMAT:wav}") String format, + @Value("${ALIYUN_TTS_SAMPLE_RATE:24000}") int sampleRate, + @Value("${ALIYUN_TTS_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${ALIYUN_TTS_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${ALIYUN_TTS_MAX_AUDIO_BYTES:10485760}") int maxAudioBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration(connectTimeoutSeconds, "Aliyun TTS connect timeout")) + .build(), + objectMapper, + apiKey, + buildEndpoint(workspaceId, region), + model, + voice, + format, + sampleRate, + positiveDuration(readTimeoutSeconds, "Aliyun TTS read timeout"), + maxAudioBytes); + } + + public AliyunTtsProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + String voice, + String format, + int sampleRate, + Duration readTimeout, + int maxAudioBytes) { + super("aliyun", Set.of(requiredText(model, "Aliyun TTS model"))); + this.httpClient = require(httpClient, "Aliyun TTS HTTP client"); + this.objectMapper = require(objectMapper, "Aliyun TTS JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = endpoint; + this.model = requiredText(model, "Aliyun TTS model"); + this.voice = requiredText(voice, "Aliyun TTS voice"); + this.format = supportedFormat(format); + this.sampleRate = supportedSampleRate(sampleRate); + this.readTimeout = requirePositive(readTimeout, "Aliyun TTS read timeout"); + this.maxAudioBytes = maxAudioBytes > 0 ? maxAudioBytes : DEFAULT_MAX_AUDIO_BYTES; } @Override - public SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request) { - throw capabilityNotConfigured(AiProviderRegistry.ALIYUN_TTS); + public Byte[] generateSpeechAudio(String text, String token) { + if (apiKey.isBlank()) { + throw retryableFailure( + "ALIYUN_TTS_CREDENTIAL_MISSING", + "Set DASHSCOPE_API_KEY before calling Aliyun TTS"); + } + return boxAudio(synthesize(text, apiKey)); + } + + private byte[] synthesize(String textValue, String credential) { + String text = trim(textValue); + if (text.isBlank()) { + throw nonRetryableFailure("INVALID_TTS_TEXT", "Speech synthesis text is required"); + } + if (text.length() > MAX_TEXT_LENGTH) { + throw nonRetryableFailure( + "TTS_TEXT_TOO_LONG", + "Speech synthesis text exceeds " + MAX_TEXT_LENGTH + " characters"); + } + requireHttpsEndpoint(endpoint, "ALIYUN_TTS_ENDPOINT_INVALID"); + + try { + Map input = Map.of( + "text", text, + "voice", voice, + "format", format, + "sample_rate", sampleRate, + "language_hints", List.of("en")); + String requestBody = objectMapper.writeValueAsString(Map.of( + "model", model, + "input", input)); + HttpRequest synthesisRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + requestBody, + StandardCharsets.UTF_8)) + .build(); + HttpResponse synthesisResponse = httpClient.send( + synthesisRequest, + limitedBodyHandler( + MAX_JSON_RESPONSE_BYTES, + "ALIYUN_TTS_RESPONSE_TOO_LARGE", + "Aliyun TTS JSON response exceeds the configured limit")); + if (!successful(synthesisResponse.statusCode())) { + throw httpError("ALIYUN_TTS_REQUEST_FAILED", synthesisResponse.statusCode()); + } + String responseBody = new String( + synthesisResponse.body(), + StandardCharsets.UTF_8); + URI audioUri = audioUri(responseBody); + HttpRequest audioRequest = HttpRequest.newBuilder() + .uri(audioUri) + .timeout(readTimeout) + .GET() + .build(); + HttpResponse audioResponse = httpClient.send( + audioRequest, + limitedBodyHandler( + maxAudioBytes, + "ALIYUN_TTS_AUDIO_TOO_LARGE", + "Aliyun TTS audio exceeds the configured limit")); + if (!successful(audioResponse.statusCode())) { + throw httpError( + "ALIYUN_TTS_AUDIO_DOWNLOAD_FAILED", + audioResponse.statusCode()); + } + byte[] audio = audioResponse.body(); + if (audio.length == 0) { + throw retryableFailure( + "ALIYUN_TTS_AUDIO_EMPTY", + "Aliyun TTS returned an empty audio file"); + } + return audio; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "ALIYUN_TTS_RESPONSE_INVALID", + "Aliyun TTS response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "ALIYUN_TTS_IO_ERROR", + "Failed to call Aliyun TTS"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "ALIYUN_TTS_INTERRUPTED", + "Aliyun TTS call was interrupted"); + } + } + + private URI audioUri(String responseBody) throws JacksonException { + JsonNode root = objectMapper.readTree(responseBody); + String audioUrl = root.path("output") + .path("audio") + .path("url") + .asString("") + .trim(); + if (audioUrl.isBlank()) { + throw retryableFailure( + "ALIYUN_TTS_AUDIO_URL_MISSING", + "Aliyun TTS response does not contain an audio URL"); + } + URI uri; + try { + uri = URI.create(audioUrl); + } + catch (IllegalArgumentException exception) { + throw retryableFailure( + "ALIYUN_TTS_AUDIO_URL_INVALID", + "Aliyun TTS returned an invalid audio URL"); + } + String host = uri.getHost() == null ? "" : uri.getHost().toLowerCase(Locale.ROOT); + boolean trustedScheme = "https".equalsIgnoreCase(uri.getScheme()) + || "http".equalsIgnoreCase(uri.getScheme()); + boolean trustedHost = host.endsWith(".aliyuncs.com"); + if (!uri.isAbsolute() + || !trustedScheme + || !trustedHost + || uri.getUserInfo() != null) { + throw retryableFailure( + "ALIYUN_TTS_AUDIO_URL_UNTRUSTED", + "Aliyun TTS returned an untrusted audio URL"); + } + if ("http".equalsIgnoreCase(uri.getScheme())) { + try { + uri = URI.create("https:" + uri.toString().substring("http:".length())); + } + catch (IllegalArgumentException exception) { + throw retryableFailure( + "ALIYUN_TTS_AUDIO_URL_INVALID", + "Aliyun TTS returned an invalid audio URL"); + } + } + return uri; + } + + private boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private BusinessException httpError(String errorCode, int statusCode) { + return retryableFailure( + errorCode, + "Aliyun TTS returned HTTP " + statusCode); + } + + private static URI buildEndpoint(String workspaceId, String region) { + String workspace = trim(workspaceId); + String endpointRegion = trim(region); + if (!safeEndpointComponent(workspace) || !safeEndpointComponent(endpointRegion)) { + return null; + } + return URI.create("https://" + workspace + "." + endpointRegion + + ".maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer"); + } + + private static void requireHttpsEndpoint(URI uri, String errorCode) { + String host = uri == null || uri.getHost() == null + ? "" + : uri.getHost().toLowerCase(Locale.ROOT); + if (uri == null + || !uri.isAbsolute() + || !"https".equalsIgnoreCase(uri.getScheme()) + || !host.endsWith(".maas.aliyuncs.com") + || uri.getUserInfo() != null + || uri.getPort() != -1 + || !"/api/v1/services/audio/tts/SpeechSynthesizer".equals(uri.getPath()) + || uri.getRawQuery() != null + || uri.getRawFragment() != null) { + throw retryableFailure( + errorCode, + "Aliyun TTS endpoint must be the trusted Aliyun speech synthesis URL"); + } + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber( + limit, + errorCode, + errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static boolean safeEndpointComponent(String value) { + return !value.isBlank() && value.matches("[A-Za-z0-9-]+"); + } + + private static String contentType(String audioFormat) { + return switch (audioFormat) { + case "mp3" -> "audio/mpeg"; + case "wav" -> "audio/wav"; + case "pcm" -> "audio/L16"; + case "opus" -> "audio/opus"; + default -> "application/octet-stream"; + }; + } + + private static String supportedFormat(String value) { + String audioFormat = trim(value).toLowerCase(Locale.ROOT); + if (!Set.of("mp3", "wav", "pcm", "opus").contains(audioFormat)) { + throw new IllegalArgumentException("Unsupported Aliyun TTS audio format: " + value); + } + return audioFormat; + } + + private static int supportedSampleRate(int value) { + if (!Set.of(8_000, 16_000, 22_050, 24_000, 44_100, 48_000).contains(value)) { + throw new IllegalArgumentException("Unsupported Aliyun TTS sample rate: " + value); + } + return value; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String result = trim(value); + if (result.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return result; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/deepseek/DeepSeekLlmProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/deepseek/DeepSeekLlmProvider.java index 3490a0b..f2915d1 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/deepseek/DeepSeekLlmProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/deepseek/DeepSeekLlmProvider.java @@ -1,22 +1,318 @@ package com.unispeaking.infrastructure.ai.deepseek; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; +import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.LlmProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; @Component public class DeepSeekLlmProvider extends LlmProvider { - public DeepSeekLlmProvider() { - super(ProviderType.DEEPSEEK, Set.of(AiProviderRegistry.DEEPSEEK_CHAT)); + private static final int DEFAULT_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final Duration readTimeout; + private final int maxResponseBytes; + + @Autowired + public DeepSeekLlmProvider( + ObjectMapper objectMapper, + @Value("${DEEPSEEK_API_KEY:}") String apiKey, + @Value("${DEEPSEEK_LLM_ENDPOINT:https://api.deepseek.com/chat/completions}") + String endpoint, + @Value("${DEEPSEEK_LLM_MODEL:deepseek-v4-flash}") String model, + @Value("${DEEPSEEK_LLM_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${DEEPSEEK_LLM_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${DEEPSEEK_LLM_MAX_RESPONSE_BYTES:2097152}") int maxResponseBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration( + connectTimeoutSeconds, + "DeepSeek LLM connect timeout")) + .build(), + objectMapper, + apiKey, + parseUri(endpoint), + model, + positiveDuration(readTimeoutSeconds, "DeepSeek LLM read timeout"), + maxResponseBytes); + } + + public DeepSeekLlmProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + Duration readTimeout, + int maxResponseBytes) { + super("deepseek", Set.of(requiredText(model, "DeepSeek LLM model"))); + this.httpClient = require(httpClient, "DeepSeek LLM HTTP client"); + this.objectMapper = require(objectMapper, "DeepSeek LLM JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = endpoint; + this.model = requiredText(model, "DeepSeek LLM model"); + this.readTimeout = requirePositive(readTimeout, "DeepSeek LLM read timeout"); + this.maxResponseBytes = maxResponseBytes > 0 + ? maxResponseBytes + : DEFAULT_MAX_RESPONSE_BYTES; } @Override - public LlmTaskResponse executeLlmTask(LlmTaskRequest request) { - throw capabilityNotConfigured(AiProviderRegistry.DEEPSEEK_CHAT); + public String executeLlmTask(String prompt, String token) { + if (apiKey.isBlank()) { + throw retryableFailure( + "DEEPSEEK_LLM_CREDENTIAL_MISSING", + "Set DEEPSEEK_API_KEY before calling DeepSeek LLM"); + } + return callForContent(prompt, apiKey); + } + + private String callForContent(String promptValue, String credential) { + String prompt = trim(promptValue); + if (prompt.isBlank()) { + throw nonRetryableFailure("INVALID_LLM_PROMPT", "LLM task prompt is required"); + } + requireTrustedEndpoint(); + + try { + Map body = Map.of( + "model", model, + "messages", List.of(Map.of("role", "user", "content", prompt)), + "thinking", Map.of("type", "disabled"), + "stream", false); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(body), + StandardCharsets.UTF_8)) + .build(); + HttpResponse response = httpClient.send( + httpRequest, + limitedBodyHandler( + maxResponseBytes, + "DEEPSEEK_LLM_RESPONSE_TOO_LARGE", + "DeepSeek LLM response exceeds the configured limit")); + if (!successful(response.statusCode())) { + throw retryableFailure( + "DEEPSEEK_LLM_REQUEST_FAILED", + "DeepSeek LLM returned HTTP " + response.statusCode()); + } + JsonNode root = objectMapper.readTree( + new String(response.body(), StandardCharsets.UTF_8)); + String content = root.path("choices") + .path(0) + .path("message") + .path("content") + .asString(""); + if (content.isBlank()) { + throw retryableFailure( + "DEEPSEEK_LLM_EMPTY_RESPONSE", + "DeepSeek LLM returned no message content"); + } + return content; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "DEEPSEEK_LLM_RESPONSE_INVALID", + "DeepSeek LLM response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "DEEPSEEK_LLM_IO_ERROR", + "Failed to call DeepSeek LLM"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "DEEPSEEK_LLM_INTERRUPTED", + "DeepSeek LLM call was interrupted"); + } + } + + private Object parseContent(String content) { + try { + return objectMapper.readTree(content); + } + catch (Exception ignored) { + return content; + } + } + + private void requireTrustedEndpoint() { + String host = endpoint == null || endpoint.getHost() == null + ? "" + : endpoint.getHost().toLowerCase(Locale.ROOT); + if (endpoint == null + || !endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !"api.deepseek.com".equals(host) + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/chat/completions".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "DEEPSEEK_LLM_ENDPOINT_INVALID", + "DeepSeek LLM endpoint must be the trusted chat completions URL"); + } + } + + private static boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private static URI parseUri(String value) { + try { + return URI.create(trim(value)); + } + catch (IllegalArgumentException exception) { + return null; + } + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber(limit, errorCode, errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String text = trim(value); + if (text.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return text; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/doubao/DoubaoAsrProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/doubao/DoubaoAsrProvider.java new file mode 100644 index 0000000..8629f5c --- /dev/null +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/doubao/DoubaoAsrProvider.java @@ -0,0 +1,378 @@ +package com.unispeaking.infrastructure.ai.doubao; + +import com.unispeaking.domain.vo.evaluation.AudioInput; +import com.unispeaking.exception.BusinessException; +import com.unispeaking.provider.AiProviderRegistry; +import com.unispeaking.provider.TranscriptionProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +@Component +public class DoubaoAsrProvider extends TranscriptionProvider { + + private static final String SUCCESS_STATUS = "20000000"; + private static final int DEFAULT_MAX_AUDIO_BYTES = 20 * 1024 * 1024; + private static final int DEFAULT_MAX_RESPONSE_BYTES = 4 * 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final String appKey; + private final String accessKey; + private final String userId; + private final URI endpoint; + private final String resourceId; + private final Duration readTimeout; + private final int maxAudioBytes; + private final int maxResponseBytes; + + @Autowired + public DoubaoAsrProvider( + ObjectMapper objectMapper, + @Value("${DOUBAO_ASR_API_KEY:}") String apiKey, + @Value("${DOUBAO_ASR_APP_KEY:}") String appKey, + @Value("${DOUBAO_ASR_ACCESS_KEY:}") String accessKey, + @Value("${DOUBAO_ASR_USER_ID:unispeaking}") String userId, + @Value("${DOUBAO_ASR_ENDPOINT:https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash}") + String endpoint, + @Value("${DOUBAO_ASR_RESOURCE_ID:volc.bigasr.auc_turbo}") String resourceId, + @Value("${DOUBAO_ASR_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${DOUBAO_ASR_READ_TIMEOUT_SECONDS:120}") int readTimeoutSeconds, + @Value("${DOUBAO_ASR_MAX_AUDIO_BYTES:20971520}") int maxAudioBytes, + @Value("${DOUBAO_ASR_MAX_RESPONSE_BYTES:4194304}") int maxResponseBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration( + connectTimeoutSeconds, + "Doubao ASR connect timeout")) + .build(), + objectMapper, + apiKey, + appKey, + accessKey, + userId, + parseUri(endpoint), + resourceId, + positiveDuration(readTimeoutSeconds, "Doubao ASR read timeout"), + maxAudioBytes, + maxResponseBytes); + } + + public DoubaoAsrProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + String appKey, + String accessKey, + String userId, + URI endpoint, + String resourceId, + Duration readTimeout, + int maxAudioBytes, + int maxResponseBytes) { + super("doubao", Set.of(requiredText(resourceId, "Doubao ASR resource ID"))); + this.httpClient = require(httpClient, "Doubao ASR HTTP client"); + this.objectMapper = require(objectMapper, "Doubao ASR JSON mapper"); + this.apiKey = trim(apiKey); + this.appKey = trim(appKey); + this.accessKey = trim(accessKey); + this.userId = requiredText(userId, "Doubao ASR user ID"); + this.endpoint = endpoint; + this.resourceId = requiredText(resourceId, "Doubao ASR resource ID"); + this.readTimeout = requirePositive(readTimeout, "Doubao ASR read timeout"); + this.maxAudioBytes = maxAudioBytes > 0 ? maxAudioBytes : DEFAULT_MAX_AUDIO_BYTES; + this.maxResponseBytes = maxResponseBytes > 0 + ? maxResponseBytes + : DEFAULT_MAX_RESPONSE_BYTES; + } + + @Override + public String convertAudioToText(Byte[] audio, String token) { + AudioInput input = new AudioInput( + unboxAudio(audio, "Doubao ASR"), + "wav"); + requireAudio(input); + requireCredentials(); + return transcribe(input); + } + + private String transcribe(AudioInput audio) { + requireTrustedEndpoint(); + + try { + Map body = Map.of( + "user", Map.of("uid", userId), + "audio", Map.of( + "data", + Base64.getEncoder().encodeToString(audio.audioData())), + "request", Map.of("model_name", "bigmodel")); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Content-Type", "application/json") + .header("X-Api-Resource-Id", resourceId) + .header("X-Api-Request-Id", UUID.randomUUID().toString()) + .header("X-Api-Sequence", "-1"); + if (!apiKey.isBlank()) { + requestBuilder.header("X-Api-Key", apiKey); + } + else { + requestBuilder + .header("X-Api-App-Key", appKey) + .header("X-Api-Access-Key", accessKey); + } + HttpResponse response = httpClient.send( + requestBuilder.POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(body), + StandardCharsets.UTF_8)).build(), + limitedBodyHandler( + maxResponseBytes, + "DOUBAO_ASR_RESPONSE_TOO_LARGE", + "Doubao ASR response exceeds the configured limit")); + if (!successful(response.statusCode())) { + throw retryableFailure( + "DOUBAO_ASR_REQUEST_FAILED", + "Doubao ASR returned HTTP " + response.statusCode()); + } + String providerStatus = response.headers() + .firstValue("X-Api-Status-Code") + .orElse(""); + if (!SUCCESS_STATUS.equals(providerStatus)) { + throw retryableFailure( + "DOUBAO_ASR_REQUEST_FAILED", + "Doubao ASR returned provider status " + + (providerStatus.isBlank() ? "missing" : providerStatus)); + } + JsonNode root = objectMapper.readTree( + new String(response.body(), StandardCharsets.UTF_8)); + String text = root.path("result").path("text").asString(""); + if (text.isBlank()) { + throw retryableFailure( + "DOUBAO_ASR_RESULT_EMPTY", + "Doubao ASR returned no transcription text"); + } + return text; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "DOUBAO_ASR_RESPONSE_INVALID", + "Doubao ASR response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "DOUBAO_ASR_IO_ERROR", + "Failed to call Doubao ASR"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "DOUBAO_ASR_INTERRUPTED", + "Doubao ASR call was interrupted"); + } + } + + private AudioInput requireAudio(AudioInput audio) { + if (audio == null || audio.audioData() == null || audio.audioData().length == 0) { + throw nonRetryableFailure( + "INVALID_TRANSCRIPTION_AUDIO", + "Audio transcription data is required"); + } + if (audio.audioData().length > maxAudioBytes) { + throw nonRetryableFailure( + "TRANSCRIPTION_AUDIO_TOO_LARGE", + "Doubao ASR audio exceeds the configured limit"); + } + requireSupportedFormat(audio.audioFormat()); + return audio; + } + + private void requireSupportedFormat(String audioFormat) { + String format = trim(audioFormat).toLowerCase(Locale.ROOT); + if (!Set.of("wav", "mp3", "ogg", "opus").contains(format)) { + throw nonRetryableFailure( + "UNSUPPORTED_TRANSCRIPTION_AUDIO_FORMAT", + "Unsupported Doubao ASR audio format: " + audioFormat); + } + } + + private void requireCredentials() { + boolean newCredential = !apiKey.isBlank(); + boolean legacyCredential = !appKey.isBlank() && !accessKey.isBlank(); + if (!newCredential && !legacyCredential) { + throw retryableFailure( + "DOUBAO_ASR_CREDENTIAL_MISSING", + "Set DOUBAO_ASR_API_KEY or both legacy Doubao ASR keys"); + } + } + + private void requireTrustedEndpoint() { + String host = endpoint == null || endpoint.getHost() == null + ? "" + : endpoint.getHost().toLowerCase(Locale.ROOT); + if (endpoint == null + || !endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !"openspeech.bytedance.com".equals(host) + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/api/v3/auc/bigmodel/recognize/flash".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "DOUBAO_ASR_ENDPOINT_INVALID", + "Doubao ASR endpoint must be the trusted BigASR flash URL"); + } + } + + private static URI parseUri(String value) { + try { + return URI.create(trim(value)); + } + catch (IllegalArgumentException exception) { + return null; + } + } + + private static boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber(limit, errorCode, errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String text = trim(value); + if (text.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return text; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } + } +} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/iflytek/IflytekScoringProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/iflytek/IflytekScoringProvider.java index b3ad179..dd26d70 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/iflytek/IflytekScoringProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/iflytek/IflytekScoringProvider.java @@ -1,25 +1,559 @@ package com.unispeaking.infrastructure.ai.iflytek; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; +import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.ScoringProvider; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.time.Duration; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Base64; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; @Component public class IflytekScoringProvider extends ScoringProvider { - public IflytekScoringProvider() { + private static final int AUDIO_FRAME_BYTES = 1_280; + private static final int DEFAULT_MAX_AUDIO_BYTES = 10 * 1024 * 1024; + private static final int MAX_PCM_SESSION_BYTES = 16_000 * 2 * 300; + private static final int MAX_REFERENCE_TEXT_CHARS = 10_000; + private static final int MAX_MESSAGE_CHARS = 1_500_000; + + private final ObjectMapper objectMapper; + private final WebSocketConnector connector; + private final String appId; + private final String apiKey; + private final String apiSecret; + private final URI endpoint; + private final String language; + private final String category; + private final Duration readTimeout; + private final int maxAudioBytes; + private final Duration frameDelay; + + @Autowired + public IflytekScoringProvider( + ObjectMapper objectMapper, + @Value("${XFYUN_APP_ID:}") String appId, + @Value("${XFYUN_API_KEY:}") String apiKey, + @Value("${XFYUN_API_SECRET:}") String apiSecret, + @Value("${XFYUN_ISE_ENDPOINT:wss://ise-api.xfyun.cn/v2/open-ise}") URI endpoint, + @Value("${XFYUN_ISE_LANGUAGE:en_vip}") String language, + @Value("${XFYUN_ISE_CATEGORY:read_sentence}") String category, + @Value("${XFYUN_ISE_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${XFYUN_ISE_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${XFYUN_ISE_MAX_AUDIO_BYTES:10485760}") int maxAudioBytes) { + this( + objectMapper, + defaultConnector(connectTimeoutSeconds), + appId, + apiKey, + apiSecret, + endpoint, + language, + category, + positiveDuration(readTimeoutSeconds, "iFlytek ISE read timeout"), + maxAudioBytes, + Duration.ofMillis(40)); + } + + public IflytekScoringProvider( + ObjectMapper objectMapper, + WebSocketConnector connector, + String appId, + String apiKey, + String apiSecret, + URI endpoint, + String language, + String category, + Duration readTimeout, + int maxAudioBytes, + Duration frameDelay) { super( - ProviderType.IFLYTEK, + "iflytek", Set.of(AiProviderRegistry.IFLYTEK_PRONUNCIATION_SCORING)); + this.objectMapper = Objects.requireNonNull(objectMapper, "iFlytek ISE JSON mapper is required"); + this.connector = Objects.requireNonNull(connector, "iFlytek ISE WebSocket connector is required"); + this.appId = trim(appId); + this.apiKey = trim(apiKey); + this.apiSecret = trim(apiSecret); + this.endpoint = Objects.requireNonNull(endpoint, "iFlytek ISE endpoint is required"); + this.language = requiredText(language, "iFlytek ISE language"); + this.category = requiredText(category, "iFlytek ISE category"); + this.readTimeout = requirePositive(readTimeout, "iFlytek ISE read timeout"); + this.maxAudioBytes = maxAudioBytes > 0 ? maxAudioBytes : DEFAULT_MAX_AUDIO_BYTES; + this.frameDelay = Objects.requireNonNull(frameDelay, "iFlytek ISE frame delay is required"); + if (frameDelay.isNegative()) { + throw new IllegalArgumentException("iFlytek ISE frame delay must not be negative"); + } + } + + @FunctionalInterface + public interface WebSocketConnector { + CompletableFuture connect(URI uri, WebSocket.Listener listener); } @Override - public PronunciationEvaluationResponse evaluatePronunciation( - PronunciationEvaluationRequest request) { - throw capabilityNotConfigured(AiProviderRegistry.IFLYTEK_PRONUNCIATION_SCORING); + public String evaluatePronunciation(String text, Byte[] audio, String token) { + String referenceText = validateReferenceText(text); + byte[] originalAudio = unboxAudio(audio, "iFlytek pronunciation evaluation"); + if (originalAudio.length > maxAudioBytes) { + throw nonRetryableFailure( + "PRONUNCIATION_AUDIO_TOO_LARGE", + "Pronunciation evaluation audio exceeds the configured limit"); + } + byte[] pcmAudio = wavPayload(originalAudio); + if (pcmAudio.length > MAX_PCM_SESSION_BYTES) { + throw nonRetryableFailure( + "PRONUNCIATION_AUDIO_TOO_LONG", + "iFlytek PCM audio must not exceed five minutes"); + } + requireCredentials(); + requireEndpoint(); + + WebSocket socket = null; + boolean completed = false; + long deadlineNanos = System.nanoTime() + readTimeout.toNanos(); + try { + CompletableFuture result = new CompletableFuture<>(); + RawIseListener listener = new RawIseListener(result); + socket = connector.connect(signedEndpoint(apiKey), listener) + .get(remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + send(socket, startFrame(referenceText, false), deadlineNanos); + for (int offset = 0; offset < pcmAudio.length; offset += AUDIO_FRAME_BYTES) { + int end = Math.min(offset + AUDIO_FRAME_BYTES, pcmAudio.length); + int audioStatus = offset == 0 ? 1 : 2; + send(socket, audioFrame( + Arrays.copyOfRange(pcmAudio, offset, end), + false, + audioStatus), + deadlineNanos); + pauseBetweenFrames(deadlineNanos); + } + send(socket, endFrame(false), deadlineNanos); + String response = result.get( + remainingMillis(deadlineNanos), + TimeUnit.MILLISECONDS); + completed = true; + return response; + } + catch (BusinessException exception) { + throw exception; + } + catch (TimeoutException exception) { + throw retryableFailure( + "IFLYTEK_ISE_TIMEOUT", + "iFlytek pronunciation evaluation timed out"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "IFLYTEK_ISE_INTERRUPTED", + "iFlytek pronunciation evaluation was interrupted"); + } + catch (ExecutionException | CompletionException exception) { + Throwable cause = exception.getCause(); + if (cause instanceof BusinessException businessException) { + throw businessException; + } + throw retryableFailure( + "IFLYTEK_ISE_CONNECTION_FAILED", + "Failed to communicate with iFlytek pronunciation evaluation"); + } + catch (JacksonException exception) { + throw retryableFailure( + "IFLYTEK_ISE_REQUEST_INVALID", + "Failed to encode the iFlytek pronunciation request"); + } + finally { + if (socket != null && !socket.isOutputClosed()) { + if (completed) { + socket.sendClose(WebSocket.NORMAL_CLOSURE, "complete"); + } + else { + socket.abort(); + } + } + } + } + + private String validateReferenceText(String text) { + String referenceText = trim(text); + if (referenceText.isBlank()) { + throw nonRetryableFailure( + "INVALID_PRONUNCIATION_REFERENCE", + "Pronunciation reference text is required"); + } + if (referenceText.length() > MAX_REFERENCE_TEXT_CHARS) { + throw nonRetryableFailure( + "PRONUNCIATION_REFERENCE_TOO_LONG", + "Pronunciation reference text exceeds the configured limit"); + } + return referenceText; + } + + private String startFrame(String referenceText, boolean mp3) throws JacksonException { + String markedText = "\uFEFF[content]\n" + referenceText; + Map business = Map.ofEntries( + Map.entry("sub", "ise"), + Map.entry("ent", language), + Map.entry("category", category), + Map.entry("cmd", "ssb"), + Map.entry("auf", "audio/L16;rate=16000"), + Map.entry("aue", mp3 ? "lame" : "raw"), + Map.entry("tte", "utf-8"), + Map.entry("rst", "entirety"), + Map.entry("ise_unite", "1"), + Map.entry("extra_ability", "multi_dimension"), + Map.entry("text", markedText), + Map.entry("ttp_skip", true)); + return objectMapper.writeValueAsString(Map.of( + "common", Map.of("app_id", appId), + "business", business, + "data", Map.of("status", 0, "data", ""))); + } + + private String audioFrame(byte[] audio, boolean mp3, int audioStatus) throws JacksonException { + return objectMapper.writeValueAsString(Map.of( + "business", Map.of( + "cmd", "auw", + "aus", audioStatus, + "aue", mp3 ? "lame" : "raw"), + "data", Map.of( + "status", 1, + "data", Base64.getEncoder().encodeToString(audio), + "data_type", 1, + "encoding", mp3 ? "lame" : "raw"))); + } + + private String endFrame(boolean mp3) throws JacksonException { + return objectMapper.writeValueAsString(Map.of( + "business", Map.of( + "cmd", "auw", + "aus", 4, + "aue", mp3 ? "lame" : "raw"), + "data", Map.of( + "status", 2, + "data", "", + "data_type", 1, + "encoding", mp3 ? "lame" : "raw"))); } + + private void send(WebSocket socket, String frame, long deadlineNanos) + throws InterruptedException, ExecutionException, TimeoutException { + socket.sendText(frame, true) + .get(remainingMillis(deadlineNanos), TimeUnit.MILLISECONDS); + } + + private void pauseBetweenFrames(long deadlineNanos) + throws InterruptedException, TimeoutException { + if (!frameDelay.isZero()) { + if (frameDelay.toNanos() >= remainingNanos(deadlineNanos)) { + throw new TimeoutException("iFlytek ISE deadline reached while pacing audio"); + } + Thread.sleep(frameDelay); + } + } + + private long remainingMillis(long deadlineNanos) throws TimeoutException { + long remainingNanos = remainingNanos(deadlineNanos); + return Math.max(1, TimeUnit.NANOSECONDS.toMillis(remainingNanos)); + } + + private long remainingNanos(long deadlineNanos) throws TimeoutException { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) { + throw new TimeoutException("iFlytek ISE deadline reached"); + } + return remaining; + } + + private URI signedEndpoint(String signingApiKey) { + try { + String host = endpoint.getHost(); + String date = ZonedDateTime.now(ZoneOffset.UTC) + .format(DateTimeFormatter.RFC_1123_DATE_TIME); + String requestLine = "GET " + endpoint.getRawPath() + " HTTP/1.1"; + String signatureOrigin = "host: " + host + "\n" + + "date: " + date + "\n" + + requestLine; + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(apiSecret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + String signature = Base64.getEncoder().encodeToString( + mac.doFinal(signatureOrigin.getBytes(StandardCharsets.UTF_8))); + String authorizationOrigin = "api_key=\"" + signingApiKey + + "\", algorithm=\"hmac-sha256\", headers=\"host date request-line\", signature=\"" + + signature + "\""; + String authorization = Base64.getEncoder().encodeToString( + authorizationOrigin.getBytes(StandardCharsets.UTF_8)); + String query = "authorization=" + encode(authorization) + + "&date=" + encode(date) + + "&host=" + encode(host); + String separator = endpoint.getRawQuery() == null ? "?" : "&"; + return URI.create(endpoint + separator + query); + } + catch (GeneralSecurityException exception) { + throw retryableFailure( + "IFLYTEK_ISE_SIGNATURE_FAILED", + "Failed to sign the iFlytek pronunciation request"); + } + } + + private byte[] wavPayload(byte[] wav) { + if (wav.length < 12 + || wav[0] != 'R' + || wav[1] != 'I' + || wav[2] != 'F' + || wav[3] != 'F' + || wav[8] != 'W' + || wav[9] != 'A' + || wav[10] != 'V' + || wav[11] != 'E') { + throw nonRetryableFailure( + "INVALID_PRONUNCIATION_WAV", + "Pronunciation WAV audio has an invalid header"); + } + int declaredRiffBytes = littleEndianInt(wav, 4); + if (declaredRiffBytes < 4 || (long) declaredRiffBytes + 8 > wav.length) { + throw invalidWav("Pronunciation WAV audio is truncated"); + } + boolean validFormat = false; + int audioDataStart = -1; + int audioDataBytes = -1; + for (int index = 12; index + 8 <= wav.length;) { + int size = littleEndianInt(wav, index + 4); + if (size < 0) { + throw invalidWav("Pronunciation WAV chunk size is invalid"); + } + int dataStart = index + 8; + long dataEnd = (long) dataStart + size; + if (dataEnd > wav.length) { + throw invalidWav("Pronunciation WAV chunk is truncated"); + } + if (wav[index] == 'f' + && wav[index + 1] == 'm' + && wav[index + 2] == 't' + && wav[index + 3] == ' ') { + if (size < 16) { + throw invalidWav("Pronunciation WAV format chunk is invalid"); + } + int encoding = littleEndianShort(wav, dataStart); + int channels = littleEndianShort(wav, dataStart + 2); + int sampleRate = littleEndianInt(wav, dataStart + 4); + int bitsPerSample = littleEndianShort(wav, dataStart + 14); + validFormat = encoding == 1 + && channels == 1 + && sampleRate == 16_000 + && bitsPerSample == 16; + } + else if (wav[index] == 'd' + && wav[index + 1] == 'a' + && wav[index + 2] == 't' + && wav[index + 3] == 'a') { + if (size <= 0) { + throw invalidWav("Pronunciation WAV audio data is empty"); + } + audioDataStart = dataStart; + audioDataBytes = size; + } + long next = dataEnd + (size & 1); + if (next <= index || next > wav.length) { + break; + } + index = (int) next; + } + if (!validFormat) { + throw invalidWav( + "Pronunciation WAV must be PCM, 16 kHz, 16-bit, and mono"); + } + if (audioDataStart < 0) { + throw invalidWav("Pronunciation WAV audio does not contain a data chunk"); + } + return Arrays.copyOfRange( + wav, + audioDataStart, + audioDataStart + audioDataBytes); + } + + private int littleEndianInt(byte[] bytes, int offset) { + return (bytes[offset] & 0xff) + | ((bytes[offset + 1] & 0xff) << 8) + | ((bytes[offset + 2] & 0xff) << 16) + | ((bytes[offset + 3] & 0xff) << 24); + } + + private int littleEndianShort(byte[] bytes, int offset) { + return (bytes[offset] & 0xff) | ((bytes[offset + 1] & 0xff) << 8); + } + + private BusinessException invalidWav(String message) { + return nonRetryableFailure("INVALID_PRONUNCIATION_WAV", message); + } + + private void requireCredentials() { + if (appId.isBlank() || apiKey.isBlank() || apiSecret.isBlank()) { + throw retryableFailure( + "IFLYTEK_ISE_CREDENTIAL_MISSING", + "Set XFYUN_APP_ID, XFYUN_API_KEY, and XFYUN_API_SECRET before calling iFlytek ISE"); + } + } + + private void requireEndpoint() { + if (!endpoint.isAbsolute() + || !"wss".equalsIgnoreCase(endpoint.getScheme()) + || !"ise-api.xfyun.cn".equalsIgnoreCase(endpoint.getHost()) + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/v2/open-ise".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "IFLYTEK_ISE_ENDPOINT_INVALID", + "iFlytek ISE endpoint must be an absolute wss URL"); + } + } + + private static WebSocketConnector defaultConnector(int connectTimeoutSeconds) { + Duration connectTimeout = positiveDuration( + connectTimeoutSeconds, + "iFlytek ISE connect timeout"); + HttpClient client = HttpClient.newBuilder() + .connectTimeout(connectTimeout) + .build(); + return (uri, listener) -> client.newWebSocketBuilder() + .connectTimeout(connectTimeout) + .buildAsync(uri, listener); + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be positive"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String trimmed = trim(value); + if (trimmed.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return trimmed; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static String encode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + + private final class RawIseListener implements WebSocket.Listener { + + private final CompletableFuture result; + private final StringBuilder message = new StringBuilder(); + + private RawIseListener(CompletableFuture result) { + this.result = result; + } + + @Override + public void onOpen(WebSocket webSocket) { + webSocket.request(1); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, + CharSequence data, + boolean last) { + if (data.length() > MAX_MESSAGE_CHARS - message.length()) { + result.completeExceptionally(retryableFailure( + "IFLYTEK_ISE_RESPONSE_TOO_LARGE", + "iFlytek pronunciation response exceeds the configured limit")); + webSocket.abort(); + return CompletableFuture.completedFuture(null); + } + message.append(data); + if (last) { + handleMessage(message.toString()); + message.setLength(0); + } + webSocket.request(1); + return CompletableFuture.completedFuture(null); + } + + private void handleMessage(String payload) { + try { + JsonNode root = objectMapper.readTree(payload); + int code = root.path("code").asInt(-1); + if (code != 0) { + result.completeExceptionally(retryableFailure( + "IFLYTEK_ISE_REQUEST_FAILED", + "iFlytek pronunciation evaluation failed with code " + code)); + return; + } + if (root.path("data").path("status").asInt(-1) == 2) { + result.complete(payload); + } + } + catch (JacksonException exception) { + result.completeExceptionally(retryableFailure( + "IFLYTEK_ISE_RESPONSE_INVALID", + "iFlytek pronunciation evaluation returned an invalid response")); + } + } + + @Override + public CompletionStage onClose( + WebSocket webSocket, + int statusCode, + String reason) { + if (!result.isDone()) { + result.completeExceptionally(retryableFailure( + "IFLYTEK_ISE_CONNECTION_CLOSED", + "iFlytek pronunciation connection closed before returning its final response")); + } + return CompletableFuture.completedFuture(null); + } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + result.completeExceptionally(retryableFailure( + "IFLYTEK_ISE_CONNECTION_FAILED", + "iFlytek pronunciation connection failed")); + } + } + } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/minimax/MiniMaxTtsProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/minimax/MiniMaxTtsProvider.java index ef1dec9..ca95f01 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/minimax/MiniMaxTtsProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/minimax/MiniMaxTtsProvider.java @@ -1,22 +1,419 @@ package com.unispeaking.infrastructure.ai.minimax; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; +import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.TtsProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; @Component public class MiniMaxTtsProvider extends TtsProvider { - public MiniMaxTtsProvider() { - super(ProviderType.MINIMAX, Set.of(AiProviderRegistry.MINIMAX_TTS)); + private static final int MAX_TEXT_LENGTH = 10_000; + private static final int DEFAULT_MAX_AUDIO_BYTES = 10 * 1024 * 1024; + private static final int JSON_OVERHEAD_BYTES = 1024 * 1024; + private static final Set TRUSTED_HOSTS = Set.of( + "api.minimaxi.com", + "api-bj.minimaxi.com", + "api.minimax.io", + "api-uw.minimax.io"); + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final String voice; + private final String format; + private final int sampleRate; + private final int bitrate; + private final Duration readTimeout; + private final int maxAudioBytes; + + @Autowired + public MiniMaxTtsProvider( + ObjectMapper objectMapper, + @Value("${MINIMAX_API_KEY:}") String apiKey, + @Value("${MINIMAX_TTS_ENDPOINT:https://api.minimaxi.com/v1/t2a_v2}") + String endpoint, + @Value("${MINIMAX_TTS_MODEL:speech-2.8-hd}") String model, + @Value("${MINIMAX_TTS_VOICE:male-qn-qingse}") String voice, + @Value("${MINIMAX_TTS_FORMAT:wav}") String format, + @Value("${MINIMAX_TTS_SAMPLE_RATE:32000}") int sampleRate, + @Value("${MINIMAX_TTS_BITRATE:128000}") int bitrate, + @Value("${MINIMAX_TTS_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${MINIMAX_TTS_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${MINIMAX_TTS_MAX_AUDIO_BYTES:10485760}") int maxAudioBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration( + connectTimeoutSeconds, + "MiniMax TTS connect timeout")) + .build(), + objectMapper, + apiKey, + parseUri(endpoint), + model, + voice, + format, + sampleRate, + bitrate, + positiveDuration(readTimeoutSeconds, "MiniMax TTS read timeout"), + maxAudioBytes); + } + + public MiniMaxTtsProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + String voice, + String format, + int sampleRate, + int bitrate, + Duration readTimeout, + int maxAudioBytes) { + super("minimax", Set.of(requiredText(model, "MiniMax TTS model"))); + this.httpClient = require(httpClient, "MiniMax TTS HTTP client"); + this.objectMapper = require(objectMapper, "MiniMax TTS JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = endpoint; + this.model = requiredText(model, "MiniMax TTS model"); + this.voice = requiredText(voice, "MiniMax TTS voice"); + this.format = supportedFormat(format); + this.sampleRate = supportedSampleRate(sampleRate); + this.bitrate = supportedBitrate(bitrate); + this.readTimeout = requirePositive(readTimeout, "MiniMax TTS read timeout"); + this.maxAudioBytes = maxAudioBytes > 0 ? maxAudioBytes : DEFAULT_MAX_AUDIO_BYTES; } @Override - public SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request) { - throw capabilityNotConfigured(AiProviderRegistry.MINIMAX_TTS); + public Byte[] generateSpeechAudio(String text, String token) { + if (apiKey.isBlank()) { + throw retryableFailure( + "MINIMAX_TTS_CREDENTIAL_MISSING", + "Set MINIMAX_API_KEY before calling MiniMax TTS"); + } + return boxAudio(synthesize(text, apiKey)); + } + + private byte[] synthesize(String textValue, String credential) { + String text = trim(textValue); + if (text.isBlank()) { + throw nonRetryableFailure("INVALID_TTS_TEXT", "Speech synthesis text is required"); + } + if (text.length() >= MAX_TEXT_LENGTH) { + throw nonRetryableFailure( + "TTS_TEXT_TOO_LONG", + "MiniMax speech synthesis text must be shorter than " + + MAX_TEXT_LENGTH + " characters"); + } + requireTrustedEndpoint(); + + try { + Map voiceSetting = Map.of( + "voice_id", voice, + "speed", 1, + "vol", 1, + "pitch", 0); + Map audioSetting = Map.of( + "sample_rate", sampleRate, + "bitrate", bitrate, + "format", format, + "channel", 1); + Map body = Map.of( + "model", model, + "text", text, + "stream", false, + "output_format", "hex", + "language_boost", "auto", + "voice_setting", voiceSetting, + "audio_setting", audioSetting); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(body), + StandardCharsets.UTF_8)) + .build(); + HttpResponse response = httpClient.send( + httpRequest, + limitedBodyHandler( + maxJsonResponseBytes(), + "MINIMAX_TTS_RESPONSE_TOO_LARGE", + "MiniMax TTS response exceeds the configured limit")); + if (!successful(response.statusCode())) { + throw retryableFailure( + "MINIMAX_TTS_REQUEST_FAILED", + "MiniMax TTS returned HTTP " + response.statusCode()); + } + byte[] audio = parseAudio(response.body()); + return audio; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "MINIMAX_TTS_RESPONSE_INVALID", + "MiniMax TTS response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "MINIMAX_TTS_IO_ERROR", + "Failed to call MiniMax TTS"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "MINIMAX_TTS_INTERRUPTED", + "MiniMax TTS call was interrupted"); + } + } + + private byte[] parseAudio(byte[] responseBody) throws JacksonException { + JsonNode root = objectMapper.readTree( + new String(responseBody, StandardCharsets.UTF_8)); + int providerStatus = root.path("base_resp") + .path("status_code") + .asInt(Integer.MIN_VALUE); + if (providerStatus != 0) { + throw retryableFailure( + "MINIMAX_TTS_REQUEST_FAILED", + "MiniMax TTS returned provider status " + providerStatus); + } + String audioHex = root.path("data").path("audio").asString("").trim(); + if (audioHex.isBlank()) { + throw retryableFailure( + "MINIMAX_TTS_AUDIO_EMPTY", + "MiniMax TTS returned no audio"); + } + byte[] audio; + try { + audio = HexFormat.of().parseHex(audioHex); + } + catch (IllegalArgumentException exception) { + throw retryableFailure( + "MINIMAX_TTS_AUDIO_INVALID", + "MiniMax TTS returned invalid hex audio"); + } + if (audio.length > maxAudioBytes) { + throw retryableFailure( + "MINIMAX_TTS_AUDIO_TOO_LARGE", + "MiniMax TTS audio exceeds the configured limit"); + } + if (audio.length == 0) { + throw retryableFailure( + "MINIMAX_TTS_AUDIO_EMPTY", + "MiniMax TTS returned an empty audio file"); + } + return audio; + } + + private int maxJsonResponseBytes() { + long limit = (long) maxAudioBytes * 2 + JSON_OVERHEAD_BYTES; + return (int) Math.min(Integer.MAX_VALUE, limit); + } + + private void requireTrustedEndpoint() { + String host = endpoint == null || endpoint.getHost() == null + ? "" + : endpoint.getHost().toLowerCase(Locale.ROOT); + if (endpoint == null + || !endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !TRUSTED_HOSTS.contains(host) + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/v1/t2a_v2".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "MINIMAX_TTS_ENDPOINT_INVALID", + "MiniMax TTS endpoint must be a trusted T2A v2 URL"); + } + } + + private static boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private static URI parseUri(String value) { + try { + return URI.create(trim(value)); + } + catch (IllegalArgumentException exception) { + return null; + } + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber(limit, errorCode, errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static String contentType(String audioFormat) { + return switch (audioFormat) { + case "mp3" -> "audio/mpeg"; + case "wav" -> "audio/wav"; + case "flac" -> "audio/flac"; + case "pcm" -> "audio/L16"; + default -> "application/octet-stream"; + }; + } + + private static String supportedFormat(String value) { + String audioFormat = trim(value).toLowerCase(Locale.ROOT); + if (!Set.of("mp3", "wav", "flac", "pcm").contains(audioFormat)) { + throw new IllegalArgumentException("Unsupported MiniMax TTS audio format: " + value); + } + return audioFormat; + } + + private static int supportedSampleRate(int value) { + if (!Set.of(8_000, 16_000, 22_050, 24_000, 32_000, 44_100).contains(value)) { + throw new IllegalArgumentException("Unsupported MiniMax TTS sample rate: " + value); + } + return value; + } + + private static int supportedBitrate(int value) { + if (!Set.of(32_000, 64_000, 128_000, 256_000).contains(value)) { + throw new IllegalArgumentException("Unsupported MiniMax TTS bitrate: " + value); + } + return value; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String text = trim(value); + if (text.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return text; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenAsrProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenAsrProvider.java new file mode 100644 index 0000000..b63d5d2 --- /dev/null +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenAsrProvider.java @@ -0,0 +1,358 @@ +package com.unispeaking.infrastructure.ai.qwen; + +import com.unispeaking.domain.vo.evaluation.AudioInput; +import com.unispeaking.exception.BusinessException; +import com.unispeaking.provider.AiProviderRegistry; +import com.unispeaking.provider.TranscriptionProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +@Component +public class QwenAsrProvider extends TranscriptionProvider { + + private static final int DEFAULT_MAX_AUDIO_BYTES = 7 * 1024 * 1024; + private static final int DEFAULT_MAX_RESPONSE_BYTES = 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final Duration readTimeout; + private final int maxAudioBytes; + private final int maxResponseBytes; + + @Autowired + public QwenAsrProvider( + ObjectMapper objectMapper, + @Value("${DASHSCOPE_API_KEY:}") String apiKey, + @Value("${BAILIAN_WORKSPACE_ID:}") String workspaceId, + @Value("${BAILIAN_REGION:cn-beijing}") String region, + @Value("${QWEN_ASR_MODEL:qwen3-asr-flash}") String model, + @Value("${QWEN_ASR_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${QWEN_ASR_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${QWEN_ASR_MAX_AUDIO_BYTES:7340032}") int maxAudioBytes, + @Value("${QWEN_ASR_MAX_RESPONSE_BYTES:1048576}") int maxResponseBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration( + connectTimeoutSeconds, + "Qwen ASR connect timeout")) + .build(), + objectMapper, + apiKey, + buildEndpoint(workspaceId, region), + model, + positiveDuration(readTimeoutSeconds, "Qwen ASR read timeout"), + maxAudioBytes, + maxResponseBytes); + } + + public QwenAsrProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + Duration readTimeout, + int maxAudioBytes, + int maxResponseBytes) { + super("qwen", Set.of(requiredText(model, "Qwen ASR model"))); + this.httpClient = require(httpClient, "Qwen ASR HTTP client"); + this.objectMapper = require(objectMapper, "Qwen ASR JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = endpoint; + this.model = requiredText(model, "Qwen ASR model"); + this.readTimeout = requirePositive(readTimeout, "Qwen ASR read timeout"); + this.maxAudioBytes = maxAudioBytes > 0 ? maxAudioBytes : DEFAULT_MAX_AUDIO_BYTES; + this.maxResponseBytes = maxResponseBytes > 0 + ? maxResponseBytes + : DEFAULT_MAX_RESPONSE_BYTES; + } + + @Override + public String convertAudioToText(Byte[] audio, String token) { + AudioInput input = new AudioInput( + unboxAudio(audio, "Qwen ASR"), + "wav"); + requireAudio(input); + if (apiKey.isBlank()) { + throw retryableFailure( + "QWEN_ASR_CREDENTIAL_MISSING", + "Set DASHSCOPE_API_KEY before calling Qwen ASR"); + } + return transcribe(input, apiKey); + } + + private String transcribe(AudioInput audio, String credential) { + requireTrustedEndpoint(); + + try { + String dataUrl = "data:" + mediaType(audio.audioFormat()) + ";base64," + + Base64.getEncoder().encodeToString(audio.audioData()); + Map audioContent = Map.of( + "type", "input_audio", + "input_audio", Map.of("data", dataUrl)); + Map body = Map.of( + "model", model, + "messages", List.of(Map.of( + "role", "user", + "content", List.of(audioContent))), + "stream", false, + "asr_options", Map.of("enable_itn", true)); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(body), + StandardCharsets.UTF_8)) + .build(); + HttpResponse response = httpClient.send( + httpRequest, + limitedBodyHandler( + maxResponseBytes, + "QWEN_ASR_RESPONSE_TOO_LARGE", + "Qwen ASR response exceeds the configured limit")); + if (!successful(response.statusCode())) { + throw retryableFailure( + "QWEN_ASR_REQUEST_FAILED", + "Qwen ASR returned HTTP " + response.statusCode()); + } + JsonNode root = objectMapper.readTree( + new String(response.body(), StandardCharsets.UTF_8)); + String text = root.path("choices") + .path(0) + .path("message") + .path("content") + .asString(""); + if (text.isBlank()) { + throw retryableFailure( + "QWEN_ASR_RESULT_EMPTY", + "Qwen ASR returned no transcription text"); + } + return text; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "QWEN_ASR_RESPONSE_INVALID", + "Qwen ASR response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "QWEN_ASR_IO_ERROR", + "Failed to call Qwen ASR"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "QWEN_ASR_INTERRUPTED", + "Qwen ASR call was interrupted"); + } + } + + private AudioInput requireAudio(AudioInput audio) { + if (audio == null || audio.audioData() == null || audio.audioData().length == 0) { + throw nonRetryableFailure( + "INVALID_TRANSCRIPTION_AUDIO", + "Audio transcription data is required"); + } + if (audio.audioData().length > maxAudioBytes) { + throw nonRetryableFailure( + "TRANSCRIPTION_AUDIO_TOO_LARGE", + "Qwen ASR audio exceeds the configured limit"); + } + mediaType(audio.audioFormat()); + return audio; + } + + private String mediaType(String audioFormat) { + return switch (trim(audioFormat).toLowerCase(Locale.ROOT)) { + case "wav" -> "audio/wav"; + case "mp3" -> "audio/mpeg"; + case "aac" -> "audio/aac"; + case "m4a" -> "audio/mp4"; + case "flac" -> "audio/flac"; + case "ogg", "opus" -> "audio/ogg"; + default -> throw nonRetryableFailure( + "UNSUPPORTED_TRANSCRIPTION_AUDIO_FORMAT", + "Unsupported Qwen ASR audio format: " + audioFormat); + }; + } + + private void requireTrustedEndpoint() { + String host = endpoint == null || endpoint.getHost() == null + ? "" + : endpoint.getHost().toLowerCase(Locale.ROOT); + if (endpoint == null + || !endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !host.endsWith(".maas.aliyuncs.com") + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/compatible-mode/v1/chat/completions".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "QWEN_ASR_ENDPOINT_INVALID", + "Qwen ASR endpoint must be the trusted Aliyun compatible-mode URL"); + } + } + + private static URI buildEndpoint(String workspaceId, String region) { + String workspace = trim(workspaceId); + String endpointRegion = trim(region); + if (!safeEndpointComponent(workspace) || !safeEndpointComponent(endpointRegion)) { + return null; + } + return URI.create("https://" + workspace + "." + endpointRegion + + ".maas.aliyuncs.com/compatible-mode/v1/chat/completions"); + } + + private static boolean safeEndpointComponent(String value) { + return !value.isBlank() && value.matches("[A-Za-z0-9-]+"); + } + + private static boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber(limit, errorCode, errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String text = trim(value); + if (text.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return text; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } + } +} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java index f776529..45ec45b 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenLlmProvider.java @@ -1,22 +1,315 @@ package com.unispeaking.infrastructure.ai.qwen; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; +import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.LlmProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; @Component public class QwenLlmProvider extends LlmProvider { - public QwenLlmProvider() { - super(ProviderType.QWEN, Set.of(AiProviderRegistry.QWEN_LLM_PLUS)); + private static final int DEFAULT_MAX_RESPONSE_BYTES = 2 * 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final Duration readTimeout; + private final int maxResponseBytes; + + @Autowired + public QwenLlmProvider( + ObjectMapper objectMapper, + @Value("${DASHSCOPE_API_KEY:}") String apiKey, + @Value("${BAILIAN_WORKSPACE_ID:}") String workspaceId, + @Value("${BAILIAN_REGION:cn-beijing}") String region, + @Value("${QWEN_LLM_MODEL:qwen3.5-plus}") String model, + @Value("${QWEN_LLM_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${QWEN_LLM_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${QWEN_LLM_MAX_RESPONSE_BYTES:2097152}") int maxResponseBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration(connectTimeoutSeconds, "Qwen LLM connect timeout")) + .build(), + objectMapper, + apiKey, + buildEndpoint(workspaceId, region), + model, + positiveDuration(readTimeoutSeconds, "Qwen LLM read timeout"), + maxResponseBytes); + } + + public QwenLlmProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + Duration readTimeout, + int maxResponseBytes) { + super("qwen", Set.of(modelOrDefault(model))); + this.httpClient = require(httpClient, "Qwen LLM HTTP client"); + this.objectMapper = require(objectMapper, "Qwen LLM JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = endpoint; + this.model = modelOrDefault(model); + this.readTimeout = requirePositive(readTimeout, "Qwen LLM read timeout"); + this.maxResponseBytes = maxResponseBytes > 0 + ? maxResponseBytes + : DEFAULT_MAX_RESPONSE_BYTES; } @Override - public LlmTaskResponse executeLlmTask(LlmTaskRequest request) { - throw capabilityNotConfigured(AiProviderRegistry.QWEN_LLM_PLUS); + public String executeLlmTask(String prompt, String token) { + if (apiKey.isBlank()) { + throw retryableFailure( + "QWEN_LLM_CREDENTIAL_MISSING", + "Set DASHSCOPE_API_KEY before calling Qwen LLM"); + } + return callForContent(prompt, apiKey); + } + + private String callForContent(String promptValue, String credential) { + String prompt = trim(promptValue); + if (prompt.isBlank()) { + throw nonRetryableFailure("INVALID_LLM_PROMPT", "LLM task prompt is required"); + } + requireHttpsEndpoint(); + + try { + Map body = Map.of( + "model", model, + "messages", List.of(Map.of("role", "user", "content", prompt)), + "enable_thinking", false); + HttpRequest httpRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(body), + StandardCharsets.UTF_8)) + .build(); + HttpResponse response = httpClient.send( + httpRequest, + limitedBodyHandler( + maxResponseBytes, + "QWEN_LLM_RESPONSE_TOO_LARGE", + "Qwen LLM response exceeds the configured limit")); + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw retryableFailure( + "QWEN_LLM_REQUEST_FAILED", + "Qwen LLM returned HTTP " + response.statusCode()); + } + String responseBody = new String(response.body(), StandardCharsets.UTF_8); + JsonNode root = objectMapper.readTree(responseBody); + String content = root.path("choices") + .path(0) + .path("message") + .path("content") + .asString(""); + if (content.isBlank()) { + throw retryableFailure( + "QWEN_LLM_EMPTY_RESPONSE", + "Qwen LLM returned no message content"); + } + return content; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "QWEN_LLM_RESPONSE_INVALID", + "Qwen LLM response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "QWEN_LLM_IO_ERROR", + "Failed to call Qwen LLM"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "QWEN_LLM_INTERRUPTED", + "Qwen LLM call was interrupted"); + } + } + + private Object parseContent(String content) { + try { + return objectMapper.readTree(content); + } + catch (Exception ignored) { + return content; + } + } + + private void requireHttpsEndpoint() { + String host = endpoint == null || endpoint.getHost() == null + ? "" + : endpoint.getHost().toLowerCase(java.util.Locale.ROOT); + if (endpoint == null + || !endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !host.endsWith(".maas.aliyuncs.com") + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/compatible-mode/v1/chat/completions".equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "QWEN_LLM_ENDPOINT_INVALID", + "Qwen LLM endpoint must be the trusted Aliyun compatible-mode URL"); + } + } + + private static URI buildEndpoint(String workspaceId, String region) { + String workspace = trim(workspaceId); + String endpointRegion = trim(region); + if (!safeEndpointComponent(workspace) || !safeEndpointComponent(endpointRegion)) { + return null; + } + return URI.create("https://" + workspace + "." + endpointRegion + + ".maas.aliyuncs.com/compatible-mode/v1/chat/completions"); + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber( + limit, + errorCode, + errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static boolean safeEndpointComponent(String value) { + return !value.isBlank() && value.matches("[A-Za-z0-9-]+"); + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static String modelOrDefault(String model) { + String configured = trim(model); + return configured.isBlank() ? AiProviderRegistry.QWEN_LLM_PLUS : configured; + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure(errorCode, errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProvider.java index 2aa9503..adf3077 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProvider.java @@ -1,10 +1,7 @@ package com.unispeaking.infrastructure.ai.qwen; import com.unispeaking.common.logging.RealtimeFlowLog; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; import com.unispeaking.domain.vo.realtime.ProviderType; -import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.provider.RealtimeProvider; import java.io.IOException; @@ -32,64 +29,58 @@ public QwenRealtimeProvider(HttpClient realtimeHttpClient, RealtimeProperties pr } @Override - public RealtimeSdpExchangeResponse exchangeRealtimeSdp(RealtimeSdpExchangeRequest request) { - if (request == null) { - throw new BusinessException("INVALID_SDP_REQUEST", "Realtime SDP exchange request is required"); - } - String offerSdp = request.offerSdp(); + public String exchangeRealtimeSdp(String modelId, String offerSdp, String token) { if (offerSdp == null || offerSdp.isBlank()) { - throw new BusinessException("INVALID_SDP", "WebRTC offer SDP is required"); + throw nonRetryableFailure("INVALID_SDP", "WebRTC offer SDP is required"); } - if (request.apiKey() == null || request.apiKey().isBlank()) { - throw new BusinessException("QWEN_CREDENTIAL_MISSING", "Qwen bearer credential is not configured"); + if (token == null || token.isBlank()) { + throw retryableFailure("QWEN_CREDENTIAL_MISSING", "Qwen bearer credential is not configured"); } - String model = request.model() == null || request.model().isBlank() + String model = modelId == null || modelId.isBlank() ? AiProviderRegistry.QWEN_REALTIME_FLASH - : request.model().trim(); + : modelId.trim(); if (!supports(model)) { - throw new BusinessException( + throw nonRetryableFailure( "QWEN_REALTIME_MODEL_NOT_SUPPORTED", "Qwen realtime model is not registered: " + model); } String sdpExchangeUrl = properties.getWebRtcSdpExchangeUrl(model); if (sdpExchangeUrl.isBlank()) { - throw new BusinessException( + throw retryableFailure( "QWEN_WORKSPACE_OR_MODEL_MISSING", "Set BAILIAN_WORKSPACE_ID before starting a Qwen realtime session"); } - String userId = request.context() == null ? null : request.context().userId(); - String localSessionId = request.context() == null ? null : request.context().businessId(); try { - RealtimeFlowLog.info("flow.3.sdp.request localSessionId={} userId={} provider={} url={} model={} temporaryToken={} offerSdp={}", - localSessionId, userId, type(), sdpExchangeUrl, model, - RealtimeFlowLog.maskSecret(request.apiKey()), + RealtimeFlowLog.info("flow.3.sdp.request provider={} url={} model={} temporaryToken={} offerSdp={}", + type(), sdpExchangeUrl, model, + RealtimeFlowLog.maskSecret(token), RealtimeFlowLog.sdpSummary(offerSdp)); HttpRequest httpRequest = HttpRequest.newBuilder() .uri(URI.create(sdpExchangeUrl)) .timeout(properties.getReadTimeout()) - .header("Authorization", "Bearer " + request.apiKey()) + .header("Authorization", "Bearer " + token) .header("Content-Type", "application/sdp") .POST(HttpRequest.BodyPublishers.ofString(offerSdp)) .build(); HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new BusinessException("QWEN_SIGNALING_FAILED", + throw retryableFailure("QWEN_SIGNALING_FAILED", "Qwen signaling returned " + response.statusCode()); } if (response.body().length() > properties.getMaxAnswerBytes()) { - throw new BusinessException("QWEN_ANSWER_TOO_LARGE", "Qwen answer SDP exceeds the configured limit"); + throw retryableFailure("QWEN_ANSWER_TOO_LARGE", "Qwen answer SDP exceeds the configured limit"); } - RealtimeFlowLog.info("flow.3.sdp.response localSessionId={} status={} answerSdp={}", - localSessionId, response.statusCode(), + RealtimeFlowLog.info("flow.3.sdp.response status={} answerSdp={}", + response.statusCode(), RealtimeFlowLog.sdpSummary(response.body())); - return new RealtimeSdpExchangeResponse(response.body(), null); + return response.body(); } catch (IOException exception) { - throw new BusinessException("QWEN_SIGNALING_IO_ERROR", "Failed to call Qwen signaling"); + throw retryableFailure("QWEN_SIGNALING_IO_ERROR", "Failed to call Qwen signaling"); } catch (InterruptedException exception) { Thread.currentThread().interrupt(); - throw new BusinessException("QWEN_SIGNALING_INTERRUPTED", "Qwen signaling call was interrupted"); + throw nonRetryableFailure("QWEN_SIGNALING_INTERRUPTED", "Qwen signaling call was interrupted"); } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenTtsProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenTtsProvider.java new file mode 100644 index 0000000..19bf201 --- /dev/null +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/infrastructure/ai/qwen/QwenTtsProvider.java @@ -0,0 +1,437 @@ +package com.unispeaking.infrastructure.ai.qwen; + +import com.unispeaking.exception.BusinessException; +import com.unispeaking.provider.AiProviderRegistry; +import com.unispeaking.provider.TtsProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +@Component +public class QwenTtsProvider extends TtsProvider { + + private static final int MAX_TEXT_LENGTH = 5_000; + private static final int DEFAULT_MAX_RESPONSE_BYTES = 1024 * 1024; + private static final int DEFAULT_MAX_AUDIO_BYTES = 10 * 1024 * 1024; + private static final String DEFAULT_ENDPOINT = + "https://dashscope.aliyuncs.com/api/v1/services/aigc/" + + "multimodal-generation/generation"; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final String apiKey; + private final URI endpoint; + private final String model; + private final String voice; + private final String languageType; + private final Duration readTimeout; + private final int maxResponseBytes; + private final int maxAudioBytes; + + @Autowired + public QwenTtsProvider( + ObjectMapper objectMapper, + @Value("${DASHSCOPE_API_KEY:}") String apiKey, + @Value("${QWEN_TTS_ENDPOINT:" + DEFAULT_ENDPOINT + "}") String endpoint, + @Value("${QWEN_TTS_MODEL:qwen3-tts-flash}") String model, + @Value("${QWEN_TTS_VOICE:Cherry}") String voice, + @Value("${QWEN_TTS_LANGUAGE_TYPE:English}") String languageType, + @Value("${QWEN_TTS_CONNECT_TIMEOUT_SECONDS:10}") int connectTimeoutSeconds, + @Value("${QWEN_TTS_READ_TIMEOUT_SECONDS:60}") int readTimeoutSeconds, + @Value("${QWEN_TTS_MAX_RESPONSE_BYTES:1048576}") int maxResponseBytes, + @Value("${QWEN_TTS_MAX_AUDIO_BYTES:10485760}") int maxAudioBytes) { + this( + HttpClient.newBuilder() + .connectTimeout(positiveDuration( + connectTimeoutSeconds, + "Qwen TTS connect timeout")) + .build(), + objectMapper, + apiKey, + parseUri(endpoint), + model, + voice, + languageType, + positiveDuration(readTimeoutSeconds, "Qwen TTS read timeout"), + maxResponseBytes, + maxAudioBytes); + } + + public QwenTtsProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + String voice, + String languageType, + Duration readTimeout, + int maxAudioBytes) { + this( + httpClient, + objectMapper, + apiKey, + endpoint, + model, + voice, + languageType, + readTimeout, + DEFAULT_MAX_RESPONSE_BYTES, + maxAudioBytes); + } + + public QwenTtsProvider( + HttpClient httpClient, + ObjectMapper objectMapper, + String apiKey, + URI endpoint, + String model, + String voice, + String languageType, + Duration readTimeout, + int maxResponseBytes, + int maxAudioBytes) { + super("qwen", Set.of(requiredText(model, "Qwen TTS model"))); + this.httpClient = require(httpClient, "Qwen TTS HTTP client"); + this.objectMapper = require(objectMapper, "Qwen TTS JSON mapper"); + this.apiKey = trim(apiKey); + this.endpoint = require(endpoint, "Qwen TTS endpoint"); + this.model = requiredText(model, "Qwen TTS model"); + this.voice = requiredText(voice, "Qwen TTS voice"); + this.languageType = requiredText(languageType, "Qwen TTS language type"); + this.readTimeout = requirePositive(readTimeout, "Qwen TTS read timeout"); + this.maxResponseBytes = positiveLimit( + maxResponseBytes, + DEFAULT_MAX_RESPONSE_BYTES); + this.maxAudioBytes = positiveLimit(maxAudioBytes, DEFAULT_MAX_AUDIO_BYTES); + } + + @Override + public Byte[] generateSpeechAudio(String text, String token) { + if (apiKey.isBlank()) { + throw retryableFailure( + "QWEN_TTS_CREDENTIAL_MISSING", + "Set DASHSCOPE_API_KEY before calling Qwen TTS"); + } + return boxAudio(synthesize(text, apiKey)); + } + + private byte[] synthesize(String textValue, String credential) { + String text = trim(textValue); + if (text.isBlank()) { + throw nonRetryableFailure( + "INVALID_TTS_TEXT", + "Speech synthesis text is required"); + } + if (text.length() > MAX_TEXT_LENGTH) { + throw nonRetryableFailure( + "TTS_TEXT_TOO_LONG", + "Qwen speech synthesis text exceeds " + + MAX_TEXT_LENGTH + " characters"); + } + requireTrustedEndpoint(); + + try { + Map input = Map.of( + "text", text, + "voice", voice, + "language_type", languageType); + HttpRequest synthesisRequest = HttpRequest.newBuilder() + .uri(endpoint) + .timeout(readTimeout) + .header("Authorization", "Bearer " + credential) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString( + objectMapper.writeValueAsString(Map.of( + "model", model, + "input", input)), + StandardCharsets.UTF_8)) + .build(); + HttpResponse synthesisResponse = httpClient.send( + synthesisRequest, + limitedBodyHandler( + maxResponseBytes, + "QWEN_TTS_RESPONSE_TOO_LARGE", + "Qwen TTS response exceeds the configured limit")); + if (!successful(synthesisResponse.statusCode())) { + throw httpError( + "QWEN_TTS_REQUEST_FAILED", + synthesisResponse.statusCode()); + } + URI audioUri = audioUri(synthesisResponse.body()); + HttpResponse audioResponse = httpClient.send( + HttpRequest.newBuilder() + .uri(audioUri) + .timeout(readTimeout) + .GET() + .build(), + limitedBodyHandler( + maxAudioBytes, + "QWEN_TTS_AUDIO_TOO_LARGE", + "Qwen TTS audio exceeds the configured limit")); + if (!successful(audioResponse.statusCode())) { + throw httpError( + "QWEN_TTS_AUDIO_DOWNLOAD_FAILED", + audioResponse.statusCode()); + } + byte[] audio = audioResponse.body(); + requireWav(audio); + return audio; + } + catch (BusinessException exception) { + throw exception; + } + catch (JacksonException exception) { + throw retryableFailure( + "QWEN_TTS_RESPONSE_INVALID", + "Qwen TTS response is not valid JSON"); + } + catch (IOException exception) { + BusinessException bodyError = businessCause(exception); + if (bodyError != null) { + throw bodyError; + } + throw retryableFailure( + "QWEN_TTS_IO_ERROR", + "Failed to call Qwen TTS"); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw nonRetryableFailure( + "QWEN_TTS_INTERRUPTED", + "Qwen TTS call was interrupted"); + } + } + + private URI audioUri(byte[] responseBody) throws JacksonException { + JsonNode root = objectMapper.readTree( + new String(responseBody, StandardCharsets.UTF_8)); + String audioUrl = root.path("output") + .path("audio") + .path("url") + .asString("") + .trim(); + if (audioUrl.isBlank()) { + throw retryableFailure( + "QWEN_TTS_AUDIO_URL_MISSING", + "Qwen TTS response does not contain an audio URL"); + } + URI uri = parseUri(audioUrl); + String host = normalizedHost(uri); + if (!uri.isAbsolute() + || !Set.of("http", "https").contains( + trim(uri.getScheme()).toLowerCase(Locale.ROOT)) + || !host.endsWith(".aliyuncs.com") + || uri.getUserInfo() != null + || uri.getPort() != -1) { + throw retryableFailure( + "QWEN_TTS_AUDIO_URL_UNTRUSTED", + "Qwen TTS returned an untrusted audio URL"); + } + if ("http".equalsIgnoreCase(uri.getScheme())) { + return parseUri("https:" + uri.toString().substring("http:".length())); + } + return uri; + } + + private void requireTrustedEndpoint() { + String host = normalizedHost(endpoint); + if (!endpoint.isAbsolute() + || !"https".equalsIgnoreCase(endpoint.getScheme()) + || !Set.of( + "dashscope.aliyuncs.com", + "dashscope-intl.aliyuncs.com").contains(host) + || endpoint.getUserInfo() != null + || endpoint.getPort() != -1 + || !"/api/v1/services/aigc/multimodal-generation/generation" + .equals(endpoint.getPath()) + || endpoint.getRawQuery() != null + || endpoint.getRawFragment() != null) { + throw retryableFailure( + "QWEN_TTS_ENDPOINT_INVALID", + "Qwen TTS endpoint must be the trusted DashScope generation URL"); + } + } + + private void requireWav(byte[] audio) { + if (audio == null + || audio.length < 12 + || audio[0] != 'R' + || audio[1] != 'I' + || audio[2] != 'F' + || audio[3] != 'F' + || audio[8] != 'W' + || audio[9] != 'A' + || audio[10] != 'V' + || audio[11] != 'E') { + throw retryableFailure( + "QWEN_TTS_AUDIO_INVALID", + "Qwen TTS did not return a WAV audio file"); + } + } + + private static HttpResponse.BodyHandler limitedBodyHandler( + int limit, + String errorCode, + String errorMessage) { + return responseInfo -> new LimitedBodySubscriber( + limit, + errorCode, + errorMessage); + } + + private static BusinessException businessCause(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof BusinessException businessException) { + return businessException; + } + } + return null; + } + + private static boolean successful(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + private static BusinessException httpError(String errorCode, int statusCode) { + return retryableFailure( + errorCode, + "Qwen TTS returned HTTP " + statusCode); + } + + private static URI parseUri(String value) { + try { + return URI.create(trim(value)); + } + catch (IllegalArgumentException exception) { + throw retryableFailure( + "QWEN_TTS_URL_INVALID", + "Qwen TTS returned or configured an invalid URL"); + } + } + + private static String normalizedHost(URI uri) { + return uri == null || uri.getHost() == null + ? "" + : uri.getHost().toLowerCase(Locale.ROOT); + } + + private static int positiveLimit(int value, int defaultValue) { + return value > 0 ? value : defaultValue; + } + + private static Duration positiveDuration(int seconds, String name) { + if (seconds <= 0) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return Duration.ofSeconds(seconds); + } + + private static Duration requirePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException(name + " must be greater than zero"); + } + return duration; + } + + private static String requiredText(String value, String name) { + String result = trim(value); + if (result.isBlank()) { + throw new IllegalArgumentException(name + " is required"); + } + return result; + } + + private static T require(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " is required"); + } + return value; + } + + private static String trim(String value) { + return value == null ? "" : value.trim(); + } + + private static final class LimitedBodySubscriber + implements HttpResponse.BodySubscriber { + + private final int limit; + private final String errorCode; + private final String errorMessage; + private final ByteArrayOutputStream bytes; + private final CompletableFuture body = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int limit, String errorCode, String errorMessage) { + this.limit = limit; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + this.bytes = new ByteArrayOutputStream(Math.min(limit, 8_192)); + } + + @Override + public CompletionStage getBody() { + return body; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List items) { + if (body.isDone()) { + return; + } + for (ByteBuffer item : items) { + if (item.remaining() > limit - bytes.size()) { + subscription.cancel(); + body.completeExceptionally(retryableFailure( + errorCode, + errorMessage)); + return; + } + byte[] chunk = new byte[item.remaining()]; + item.get(chunk); + bytes.writeBytes(chunk); + } + subscription.request(1); + } + + @Override + public void onError(Throwable throwable) { + body.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + body.complete(bytes.toByteArray()); + } + } +} diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AbstractAiProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AbstractAiProvider.java index 97d9338..c9a7442 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AbstractAiProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AbstractAiProvider.java @@ -1,16 +1,5 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.AudioTranscriptionRequest; -import com.unispeaking.domain.dto.ai.AudioTranscriptionResponse; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; -import com.unispeaking.domain.vo.realtime.ProviderType; import com.unispeaking.exception.BusinessException; import java.util.Locale; import java.util.Set; @@ -18,19 +7,22 @@ public abstract class AbstractAiProvider implements AiProvider { - private final ProviderType providerType; + private final String providerId; private final Set supportedModels; - protected AbstractAiProvider(ProviderType providerType, Set supportedModels) { - this.providerType = providerType; + protected AbstractAiProvider(String providerId, Set supportedModels) { + this.providerId = normalizeProviderId(providerId); + if (this.providerId.isBlank()) { + throw new IllegalArgumentException("AI provider ID is required"); + } this.supportedModels = supportedModels.stream() .map(AbstractAiProvider::normalizeModelId) .collect(Collectors.toUnmodifiableSet()); } @Override - public final ProviderType type() { - return providerType; + public final String providerId() { + return providerId; } @Override @@ -42,46 +34,78 @@ public final boolean supports(String modelId) { return supportedModels.contains(normalizeModelId(modelId)); } - @Override - public RealtimeSdpExchangeResponse exchangeRealtimeSdp(RealtimeSdpExchangeRequest request) { - throw unsupportedCapability("realtime SDP exchange"); - } - - @Override - public SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request) { - throw unsupportedCapability("speech synthesis"); + protected final BusinessException capabilityNotConfigured(String modelId) { + return retryableFailure( + "AI_PROVIDER_CAPABILITY_NOT_CONFIGURED", + providerId() + " provider is registered for " + modelId + + " but its " + capability() + " API is not configured"); } - @Override - public LlmTaskResponse executeLlmTask(LlmTaskRequest request) { - throw unsupportedCapability("LLM task execution"); + protected static final BusinessException retryableFailure(String code, String message) { + return new ClassifiedProviderException(code, message, true); } - @Override - public AudioTranscriptionResponse convertAudioToText(AudioTranscriptionRequest request) { - throw unsupportedCapability("audio transcription"); + /** + * Marks a request or interruption failure that must not be sent to another + * provider. Provider/network/configuration failures use + * {@link #retryableFailure(String, String)} instead. + */ + protected static final BusinessException nonRetryableFailure(String code, String message) { + return new ClassifiedProviderException(code, message, false); } - @Override - public PronunciationEvaluationResponse evaluatePronunciation( - PronunciationEvaluationRequest request) { - throw unsupportedCapability("pronunciation evaluation"); + protected static byte[] unboxAudio(Byte[] audio, String operationName) { + if (audio == null || audio.length == 0) { + throw nonRetryableFailure( + "INVALID_AUDIO", + operationName + " WAV audio is required"); + } + byte[] bytes = new byte[audio.length]; + for (int index = 0; index < audio.length; index++) { + if (audio[index] == null) { + throw nonRetryableFailure( + "INVALID_AUDIO", + operationName + " WAV audio contains a null byte"); + } + bytes[index] = audio[index]; + } + return bytes; } - protected final BusinessException capabilityNotConfigured(String modelId) { - return new BusinessException( - "AI_PROVIDER_CAPABILITY_NOT_CONFIGURED", - type() + " provider is registered for " + modelId - + " but its " + capability() + " API is not configured"); + protected static Byte[] boxAudio(byte[] audio) { + Byte[] boxed = new Byte[audio.length]; + for (int index = 0; index < audio.length; index++) { + boxed[index] = audio[index]; + } + return boxed; } - private BusinessException unsupportedCapability(String operation) { - return new BusinessException( - "AI_PROVIDER_CAPABILITY_NOT_SUPPORTED", - getClass().getSimpleName() + " does not support " + operation); + static Boolean retryable(BusinessException exception) { + if (exception instanceof ClassifiedProviderException classified) { + return classified.retryable(); + } + return null; } static String normalizeModelId(String modelId) { return modelId == null ? "" : modelId.trim().toLowerCase(Locale.ROOT); } + + private static String normalizeProviderId(String providerId) { + return providerId == null ? "" : providerId.trim().toLowerCase(Locale.ROOT); + } + + private static final class ClassifiedProviderException extends BusinessException { + + private final boolean retryable; + + private ClassifiedProviderException(String code, String message, boolean retryable) { + super(code, message); + this.retryable = retryable; + } + + private boolean retryable() { + return retryable; + } + } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProvider.java index e7e8309..0a1700a 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProvider.java @@ -1,35 +1,81 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.AudioTranscriptionRequest; -import com.unispeaking.domain.dto.ai.AudioTranscriptionResponse; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; +/** + * Common contract exposed by every pluggable AI provider adapter. + * + *

The Registry invokes only the operation matching {@link #capability()}. + * Default methods make unsupported operations explicit without forcing an + * adapter to implement capabilities it does not provide.

+ */ public interface AiProvider { - ProviderType type(); + String providerId(); AiCapability capability(); Set supportedModels(); - RealtimeSdpExchangeResponse exchangeRealtimeSdp(RealtimeSdpExchangeRequest request); + /** + * Completes the WebRTC Offer SDP / Answer SDP exchange. + * + * @param offerSdp browser-generated Offer SDP + * @param token short-lived provider credential + * @return provider Answer SDP without modification + */ + default String exchangeRealtimeSdp(String offerSdp, String token) { + throw unsupported("Realtime"); + } - SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request); + /** + * Converts text to the provider's configured WAV audio. + * + * @param text text to synthesize + * @param token reserved for interface compatibility; TTS uses server credentials + * @return WAV audio bytes + */ + default Byte[] generateSpeechAudio(String text, String token) { + throw unsupported("TTS"); + } - LlmTaskResponse executeLlmTask(LlmTaskRequest request); + /** + * Executes a text-generation task. + * + * @param prompt prompt sent to the model + * @param token reserved for interface compatibility; LLM uses server credentials + * @return only the model message content + */ + default String executeLlmTask(String prompt, String token) { + throw unsupported("LLM"); + } - AudioTranscriptionResponse convertAudioToText(AudioTranscriptionRequest request); + /** + * Converts WAV audio to transcription text. + * + * @param audio complete WAV file bytes + * @param token reserved for interface compatibility; ASR uses server credentials + * @return only the transcription text + */ + default String convertAudioToText(Byte[] audio, String token) { + throw unsupported("ASR"); + } - PronunciationEvaluationResponse evaluatePronunciation( - PronunciationEvaluationRequest request); + /** + * Sends WAV audio for pronunciation evaluation. + * + * @param text reference text + * @param audio complete WAV file bytes + * @param token reserved for interface compatibility; scoring uses server credentials + * @return the complete final provider response without score parsing + */ + default String evaluatePronunciation(String text, Byte[] audio, String token) { + throw unsupported("pronunciation scoring"); + } + + private UnsupportedOperationException unsupported(String operation) { + return new UnsupportedOperationException( + providerId() + " provider does not support " + operation); + } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java index c5388de..9f38f31 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/AiProviderRegistry.java @@ -1,65 +1,127 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.AudioTranscriptionRequest; -import com.unispeaking.domain.dto.ai.AudioTranscriptionResponse; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; import com.unispeaking.domain.vo.ai.AiCapability; import com.unispeaking.domain.vo.ai.AiModelDefinition; import com.unispeaking.domain.vo.realtime.ProviderType; import com.unispeaking.exception.BusinessException; +import java.util.Collections; import java.util.EnumMap; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component public class AiProviderRegistry { + private static final Logger LOGGER = LoggerFactory.getLogger(AiProviderRegistry.class); + + /** + * Internal routing result for audit, metering, and diagnostics. User-facing + * business methods can continue returning only the response payload. + */ + public record RoutedResult( + String modelId, + String providerId, + AiCapability capability, + T response) { + } + public static final String QWEN_REALTIME_FLASH = "qwen3.5-omni-flash-realtime"; public static final String QWEN_REALTIME_PLUS = "qwen3.5-omni-plus-realtime"; public static final String QWEN_LLM_PLUS = "qwen3.5-plus"; - public static final String DEEPSEEK_CHAT = "deepseek-chat"; - public static final String IFLYTEK_PRONUNCIATION_SCORING = "iflytek-pronunciation-evaluation"; - public static final String ALIYUN_TTS = "aliyun-tts"; - public static final String MINIMAX_TTS = "minimax-tts"; - - private static final List MODEL_CATALOG = List.of( - new AiModelDefinition(QWEN_REALTIME_FLASH, ProviderType.QWEN, AiCapability.REALTIME, true), - new AiModelDefinition(QWEN_REALTIME_PLUS, ProviderType.QWEN, AiCapability.REALTIME, false), - new AiModelDefinition(QWEN_LLM_PLUS, ProviderType.QWEN, AiCapability.LLM, true), - new AiModelDefinition(DEEPSEEK_CHAT, ProviderType.DEEPSEEK, AiCapability.LLM, false), - new AiModelDefinition( - IFLYTEK_PRONUNCIATION_SCORING, - ProviderType.IFLYTEK, - AiCapability.SCORING, - true), - new AiModelDefinition(ALIYUN_TTS, ProviderType.ALIYUN, AiCapability.TTS, true), - new AiModelDefinition(MINIMAX_TTS, ProviderType.MINIMAX, AiCapability.TTS, false)); + public static final String DEEPSEEK_CHAT = "deepseek-v4-flash"; + public static final String QWEN_ASR = "qwen3-asr-flash"; + public static final String DOUBAO_ASR = "volc.bigasr.auc_turbo"; + public static final String IFLYTEK_PRONUNCIATION_SCORING = "iflytek-open-ise"; + public static final String QWEN_TTS = "qwen3-tts-flash"; + public static final String ALIYUN_TTS = "cosyvoice-v3-flash"; + public static final String MINIMAX_TTS = "speech-2.8-hd"; + private static final Map> DEFAULT_MODEL_ROUTES = Map.of( + AiCapability.REALTIME, List.of(QWEN_REALTIME_FLASH), + AiCapability.LLM, List.of(QWEN_LLM_PLUS, DEEPSEEK_CHAT), + AiCapability.SCORING, List.of(IFLYTEK_PRONUNCIATION_SCORING), + AiCapability.TTS, List.of(QWEN_TTS, ALIYUN_TTS, MINIMAX_TTS), + AiCapability.TRANSCRIPTION, List.of(QWEN_ASR, DOUBAO_ASR)); + + private static final Map> DEFAULT_PROVIDER_ROUTES = Map.of( + AiCapability.REALTIME, List.of("qwen"), + AiCapability.LLM, List.of("qwen", "deepseek"), + AiCapability.SCORING, List.of("iflytek"), + AiCapability.TTS, List.of("qwen", "aliyun", "minimax"), + AiCapability.TRANSCRIPTION, List.of("qwen", "doubao")); + + private final List models; private final Map modelDefinitions; - private final Map defaultModels; + private final Map> modelRoutes; private final Map realtimeProviders; private final Map llmProviders; private final Map scoringProviders; private final Map ttsProviders; private final Map transcriptionProviders; + @Autowired + public AiProviderRegistry( + List realtimeProviders, + List llmProviders, + List scoringProviders, + List ttsProviders, + List transcriptionProviders, + @Value("${AI_PROVIDER_ROUTE_REALTIME:}") + String realtimeRoute, + @Value("${AI_PROVIDER_ROUTE_LLM:}") + String llmRoute, + @Value("${AI_PROVIDER_ROUTE_SCORING:}") + String scoringRoute, + @Value("${AI_PROVIDER_ROUTE_TTS:}") + String ttsRoute, + @Value("${AI_PROVIDER_ROUTE_TRANSCRIPTION:}") + String transcriptionRoute) { + this( + realtimeProviders, + llmProviders, + scoringProviders, + ttsProviders, + transcriptionProviders, + Map.of( + AiCapability.REALTIME, parseRoute(realtimeRoute), + AiCapability.LLM, parseRoute(llmRoute), + AiCapability.SCORING, parseRoute(scoringRoute), + AiCapability.TTS, parseRoute(ttsRoute), + AiCapability.TRANSCRIPTION, parseRoute(transcriptionRoute))); + } + public AiProviderRegistry( List realtimeProviders, List llmProviders, List scoringProviders, List ttsProviders, List transcriptionProviders) { - this.modelDefinitions = buildModelDefinitions(); - this.defaultModels = buildDefaultModels(); + this( + realtimeProviders, + llmProviders, + scoringProviders, + ttsProviders, + transcriptionProviders, + Map.of()); + } + + public AiProviderRegistry( + List realtimeProviders, + List llmProviders, + List scoringProviders, + List ttsProviders, + List transcriptionProviders, + Map> configuredRoutes) { this.realtimeProviders = registerProviders(realtimeProviders, AiCapability.REALTIME); this.llmProviders = registerProviders(llmProviders, AiCapability.LLM); this.scoringProviders = registerProviders(scoringProviders, AiCapability.SCORING); @@ -67,11 +129,13 @@ public AiProviderRegistry( this.transcriptionProviders = registerProviders( transcriptionProviders, AiCapability.TRANSCRIPTION); - validateCatalogRegistrations(); + this.modelRoutes = buildModelRoutes(configuredRoutes); + this.modelDefinitions = buildModelDefinitions(); + this.models = List.copyOf(modelDefinitions.values()); } public List models() { - return MODEL_CATALOG; + return models; } public AiModelDefinition getModel(String modelId) { @@ -83,19 +147,58 @@ public AiModelDefinition getModel(String modelId) { } public String defaultModel(AiCapability capability) { - String modelId = defaultModels.get(capability); - if (modelId == null) { + List route = modelRoutes.get(capability); + if (route == null || route.isEmpty()) { throw new BusinessException( "AI_DEFAULT_MODEL_NOT_FOUND", "No default AI model is registered for " + capability); } - return modelId; + return route.getFirst(); + } + + public List route(AiCapability capability) { + List route = modelRoutes.get(capability); + if (route == null || route.isEmpty()) { + throw new BusinessException( + "AI_PROVIDER_ROUTE_NOT_FOUND", + "No AI provider route is registered for " + capability); + } + return route; } public RealtimeProvider getRealtimeProvider(String modelId) { return requiredProvider(realtimeProviders, modelId, AiCapability.REALTIME); } + /** + * Routes Realtime SDP exchange without exposing the selected adapter to the + * user. When a model is explicit, its provider must match the provider hint. + * When no model is explicit, the Registry route is authoritative so changing + * the configured route transparently replaces the Realtime model. + */ + public T routeRealtime( + ProviderType requestedProvider, + String requestedModel, + BiFunction operation) { + String model = AbstractAiProvider.normalizeModelId(requestedModel); + List models = model.isBlank() + ? route(AiCapability.REALTIME) + : List.of(model); + if (!model.isBlank()) { + RealtimeProvider provider = getRealtimeProvider(model); + if (requestedProvider != null && requestedProvider != provider.type()) { + throw new BusinessException( + "AI_PROVIDER_MODEL_MISMATCH", + "Requested provider " + requestedProvider + + " does not own realtime model " + model); + } + } + return invokeModels( + AiCapability.REALTIME, + models, + modelId -> operation.apply(modelId, getRealtimeProvider(modelId))); + } + public LlmProvider getLlmProvider(String modelId) { return requiredProvider(llmProviders, modelId, AiCapability.LLM); } @@ -112,99 +215,258 @@ public TranscriptionProvider getTranscriptionProvider(String modelId) { return requiredProvider(transcriptionProviders, modelId, AiCapability.TRANSCRIPTION); } - public RealtimeSdpExchangeResponse exchangeRealtimeSdp( - String modelId, - RealtimeSdpExchangeRequest request) { - if (request == null) { - throw new BusinessException( - "INVALID_SDP_REQUEST", - "Realtime SDP exchange request is required"); - } - String normalizedModelId = AbstractAiProvider.normalizeModelId(modelId); - if (request.model() != null - && !request.model().isBlank() - && !normalizedModelId.equals(AbstractAiProvider.normalizeModelId(request.model()))) { - throw new BusinessException( - "AI_MODEL_REQUEST_MISMATCH", - "Registry model " + modelId + " does not match request model " + request.model()); - } - RealtimeSdpExchangeRequest routedRequest = new RealtimeSdpExchangeRequest( - request.context(), - getModel(modelId).modelId(), - request.offerSdp(), - request.apiKey()); - return getRealtimeProvider(modelId).exchangeRealtimeSdp(routedRequest); + public String exchangeRealtimeSdp(String modelId, String offerSdp, String token) { + String registeredModelId = getModel(modelId).modelId(); + return getRealtimeProvider(registeredModelId) + .exchangeRealtimeSdp(registeredModelId, offerSdp, token); } - public SpeechAudioResponse generateSpeechAudio(String modelId, SpeechAudioRequest request) { - return getTtsProvider(modelId).generateSpeechAudio(request); + public String exchangeRealtimeSdp(String offerSdp, String token) { + return routeRealtime( + null, + null, + (modelId, provider) -> provider.exchangeRealtimeSdp( + modelId, + offerSdp, + token)); } - public LlmTaskResponse executeLlmTask(String modelId, LlmTaskRequest request) { - return getLlmProvider(modelId).executeLlmTask(request); + public Byte[] generateSpeechAudio(String modelId, String text, String token) { + return getTtsProvider(modelId).generateSpeechAudio(text, token); } - public AudioTranscriptionResponse convertAudioToText( - String modelId, - AudioTranscriptionRequest request) { - return getTranscriptionProvider(modelId).convertAudioToText(request); + public Byte[] generateSpeechAudio(String text, String token) { + return generateSpeechAudioRouted(text, token).response(); + } + + public RoutedResult generateSpeechAudioRouted(String text, String token) { + return invokeRouteWithResult( + AiCapability.TTS, + modelId -> generateSpeechAudio(modelId, text, token)); + } + + public String executeLlmTask(String modelId, String prompt, String token) { + return getLlmProvider(modelId).executeLlmTask(prompt, token); + } + + public String executeLlmTask(String prompt, String token) { + return executeLlmTaskRouted(prompt, token).response(); + } + + public RoutedResult executeLlmTaskRouted(String prompt, String token) { + return invokeRouteWithResult( + AiCapability.LLM, + modelId -> executeLlmTask(modelId, prompt, token)); } - public PronunciationEvaluationResponse evaluatePronunciation( + public String convertAudioToText(String modelId, Byte[] audio, String token) { + return getTranscriptionProvider(modelId).convertAudioToText(audio, token); + } + + public String convertAudioToText(Byte[] audio, String token) { + return convertAudioToTextRouted(audio, token).response(); + } + + public RoutedResult convertAudioToTextRouted(Byte[] audio, String token) { + return invokeRouteWithResult( + AiCapability.TRANSCRIPTION, + modelId -> convertAudioToText(modelId, audio, token)); + } + + public String evaluatePronunciation( String modelId, - PronunciationEvaluationRequest request) { - return getScoringProvider(modelId).evaluatePronunciation(request); + String text, + Byte[] audio, + String token) { + return getScoringProvider(modelId).evaluatePronunciation(text, audio, token); + } + + public String evaluatePronunciation(String text, Byte[] audio, String token) { + return evaluatePronunciationRouted(text, audio, token).response(); + } + + public RoutedResult evaluatePronunciationRouted( + String text, + Byte[] audio, + String token) { + return invokeRouteWithResult( + AiCapability.SCORING, + modelId -> evaluatePronunciation(modelId, text, audio, token)); } private Map buildModelDefinitions() { Map definitions = new LinkedHashMap<>(); - for (AiModelDefinition definition : MODEL_CATALOG) { - String modelId = AbstractAiProvider.normalizeModelId(definition.modelId()); - if (definitions.putIfAbsent(modelId, definition) != null) { - throw new IllegalStateException("Duplicate AI model definition: " + modelId); + for (AiCapability capability : AiCapability.values()) { + Map providers = providers(capability); + if (providers.isEmpty()) { + continue; + } + for (String modelId : orderedModelIds(capability, providers.keySet())) { + AiProvider provider = providers.get(modelId); + AiModelDefinition definition = new AiModelDefinition( + modelId, + provider.providerId(), + capability, + modelId.equals(defaultModel(capability))); + if (definitions.putIfAbsent(modelId, definition) != null) { + throw new IllegalStateException("Duplicate AI model definition: " + modelId); + } } } - return Map.copyOf(definitions); + return Collections.unmodifiableMap(new LinkedHashMap<>(definitions)); } - private Map buildDefaultModels() { - Map defaults = new EnumMap<>(AiCapability.class); - for (AiModelDefinition definition : MODEL_CATALOG) { - if (definition.defaultModel() - && defaults.putIfAbsent(definition.capability(), definition.modelId()) != null) { - throw new IllegalStateException( - "Duplicate default AI model for " + definition.capability()); + private Map> buildModelRoutes( + Map> configuredRoutes) { + Map> routes = new EnumMap<>(AiCapability.class); + Map> safeRoutes = configuredRoutes == null + ? Map.of() + : configuredRoutes; + for (AiCapability capability : AiCapability.values()) { + Map registered = providers(capability); + if (registered.isEmpty()) { + continue; + } + List configured = safeRoutes.get(capability); + List route = configured == null || configured.isEmpty() + ? defaultRoute(capability, registered) + : normalizeRoute(configured); + for (String modelId : route) { + if (!registered.containsKey(modelId)) { + throw new IllegalStateException( + "AI provider route references an unavailable " + + capability + " model: " + modelId); + } + } + routes.put(capability, route); + } + return Map.copyOf(routes); + } + + private List defaultRoute( + AiCapability capability, + Map registeredProviders) { + LinkedHashSet route = new LinkedHashSet<>(); + for (String modelId : DEFAULT_MODEL_ROUTES.getOrDefault(capability, List.of())) { + if (registeredProviders.containsKey(modelId)) { + route.add(modelId); } } - return Map.copyOf(defaults); + for (String providerId : DEFAULT_PROVIDER_ROUTES.getOrDefault(capability, List.of())) { + addProviderModelsIfAbsent(route, registeredProviders, providerId); + } + for (AiProvider provider : registeredProviders.values()) { + addProviderModelsIfAbsent(route, registeredProviders, provider.providerId()); + } + return List.copyOf(route); + } + + private void addProviderModelsIfAbsent( + LinkedHashSet route, + Map registeredProviders, + String providerId) { + boolean alreadyRouted = route.stream() + .map(registeredProviders::get) + .anyMatch(provider -> provider != null + && provider.providerId().equals(providerId)); + if (alreadyRouted) { + return; + } + registeredProviders.forEach((modelId, provider) -> { + if (provider.providerId().equals(providerId)) { + route.add(modelId); + } + }); } private Map registerProviders( List providers, AiCapability capability) { Map registered = new LinkedHashMap<>(); - for (T provider : providers) { + for (T provider : providers == null ? List.of() : providers) { if (provider.capability() != capability) { throw new IllegalStateException( "Provider capability mismatch: " + provider.getClass().getName()); } for (String modelId : provider.supportedModels()) { - AiModelDefinition definition = modelDefinitions.get(modelId); - if (definition == null) { - throw new IllegalStateException( - "Provider references an unregistered AI model: " + modelId); - } - if (definition.capability() != capability || definition.providerType() != provider.type()) { - throw new IllegalStateException( - "Provider does not match AI model definition: " + modelId); - } if (registered.putIfAbsent(modelId, provider) != null) { throw new IllegalStateException( "Duplicate AI provider registration for model " + modelId); } } } - return Map.copyOf(registered); + return Collections.unmodifiableMap(new LinkedHashMap<>(registered)); + } + + private RoutedResult invokeRouteWithResult( + AiCapability capability, + Function operation) { + return invokeModels( + capability, + route(capability), + modelId -> { + AiModelDefinition definition = getModel(modelId); + return new RoutedResult<>( + definition.modelId(), + definition.providerId(), + capability, + operation.apply(modelId)); + }); + } + + private T invokeModels( + AiCapability capability, + List models, + Function operation) { + BusinessException lastFailure = null; + for (int index = 0; index < models.size(); index++) { + String modelId = models.get(index); + try { + T response = operation.apply(modelId); + AiModelDefinition definition = getModel(modelId); + LOGGER.info( + "AI provider selected capability={} model={} provider={}", + capability, + definition.modelId(), + definition.providerId()); + return response; + } + catch (BusinessException exception) { + if (!shouldFailOver(exception)) { + throw exception; + } + lastFailure = exception; + if (index + 1 < models.size()) { + LOGGER.warn( + "AI provider failover capability={} failedModel={} errorCode={} nextModel={}", + capability, + modelId, + exception.code(), + models.get(index + 1)); + } + } + } + if (lastFailure != null) { + throw lastFailure; + } + throw new BusinessException( + "AI_PROVIDER_ROUTE_EXHAUSTED", + "No AI provider completed the " + capability + " request"); + } + + private boolean shouldFailOver(BusinessException exception) { + Boolean classifiedRetryable = AbstractAiProvider.retryable(exception); + if (classifiedRetryable != null) { + return classifiedRetryable; + } + String code = exception.code() == null ? "" : exception.code(); + return !code.startsWith("INVALID_") + && !code.startsWith("UNSUPPORTED_") + && !code.endsWith("_INTERRUPTED") + && !"TTS_TEXT_TOO_LONG".equals(code) + && !"PRONUNCIATION_AUDIO_TOO_LARGE".equals(code) + && !"PRONUNCIATION_AUDIO_TOO_LONG".equals(code) + && !"PRONUNCIATION_REFERENCE_TOO_LONG".equals(code); } private T requiredProvider( @@ -226,19 +488,39 @@ private T requiredProvider( return provider; } - private void validateCatalogRegistrations() { - for (AiModelDefinition definition : MODEL_CATALOG) { - Map providers = switch (definition.capability()) { - case REALTIME -> realtimeProviders; - case LLM -> llmProviders; - case SCORING -> scoringProviders; - case TTS -> ttsProviders; - case TRANSCRIPTION -> transcriptionProviders; - }; - if (!providers.containsKey(AbstractAiProvider.normalizeModelId(definition.modelId()))) { - throw new IllegalStateException( - "No provider implementation registered for AI model " + definition.modelId()); + private Map providers(AiCapability capability) { + return switch (capability) { + case REALTIME -> realtimeProviders; + case LLM -> llmProviders; + case SCORING -> scoringProviders; + case TTS -> ttsProviders; + case TRANSCRIPTION -> transcriptionProviders; + }; + } + + private List orderedModelIds( + AiCapability capability, + Set registeredModelIds) { + LinkedHashSet ordered = new LinkedHashSet<>(route(capability)); + ordered.addAll(registeredModelIds); + return List.copyOf(ordered); + } + + private static List parseRoute(String value) { + if (value == null || value.isBlank()) { + return List.of(); + } + return normalizeRoute(List.of(value.split(","))); + } + + private static List normalizeRoute(List modelIds) { + LinkedHashSet normalized = new LinkedHashSet<>(); + for (String modelId : modelIds) { + String value = AbstractAiProvider.normalizeModelId(modelId); + if (!value.isBlank()) { + normalized.add(value); } } + return List.copyOf(normalized); } } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java index 434a6d6..403b86b 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/LlmProvider.java @@ -1,21 +1,16 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; public abstract class LlmProvider extends AbstractAiProvider { - protected LlmProvider(ProviderType providerType, Set supportedModels) { - super(providerType, supportedModels); + protected LlmProvider(String providerId, Set supportedModels) { + super(providerId, supportedModels); } @Override public final AiCapability capability() { return AiCapability.LLM; } - - public abstract LlmTaskResponse executeLlmTask(LlmTaskRequest request); } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/RealtimeProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/RealtimeProvider.java index 097cad3..86bf01e 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/RealtimeProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/RealtimeProvider.java @@ -1,15 +1,20 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; import com.unispeaking.domain.vo.ai.AiCapability; import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; public abstract class RealtimeProvider extends AbstractAiProvider { + private final ProviderType providerType; + protected RealtimeProvider(ProviderType providerType, Set supportedModels) { - super(providerType, supportedModels); + super(providerType == null ? null : providerType.name(), supportedModels); + this.providerType = providerType; + } + + public final ProviderType type() { + return providerType; } @Override @@ -17,6 +22,16 @@ public final AiCapability capability() { return AiCapability.REALTIME; } - public abstract RealtimeSdpExchangeResponse exchangeRealtimeSdp( - RealtimeSdpExchangeRequest request); + @Override + public final String exchangeRealtimeSdp(String offerSdp, String token) { + return exchangeRealtimeSdp(null, offerSdp, token); + } + + /** + * Exchanges a WebRTC Offer SDP for the provider's Answer SDP. + */ + public abstract String exchangeRealtimeSdp( + String modelId, + String offerSdp, + String token); } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/ScoringProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/ScoringProvider.java index 2185b36..303cb04 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/ScoringProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/ScoringProvider.java @@ -1,22 +1,16 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; public abstract class ScoringProvider extends AbstractAiProvider { - protected ScoringProvider(ProviderType providerType, Set supportedModels) { - super(providerType, supportedModels); + protected ScoringProvider(String providerId, Set supportedModels) { + super(providerId, supportedModels); } @Override public final AiCapability capability() { return AiCapability.SCORING; } - - public abstract PronunciationEvaluationResponse evaluatePronunciation( - PronunciationEvaluationRequest request); } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TranscriptionProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TranscriptionProvider.java index 3e8f084..ac003ff 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TranscriptionProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TranscriptionProvider.java @@ -1,22 +1,16 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.AudioTranscriptionRequest; -import com.unispeaking.domain.dto.ai.AudioTranscriptionResponse; import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; public abstract class TranscriptionProvider extends AbstractAiProvider { - protected TranscriptionProvider(ProviderType providerType, Set supportedModels) { - super(providerType, supportedModels); + protected TranscriptionProvider(String providerId, Set supportedModels) { + super(providerId, supportedModels); } @Override public final AiCapability capability() { return AiCapability.TRANSCRIPTION; } - - public abstract AudioTranscriptionResponse convertAudioToText( - AudioTranscriptionRequest request); } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TtsProvider.java b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TtsProvider.java index ea33d1c..e3bf514 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TtsProvider.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/provider/TtsProvider.java @@ -1,21 +1,16 @@ package com.unispeaking.provider; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.realtime.ProviderType; import java.util.Set; public abstract class TtsProvider extends AbstractAiProvider { - protected TtsProvider(ProviderType providerType, Set supportedModels) { - super(providerType, supportedModels); + protected TtsProvider(String providerId, Set supportedModels) { + super(providerId, supportedModels); } @Override public final AiCapability capability() { return AiCapability.TTS; } - - public abstract SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request); } diff --git a/backend/unispeaking-server/src/main/java/com/unispeaking/service/realtime/impl/RealtimeConnectionServiceImpl.java b/backend/unispeaking-server/src/main/java/com/unispeaking/service/realtime/impl/RealtimeConnectionServiceImpl.java index 9bd9ae4..2b76efa 100644 --- a/backend/unispeaking-server/src/main/java/com/unispeaking/service/realtime/impl/RealtimeConnectionServiceImpl.java +++ b/backend/unispeaking-server/src/main/java/com/unispeaking/service/realtime/impl/RealtimeConnectionServiceImpl.java @@ -1,15 +1,11 @@ package com.unispeaking.service.realtime.impl; import com.unispeaking.common.logging.RealtimeFlowLog; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.vo.ai.AiCapability; -import com.unispeaking.domain.vo.ai.AiCallContext; import com.unispeaking.domain.vo.prompt.SessionPrompt; import com.unispeaking.domain.vo.realtime.ProviderType; import com.unispeaking.domain.vo.realtime.RealtimeConnectionResult; import com.unispeaking.domain.po.session.AbstractSceneSession; import com.unispeaking.domain.dto.command.StartCommand; -import com.unispeaking.exception.BusinessException; import com.unispeaking.provider.AiProviderRegistry; import com.unispeaking.service.realtime.RealtimeConnectionService; import com.unispeaking.service.realtime.RealtimeCredentialService; @@ -31,31 +27,31 @@ public RealtimeConnectionServiceImpl( @Override public RealtimeConnectionResult connect( ProviderType type, AbstractSceneSession session, SessionPrompt prompt, StartCommand command) { - String model = command.model() == null || command.model().isBlank() - ? providerRegistry.defaultModel(AiCapability.REALTIME) - : command.model().trim(); - var provider = providerRegistry.getRealtimeProvider(model); - if (type != null && type != provider.type()) { - throw new BusinessException( - "AI_PROVIDER_MODEL_MISMATCH", - "Requested provider " + type + " does not own realtime model " + model); - } - var credential = credentialService.getCredential(provider.type()); - var exchangeResult = provider.exchangeRealtimeSdp( - new RealtimeSdpExchangeRequest( - new AiCallContext(session.getUserId(), session.getId()), - model, - command.offerSdp(), - credential.bearerToken())); - RealtimeFlowLog.info( - "flow.3.sdp.completed localSessionId={} provider={} model={} credentialExpiresAt={}", - session.getId(), - provider.type(), - model, - credential.expiresAt()); + RealtimeAttempt attempt = providerRegistry.routeRealtime( + type, + command.model(), + (model, provider) -> { + var credential = credentialService.getCredential(provider.type()); + String answerSdp = provider.exchangeRealtimeSdp( + model, + command.offerSdp(), + credential.bearerToken()); + RealtimeFlowLog.info( + "flow.3.sdp.completed localSessionId={} provider={} model={} credentialExpiresAt={}", + session.getId(), + provider.type(), + model, + credential.expiresAt()); + return new RealtimeAttempt(answerSdp, credential.expiresAt()); + }); return new RealtimeConnectionResult( - exchangeResult.aiCallId(), - exchangeResult.answerSdp(), - credential.expiresAt()); + null, + attempt.answerSdp(), + attempt.credentialExpiresAt()); + } + + private record RealtimeAttempt( + String answerSdp, + java.time.Instant credentialExpiresAt) { } } diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java index 619f9d7..8ce8eab 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/infrastructure/ai/qwen/QwenRealtimeProviderTest.java @@ -1,9 +1,16 @@ package com.unispeaking.infrastructure.ai.qwen; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.vo.ai.AiCallContext; +import com.unispeaking.exception.BusinessException; +import com.unispeaking.infrastructure.ai.aliyun.AliyunTtsProvider; +import com.unispeaking.infrastructure.ai.deepseek.DeepSeekLlmProvider; +import com.unispeaking.infrastructure.ai.doubao.DoubaoAsrProvider; +import com.unispeaking.infrastructure.ai.iflytek.IflytekScoringProvider; +import com.unispeaking.infrastructure.ai.minimax.MiniMaxTtsProvider; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.Authenticator; @@ -15,23 +22,132 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLSession; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; class QwenRealtimeProviderTest { + @Test + void registersTheActuallyConfiguredModelForEveryReplaceableAdapter() { + RecordingHttpClient httpClient = new RecordingHttpClient(); + ObjectMapper objectMapper = new ObjectMapper(); + + assertEquals( + Set.of("qwen-custom-llm"), + new QwenLlmProvider( + httpClient, + objectMapper, + "key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen-custom-llm", + Duration.ofSeconds(20), + 1_048_576) + .supportedModels()); + assertEquals( + Set.of("deepseek-custom-llm"), + new DeepSeekLlmProvider( + httpClient, + objectMapper, + "key", + URI.create("https://api.deepseek.com/chat/completions"), + "deepseek-custom-llm", + Duration.ofSeconds(20), + 1_048_576) + .supportedModels()); + assertEquals( + Set.of("qwen-custom-asr"), + new QwenAsrProvider( + httpClient, + objectMapper, + "key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen-custom-asr", + Duration.ofSeconds(20), + 7_340_032, + 1_048_576) + .supportedModels()); + assertEquals( + Set.of("doubao-custom-asr"), + new DoubaoAsrProvider( + httpClient, + objectMapper, + "key", + "", + "", + "unispeaking", + URI.create("https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash"), + "doubao-custom-asr", + Duration.ofSeconds(20), + 20_971_520, + 4_194_304) + .supportedModels()); + assertEquals( + Set.of("qwen-custom-tts"), + new QwenTtsProvider( + httpClient, + objectMapper, + "key", + URI.create( + "https://dashscope.aliyuncs.com/api/v1/services/aigc/" + + "multimodal-generation/generation"), + "qwen-custom-tts", + "Cherry", + "English", + Duration.ofSeconds(20), + 10_485_760) + .supportedModels()); + assertEquals( + Set.of("aliyun-custom-tts"), + new AliyunTtsProvider( + httpClient, + objectMapper, + "key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer"), + "aliyun-custom-tts", + "loongemily_v3", + "wav", + 24_000, + Duration.ofSeconds(20), + 10_485_760) + .supportedModels()); + assertEquals( + Set.of("minimax-custom-tts"), + new MiniMaxTtsProvider( + httpClient, + objectMapper, + "key", + URI.create("https://api.minimaxi.com/v1/t2a_v2"), + "minimax-custom-tts", + "male-qn-qingse", + "wav", + 32_000, + 128_000, + Duration.ofSeconds(20), + 10_485_760) + .supportedModels()); + } + @Test void exchangesOfferAndAnswerSdpWithTheTemporaryBearerCredential() throws IOException, InterruptedException { - RecordingHttpClient httpClient = new RecordingHttpClient("answer-sdp"); + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse(200, "answer-sdp")); RealtimeProperties properties = new RealtimeProperties( "", "workspace-123", @@ -44,14 +160,12 @@ void exchangesOfferAndAnswerSdpWithTheTemporaryBearerCredential() 1_048_576); properties.validate(); QwenRealtimeProvider provider = new QwenRealtimeProvider(httpClient, properties); - var result = provider.exchangeRealtimeSdp( - new RealtimeSdpExchangeRequest( - new AiCallContext("user-1", "session-1"), - "qwen3.5-omni-flash-realtime", - "offer-sdp", - "temporary-token")); - - HttpRequest request = httpClient.request; + String result = provider.exchangeRealtimeSdp( + "qwen3.5-omni-flash-realtime", + "offer-sdp", + "temporary-token"); + + HttpRequest request = httpClient.requests.getFirst(); assertEquals( "https://workspace-123.cn-beijing.maas.aliyuncs.com/api/v1/webrtc/realtime?model=qwen3.5-omni-flash-realtime", request.uri().toString()); @@ -60,7 +174,608 @@ void exchangesOfferAndAnswerSdpWithTheTemporaryBearerCredential() assertEquals("application/sdp", request.headers().firstValue("Content-Type").orElseThrow()); assertEquals("offer-sdp", readBody(request)); - assertEquals("answer-sdp", result.answerSdp()); + assertEquals("answer-sdp", result); + } + + @Test + void executesQwenLlmTaskWithTheServerConfiguredCredential() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + {"choices":[{"message":{"content":"{\\"answer\\":\\"ok\\"}"}}]} + """))); + QwenLlmProvider provider = new QwenLlmProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 1_048_576); + + String response = provider.executeLlmTask("Return JSON.", null); + + assertEquals("{\"answer\":\"ok\"}", response); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals("Bearer dashscope-key", + request.headers().firstValue("Authorization").orElseThrow()); + String body = readBody(request); + assertTrue(body.contains("\"model\":\"qwen3.5-plus\"")); + assertTrue(body.contains("\"content\":\"Return JSON.\"")); + assertTrue(body.contains("\"enable_thinking\":false")); + assertFalse(body.contains("dashscope-key")); + assertFalse(httpClient.bodyCompletedOnSubscribe); + } + + @Test + void mapsMalformedQwenResponseToABusinessError() { + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse(200, utf8("not-json"))); + QwenLlmProvider provider = new QwenLlmProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 1_048_576); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.executeLlmTask("Return JSON.", null)); + + assertEquals("QWEN_LLM_RESPONSE_INVALID", exception.code()); + } + + @Test + void rejectsAnUntrustedQwenEndpointBeforeSendingCredentials() { + RecordingHttpClient httpClient = new RecordingHttpClient(); + QwenLlmProvider provider = new QwenLlmProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://evil.example/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 1_048_576); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.executeLlmTask("Return JSON.", null)); + + assertEquals("QWEN_LLM_ENDPOINT_INVALID", exception.code()); + assertTrue(httpClient.requests.isEmpty()); + } + + @Test + void rejectsAnOversizedQwenResponseWhileReadingIt() { + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse(200, new byte[11])); + QwenLlmProvider provider = new QwenLlmProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3.5-plus", + Duration.ofSeconds(20), + 10); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.executeLlmTask("Return JSON.", null)); + + assertEquals("QWEN_LLM_RESPONSE_TOO_LARGE", exception.code()); + } + + @Test + void generatesAliyunSpeechAndDownloadsTheReturnedAudio() { + byte[] audio = new byte[] {1, 2, 3, 4}; + String audioUrl = "http://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/test/audio%2Bfile.mp3?Expires=1&Signature=a%2Fb%3D"; + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse( + 200, + utf8(""" + {"request_id":"req-1","output":{"finish_reason":"stop","audio":{"url":"%s"}}} + """.formatted(audioUrl))), + new QueuedResponse(200, audio)); + AliyunTtsProvider provider = new AliyunTtsProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer"), + "cosyvoice-v3-flash", + "loongemily_v3", + "mp3", + 24_000, + Duration.ofSeconds(20), + 1_048_576); + + Byte[] response = provider.generateSpeechAudio("Practice makes progress.", null); + + assertEquals(List.of((byte) 1, (byte) 2, (byte) 3, (byte) 4), + List.of(response)); + assertEquals(2, httpClient.requests.size()); + String requestBody = readBody(httpClient.requests.getFirst()); + assertTrue(requestBody.contains("\"model\":\"cosyvoice-v3-flash\"")); + assertTrue(requestBody.contains("\"voice\":\"loongemily_v3\"")); + assertTrue(requestBody.contains("\"language_hints\":[\"en\"]")); + assertEquals( + "https://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/test/audio%2Bfile.mp3?Expires=1&Signature=a%2Fb%3D", + httpClient.requests.get(1).uri().toString()); + assertFalse(httpClient.bodyCompletedOnSubscribe); + } + + @Test + void generatesQwenSpeechWithTheServerConfiguredCredential() { + byte[] wav = wavWithSampleRate(24_000); + String audioUrl = + "https://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/test/qwen.wav"; + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse( + 200, + utf8(""" + {"output":{"finish_reason":"stop","audio":{"url":"%s"}}} + """.formatted(audioUrl))), + new QueuedResponse(200, wav)); + QwenTtsProvider provider = new QwenTtsProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create( + "https://dashscope.aliyuncs.com/api/v1/services/aigc/" + + "multimodal-generation/generation"), + "qwen3-tts-flash", + "Cherry", + "English", + Duration.ofSeconds(20), + 1_048_576); + + Byte[] response = provider.generateSpeechAudio( + "Practice makes progress.", + "must-not-be-used"); + + assertEquals(List.of(box(wav)), List.of(response)); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals( + "Bearer dashscope-key", + request.headers().firstValue("Authorization").orElseThrow()); + assertEquals( + "https://dashscope.aliyuncs.com/api/v1/services/aigc/" + + "multimodal-generation/generation", + request.uri().toString()); + String body = readBody(request); + assertTrue(body.contains("\"model\":\"qwen3-tts-flash\"")); + assertTrue(body.contains("\"text\":\"Practice makes progress.\"")); + assertTrue(body.contains("\"voice\":\"Cherry\"")); + assertTrue(body.contains("\"language_type\":\"English\"")); + assertFalse(body.contains("dashscope-key")); + assertFalse(body.contains("must-not-be-used")); + } + + @Test + void rejectsOversizedAliyunAudioWhileReadingIt() { + String audioUrl = "https://dashscope-result-bj.oss-cn-beijing.aliyuncs.com/test/audio.mp3"; + RecordingHttpClient httpClient = new RecordingHttpClient( + new QueuedResponse( + 200, + utf8(""" + {"output":{"audio":{"url":"%s"}}} + """.formatted(audioUrl))), + new QueuedResponse(200, new byte[4])); + AliyunTtsProvider provider = new AliyunTtsProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/api/v1/services/audio/tts/SpeechSynthesizer"), + "cosyvoice-v3-flash", + "loongemily_v3", + "mp3", + 24_000, + Duration.ofSeconds(20), + 3); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.generateSpeechAudio("Practice makes progress.", null)); + + assertEquals("ALIYUN_TTS_AUDIO_TOO_LARGE", exception.code()); + } + + @Test + void streamsWavToIflytekWithTheServerConfiguredCredential() throws Exception { + String xml = """ + + + + + + + """; + String finalMessage = """ + {"code":0,"message":"success","data":{"status":2,"data":"%s"}} + """.formatted(Base64.getEncoder().encodeToString(xml.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + RecordingWebSocketConnector connector = new RecordingWebSocketConnector(finalMessage); + IflytekScoringProvider provider = new IflytekScoringProvider( + new ObjectMapper(), + connector, + "app-id", + "api-key", + "api-secret", + URI.create("wss://ise-api.xfyun.cn/v2/open-ise"), + "en_vip", + "read_sentence", + Duration.ofSeconds(2), + 1_048_576, + Duration.ZERO); + + String response = provider.evaluatePronunciation( + "Practice makes progress.", + box(wavWithSampleRate(16_000)), + null); + + assertEquals(finalMessage, response); + assertTrue(connector.uri.getQuery().contains("authorization=")); + assertTrue(connector.uri.getQuery().contains("host=ise-api.xfyun.cn")); + assertTrue(connector.frames.stream().anyMatch(frame -> frame.contains("\"cmd\":\"ssb\""))); + assertTrue(connector.frames.stream().anyMatch(frame -> frame.contains("\"aus\":1"))); + assertTrue(connector.frames.stream().anyMatch(frame -> frame.contains("\"aus\":4"))); + assertTrue(connector.frames.stream().noneMatch(frame -> frame.contains("api-secret"))); + JsonNode startFrame = new ObjectMapper().readTree(connector.frames.getFirst()); + assertEquals( + "\uFEFF[content]\nPractice makes progress.", + startFrame.path("business").path("text").asString()); + assertTrue(startFrame.path("business").path("ttp_skip").asBoolean(false)); + } + + @Test + void rejectsAnUntrustedIflytekEndpointBeforeConnecting() { + RecordingWebSocketConnector connector = new RecordingWebSocketConnector("{}"); + IflytekScoringProvider provider = iflytekProvider( + connector, + URI.create("wss://evil.example/v2/open-ise"), + Duration.ofSeconds(2)); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.evaluatePronunciation( + "Practice makes progress.", + box(wavWithSampleRate(16_000)), + null)); + + assertEquals("IFLYTEK_ISE_ENDPOINT_INVALID", exception.code()); + assertEquals(null, connector.uri); + } + + @Test + void appliesTheIflytekDeadlineToAudioFrameSends() { + String xml = ""; + String finalMessage = """ + {"code":0,"data":{"status":2,"data":"%s"}} + """.formatted(Base64.getEncoder().encodeToString( + xml.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + RecordingWebSocketConnector connector = + new RecordingWebSocketConnector(finalMessage, true); + IflytekScoringProvider provider = iflytekProvider( + connector, + URI.create("wss://ise-api.xfyun.cn/v2/open-ise"), + Duration.ofMillis(10)); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.evaluatePronunciation( + "Practice makes progress.", + box(wavWithSampleRate(16_000)), + null)); + + assertEquals("IFLYTEK_ISE_TIMEOUT", exception.code()); + } + + @Test + void rejectsWavAudioThatDoesNotMatchTheIflytekPcmContract() { + RecordingWebSocketConnector connector = new RecordingWebSocketConnector("{}"); + IflytekScoringProvider provider = iflytekProvider( + connector, + URI.create("wss://ise-api.xfyun.cn/v2/open-ise"), + Duration.ofSeconds(2)); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.evaluatePronunciation( + "Practice makes progress.", + box(wavWithSampleRate(44_100)), + null)); + + assertEquals("INVALID_PRONUNCIATION_WAV", exception.code()); + assertEquals(null, connector.uri); + } + + @Test + void returnsTheFinalIflytekResponseWithoutParsingScores() { + String xml = ""; + String finalMessage = """ + {"code":0,"data":{"status":2,"data":"%s"}} + """.formatted(Base64.getEncoder().encodeToString( + xml.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + RecordingWebSocketConnector connector = new RecordingWebSocketConnector(finalMessage); + IflytekScoringProvider provider = iflytekProvider( + connector, + URI.create("wss://ise-api.xfyun.cn/v2/open-ise"), + Duration.ofSeconds(2)); + + String response = provider.evaluatePronunciation( + "Practice makes progress.", + box(wavWithSampleRate(16_000)), + null); + + assertEquals(finalMessage, response); + } + + @Test + void executesDeepSeekLlmTaskWithTheServerConfiguredCredential() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + {"choices":[{"message":{"content":"{\\"answer\\":\\"deepseek-ok\\"}"}}]} + """))); + DeepSeekLlmProvider provider = new DeepSeekLlmProvider( + httpClient, + new ObjectMapper(), + "deepseek-key", + URI.create("https://api.deepseek.com/chat/completions"), + "deepseek-v4-flash", + Duration.ofSeconds(20), + 1_048_576); + + String response = provider.executeLlmTask("Return JSON.", null); + + assertEquals("{\"answer\":\"deepseek-ok\"}", response); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals("Bearer deepseek-key", + request.headers().firstValue("Authorization").orElseThrow()); + String body = readBody(request); + assertTrue(body.contains("\"model\":\"deepseek-v4-flash\"")); + assertTrue(body.contains("\"content\":\"Return JSON.\"")); + assertTrue(body.contains("\"thinking\":{\"type\":\"disabled\"}")); + assertFalse(body.contains("deepseek-key")); + assertFalse(httpClient.bodyCompletedOnSubscribe); + } + + @Test + void rejectsAnUntrustedDeepSeekEndpointBeforeSendingCredentials() { + RecordingHttpClient httpClient = new RecordingHttpClient(); + DeepSeekLlmProvider provider = new DeepSeekLlmProvider( + httpClient, + new ObjectMapper(), + "deepseek-key", + URI.create("https://evil.example/chat/completions"), + "deepseek-v4-flash", + Duration.ofSeconds(20), + 1_048_576); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.executeLlmTask("Return JSON.", null)); + + assertEquals("DEEPSEEK_LLM_ENDPOINT_INVALID", exception.code()); + assertTrue(httpClient.requests.isEmpty()); + } + + @Test + void generatesMiniMaxSpeechFromHexAudio() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + { + "data":{"audio":"494433040000","status":2}, + "base_resp":{"status_code":0,"status_msg":"success"} + } + """))); + MiniMaxTtsProvider provider = new MiniMaxTtsProvider( + httpClient, + new ObjectMapper(), + "minimax-key", + URI.create("https://api.minimaxi.com/v1/t2a_v2"), + "speech-2.8-hd", + "male-qn-qingse", + "mp3", + 32_000, + 128_000, + Duration.ofSeconds(20), + 1_048_576); + + Byte[] response = provider.generateSpeechAudio("Practice makes progress.", null); + + assertEquals(List.of((byte) 0x49, (byte) 0x44, (byte) 0x33, (byte) 0x04, (byte) 0, (byte) 0), + List.of(response)); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals("Bearer minimax-key", + request.headers().firstValue("Authorization").orElseThrow()); + String body = readBody(request); + assertTrue(body.contains("\"model\":\"speech-2.8-hd\"")); + assertTrue(body.contains("\"voice_id\":\"male-qn-qingse\"")); + assertTrue(body.contains("\"sample_rate\":32000")); + assertTrue(body.contains("\"bitrate\":128000")); + assertTrue(body.contains("\"output_format\":\"hex\"")); + assertFalse(body.contains("minimax-key")); + assertFalse(httpClient.bodyCompletedOnSubscribe); + } + + @Test + void mapsMiniMaxProviderErrorsToABusinessError() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + { + "data":null, + "trace_id":"trace-123", + "base_resp":{"status_code":1008,"status_msg":"insufficient balance"} + } + """))); + MiniMaxTtsProvider provider = new MiniMaxTtsProvider( + httpClient, + new ObjectMapper(), + "minimax-key", + URI.create("https://api.minimaxi.com/v1/t2a_v2"), + "speech-2.8-hd", + "male-qn-qingse", + "mp3", + 32_000, + 128_000, + Duration.ofSeconds(20), + 1_048_576); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.generateSpeechAudio("Practice makes progress.", null)); + + assertEquals("MINIMAX_TTS_REQUEST_FAILED", exception.code()); + assertTrue(exception.getMessage().contains("1008")); + assertFalse(exception.getMessage().contains("minimax-key")); + } + + @Test + void transcribesAudioWithQwenAsrUsingAnInlineDataUrl() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + {"choices":[{"message":{"content":"Practice makes progress."}}]} + """))); + QwenAsrProvider provider = new QwenAsrProvider( + httpClient, + new ObjectMapper(), + "dashscope-key", + URI.create("https://workspace-123.cn-beijing.maas.aliyuncs.com/compatible-mode/v1/chat/completions"), + "qwen3-asr-flash", + Duration.ofSeconds(20), + 7_340_032, + 1_048_576); + + String response = provider.convertAudioToText( + box(new byte[] {1, 2, 3}), + null); + + assertEquals("Practice makes progress.", response); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals("Bearer dashscope-key", + request.headers().firstValue("Authorization").orElseThrow()); + String body = readBody(request); + assertTrue(body.contains("\"model\":\"qwen3-asr-flash\"")); + assertTrue(body.contains("\"type\":\"input_audio\"")); + assertTrue(body.contains("data:audio/wav;base64,AQID")); + assertTrue(body.contains("\"enable_itn\":true")); + assertFalse(body.contains("dashscope-key")); + } + + @Test + void transcribesAudioWithDoubaoBigAsr() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8(""" + {"result":{"text":"Practice makes progress."}} + """), + Map.of("X-Api-Status-Code", List.of("20000000")))); + DoubaoAsrProvider provider = new DoubaoAsrProvider( + httpClient, + new ObjectMapper(), + "doubao-api-key", + "", + "", + "unispeaking", + URI.create("https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash"), + "volc.bigasr.auc_turbo", + Duration.ofSeconds(20), + 20_971_520, + 4_194_304); + + String response = provider.convertAudioToText( + box(new byte[] {1, 2, 3}), + null); + + assertEquals("Practice makes progress.", response); + HttpRequest request = httpClient.requests.getFirst(); + assertEquals("doubao-api-key", + request.headers().firstValue("X-Api-Key").orElseThrow()); + assertEquals("volc.bigasr.auc_turbo", + request.headers().firstValue("X-Api-Resource-Id").orElseThrow()); + assertEquals("-1", + request.headers().firstValue("X-Api-Sequence").orElseThrow()); + assertFalse(request.headers().firstValue("X-Api-Request-Id").orElseThrow().isBlank()); + String body = readBody(request); + assertTrue(body.contains("\"uid\":\"unispeaking\"")); + assertTrue(body.contains("\"data\":\"AQID\"")); + assertTrue(body.contains("\"model_name\":\"bigmodel\"")); + assertFalse(body.contains("doubao-api-key")); + } + + @Test + void mapsDoubaoProviderStatusToARetryableBusinessError() { + RecordingHttpClient httpClient = new RecordingHttpClient(new QueuedResponse( + 200, + utf8("{}"), + Map.of("X-Api-Status-Code", List.of("55000031")))); + DoubaoAsrProvider provider = new DoubaoAsrProvider( + httpClient, + new ObjectMapper(), + "doubao-api-key", + "", + "", + "unispeaking", + URI.create("https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash"), + "volc.bigasr.auc_turbo", + Duration.ofSeconds(20), + 20_971_520, + 4_194_304); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> provider.convertAudioToText( + box(new byte[] {1, 2, 3}), + null)); + + assertEquals("DOUBAO_ASR_REQUEST_FAILED", exception.code()); + assertTrue(exception.getMessage().contains("55000031")); + } + + private IflytekScoringProvider iflytekProvider( + RecordingWebSocketConnector connector, + URI endpoint, + Duration readTimeout) { + return new IflytekScoringProvider( + new ObjectMapper(), + connector, + "app-id", + "api-key", + "api-secret", + endpoint, + "en_vip", + "read_sentence", + readTimeout, + 1_048_576, + Duration.ZERO); + } + + private static byte[] utf8(String text) { + return text.getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + private byte[] wavWithSampleRate(int sampleRate) { + ByteBuffer wav = ByteBuffer.allocate(46).order(ByteOrder.LITTLE_ENDIAN); + wav.put("RIFF".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + wav.putInt(38); + wav.put("WAVE".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + wav.put("fmt ".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + wav.putInt(16); + wav.putShort((short) 1); + wav.putShort((short) 1); + wav.putInt(sampleRate); + wav.putInt(sampleRate * 2); + wav.putShort((short) 2); + wav.putShort((short) 16); + wav.put("data".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + wav.putInt(2); + wav.putShort((short) 0); + return wav.array(); } private String readBody(HttpRequest request) { @@ -93,13 +808,30 @@ public void onComplete() { return bytes.toString(java.nio.charset.StandardCharsets.UTF_8); } + private List toByteList(byte[] bytes) { + List result = new ArrayList<>(); + for (byte value : bytes) { + result.add(value); + } + return result; + } + + private Byte[] box(byte[] bytes) { + Byte[] result = new Byte[bytes.length]; + for (int index = 0; index < bytes.length; index++) { + result[index] = bytes[index]; + } + return result; + } + private static final class RecordingHttpClient extends HttpClient { - private final String answerSdp; - private HttpRequest request; + private final List responses; + private final List requests = new ArrayList<>(); + private boolean bodyCompletedOnSubscribe; - private RecordingHttpClient(String answerSdp) { - this.answerSdp = answerSdp; + private RecordingHttpClient(QueuedResponse... responses) { + this.responses = new ArrayList<>(List.of(responses)); } @Override @@ -107,15 +839,42 @@ private RecordingHttpClient(String answerSdp) { public HttpResponse send( HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) { - this.request = request; - return (HttpResponse) new StringHttpResponse(request, answerSdp); + requests.add(request); + QueuedResponse response = responses.removeFirst(); + byte[] rawBody = response.body() instanceof byte[] bytes + ? bytes + : response.body().toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + HttpResponse.BodySubscriber subscriber = responseBodyHandler.apply( + new RecordedResponseInfo(response.statusCode(), response.headers())); + subscriber.onSubscribe(new Flow.Subscription() { + @Override public void request(long n) { } + @Override public void cancel() { } + }); + bodyCompletedOnSubscribe |= subscriber.getBody().toCompletableFuture().isDone(); + subscriber.onNext(List.of(ByteBuffer.wrap(rawBody))); + subscriber.onComplete(); + T handledBody; + try { + handledBody = subscriber.getBody().toCompletableFuture().join(); + } + catch (java.util.concurrent.CompletionException exception) { + if (exception.getCause() instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw exception; + } + return (HttpResponse) new RecordedHttpResponse<>( + request, + response.statusCode(), + handledBody, + response.headers()); } @Override public CompletableFuture> sendAsync( HttpRequest request, HttpResponse.BodyHandler responseBodyHandler) { - throw new UnsupportedOperationException(); + return CompletableFuture.completedFuture(send(request, responseBodyHandler)); } @Override @@ -123,7 +882,7 @@ public CompletableFuture> sendAsync( HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, HttpResponse.PushPromiseHandler pushPromiseHandler) { - throw new UnsupportedOperationException(); + return CompletableFuture.completedFuture(send(request, responseBodyHandler)); } @Override public Optional cookieHandler() { return Optional.empty(); } @@ -137,16 +896,143 @@ public CompletableFuture> sendAsync( @Override public Optional executor() { return Optional.empty(); } } - private record StringHttpResponse(HttpRequest request, String body) - implements HttpResponse { + private record QueuedResponse( + int statusCode, + Object body, + Map> headers) { - @Override public int statusCode() { return 200; } - @Override public Optional> previousResponse() { return Optional.empty(); } + private QueuedResponse(int statusCode, Object body) { + this(statusCode, body, Map.of()); + } + } + + private record RecordedResponseInfo( + int statusCode, + Map> responseHeaders) + implements HttpResponse.ResponseInfo { @Override public HttpHeaders headers() { - return HttpHeaders.of(Map.of(), (name, value) -> true); + return HttpHeaders.of(responseHeaders, (name, value) -> true); + } + @Override public HttpClient.Version version() { + return HttpClient.Version.HTTP_1_1; + } + } + + private record RecordedHttpResponse( + HttpRequest request, + int statusCode, + T body, + Map> responseHeaders) + implements HttpResponse { + + @Override public Optional> previousResponse() { return Optional.empty(); } + @Override public HttpHeaders headers() { + return HttpHeaders.of(responseHeaders, (name, value) -> true); } @Override public Optional sslSession() { return Optional.empty(); } @Override public URI uri() { return request.uri(); } @Override public HttpClient.Version version() { return HttpClient.Version.HTTP_1_1; } } + + private static final class RecordingWebSocketConnector + implements IflytekScoringProvider.WebSocketConnector { + + private final String finalMessage; + private final boolean delayFirstSend; + private final List frames = new ArrayList<>(); + private URI uri; + + private RecordingWebSocketConnector(String finalMessage) { + this(finalMessage, false); + } + + private RecordingWebSocketConnector(String finalMessage, boolean delayFirstSend) { + this.finalMessage = finalMessage; + this.delayFirstSend = delayFirstSend; + } + + @Override + public CompletableFuture connect( + URI uri, + java.net.http.WebSocket.Listener listener) { + this.uri = uri; + RecordingWebSocket socket = new RecordingWebSocket( + listener, + frames, + finalMessage, + delayFirstSend); + listener.onOpen(socket); + return CompletableFuture.completedFuture(socket); + } + } + + private static final class RecordingWebSocket implements java.net.http.WebSocket { + + private final Listener listener; + private final List frames; + private final String finalMessage; + private final boolean delayFirstSend; + private boolean inputClosed; + private boolean outputClosed; + + private RecordingWebSocket( + Listener listener, + List frames, + String finalMessage, + boolean delayFirstSend) { + this.listener = listener; + this.frames = frames; + this.finalMessage = finalMessage; + this.delayFirstSend = delayFirstSend; + } + + @Override + public CompletableFuture sendText( + CharSequence data, + boolean last) { + String frame = data.toString(); + frames.add(frame); + if (delayFirstSend && frames.size() == 1) { + CompletableFuture delayed = new CompletableFuture<>(); + CompletableFuture.delayedExecutor(100, TimeUnit.MILLISECONDS) + .execute(() -> delayed.complete(this)); + return delayed; + } + if (frame.contains("\"status\":2")) { + listener.onText(this, finalMessage, true); + } + return CompletableFuture.completedFuture(this); + } + + @Override + public CompletableFuture sendBinary(ByteBuffer data, boolean last) { + return CompletableFuture.completedFuture(this); + } + + @Override + public CompletableFuture sendPing(ByteBuffer message) { + return CompletableFuture.completedFuture(this); + } + + @Override + public CompletableFuture sendPong(ByteBuffer message) { + return CompletableFuture.completedFuture(this); + } + + @Override + public CompletableFuture sendClose(int statusCode, String reason) { + outputClosed = true; + inputClosed = true; + return CompletableFuture.completedFuture(this); + } + + @Override public void request(long n) { } + @Override public String getSubprotocol() { return ""; } + @Override public boolean isOutputClosed() { return outputClosed; } + @Override public boolean isInputClosed() { return inputClosed; } + @Override public void abort() { + outputClosed = true; + inputClosed = true; + } + } } diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/provider/AiProviderRegistryTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/provider/AiProviderRegistryTest.java index c16832c..a65ec65 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/provider/AiProviderRegistryTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/provider/AiProviderRegistryTest.java @@ -3,24 +3,31 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; -import com.unispeaking.domain.dto.ai.LlmTaskRequest; -import com.unispeaking.domain.dto.ai.LlmTaskResponse; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationRequest; -import com.unispeaking.domain.dto.ai.PronunciationEvaluationResponse; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeRequest; -import com.unispeaking.domain.dto.ai.RealtimeSdpExchangeResponse; -import com.unispeaking.domain.dto.ai.SpeechAudioRequest; -import com.unispeaking.domain.dto.ai.SpeechAudioResponse; import com.unispeaking.domain.vo.ai.AiCapability; +import com.unispeaking.domain.vo.ai.AiModelDefinition; import com.unispeaking.domain.vo.realtime.ProviderType; import com.unispeaking.exception.BusinessException; import java.util.List; +import java.util.Map; import java.util.Set; import org.junit.jupiter.api.Test; class AiProviderRegistryTest { + @Test + void exposesVendorNeutralProviderIdentifiers() { + assertTrue( + List.of(AiProvider.class.getMethods()).stream() + .anyMatch(method -> method.getName().equals("providerId") + && method.getReturnType() == String.class)); + assertTrue( + List.of(AiModelDefinition.class.getRecordComponents()).stream() + .anyMatch(component -> component.getName().equals("providerId") + && component.getType() == String.class)); + } + @Test void selectsProvidersByCapabilityAndModel() { StubRealtimeProvider realtime = new StubRealtimeProvider(); @@ -37,8 +44,8 @@ void selectsProvidersByCapabilityAndModel() { "answer", registry.exchangeRealtimeSdp( AiProviderRegistry.QWEN_REALTIME_FLASH, - new RealtimeSdpExchangeRequest(null, AiProviderRegistry.QWEN_REALTIME_FLASH, "offer", "key")) - .answerSdp()); + "offer", + "key")); } @Test @@ -60,13 +67,155 @@ void rejectsCapabilityMismatchAndDuplicateModelRegistration() { List.of())); } + @Test + void routesAFeatureCallThroughTheConfiguredPrimaryModel() { + StubQwenLlmProvider qwen = new StubQwenLlmProvider(); + StubDeepSeekLlmProvider deepSeek = new StubDeepSeekLlmProvider(); + AiProviderRegistry registry = registry( + List.of(qwen, deepSeek), + Map.of(AiCapability.LLM, List.of(AiProviderRegistry.DEEPSEEK_CHAT))); + + String response = registry.executeLlmTask("hello", null); + + assertEquals("deepseek", response); + assertEquals(0, qwen.calls); + assertEquals(1, deepSeek.calls); + assertEquals(AiProviderRegistry.DEEPSEEK_CHAT, registry.defaultModel(AiCapability.LLM)); + } + + @Test + void preservesProviderOrderWhenConfiguredModelsReplaceDefaultModelNames() { + LlmProvider qwen = new ConfiguredModelLlmProvider("qwen", "qwen-custom-llm"); + LlmProvider deepSeek = new ConfiguredModelLlmProvider( + "deepseek", + "deepseek-custom-llm"); + + AiProviderRegistry registry = registry(List.of(deepSeek, qwen), Map.of()); + + assertEquals("qwen-custom-llm", registry.defaultModel(AiCapability.LLM)); + assertEquals( + List.of("qwen-custom-llm", "deepseek-custom-llm"), + registry.route(AiCapability.LLM)); + assertSame(qwen, registry.getLlmProvider("qwen-custom-llm")); + assertSame(deepSeek, registry.getLlmProvider("deepseek-custom-llm")); + } + + @Test + void usesQwenAsThePrimaryModelForEveryCapabilityExceptScoring() { + AiProviderRegistry registry = registry(new StubRealtimeProvider()); + + assertEquals( + AiProviderRegistry.QWEN_REALTIME_FLASH, + registry.defaultModel(AiCapability.REALTIME)); + assertEquals( + AiProviderRegistry.QWEN_LLM_PLUS, + registry.defaultModel(AiCapability.LLM)); + assertEquals( + StubTranscriptionProvider.MODEL_ID, + registry.defaultModel(AiCapability.TRANSCRIPTION)); + assertEquals( + AiProviderRegistry.QWEN_TTS, + registry.defaultModel(AiCapability.TTS)); + assertEquals( + AiProviderRegistry.IFLYTEK_PRONUNCIATION_SCORING, + registry.defaultModel(AiCapability.SCORING)); + } + + @Test + void failsOverToTheNextConfiguredModelWhenThePrimaryProviderIsUnavailable() { + StubQwenLlmProvider qwen = new StubQwenLlmProvider(); + qwen.failure = new BusinessException( + "QWEN_LLM_IO_ERROR", + "primary unavailable"); + StubDeepSeekLlmProvider deepSeek = new StubDeepSeekLlmProvider(); + AiProviderRegistry registry = registry( + List.of(qwen, deepSeek), + Map.of( + AiCapability.LLM, + List.of( + AiProviderRegistry.QWEN_LLM_PLUS, + AiProviderRegistry.DEEPSEEK_CHAT))); + + String response = registry.executeLlmTask("hello", null); + + assertEquals("deepseek", response); + assertEquals(1, qwen.calls); + assertEquals(1, deepSeek.calls); + } + + @Test + void doesNotFailOverWhenTheRequestItselfIsInvalid() { + StubQwenLlmProvider qwen = new StubQwenLlmProvider(); + qwen.failure = new BusinessException("INVALID_LLM_PROMPT", "prompt is required"); + StubDeepSeekLlmProvider deepSeek = new StubDeepSeekLlmProvider(); + AiProviderRegistry registry = registry( + List.of(qwen, deepSeek), + Map.of( + AiCapability.LLM, + List.of( + AiProviderRegistry.QWEN_LLM_PLUS, + AiProviderRegistry.DEEPSEEK_CHAT))); + + BusinessException exception = assertThrows( + BusinessException.class, + () -> registry.executeLlmTask("", null)); + + assertEquals("INVALID_LLM_PROMPT", exception.code()); + assertEquals(1, qwen.calls); + assertEquals(0, deepSeek.calls); + } + + @Test + void switchesScoringProviderByChangingOnlyTheConfiguredModelRoute() { + StubScoringProvider iflytek = new StubScoringProvider(); + AlternativeScoringProvider alternative = new AlternativeScoringProvider(); + AiProviderRegistry registry = new AiProviderRegistry( + List.of(new StubRealtimeProvider()), + llmProviders(), + List.of(iflytek, alternative), + ttsProviders(), + List.of(new StubTranscriptionProvider()), + Map.of( + AiCapability.REALTIME, + List.of(AiProviderRegistry.QWEN_REALTIME_FLASH), + AiCapability.LLM, + List.of(AiProviderRegistry.QWEN_LLM_PLUS), + AiCapability.SCORING, + List.of(AlternativeScoringProvider.MODEL_ID), + AiCapability.TTS, + List.of(AiProviderRegistry.ALIYUN_TTS), + AiCapability.TRANSCRIPTION, + List.of(StubTranscriptionProvider.MODEL_ID))); + + String response = registry.evaluatePronunciation( + "hello", + new Byte[] {0}, + null); + + assertEquals("{\"totalScore\":99}", response); + assertEquals(0, iflytek.calls); + assertEquals(1, alternative.calls); + } + private AiProviderRegistry registry(RealtimeProvider realtimeProvider) { return new AiProviderRegistry( List.of(realtimeProvider), llmProviders(), List.of(new StubScoringProvider()), ttsProviders(), - List.of()); + List.of(new StubTranscriptionProvider())); + } + + private AiProviderRegistry registry( + List providers, + Map> routeOverrides) { + return new AiProviderRegistry( + List.of(new StubRealtimeProvider()), + providers, + List.of(new StubScoringProvider()), + ttsProviders(), + List.of(new StubTranscriptionProvider()), + routeOverrides); } private List llmProviders() { @@ -74,7 +223,10 @@ private List llmProviders() { } private List ttsProviders() { - return List.of(new StubAliyunTtsProvider(), new StubMiniMaxTtsProvider()); + return List.of( + new StubQwenTtsProvider(), + new StubAliyunTtsProvider(), + new StubMiniMaxTtsProvider()); } private static final class StubRealtimeProvider extends RealtimeProvider { @@ -87,66 +239,121 @@ private StubRealtimeProvider() { } @Override - public RealtimeSdpExchangeResponse exchangeRealtimeSdp(RealtimeSdpExchangeRequest request) { - return new RealtimeSdpExchangeResponse("answer", null); + public String exchangeRealtimeSdp( + String modelId, + String offerSdp, + String token) { + return "answer"; } } private static final class StubQwenLlmProvider extends LlmProvider { + private int calls; + private BusinessException failure; + private StubQwenLlmProvider() { - super(ProviderType.QWEN, Set.of(AiProviderRegistry.QWEN_LLM_PLUS)); + super("qwen", Set.of(AiProviderRegistry.QWEN_LLM_PLUS)); } @Override - public LlmTaskResponse executeLlmTask(LlmTaskRequest request) { - return null; + public String executeLlmTask(String prompt, String token) { + calls++; + if (failure != null) { + throw failure; + } + return "qwen"; } } private static final class StubDeepSeekLlmProvider extends LlmProvider { + private int calls; + private StubDeepSeekLlmProvider() { - super(ProviderType.DEEPSEEK, Set.of(AiProviderRegistry.DEEPSEEK_CHAT)); + super("deepseek", Set.of(AiProviderRegistry.DEEPSEEK_CHAT)); + } + + @Override + public String executeLlmTask(String prompt, String token) { + calls++; + return "deepseek"; + } + } + + private static final class ConfiguredModelLlmProvider extends LlmProvider { + + private ConfiguredModelLlmProvider(String providerId, String modelId) { + super(providerId, Set.of(modelId)); } @Override - public LlmTaskResponse executeLlmTask(LlmTaskRequest request) { - return null; + public String executeLlmTask(String prompt, String token) { + return providerId(); } } private static final class StubScoringProvider extends ScoringProvider { + private int calls; + private StubScoringProvider() { super( - ProviderType.IFLYTEK, + "iflytek", Set.of(AiProviderRegistry.IFLYTEK_PRONUNCIATION_SCORING)); } @Override - public PronunciationEvaluationResponse evaluatePronunciation( - PronunciationEvaluationRequest request) { - return null; + public String evaluatePronunciation(String text, Byte[] audio, String token) { + calls++; + return "{}"; + } + } + + private static final class AlternativeScoringProvider extends ScoringProvider { + + private static final String MODEL_ID = "alternative-pronunciation-scoring"; + private int calls; + + private AlternativeScoringProvider() { + super("future-vendor", Set.of(MODEL_ID)); + } + + @Override + public String evaluatePronunciation(String text, Byte[] audio, String token) { + calls++; + return "{\"totalScore\":99}"; } } private static final class StubAliyunTtsProvider extends TtsProvider { private StubAliyunTtsProvider() { - super(ProviderType.ALIYUN, Set.of(AiProviderRegistry.ALIYUN_TTS)); + super("aliyun", Set.of(AiProviderRegistry.ALIYUN_TTS)); } - @Override - public SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request) { - return null; + } + + private static final class StubQwenTtsProvider extends TtsProvider { + private StubQwenTtsProvider() { + super("qwen", Set.of(AiProviderRegistry.QWEN_TTS)); } } private static final class StubMiniMaxTtsProvider extends TtsProvider { private StubMiniMaxTtsProvider() { - super(ProviderType.MINIMAX, Set.of(AiProviderRegistry.MINIMAX_TTS)); + super("minimax", Set.of(AiProviderRegistry.MINIMAX_TTS)); + } + + } + + private static final class StubTranscriptionProvider extends TranscriptionProvider { + + private static final String MODEL_ID = "stub-asr"; + + private StubTranscriptionProvider() { + super("qwen", Set.of(MODEL_ID)); } @Override - public SpeechAudioResponse generateSpeechAudio(SpeechAudioRequest request) { - return null; + public String convertAudioToText(Byte[] audio, String token) { + return "transcript"; } } } diff --git a/backend/unispeaking-server/src/test/java/com/unispeaking/service/SessionRealtimeConnectionTest.java b/backend/unispeaking-server/src/test/java/com/unispeaking/service/SessionRealtimeConnectionTest.java index 811a729..f6cfeed 100644 --- a/backend/unispeaking-server/src/test/java/com/unispeaking/service/SessionRealtimeConnectionTest.java +++ b/backend/unispeaking-server/src/test/java/com/unispeaking/service/SessionRealtimeConnectionTest.java @@ -11,15 +11,24 @@ import com.unispeaking.domain.vo.prompt.SessionPrompt; import com.unispeaking.domain.vo.realtime.ProviderType; import com.unispeaking.domain.vo.realtime.RealtimeConnectionResult; +import com.unispeaking.domain.vo.realtime.RealtimeCredential; import com.unispeaking.domain.vo.scene.SceneType; import com.unispeaking.domain.vo.session.SessionStatus; import com.unispeaking.repository.SessionStateStore; +import com.unispeaking.exception.BusinessException; +import com.unispeaking.provider.AiProviderRegistry; +import com.unispeaking.provider.RealtimeProvider; import com.unispeaking.service.quota.UsageQuotaService; import com.unispeaking.service.realtime.RealtimeConnectionService; +import com.unispeaking.service.realtime.RealtimeCredentialService; +import com.unispeaking.service.realtime.impl.RealtimeConnectionServiceImpl; import com.unispeaking.service.session.SessionService; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.junit.jupiter.api.Test; @@ -81,6 +90,51 @@ protected void handleSessionCompleted(AbstractSceneSession session) { assertEquals(SessionStatus.WAITING_CLIENT, response.status()); } + @Test + void realtimeConnectionUsesTheConfiguredFallbackProviderWithoutExposingItToTheCaller() { + FailingRealtimeProvider primary = new FailingRealtimeProvider(); + SuccessfulRealtimeProvider fallback = new SuccessfulRealtimeProvider(); + AiProviderRegistry registry = new AiProviderRegistry( + List.of(primary, fallback), + List.of(), + List.of(), + List.of(), + List.of(), + Map.of( + com.unispeaking.domain.vo.ai.AiCapability.REALTIME, + List.of( + FailingRealtimeProvider.MODEL_ID, + SuccessfulRealtimeProvider.MODEL_ID))); + RecordingCredentialService credentials = new RecordingCredentialService(); + RealtimeConnectionService service = new RealtimeConnectionServiceImpl( + registry, + credentials); + FreeChatSceneSession session = new FreeChatSceneSession("session-1", "user-1"); + + RealtimeConnectionResult result = service.connect( + null, + session, + new SessionPrompt("system prompt"), + new StartCommand( + SceneType.FREE_CHAT, + "user-1", + "scene-1", + "offer-sdp", + null, + null, + null, + null, + false)); + + assertEquals("fallback-answer", result.answerSdp()); + assertEquals( + List.of(ProviderType.QWEN, ProviderType.OPENAI), + credentials.requestedProviders); + assertEquals(1, primary.calls); + assertEquals(1, fallback.calls); + assertEquals(SuccessfulRealtimeProvider.MODEL_ID, fallback.modelId); + } + private static final class RecordingConnectionService implements RealtimeConnectionService { private ProviderType providerType; private StartCommand command; @@ -98,6 +152,59 @@ public RealtimeConnectionResult connect( } } + private static final class FailingRealtimeProvider extends RealtimeProvider { + + private static final String MODEL_ID = "primary-realtime"; + private int calls; + + private FailingRealtimeProvider() { + super(ProviderType.QWEN, Set.of(MODEL_ID)); + } + + @Override + public String exchangeRealtimeSdp( + String modelId, + String offerSdp, + String token) { + calls++; + throw new BusinessException( + "QWEN_SIGNALING_IO_ERROR", + "primary unavailable"); + } + } + + private static final class SuccessfulRealtimeProvider extends RealtimeProvider { + + private static final String MODEL_ID = "fallback-realtime"; + private int calls; + private String modelId; + + private SuccessfulRealtimeProvider() { + super(ProviderType.OPENAI, Set.of(MODEL_ID)); + } + + @Override + public String exchangeRealtimeSdp( + String modelId, + String offerSdp, + String token) { + calls++; + this.modelId = modelId; + return "fallback-answer"; + } + } + + private static final class RecordingCredentialService implements RealtimeCredentialService { + + private final List requestedProviders = new ArrayList<>(); + + @Override + public RealtimeCredential getCredential(ProviderType providerType) { + requestedProviders.add(providerType); + return new RealtimeCredential(providerType.name().toLowerCase() + "-token", null); + } + } + private static final class FixedSessionIdGenerator extends SessionIdGenerator { @Override public String generate() { diff --git a/deploy/env/.env.example b/deploy/env/.env.example index cd0ea3d..dac2bf7 100644 --- a/deploy/env/.env.example +++ b/deploy/env/.env.example @@ -1,13 +1,128 @@ -# Copy this file to deploy/env/.env and set the real secret there. -# Never commit deploy/env/.env. +# Copy this file to deploy/env/.env and replace the credential placeholders. +# Never commit deploy/env/.env or expose any secret through a VITE_ variable. +# ============================================================================= +# Application +# ============================================================================= SERVER_PORT=8080 +VITE_BACKEND_URL=/backend +REALTIME_FLOW_LOG_FILE=logs/realtime-flow.log + +# ============================================================================= +# AI model routes +# The first model is primary; later models are fallbacks. +# If a model ID below is changed, update the corresponding route as well. +# ============================================================================= +AI_PROVIDER_ROUTE_REALTIME=qwen3.5-omni-flash-realtime +AI_PROVIDER_ROUTE_LLM=qwen3.5-plus,deepseek-v4-flash +AI_PROVIDER_ROUTE_TRANSCRIPTION=qwen3-asr-flash,volc.bigasr.auc_turbo +AI_PROVIDER_ROUTE_TTS=qwen3-tts-flash,cosyvoice-v3-flash,speech-2.8-hd +AI_PROVIDER_ROUTE_SCORING=iflytek-open-ise + +# ============================================================================= +# Alibaba Cloud Model Studio / Qwen shared permanent credentials +# DASHSCOPE_API_KEY is a server-side permanent key. It is never returned to +# clients. Only Realtime exchanges it for a short-lived temporary key. +# ============================================================================= +DASHSCOPE_API_KEY=replace-with-your-dashscope-api-key +BAILIAN_WORKSPACE_ID=replace-with-your-bailian-workspace-id +BAILIAN_REGION=cn-beijing -DASHSCOPE_API_KEY=replace-with-your-api-key -BAILIAN_WORKSPACE_ID=replace-with-your-workspace-id +# ============================================================================= +# Qwen Realtime - primary +# The backend creates a temporary key with the permanent DASHSCOPE_API_KEY and +# passes only that temporary key to the Realtime SDP exchange. +# ============================================================================= BAILIAN_MODEL=qwen3.5-omni-flash-realtime -# BAILIAN_REGION=cn-beijing -# REALTIME_QWEN_TEMPORARY_KEY_TTL_SECONDS=300 +REALTIME_QWEN_TEMPORARY_KEY_ENDPOINT=https://dashscope.aliyuncs.com/api/v1/tokens +REALTIME_QWEN_TEMPORARY_KEY_TTL_SECONDS=300 +REALTIME_QWEN_CONNECT_TIMEOUT=10s +REALTIME_QWEN_READ_TIMEOUT=20s +REALTIME_QWEN_MAX_ANSWER_BYTES=1048576 -# Public frontend setting; never put a secret in a VITE_ variable. -VITE_BACKEND_URL=/backend +# ============================================================================= +# Qwen LLM - primary +# ============================================================================= +QWEN_LLM_MODEL=qwen3.5-plus +QWEN_LLM_CONNECT_TIMEOUT_SECONDS=10 +QWEN_LLM_READ_TIMEOUT_SECONDS=60 +QWEN_LLM_MAX_RESPONSE_BYTES=2097152 + +# DeepSeek LLM - fallback +DEEPSEEK_API_KEY= +DEEPSEEK_LLM_ENDPOINT=https://api.deepseek.com/chat/completions +DEEPSEEK_LLM_MODEL=deepseek-v4-flash +DEEPSEEK_LLM_CONNECT_TIMEOUT_SECONDS=10 +DEEPSEEK_LLM_READ_TIMEOUT_SECONDS=60 +DEEPSEEK_LLM_MAX_RESPONSE_BYTES=2097152 + +# ============================================================================= +# Qwen ASR - primary, WAV input +# ============================================================================= +QWEN_ASR_MODEL=qwen3-asr-flash +QWEN_ASR_CONNECT_TIMEOUT_SECONDS=10 +QWEN_ASR_READ_TIMEOUT_SECONDS=60 +QWEN_ASR_MAX_AUDIO_BYTES=7340032 +QWEN_ASR_MAX_RESPONSE_BYTES=1048576 + +# Doubao ASR - fallback +# Prefer DOUBAO_ASR_API_KEY. APP_KEY + ACCESS_KEY are legacy alternatives. +DOUBAO_ASR_API_KEY= +DOUBAO_ASR_APP_KEY= +DOUBAO_ASR_ACCESS_KEY= +DOUBAO_ASR_USER_ID=unispeaking +DOUBAO_ASR_ENDPOINT=https://openspeech.bytedance.com/api/v3/auc/bigmodel/recognize/flash +DOUBAO_ASR_RESOURCE_ID=volc.bigasr.auc_turbo +DOUBAO_ASR_CONNECT_TIMEOUT_SECONDS=10 +DOUBAO_ASR_READ_TIMEOUT_SECONDS=120 +DOUBAO_ASR_MAX_AUDIO_BYTES=20971520 +DOUBAO_ASR_MAX_RESPONSE_BYTES=4194304 + +# ============================================================================= +# Qwen3-TTS - primary, WAV output +# Uses the permanent server-side DASHSCOPE_API_KEY and ignores the token +# parameter from AiProvider. +# ============================================================================= +QWEN_TTS_ENDPOINT=https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation +QWEN_TTS_MODEL=qwen3-tts-flash +QWEN_TTS_VOICE=Cherry +QWEN_TTS_LANGUAGE_TYPE=English +QWEN_TTS_CONNECT_TIMEOUT_SECONDS=10 +QWEN_TTS_READ_TIMEOUT_SECONDS=60 +QWEN_TTS_MAX_RESPONSE_BYTES=1048576 +QWEN_TTS_MAX_AUDIO_BYTES=10485760 + +# CosyVoice TTS - first fallback, shares the DashScope permanent credential +ALIYUN_TTS_MODEL=cosyvoice-v3-flash +ALIYUN_TTS_VOICE=loongemily_v3 +ALIYUN_TTS_FORMAT=wav +ALIYUN_TTS_SAMPLE_RATE=24000 +ALIYUN_TTS_CONNECT_TIMEOUT_SECONDS=10 +ALIYUN_TTS_READ_TIMEOUT_SECONDS=60 +ALIYUN_TTS_MAX_AUDIO_BYTES=10485760 + +# MiniMax TTS - second fallback +MINIMAX_API_KEY= +MINIMAX_TTS_ENDPOINT=https://api.minimaxi.com/v1/t2a_v2 +MINIMAX_TTS_MODEL=speech-2.8-hd +MINIMAX_TTS_VOICE=male-qn-qingse +MINIMAX_TTS_FORMAT=wav +MINIMAX_TTS_SAMPLE_RATE=32000 +MINIMAX_TTS_BITRATE=128000 +MINIMAX_TTS_CONNECT_TIMEOUT_SECONDS=10 +MINIMAX_TTS_READ_TIMEOUT_SECONDS=60 +MINIMAX_TTS_MAX_AUDIO_BYTES=10485760 + +# ============================================================================= +# iFlytek pronunciation scoring - the only scoring provider +# Uses server-side permanent credentials and returns the final raw response. +# ============================================================================= +XFYUN_APP_ID=replace-with-your-xfyun-app-id +XFYUN_API_KEY=replace-with-your-xfyun-api-key +XFYUN_API_SECRET=replace-with-your-xfyun-api-secret +XFYUN_ISE_ENDPOINT=wss://ise-api.xfyun.cn/v2/open-ise +XFYUN_ISE_LANGUAGE=en_vip +XFYUN_ISE_CATEGORY=read_sentence +XFYUN_ISE_CONNECT_TIMEOUT_SECONDS=10 +XFYUN_ISE_READ_TIMEOUT_SECONDS=60 +XFYUN_ISE_MAX_AUDIO_BYTES=10485760