From 060833edad89ea619ca8defbed48b78a066721f5 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:11:43 +0800 Subject: [PATCH] feat: add MiniMax speech provider --- src/io/speech/SpeechProviderResolver.ts | 1 + src/io/speech/SpeechRuntime.ts | 13 + .../MiniMaxTextToSpeechProvider.test.ts | 219 +++++++++++ src/io/speech/__tests__/SpeechRuntime.test.ts | 2 + .../speech/__tests__/providerCatalog.test.ts | 11 + src/io/speech/index.ts | 1 + src/io/speech/providerCatalog.ts | 11 + .../providers/MiniMaxTextToSpeechProvider.ts | 345 ++++++++++++++++++ 8 files changed, 603 insertions(+) create mode 100644 src/io/speech/__tests__/MiniMaxTextToSpeechProvider.test.ts create mode 100644 src/io/speech/providers/MiniMaxTextToSpeechProvider.ts diff --git a/src/io/speech/SpeechProviderResolver.ts b/src/io/speech/SpeechProviderResolver.ts index cecec4d0083..3b7e1764ab1 100644 --- a/src/io/speech/SpeechProviderResolver.ts +++ b/src/io/speech/SpeechProviderResolver.ts @@ -447,6 +447,7 @@ export class SpeechProviderResolver extends EventEmitter { { id: 'openai-tts', kind: 'tts' as const, envVars: ['OPENAI_API_KEY'] }, { id: 'elevenlabs', kind: 'tts' as const, envVars: ['ELEVENLABS_API_KEY'] }, { id: 'deepgram-aura', kind: 'tts' as const, envVars: ['DEEPGRAM_API_KEY'] }, + { id: 'minimax-tts', kind: 'tts' as const, envVars: ['MINIMAX_API_KEY'] }, { id: 'azure-speech-tts', kind: 'tts' as const, diff --git a/src/io/speech/SpeechRuntime.ts b/src/io/speech/SpeechRuntime.ts index 399d5d24bdd..5bca6f7f618 100644 --- a/src/io/speech/SpeechRuntime.ts +++ b/src/io/speech/SpeechRuntime.ts @@ -12,6 +12,7 @@ import { SpeechSession } from './SpeechSession.js'; import { BuiltInAdaptiveVadProvider } from '../hearing/providers/BuiltInAdaptiveVadProvider.js'; import { ElevenLabsTextToSpeechProvider } from './providers/ElevenLabsTextToSpeechProvider.js'; import { OpenAITextToSpeechProvider } from './providers/OpenAITextToSpeechProvider.js'; +import { MiniMaxTextToSpeechProvider } from './providers/MiniMaxTextToSpeechProvider.js'; import { OpenAIWhisperSpeechToTextProvider } from '../hearing/providers/OpenAIWhisperSpeechToTextProvider.js'; import type { ProviderRequirements, @@ -90,6 +91,18 @@ export class SpeechRuntime { this.registry.registerTtsProvider(tts); this.registerProviderInResolver(tts, 'tts'); } + + const miniMaxApiKey = env['MINIMAX_API_KEY']; + if (miniMaxApiKey) { + const tts = new MiniMaxTextToSpeechProvider({ + apiKey: miniMaxApiKey, + region: env['MINIMAX_REGION'] === 'china' ? 'china' : 'global', + model: env['MINIMAX_TTS_MODEL'] ?? 'speech-2.8-hd', + voice: env['MINIMAX_TTS_VOICE'], + }); + this.registry.registerTtsProvider(tts); + this.registerProviderInResolver(tts, 'tts'); + } } } diff --git a/src/io/speech/__tests__/MiniMaxTextToSpeechProvider.test.ts b/src/io/speech/__tests__/MiniMaxTextToSpeechProvider.test.ts new file mode 100644 index 00000000000..141c919616d --- /dev/null +++ b/src/io/speech/__tests__/MiniMaxTextToSpeechProvider.test.ts @@ -0,0 +1,219 @@ +import { EventEmitter } from "node:events"; +import WebSocket from "ws"; +import { describe, expect, it, vi } from "vitest"; +import { MiniMaxTextToSpeechProvider } from "../providers/MiniMaxTextToSpeechProvider.js"; + +function response(body: unknown): Response { + return { + ok: true, + status: 200, + statusText: "OK", + json: vi.fn(async () => body), + } as unknown as Response; +} + +describe("MiniMaxTextToSpeechProvider", () => { + it("synthesizes hex audio through the global HTTP endpoint", async () => { + const fetchImpl = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + response({ + data: { audio: "000102ff", status: 2 }, + extra_info: { audio_length: 1250, usage_characters: 5 }, + base_resp: { status_code: 0, status_msg: "success" }, + }), + ); + const provider = new MiniMaxTextToSpeechProvider({ + apiKey: "test-key", + fetchImpl, + }); + + const result = await provider.synthesize("hello", { + voice: "English_expressive_narrator", + providerSpecificOptions: { + languageBoost: "English", + pronunciationDict: { tone: ["hello/hello"] }, + audioSetting: { sample_rate: 32000 }, + voiceModify: { pitch: 1 }, + subtitleEnable: true, + }, + }); + + expect(result.audioBuffer).toEqual(Buffer.from([0, 1, 2, 255])); + expect(result.durationSeconds).toBe(1.25); + const [url, init] = fetchImpl.mock.calls[0]!; + expect(url).toBe("https://api.minimax.io/v1/t2a_v2"); + expect((init?.headers as Record).Authorization).toBe( + "Bearer test-key", + ); + expect(JSON.parse(init?.body as string)).toMatchObject({ + model: "speech-2.8-hd", + text: "hello", + stream: false, + output_format: "hex", + language_boost: "English", + voice_setting: { voice_id: "English_expressive_narrator" }, + audio_setting: { sample_rate: 32000, format: "mp3" }, + pronunciation_dict: { tone: ["hello/hello"] }, + voice_modify: { pitch: 1 }, + subtitle_enable: true, + }); + }); + + it("uses the China endpoint and downloads URL output", async () => { + const fetchImpl = vi + .fn( + ( + _input: string | URL | Request, + _init?: RequestInit, + ): Promise => + Promise.resolve(undefined as unknown as Response), + ) + .mockResolvedValueOnce( + response({ + data: { audio: "https://cdn.example.com/audio.wav", status: 2 }, + base_resp: { status_code: 0 }, + }), + ) + .mockResolvedValueOnce({ + arrayBuffer: vi.fn(async () => Buffer.from("audio")), + } as unknown as Response); + const provider = new MiniMaxTextToSpeechProvider({ + apiKey: "test-key", + region: "china", + fetchImpl, + }); + + const result = await provider.synthesize("hello", { + outputFormat: "wav", + providerSpecificOptions: { outputFormat: "url" }, + }); + + expect(fetchImpl.mock.calls[0][0]).toBe( + "https://api.minimaxi.com/v1/t2a_v2", + ); + expect(result.audioBuffer.toString()).toBe("audio"); + expect(result.mimeType).toBe("audio/wav"); + }); + + it("creates and queries asynchronous speech tasks", async () => { + const fetchImpl = vi + .fn( + ( + _input: string | URL | Request, + _init?: RequestInit, + ): Promise => + Promise.resolve(undefined as unknown as Response), + ) + .mockResolvedValueOnce( + response({ + task_id: "task-1", + file_id: 42, + base_resp: { status_code: 0 }, + }), + ) + .mockResolvedValueOnce( + response({ + task_id: "task-1", + status: "success", + file_id: 42, + base_resp: { status_code: 0 }, + }), + ); + const provider = new MiniMaxTextToSpeechProvider({ + apiKey: "test-key", + fetchImpl, + }); + + await expect(provider.createAsync("long text")).resolves.toMatchObject({ + task_id: "task-1", + }); + await expect(provider.queryAsync("task-1")).resolves.toMatchObject({ + status: "success", + }); + expect(fetchImpl.mock.calls.map((call) => call[0])).toEqual([ + "https://api.minimax.io/v1/t2a_async_v2", + "https://api.minimax.io/v1/query/t2a_async_query_v2", + ]); + expect(JSON.parse(fetchImpl.mock.calls[1]![1]?.body as string)).toEqual({ + task_id: "task-1", + }); + }); + + it("runs the WebSocket start, continue, and finish protocol", async () => { + class Socket extends EventEmitter { + sent: string[] = []; + send(value: string) { + this.sent.push(value); + } + close() {} + } + const socket = new Socket(); + const provider = new MiniMaxTextToSpeechProvider({ + apiKey: "test-key", + webSocketFactory: (url, headers) => { + expect(url).toBe("wss://api.minimax.io/ws/v1/t2a_v2"); + expect(headers.Authorization).toBe("Bearer test-key"); + return socket as unknown as WebSocket; + }, + }); + + const resultPromise = provider.synthesizeWebSocket("hello"); + socket.emit( + "message", + JSON.stringify({ + event: "connected_success", + base_resp: { status_code: 0 }, + }), + ); + socket.emit( + "message", + JSON.stringify({ + event: "task_started", + base_resp: { status_code: 0 }, + }), + ); + socket.emit( + "message", + JSON.stringify({ + event: "task_continued", + data: { audio: "0001ff" }, + base_resp: { status_code: 0 }, + }), + ); + socket.emit( + "message", + JSON.stringify({ + event: "task_finished", + base_resp: { status_code: 0 }, + }), + ); + + await expect(resultPromise).resolves.toMatchObject({ + audioBuffer: Buffer.from([0, 1, 255]), + mimeType: "audio/mpeg", + }); + expect(socket.sent.map((value) => JSON.parse(value).event)).toEqual([ + "task_start", + "task_continue", + "task_finish", + ]); + }); + + it("rejects invalid hex audio", async () => { + const fetchImpl = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + response({ + data: { audio: "not-hex", status: 2 }, + base_resp: { status_code: 0 }, + }), + ); + const provider = new MiniMaxTextToSpeechProvider({ + apiKey: "test-key", + fetchImpl, + }); + + await expect(provider.synthesize("hello")).rejects.toThrow( + "invalid hex audio", + ); + }); +}); diff --git a/src/io/speech/__tests__/SpeechRuntime.test.ts b/src/io/speech/__tests__/SpeechRuntime.test.ts index 22467423404..764f652e3d7 100644 --- a/src/io/speech/__tests__/SpeechRuntime.test.ts +++ b/src/io/speech/__tests__/SpeechRuntime.test.ts @@ -16,6 +16,7 @@ describe('SpeechRuntime', () => { env: { OPENAI_API_KEY: 'sk-openai', ELEVENLABS_API_KEY: 'sk-elevenlabs', + MINIMAX_API_KEY: 'sk-minimax', }, }); @@ -25,6 +26,7 @@ describe('SpeechRuntime', () => { expect(runtime.getProvider('openai-whisper')).toBeDefined(); expect(runtime.getProvider('openai-tts')).toBeDefined(); expect(runtime.getProvider('elevenlabs')).toBeDefined(); + expect(runtime.getProvider('minimax-tts')).toBeDefined(); }); it('should hydrate speech providers from the extension manager', async () => { diff --git a/src/io/speech/__tests__/providerCatalog.test.ts b/src/io/speech/__tests__/providerCatalog.test.ts index 6dd7824755a..33f92d1a366 100644 --- a/src/io/speech/__tests__/providerCatalog.test.ts +++ b/src/io/speech/__tests__/providerCatalog.test.ts @@ -23,6 +23,17 @@ describe('providerCatalog', () => { expect(entry!.kind).toBe('stt'); }); + it('should expose MiniMax HTTP, async, and WebSocket TTS capabilities', () => { + const entry = findSpeechProviderCatalogEntry('minimax-tts'); + expect(entry).toMatchObject({ + kind: 'tts', + streaming: true, + defaultModel: 'speech-2.8-hd', + envVars: ['MINIMAX_API_KEY'], + }); + expect(entry?.features).toEqual(expect.arrayContaining(['async', 'websocket'])); + }); + it('should mark nvidia-nemo as unavailable (planned but not implemented)', () => { const entry = findSpeechProviderCatalogEntry('nvidia-nemo'); expect(entry).toBeDefined(); diff --git a/src/io/speech/index.ts b/src/io/speech/index.ts index 4619c38ba99..7ea76a6205a 100644 --- a/src/io/speech/index.ts +++ b/src/io/speech/index.ts @@ -10,6 +10,7 @@ export * from './providers/OpenAITextToSpeechProvider.js'; export * from './providers/ElevenLabsTextToSpeechProvider.js'; export * from './providers/DeepgramTextToSpeechProvider.js'; export * from './providers/AzureSpeechTTSProvider.js'; +export * from './providers/MiniMaxTextToSpeechProvider.js'; // STT/VAD providers have moved to the hearing/ module export * from '../hearing/providers/OpenAIWhisperSpeechToTextProvider.js'; export * from '../hearing/providers/BuiltInAdaptiveVadProvider.js'; diff --git a/src/io/speech/providerCatalog.ts b/src/io/speech/providerCatalog.ts index 0115c43e146..028fd1514a4 100644 --- a/src/io/speech/providerCatalog.ts +++ b/src/io/speech/providerCatalog.ts @@ -157,6 +157,17 @@ export const SPEECH_PROVIDER_CATALOG: readonly SpeechProviderCatalogEntry[] = [ defaultVoice: 'aura-2-thalia-en', features: ['cloud', 'tts', 'streaming', 'low-latency'], }, + { + id: 'minimax-tts', + kind: 'tts', + label: 'MiniMax TTS', + envVars: ['MINIMAX_API_KEY'], + local: false, + streaming: true, + description: 'Speech synthesis via MiniMax HTTP, async, and WebSocket APIs.', + defaultModel: 'speech-2.8-hd', + features: ['cloud', 'tts', 'streaming', 'websocket', 'async'], + }, { id: 'google-cloud-tts', kind: 'tts', diff --git a/src/io/speech/providers/MiniMaxTextToSpeechProvider.ts b/src/io/speech/providers/MiniMaxTextToSpeechProvider.ts new file mode 100644 index 00000000000..7e5ef399234 --- /dev/null +++ b/src/io/speech/providers/MiniMaxTextToSpeechProvider.ts @@ -0,0 +1,345 @@ +import WebSocket from "ws"; +import { ApiKeyPool } from "../../../core/providers/ApiKeyPool.js"; +import type { + SpeechSynthesisOptions, + SpeechSynthesisResult, + TextToSpeechProvider, +} from "../types.js"; + +export type MiniMaxSpeechRegion = "global" | "china"; +export type MiniMaxSpeechAudioFormat = "mp3" | "wav" | "flac" | "pcm"; + +export const MINIMAX_SPEECH_MODELS = [ + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", +] as const; + +const REGION_HOSTS: Record = { + global: "api.minimax.io", + china: "api.minimaxi.com", +}; + +const MIME_TYPES: Record = { + mp3: "audio/mpeg", + wav: "audio/wav", + flac: "audio/flac", + pcm: "audio/L16", +}; + +export interface MiniMaxTextToSpeechProviderConfig { + apiKey: string; + region?: MiniMaxSpeechRegion; + model?: string; + voice?: string; + baseUrl?: string; + webSocketUrl?: string; + fetchImpl?: typeof fetch; + webSocketFactory?: ( + url: string, + headers: Record, + ) => WebSocket; +} + +export interface MiniMaxSpeechProviderOptions { + languageBoost?: string; + outputFormat?: "hex" | "url"; + pronunciationDict?: Record; + audioSetting?: Record; + voiceSetting?: Record; + voiceModify?: Record; + subtitleEnable?: boolean; +} + +interface MiniMaxBaseResponse { + status_code?: number; + status_msg?: string; +} + +interface MiniMaxSpeechResponse { + data?: { audio?: string; status?: number }; + extra_info?: { audio_length?: number; usage_characters?: number }; + base_resp?: MiniMaxBaseResponse; +} + +export interface MiniMaxAsyncSpeechResponse { + task_id?: string; + file_id?: number; + status?: string; + base_resp?: MiniMaxBaseResponse; +} + +function decodeHexAudio(value: string): Buffer { + if (value.length % 2 !== 0 || !/^[0-9a-f]*$/i.test(value)) { + throw new Error("MiniMax speech returned invalid hex audio."); + } + return Buffer.from(value, "hex"); +} + +/** Text-to-speech provider for MiniMax HTTP, async, and WebSocket APIs. */ +export class MiniMaxTextToSpeechProvider implements TextToSpeechProvider { + public readonly id = "minimax-tts"; + public readonly displayName = "MiniMax TTS"; + public readonly supportsStreaming = true; + + private readonly fetchImpl: typeof fetch; + private readonly keyPool: ApiKeyPool; + private readonly region: MiniMaxSpeechRegion; + private readonly host: string; + + constructor(private readonly config: MiniMaxTextToSpeechProviderConfig) { + this.region = config.region ?? "global"; + this.host = REGION_HOSTS[this.region]; + this.fetchImpl = config.fetchImpl ?? fetch; + this.keyPool = new ApiKeyPool(config.apiKey); + } + + getProviderName(): string { + return this.displayName; + } + + async synthesize( + text: string, + options: SpeechSynthesisOptions = {}, + ): Promise { + const providerOptions = this.providerOptions(options); + const audioFormat = this.audioFormat(options, providerOptions); + const outputFormat = providerOptions.outputFormat ?? "hex"; + const response = await this.request("/v1/t2a_v2", { + ...this.buildRequest(text, options, providerOptions, audioFormat), + stream: false, + output_format: outputFormat, + subtitle_enable: providerOptions.subtitleEnable, + }); + const audio = response.data?.audio; + if (!audio || response.data?.status !== 2) { + throw new Error( + "MiniMax speech synthesis completed without audio output.", + ); + } + + const audioBuffer = + outputFormat === "url" + ? Buffer.from(await (await this.fetchImpl(audio)).arrayBuffer()) + : decodeHexAudio(audio); + return this.result(audioBuffer, text, options, audioFormat, response); + } + + async createAsync( + text: string, + options: SpeechSynthesisOptions = {}, + ): Promise { + const providerOptions = this.providerOptions(options); + return this.request( + "/v1/t2a_async_v2", + this.buildRequest( + text, + options, + providerOptions, + this.audioFormat(options, providerOptions), + ), + ); + } + + async queryAsync(taskId: string): Promise { + return this.request("/v1/query/t2a_async_query_v2", { task_id: taskId }); + } + + async synthesizeWebSocket( + text: string, + options: SpeechSynthesisOptions = {}, + ): Promise { + const providerOptions = this.providerOptions(options); + const audioFormat = this.audioFormat(options, providerOptions); + const socketUrl = + this.config.webSocketUrl ?? `wss://${this.host}/ws/v1/t2a_v2`; + const headers = { Authorization: `Bearer ${this.keyPool.next()}` }; + const socket = this.config.webSocketFactory + ? this.config.webSocketFactory(socketUrl, headers) + : new WebSocket(socketUrl, { headers }); + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let settled = false; + const fail = (error: Error) => { + if (settled) return; + settled = true; + socket.close(); + reject(error); + }; + + socket.on("message", (raw) => { + try { + const message = JSON.parse(raw.toString()) as { + event?: string; + data?: { audio?: string }; + base_resp?: MiniMaxBaseResponse; + }; + if ( + message.base_resp?.status_code && + message.base_resp.status_code !== 0 + ) { + fail( + new Error( + message.base_resp.status_msg ?? + "MiniMax WebSocket synthesis failed.", + ), + ); + return; + } + if (message.event === "connected_success") { + socket.send( + JSON.stringify({ + event: "task_start", + ...this.buildRequest("", options, providerOptions, audioFormat), + text: undefined, + }), + ); + } else if (message.event === "task_started") { + socket.send(JSON.stringify({ event: "task_continue", text })); + socket.send(JSON.stringify({ event: "task_finish" })); + } else if ( + message.event === "task_continued" && + message.data?.audio + ) { + chunks.push(decodeHexAudio(message.data.audio)); + } else if (message.event === "task_finished") { + settled = true; + socket.close(); + resolve( + this.result(Buffer.concat(chunks), text, options, audioFormat), + ); + } else if (message.event === "task_failed") { + fail(new Error("MiniMax WebSocket synthesis failed.")); + } + } catch (error) { + fail(error instanceof Error ? error : new Error(String(error))); + } + }); + socket.on("error", (error) => fail(error)); + socket.on("close", () => { + if (!settled) + fail( + new Error("MiniMax WebSocket closed before synthesis completed."), + ); + }); + }); + } + + private providerOptions( + options: SpeechSynthesisOptions, + ): MiniMaxSpeechProviderOptions { + return (options.providerSpecificOptions ?? + {}) as MiniMaxSpeechProviderOptions; + } + + private audioFormat( + options: SpeechSynthesisOptions, + providerOptions: MiniMaxSpeechProviderOptions, + ): MiniMaxSpeechAudioFormat { + const requested = + providerOptions.audioSetting?.["format"] ?? options.outputFormat ?? "mp3"; + if ( + requested !== "mp3" && + requested !== "wav" && + requested !== "flac" && + requested !== "pcm" + ) { + throw new Error("MiniMax speech format must be mp3, wav, flac, or pcm."); + } + return requested; + } + + private buildRequest( + text: string, + options: SpeechSynthesisOptions, + providerOptions: MiniMaxSpeechProviderOptions, + audioFormat: MiniMaxSpeechAudioFormat, + ): Record { + return { + model: options.model ?? this.config.model ?? "speech-2.8-hd", + text, + language_boost: providerOptions.languageBoost ?? options.languageCode, + voice_setting: { + voice_id: + options.voice ?? this.config.voice ?? "English_expressive_narrator", + ...(options.speed !== undefined ? { speed: options.speed } : {}), + ...(options.volume !== undefined ? { vol: options.volume } : {}), + ...(options.pitch !== undefined ? { pitch: options.pitch } : {}), + ...providerOptions.voiceSetting, + }, + pronunciation_dict: providerOptions.pronunciationDict, + audio_setting: { ...providerOptions.audioSetting, format: audioFormat }, + voice_modify: providerOptions.voiceModify, + }; + } + + private async request( + path: string, + body: Record, + ): Promise { + const baseUrl = this.config.baseUrl ?? `https://${this.host}`; + const response = await this.fetchImpl(`${baseUrl}${path}`, { + method: "POST", + headers: { + Authorization: `Bearer ${this.keyPool.next()}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + let payload: T & { base_resp?: MiniMaxBaseResponse }; + try { + payload = (await response.json()) as T & { + base_resp?: MiniMaxBaseResponse; + }; + } catch { + throw new Error( + `MiniMax speech returned invalid JSON (${response.status}).`, + ); + } + if (!response.ok || payload.base_resp?.status_code !== 0) { + const message = + payload.base_resp?.status_msg ?? + response.statusText ?? + "request failed"; + throw new Error( + `MiniMax speech request failed (${response.status}): ${message}`, + ); + } + return payload; + } + + private result( + audioBuffer: Buffer, + text: string, + options: SpeechSynthesisOptions, + audioFormat: MiniMaxSpeechAudioFormat, + response?: MiniMaxSpeechResponse, + ): SpeechSynthesisResult { + const voice = + options.voice ?? this.config.voice ?? "English_expressive_narrator"; + const model = options.model ?? this.config.model ?? "speech-2.8-hd"; + return { + audioBuffer, + mimeType: MIME_TYPES[audioFormat], + cost: 0, + durationSeconds: + response?.extra_info?.audio_length !== undefined + ? response.extra_info.audio_length / 1000 + : undefined, + providerResponse: response, + voiceUsed: voice, + providerName: this.displayName, + usage: { + characters: response?.extra_info?.usage_characters ?? text.length, + modelUsed: model, + region: this.region, + }, + }; + } +}