diff --git a/.changeset/calm-spoons-listen.md b/.changeset/calm-spoons-listen.md new file mode 100644 index 000000000..5c0ca3438 --- /dev/null +++ b/.changeset/calm-spoons-listen.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-elevenlabs': patch +--- + +Populate ElevenLabs STT confidence from Scribe word log probabilities. diff --git a/plugins/elevenlabs/src/stt.test.ts b/plugins/elevenlabs/src/stt.test.ts index 449af9300..81583c714 100644 --- a/plugins/elevenlabs/src/stt.test.ts +++ b/plugins/elevenlabs/src/stt.test.ts @@ -17,6 +17,21 @@ function makeFrame(samplesPerChannel = 800, sampleRate = 16000): AudioFrame { return new AudioFrame(data, sampleRate, 1, samplesPerChannel); } +async function recognizeWords(words?: Record[]) { + const { server, baseURL } = await startHttpServer((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ text: 'hello', language_code: 'en', words })); + }); + + try { + return await new STT({ apiKey: 'test-key', baseURL }).recognize(makeFrame(), { + connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + } finally { + await closeHttpServer(server); + } +} + async function startHttpServer(handler: RequestListener) { const server = createServer(handler); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); @@ -135,6 +150,73 @@ describe('ElevenLabs STT integration', () => { }); describe('ElevenLabs STT', () => { + it('calculates confidence from spoken-word logprobs', async () => { + const event = await recognizeWords([ + { type: 'word', logprob: -0.01 }, + { type: 'spacing', logprob: -2 }, + { type: 'word', logprob: -0.05 }, + ]); + + expect(event.alternatives?.[0]?.confidence).toBeGreaterThan(0.9); + expect(event.alternatives?.[0]?.confidence).toBeLessThanOrEqual(1); + }); + + it('flags low-quality transcription confidence', async () => { + const event = await recognizeWords([ + { type: 'word', logprob: -2.5 }, + { type: 'word', logprob: -3 }, + ]); + + expect(event.alternatives?.[0]?.confidence).toBeLessThan(0.2); + }); + + it('defaults confidence to zero without logprobs', async () => { + const withoutWords = await recognizeWords(); + const withoutLogprobs = await recognizeWords([{ text: 'hi', start: 0.1, end: 0.4 }]); + + expect(withoutWords.alternatives?.[0]?.confidence).toBe(0); + expect(withoutLogprobs.alternatives?.[0]?.confidence).toBe(0); + }); + + it('sets confidence on committed transcripts', async () => { + const { wss, baseURL } = await startWebSocketServer(); + let connected = false; + + wss.on('connection', (ws) => { + connected = true; + ws.on('message', () => { + ws.send( + JSON.stringify({ + message_type: 'committed_transcript', + text: 'hello', + words: [{ text: 'hello', start: 0.1, end: 0.4, type: 'word', logprob: -0.02 }], + }), + ); + ws.send(JSON.stringify({ message_type: 'committed_transcript', text: '' })); + }); + }); + + try { + const stream = new STT({ + apiKey: 'test-key', + baseURL, + model: 'scribe_v2_realtime', + serverVad: { vadSilenceThresholdSecs: 0.5 }, + }).stream(); + await waitUntil(() => connected); + stream.pushFrame(makeFrame()); + stream.flush(); + stream.endInput(); + + const events = await collectUntilEnd(stream); + stream.close(); + const final = events.find((event) => event.type === sttLib.SpeechEventType.FINAL_TRANSCRIPT); + expect(final?.alternatives?.[0]?.confidence).toBeGreaterThan(0.9); + } finally { + await closeWebSocketServer(wss); + } + }); + it('defaults to Scribe v1 batch recognition', () => { const stt = new STT({ apiKey: 'test-key' }); diff --git a/plugins/elevenlabs/src/stt.ts b/plugins/elevenlabs/src/stt.ts index b279dd084..145add2c6 100644 --- a/plugins/elevenlabs/src/stt.ts +++ b/plugins/elevenlabs/src/stt.ts @@ -88,6 +88,8 @@ interface ElevenLabsWord { start?: number; end?: number; speaker_id?: string | null; + type?: string; + logprob?: number; } interface ElevenLabsBatchResponse { @@ -158,10 +160,26 @@ function asWords(value: unknown): ElevenLabsWord[] { start: asNumber(record.start), end: asNumber(record.end), speaker_id: asString(record.speaker_id) ?? null, + type: asString(record.type), + logprob: asNumber(record.logprob), }; }); } +function speechConfidence(words?: ElevenLabsWord[]): number { + if (!words) return 0; + + const logprobs = words.flatMap((word) => + word.type === 'word' && word.logprob !== undefined ? [word.logprob] : [], + ); + if (logprobs.length === 0) return 0; + + return Math.min( + 1, + Math.max(0, Math.exp(logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length)), + ); +} + function parseBatchResponse(value: unknown): ElevenLabsBatchResponse { const record = toRecord(value); return { @@ -428,7 +446,7 @@ export class STT extends stt.STT { speakerId, startTime, endTime, - confidence: 0, + confidence: speechConfidence(words), words: words?.map((word) => createTimedString({ text: word.text ?? '', @@ -798,7 +816,7 @@ export class SpeechStream extends stt.SpeechStream { text, startTime: startTime + this.startTimeOffset, endTime: endTime + this.startTimeOffset, - confidence: 0, + confidence: speechConfidence(words), }; if (words.length > 0) { speechData.words = words.map((word) =>