fix(elevenlabs): populate SpeechData.confidence from Scribe logprobs

.changeset/calm-spoons-listen.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-elevenlabs': patch
+---
+
+Populate ElevenLabs STT confidence from Scribe word log probabilities.

plugins/elevenlabs/src/stt.test.ts

@@ -17,6 +17,21 @@ function makeFrame(samplesPerChannel = 800, sampleRate = 16000): AudioFrame {
   return new AudioFrame(data, sampleRate, 1, samplesPerChannel);
 }
 
+async function recognizeWords(words?: Record<string, unknown>[]) {
+  const { server, baseURL } = await startHttpServer((_req, res) => {
+    res.writeHead(200, { 'content-type': 'application/json' });
+    res.end(JSON.stringify({ text: 'hello', language_code: 'en', words }));
+  });
+
+  try {
+    return await new STT({ apiKey: 'test-key', baseURL }).recognize(makeFrame(), {
+      connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 1000 },
+    });
+  } finally {
+    await closeHttpServer(server);
+  }
+}
+
 async function startHttpServer(handler: RequestListener) {
   const server = createServer(handler);
   await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
@@ -135,6 +150,73 @@ describe('ElevenLabs STT integration', () => {
 });
 
 describe('ElevenLabs STT', () => {
+  it('calculates confidence from spoken-word logprobs', async () => {
+    const event = await recognizeWords([
+      { type: 'word', logprob: -0.01 },
+      { type: 'spacing', logprob: -2 },
+      { type: 'word', logprob: -0.05 },
+    ]);
+
+    expect(event.alternatives?.[0]?.confidence).toBeGreaterThan(0.9);
+    expect(event.alternatives?.[0]?.confidence).toBeLessThanOrEqual(1);
+  });
+
+  it('flags low-quality transcription confidence', async () => {
+    const event = await recognizeWords([
+      { type: 'word', logprob: -2.5 },
+      { type: 'word', logprob: -3 },
+    ]);
+
+    expect(event.alternatives?.[0]?.confidence).toBeLessThan(0.2);
+  });
+
+  it('defaults confidence to zero without logprobs', async () => {
+    const withoutWords = await recognizeWords();
+    const withoutLogprobs = await recognizeWords([{ text: 'hi', start: 0.1, end: 0.4 }]);
+
+    expect(withoutWords.alternatives?.[0]?.confidence).toBe(0);
+    expect(withoutLogprobs.alternatives?.[0]?.confidence).toBe(0);
+  });
+
+  it('sets confidence on committed transcripts', async () => {
+    const { wss, baseURL } = await startWebSocketServer();
+    let connected = false;
+
+    wss.on('connection', (ws) => {
+      connected = true;
+      ws.on('message', () => {
+        ws.send(
+          JSON.stringify({
+            message_type: 'committed_transcript',
+            text: 'hello',
+            words: [{ text: 'hello', start: 0.1, end: 0.4, type: 'word', logprob: -0.02 }],
+          }),
+        );
+        ws.send(JSON.stringify({ message_type: 'committed_transcript', text: '' }));
+      });
+    });
+
+    try {
+      const stream = new STT({
+        apiKey: 'test-key',
+        baseURL,
+        model: 'scribe_v2_realtime',
+        serverVad: { vadSilenceThresholdSecs: 0.5 },
+      }).stream();
+      await waitUntil(() => connected);
+      stream.pushFrame(makeFrame());
+      stream.flush();
+      stream.endInput();
+
+      const events = await collectUntilEnd(stream);
+      stream.close();
+      const final = events.find((event) => event.type === sttLib.SpeechEventType.FINAL_TRANSCRIPT);
+      expect(final?.alternatives?.[0]?.confidence).toBeGreaterThan(0.9);
+    } finally {
+      await closeWebSocketServer(wss);
+    }
+  });
+
   it('defaults to Scribe v1 batch recognition', () => {
     const stt = new STT({ apiKey: 'test-key' });
 

plugins/elevenlabs/src/stt.ts

@@ -88,6 +88,8 @@ interface ElevenLabsWord {
   start?: number;
   end?: number;
   speaker_id?: string | null;
+  type?: string;
+  logprob?: number;
 }
 
 interface ElevenLabsBatchResponse {
@@ -158,10 +160,26 @@ function asWords(value: unknown): ElevenLabsWord[] {
       start: asNumber(record.start),
       end: asNumber(record.end),
       speaker_id: asString(record.speaker_id) ?? null,
+      type: asString(record.type),
+      logprob: asNumber(record.logprob),
     };
   });
 }
 
+function speechConfidence(words?: ElevenLabsWord[]): number {
+  if (!words) return 0;
+
+  const logprobs = words.flatMap((word) =>
+    word.type === 'word' && word.logprob !== undefined ? [word.logprob] : [],
+  );
+  if (logprobs.length === 0) return 0;
+
+  return Math.min(
+    1,
+    Math.max(0, Math.exp(logprobs.reduce((sum, value) => sum + value, 0) / logprobs.length)),
+  );
+}
+
 function parseBatchResponse(value: unknown): ElevenLabsBatchResponse {
   const record = toRecord(value);
   return {
@@ -428,7 +446,7 @@ export class STT extends stt.STT {
           speakerId,
           startTime,
           endTime,
-          confidence: 0,
+          confidence: speechConfidence(words),
           words: words?.map((word) =>
             createTimedString({
               text: word.text ?? '',
@@ -798,7 +816,7 @@ export class SpeechStream extends stt.SpeechStream {
       text,
       startTime: startTime + this.startTimeOffset,
       endTime: endTime + this.startTimeOffset,
-      confidence: 0,
+      confidence: speechConfidence(words),
     };
     if (words.length > 0) {
       speechData.words = words.map((word) =>