From c8849faf84698352ce531926b325d1b2847f83a5 Mon Sep 17 00:00:00 2001 From: Tina Nguyen Date: Fri, 31 Jul 2026 16:15:55 -0400 Subject: [PATCH] add keyterms --- .changeset/cartesia-stt-keyterms.md | 5 ++ agents/src/inference/stt.test.ts | 9 ++++ plugins/cartesia/src/stt.test.ts | 51 +++++++++++++++++++- plugins/cartesia/src/stt.ts | 75 +++++++++++++++++++++++------ 4 files changed, 125 insertions(+), 15 deletions(-) create mode 100644 .changeset/cartesia-stt-keyterms.md diff --git a/.changeset/cartesia-stt-keyterms.md b/.changeset/cartesia-stt-keyterms.md new file mode 100644 index 000000000..be0b06458 --- /dev/null +++ b/.changeset/cartesia-stt-keyterms.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-cartesia': patch +--- + +Expose Cartesia STT keyterm prompting: pass `keyterm` to bias recognition toward specific words and phrases. Turn-detecting models (e.g. `ink-2`) only. diff --git a/agents/src/inference/stt.test.ts b/agents/src/inference/stt.test.ts index e79bad61f..81fb3196d 100644 --- a/agents/src/inference/stt.test.ts +++ b/agents/src/inference/stt.test.ts @@ -544,6 +544,15 @@ describe('STT session keyterms', () => { stream.close(); }); + + it('passes Cartesia keyterms through to the gateway extra', () => { + const stt = makeStt({ model: 'cartesia/ink-2', modelOptions: { keyterm: ['Acme'] } }); + const stream = stt.stream(); + + expect(stream['opts'].modelOptions).toHaveProperty('keyterm', ['Acme']); + + stream.close(); + }); }); describe('STT VAD handling for Speechmatics models', () => { diff --git a/plugins/cartesia/src/stt.test.ts b/plugins/cartesia/src/stt.test.ts index f2e5f39c7..fb073575c 100644 --- a/plugins/cartesia/src/stt.test.ts +++ b/plugins/cartesia/src/stt.test.ts @@ -4,7 +4,7 @@ import { VAD } from '@livekit/agents-plugin-silero'; import { stt } from '@livekit/agents-plugins-test'; import { describe, expect, it } from 'vitest'; -import { STT } from './stt.js'; +import { STT, buildSTTWebsocketUrl } from './stt.js'; const hasCartesiaApiKey = Boolean(process.env.CARTESIA_API_KEY); @@ -16,6 +16,55 @@ describe('Cartesia STT capabilities', () => { }); }); +describe('Cartesia STT keyterms', () => { + const baseOpts = { + apiKey: 'test-key', + model: 'ink-2', + sampleRate: 16_000, + baseUrl: 'https://api.cartesia.ai', + audioChunkDurationMS: 160, + language: 'en', + }; + + it('sends one keyterm query param per term', () => { + const url = new URL(buildSTTWebsocketUrl({ ...baseOpts, keyterm: ['LiveKit', 'Cartesia'] })); + + expect(url.searchParams.getAll('keyterm')).toEqual(['LiveKit', 'Cartesia']); + expect(url.searchParams.get('model')).toBe('ink-2'); + }); + + it('omits the keyterm param when no terms are set', () => { + const url = new URL(buildSTTWebsocketUrl(baseOpts)); + + expect(url.searchParams.has('keyterm')).toBe(false); + }); + + it('rejects keyterms on non turn-detecting models', () => { + expect( + () => new STT({ apiKey: 'test-key', model: 'ink-whisper', keyterm: ['LiveKit'] }), + ).toThrow(/only supported by turn-detecting models/); + }); + + it('rejects a language switch that would route keyterms to ink-whisper', () => { + const instance = new STT({ apiKey: 'test-key', keyterm: ['LiveKit'] }); + expect(instance.model).toBe('ink-2'); + + // 'fr' resolves to the multilingual ink-whisper, which does not take keyterms + expect(() => instance.updateOptions({ language: 'fr' })).toThrow( + /only supported by turn-detecting models/, + ); + // the rejected update must not have been committed + expect(instance.model).toBe('ink-2'); + }); + + it('allows switching to ink-whisper once keyterms are cleared', () => { + const instance = new STT({ apiKey: 'test-key', keyterm: ['LiveKit'] }); + + instance.updateOptions({ keyterm: [], language: 'fr' }); + expect(instance.model).toBe('ink-whisper'); + }); +}); + if (hasCartesiaApiKey) { describe('Cartesia STT', async () => { await stt(new STT(), await VAD.load(), { nonStreaming: false }); diff --git a/plugins/cartesia/src/stt.ts b/plugins/cartesia/src/stt.ts index 9cb8ddd35..6bdc38bb1 100644 --- a/plugins/cartesia/src/stt.ts +++ b/plugins/cartesia/src/stt.ts @@ -143,6 +143,14 @@ export type STTOptions = { baseUrl: string; audioChunkDurationMS: number; language: string; + /** + * Key terms to improve recall of specific words and phrases (up to 100 terms + * totaling 1200 characters). + * + * Turn-detecting models (e.g. `ink-2`) only — passing this with `ink-whisper` + * throws. + */ + keyterm?: string[]; }; const defaultSTTOptions = { @@ -164,6 +172,7 @@ function mergeSTTOptions(base: STTOptions, override: Partial): STTOp audioChunkDurationMS: override.audioChunkDurationMS ?? base.audioChunkDurationMS, language: override.language !== undefined ? normalizeLanguage(override.language) : base.language, + keyterm: override.keyterm ?? base.keyterm, }; } @@ -176,6 +185,49 @@ function resolveSTTModel(language: string): STTModel { return getBaseLanguage(language) === 'en' ? 'ink-2' : 'ink-whisper'; } +function isWhisperModel(model: string): boolean { + return model.startsWith('ink-whisper'); +} + +/** + * Keyterm prompting is only offered by the turn-detecting models (e.g. `ink-2`); + * `ink-whisper` does not accept it. Mirrors the Python plugin, which raises on + * the same combination rather than silently dropping the terms. + */ +function validateKeyterm(model: string, keyterm: string[] | undefined) { + if (keyterm?.length && isWhisperModel(model)) { + throw new Error( + `The 'keyterm' parameter is only supported by turn-detecting models (e.g. ink-2); model '${model}' does not support it.`, + ); + } +} + +/** + * Build the `/stt/turns/websocket` URL for a set of options. + * + * The Cartesia endpoint only accepts model, sample_rate, encoding and keyterm — + * there is no `language` query param. Language selection is expressed through + * the model (ink-2 for English, ink-whisper otherwise), so `language` is used + * only to tag emitted transcripts. + * + * @internal + */ +export function buildSTTWebsocketUrl(opts: STTOptions): string { + const params = new URLSearchParams({ + model: opts.model, + sample_rate: opts.sampleRate.toString(), + encoding: AUDIO_ENCODING, + }); + + // keyterm repeats, one query param per term + for (const term of opts.keyterm ?? []) { + params.append('keyterm', term); + } + + const wsBase = opts.baseUrl.replace(/^http/, 'ws'); + return `${wsBase}/stt/turns/websocket?${params.toString()}`; +} + /** * Cartesia speech to text. * @@ -224,6 +276,8 @@ export class STT extends stt.STT { if (opts.model === undefined) { this.#opts.model = resolveSTTModel(this.#opts.language); } + + validateKeyterm(this.#opts.model, this.#opts.keyterm); } override get label(): string { @@ -247,14 +301,18 @@ export class STT extends stt.STT { } updateOptions(opts: Partial) { - this.#opts = mergeSTTOptions(this.#opts, opts); + const nextOpts = mergeSTTOptions(this.#opts, opts); // Keep the model in sync with a newly set language (e.g. switching to a // non-English language must move off the English-only ink-2), unless the // caller pinned a model in the same call. Mirrors the constructor. if (opts.language !== undefined && opts.model === undefined) { - this.#opts.model = resolveSTTModel(this.#opts.language); + nextOpts.model = resolveSTTModel(nextOpts.language); } + + // Validate before committing so a rejected update leaves the STT usable. + validateKeyterm(nextOpts.model, nextOpts.keyterm); + this.#opts = nextOpts; } } @@ -632,18 +690,7 @@ export class SpeechStream extends stt.SpeechStream { } #getCartesiaUrl(): string { - // The Cartesia /stt/turns/websocket endpoint only accepts model, sample_rate - // and encoding — there is no `language` query param. Language selection is - // expressed through the model (ink-2 for English, ink-whisper otherwise), - // so #opts.language is used only to tag emitted transcripts. - const params = new URLSearchParams({ - model: this.#opts.model, - sample_rate: this.#opts.sampleRate.toString(), - encoding: AUDIO_ENCODING, - }); - - const wsBase = this.#opts.baseUrl.replace(/^http/, 'ws'); - return `${wsBase}/stt/turns/websocket?${params.toString()}`; + return buildSTTWebsocketUrl(this.#opts); } override close() {