diff --git a/dashscope/audio/qwen_omni/__init__.py b/dashscope/audio/qwen_omni/__init__.py index 629fba1..a371cb8 100644 --- a/dashscope/audio/qwen_omni/__init__.py +++ b/dashscope/audio/qwen_omni/__init__.py @@ -3,6 +3,9 @@ from .omni_realtime import ( AudioFormat, + AudioFormatConfig, + AudioFormatType, + AudioSampleRate, MultiModality, OmniRealtimeCallback, OmniRealtimeConversation, @@ -11,6 +14,9 @@ __all__ = [ "OmniRealtimeCallback", "AudioFormat", + "AudioFormatConfig", + "AudioFormatType", + "AudioSampleRate", "MultiModality", "OmniRealtimeConversation", ] diff --git a/dashscope/audio/qwen_omni/omni_realtime.py b/dashscope/audio/qwen_omni/omni_realtime.py index e3a1ad7..4bddc53 100644 --- a/dashscope/audio/qwen_omni/omni_realtime.py +++ b/dashscope/audio/qwen_omni/omni_realtime.py @@ -5,7 +5,7 @@ import threading import time from dataclasses import field, dataclass -from typing import List, Any, Dict +from typing import List, Any, Dict, Union import uuid from enum import Enum, unique @@ -87,6 +87,99 @@ def __str__(self): return f"{self.format.upper()} with {self.sample_rate}Hz sample rate, {self.channels} channel, {self.bit_rate} bit rate: {self.format_str}" # noqa: E501 # pylint: disable=line-too-long +@unique +class AudioFormatType(Enum): + """ + Preset audio format types for omni realtime input/output audio. + + These are provided for convenient reference only. ``AudioFormatConfig`` + also accepts any raw string value (e.g. ``"mp3"``) without validation, + so new formats can be used without upgrading the SDK. + """ + + PCM = "pcm" + WAV = "wav" + + def __str__(self): + return self.value + + +@unique +class AudioSampleRate(Enum): + """ + Preset audio sample rates(Hz) for omni realtime input/output audio. + + These are provided for convenient reference only. ``AudioFormatConfig`` + also accepts any raw int value without validation. + """ + + SAMPLE_RATE_8K = 8000 + SAMPLE_RATE_16K = 16000 + SAMPLE_RATE_24K = 24000 + SAMPLE_RATE_48K = 48000 + + def __int__(self): + return self.value + + +@dataclass +class AudioFormatConfig: + """ + Audio format config for input(uplink) / output(downlink) audio in a + omni realtime session. + + No strict validation is performed: the preset enums are only for + convenient reference, while any raw value is also accepted so that new + formats or sample rates can be used without upgrading the SDK. + + Parameters + ---------- + type: str + audio format type. Presets ``AudioFormatType.PCM`` / ``.WAV`` are + provided for convenience; accepts an ``AudioFormatType`` enum or any + raw string such as ``"pcm"`` / ``"wav"`` / ``"mp3"``. + sample_rate: int + audio sample rate in Hz. Presets ``AudioSampleRate`` (8000 / 16000 / + 24000 / 48000) are provided for convenience; accepts an + ``AudioSampleRate`` enum or any raw int value. Defaults to 16000, + which suits input audio; for output audio the server-side default + is typically 24000, set it explicitly if needed. + extra_params: Dict[str, Any] + free extension parameters that will be merged into the ``format`` + dict. Reserved for future parameters such as speech rate, e.g. + ``{"speech_rate": 1.2}``. Merged last, so identical keys (including + ``type`` / ``sample_rate``) override the dataclass fields. + """ + + type: Union[str, AudioFormatType] = AudioFormatType.PCM.value + sample_rate: Union[ + int, + AudioSampleRate, + ] = AudioSampleRate.SAMPLE_RATE_16K.value + extra_params: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """ + convert to the ``format`` dict used in the session.update request. + + Preset enums are unwrapped to their underlying value; any raw value + is passed through as-is without validation. + """ + format_type = self.type + if isinstance(format_type, AudioFormatType): + format_type = format_type.value + sample_rate = self.sample_rate + if isinstance(sample_rate, AudioSampleRate): + sample_rate = sample_rate.value + result: Dict[str, Any] = { + "type": format_type, + "sample_rate": sample_rate, + } + if self.extra_params: + result.update(self.extra_params) + return result + + class MultiModality(Enum): """ MultiModality @@ -232,6 +325,76 @@ def create_item(self, item: dict): enable_log=True, ) + def _apply_audio_format( + self, + input_audio_format: AudioFormat, + output_audio_format: AudioFormat, + input_audio_config: AudioFormatConfig, + output_audio_config: AudioFormatConfig, + ) -> None: + """ + apply audio format config into ``self.config``. + + Prefer the new-style ``session.audio.{input,output}.format`` structure + when ``input_audio_config`` / ``output_audio_config`` is provided, + otherwise keep the old-style top-level fields for backward + compatibility. + + When only one side is provided, the other side is omitted from the + request and the server-side default applies (input: pcm/16000, + output: pcm/24000); the legacy top-level fields are NOT mixed in, + to avoid sending both structures in one request. + """ + if input_audio_config is None and output_audio_config is None: + # old-style: keep backward compatibility + self.config["input_audio_format"] = input_audio_format.format_str + self.config["output_audio_format"] = output_audio_format.format_str + return + # new-style: separate audio format type and sample rate under + # session.audio.{input,output}.format + audio: Dict[str, Any] = {} + if input_audio_config is not None: + audio["input"] = {"format": input_audio_config.to_dict()} + if output_audio_config is not None: + audio["output"] = {"format": output_audio_config.to_dict()} + self.config["audio"] = audio + + def _apply_transcription_params( + self, + transcription_params: TranscriptionParams, + ) -> None: + """ + apply transcription params into ``self.config``. + + The legacy top-level ``input_audio_format`` / ``sample_rate`` fields + are skipped when the new-style ``audio`` structure is present, to + avoid sending conflicting audio format fields in one request. + """ + self.config["input_audio_transcription"] = {} + if transcription_params.language is not None: + self.config["input_audio_transcription"].update( + {"language": transcription_params.language}, + ) + if transcription_params.corpus_text is not None: + transcription_params.corpus = { + "text": transcription_params.corpus_text, + } + if transcription_params.corpus is not None: + self.config["input_audio_transcription"].update( + {"corpus": transcription_params.corpus}, + ) + if "audio" in self.config: + logger.warning( + "[omni realtime] input_audio_config/output_audio_config is " + "set, skip top-level input_audio_format/sample_rate from " + "transcription_params to avoid conflicting audio formats", + ) + return + self.config[ + "input_audio_format" + ] = transcription_params.input_audio_format + self.config["sample_rate"] = transcription_params.sample_rate + def update_session( self, output_modalities: List[MultiModality], @@ -248,6 +411,8 @@ def update_session( turn_detection_param: dict = None, translation_params: TranslationParams = None, transcription_params: TranscriptionParams = None, + input_audio_config: AudioFormatConfig = None, + output_audio_config: AudioFormatConfig = None, **kwargs, ) -> None: """ @@ -260,9 +425,15 @@ def update_session( voice: str voice to be used in session input_audio_format: AudioFormat - input audio format + input audio format. Legacy parameter, kept for backward + compatibility and still the default path. Prefer + ``input_audio_config`` which supports separate audio format + type and sample rate. output_audio_format: AudioFormat - output audio format + output audio format. Legacy parameter, kept for backward + compatibility and still the default path. Prefer + ``output_audio_config`` which supports separate audio format + type and sample rate. enable_turn_detection: bool enable turn detection turn_detection_threshold: float @@ -278,13 +449,35 @@ def update_session( transcription params, include language, sample_rate, input_audio_format, corpus. # noqa: E501 # pylint: disable=line-too-long Only effective with qwen3-asr-flash-realtime model or further models. Do not set this parameter for other models. + input_audio_config: AudioFormatConfig + input(uplink) audio format config. Supports separate format type + (pcm/wav) and sample rate (8000/16000/24000/48000), as well as + free extension parameters via ``extra_params``. When provided, + the request emits the ``session.audio.input.format`` structure. + output_audio_config: AudioFormatConfig + output(downlink) audio format config. Supports separate format + type (pcm/wav) and sample rate (8000/16000/24000/48000), as well + as free extension parameters via ``extra_params``. When provided, + the request emits the ``session.audio.output.format`` structure. + + Notes + ----- + Once either ``input_audio_config`` or ``output_audio_config`` is + provided, the legacy top-level ``input_audio_format`` / + ``output_audio_format`` fields are not sent; an unspecified side + falls back to the server-side default. The top-level format fields + from ``transcription_params`` are also skipped in this case. """ self.config = { "modalities": [m.value for m in output_modalities], "voice": voice, - "input_audio_format": input_audio_format.format_str, - "output_audio_format": output_audio_format.format_str, } + self._apply_audio_format( + input_audio_format, + output_audio_format, + input_audio_config, + output_audio_config, + ) if enable_input_audio_transcription: self.config["input_audio_transcription"] = { "model": input_audio_transcription_model, @@ -315,23 +508,7 @@ def update_session( "phrases": translation_params.corpus.phrases, } if transcription_params is not None: - self.config["input_audio_transcription"] = {} - if transcription_params.language is not None: - self.config["input_audio_transcription"].update( - {"language": transcription_params.language}, - ) - if transcription_params.corpus_text is not None: - transcription_params.corpus = { - "text": transcription_params.corpus_text, - } - if transcription_params.corpus is not None: - self.config["input_audio_transcription"].update( - {"corpus": transcription_params.corpus}, - ) - self.config[ - "input_audio_format" - ] = transcription_params.input_audio_format - self.config["sample_rate"] = transcription_params.sample_rate + self._apply_transcription_params(transcription_params) self.config.update(kwargs) self.__send_str( json.dumps(