feat(model/omni):support session.audio.input/output

dashscope/audio/qwen_omni/__init__.py

@@ -3,6 +3,9 @@
 
 from .omni_realtime import (
     AudioFormat,
+    AudioFormatConfig,
+    AudioFormatType,
+    AudioSampleRate,
     MultiModality,
     OmniRealtimeCallback,
     OmniRealtimeConversation,
@@ -11,6 +14,9 @@
 __all__ = [
     "OmniRealtimeCallback",
     "AudioFormat",
+    "AudioFormatConfig",
+    "AudioFormatType",
+    "AudioSampleRate",
     "MultiModality",
     "OmniRealtimeConversation",
 ]

dashscope/audio/qwen_omni/omni_realtime.py

@@ -5,7 +5,7 @@
 import threading
 import time
 from dataclasses import field, dataclass
-from typing import List, Any, Dict
+from typing import List, Any, Dict, Union
 import uuid
 from enum import Enum, unique
 
@@ -87,6 +87,99 @@ def __str__(self):
         return f"{self.format.upper()} with {self.sample_rate}Hz sample rate, {self.channels} channel, {self.bit_rate} bit rate:  {self.format_str}"  # noqa: E501  # pylint: disable=line-too-long
 
 
+@unique
+class AudioFormatType(Enum):
+    """
+    Preset audio format types for omni realtime input/output audio.
+
+    These are provided for convenient reference only. ``AudioFormatConfig``
+    also accepts any raw string value (e.g. ``"mp3"``) without validation,
+    so new formats can be used without upgrading the SDK.
+    """
+
+    PCM = "pcm"
+    WAV = "wav"
+
+    def __str__(self):
+        return self.value
+
+
+@unique
+class AudioSampleRate(Enum):
+    """
+    Preset audio sample rates(Hz) for omni realtime input/output audio.
+
+    These are provided for convenient reference only. ``AudioFormatConfig``
+    also accepts any raw int value without validation.
+    """
+
+    SAMPLE_RATE_8K = 8000
+    SAMPLE_RATE_16K = 16000
+    SAMPLE_RATE_24K = 24000
+    SAMPLE_RATE_48K = 48000
+
+    def __int__(self):
+        return self.value
+
+
+@dataclass
+class AudioFormatConfig:
+    """
+    Audio format config for input(uplink) / output(downlink) audio in a
+    omni realtime session.
+
+    No strict validation is performed: the preset enums are only for
+    convenient reference, while any raw value is also accepted so that new
+    formats or sample rates can be used without upgrading the SDK.
+
+    Parameters
+    ----------
+    type: str
+        audio format type. Presets ``AudioFormatType.PCM`` / ``.WAV`` are
+        provided for convenience; accepts an ``AudioFormatType`` enum or any
+        raw string such as ``"pcm"`` / ``"wav"`` / ``"mp3"``.
+    sample_rate: int
+        audio sample rate in Hz. Presets ``AudioSampleRate`` (8000 / 16000 /
+        24000 / 48000) are provided for convenience; accepts an
+        ``AudioSampleRate`` enum or any raw int value. Defaults to 16000,
+        which suits input audio; for output audio the server-side default
+        is typically 24000, set it explicitly if needed.
+    extra_params: Dict[str, Any]
+        free extension parameters that will be merged into the ``format``
+        dict. Reserved for future parameters such as speech rate, e.g.
+        ``{"speech_rate": 1.2}``. Merged last, so identical keys (including
+        ``type`` / ``sample_rate``) override the dataclass fields.
+    """
+
+    type: Union[str, AudioFormatType] = AudioFormatType.PCM.value
+    sample_rate: Union[
+        int,
+        AudioSampleRate,
+    ] = AudioSampleRate.SAMPLE_RATE_16K.value
+    extra_params: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        convert to the ``format`` dict used in the session.update request.
+
+        Preset enums are unwrapped to their underlying value; any raw value
+        is passed through as-is without validation.
+        """
+        format_type = self.type
+        if isinstance(format_type, AudioFormatType):
+            format_type = format_type.value
+        sample_rate = self.sample_rate
+        if isinstance(sample_rate, AudioSampleRate):
+            sample_rate = sample_rate.value
+        result: Dict[str, Any] = {
+            "type": format_type,
+            "sample_rate": sample_rate,
+        }
+        if self.extra_params:
+            result.update(self.extra_params)
+        return result
+
+
 class MultiModality(Enum):
     """
     MultiModality
@@ -232,6 +325,76 @@ def create_item(self, item: dict):
             enable_log=True,
         )
 
+    def _apply_audio_format(
+        self,
+        input_audio_format: AudioFormat,
+        output_audio_format: AudioFormat,
+        input_audio_config: AudioFormatConfig,
+        output_audio_config: AudioFormatConfig,
+    ) -> None:
+        """
+        apply audio format config into ``self.config``.
+
+        Prefer the new-style ``session.audio.{input,output}.format`` structure
+        when ``input_audio_config`` / ``output_audio_config`` is provided,
+        otherwise keep the old-style top-level fields for backward
+        compatibility.
+
+        When only one side is provided, the other side is omitted from the
+        request and the server-side default applies (input: pcm/16000,
+        output: pcm/24000); the legacy top-level fields are NOT mixed in,
+        to avoid sending both structures in one request.
+        """
+        if input_audio_config is None and output_audio_config is None:
+            # old-style: keep backward compatibility
+            self.config["input_audio_format"] = input_audio_format.format_str
+            self.config["output_audio_format"] = output_audio_format.format_str
+            return
+        # new-style: separate audio format type and sample rate under
+        # session.audio.{input,output}.format
+        audio: Dict[str, Any] = {}
+        if input_audio_config is not None:
+            audio["input"] = {"format": input_audio_config.to_dict()}
+        if output_audio_config is not None:
+            audio["output"] = {"format": output_audio_config.to_dict()}
+        self.config["audio"] = audio
+
+    def _apply_transcription_params(
+        self,
+        transcription_params: TranscriptionParams,
+    ) -> None:
+        """
+        apply transcription params into ``self.config``.
+
+        The legacy top-level ``input_audio_format`` / ``sample_rate`` fields
+        are skipped when the new-style ``audio`` structure is present, to
+        avoid sending conflicting audio format fields in one request.
+        """
+        self.config["input_audio_transcription"] = {}
+        if transcription_params.language is not None:
+            self.config["input_audio_transcription"].update(
+                {"language": transcription_params.language},
+            )
+        if transcription_params.corpus_text is not None:
+            transcription_params.corpus = {
+                "text": transcription_params.corpus_text,
+            }
+        if transcription_params.corpus is not None:
+            self.config["input_audio_transcription"].update(
+                {"corpus": transcription_params.corpus},
+            )
+        if "audio" in self.config:
+            logger.warning(
+                "[omni realtime] input_audio_config/output_audio_config is "
+                "set, skip top-level input_audio_format/sample_rate from "
+                "transcription_params to avoid conflicting audio formats",
+            )
+            return
+        self.config[
+            "input_audio_format"
+        ] = transcription_params.input_audio_format
+        self.config["sample_rate"] = transcription_params.sample_rate
+
     def update_session(
         self,
         output_modalities: List[MultiModality],
@@ -248,6 +411,8 @@ def update_session(
         turn_detection_param: dict = None,
         translation_params: TranslationParams = None,
         transcription_params: TranscriptionParams = None,
+        input_audio_config: AudioFormatConfig = None,
+        output_audio_config: AudioFormatConfig = None,
         **kwargs,
     ) -> None:
         """
@@ -260,9 +425,15 @@ def update_session(
         voice: str
             voice to be used in session
         input_audio_format: AudioFormat
-            input audio format
+            input audio format. Legacy parameter, kept for backward
+            compatibility and still the default path. Prefer
+            ``input_audio_config`` which supports separate audio format
+            type and sample rate.
         output_audio_format: AudioFormat
-            output audio format
+            output audio format. Legacy parameter, kept for backward
+            compatibility and still the default path. Prefer
+            ``output_audio_config`` which supports separate audio format
+            type and sample rate.
         enable_turn_detection: bool
             enable turn detection
         turn_detection_threshold: float
@@ -278,13 +449,35 @@ def update_session(
             transcription params, include language, sample_rate, input_audio_format, corpus.  # noqa: E501  # pylint: disable=line-too-long
             Only effective with qwen3-asr-flash-realtime model or
             further models. Do not set this parameter for other models.
+        input_audio_config: AudioFormatConfig
+            input(uplink) audio format config. Supports separate format type
+            (pcm/wav) and sample rate (8000/16000/24000/48000), as well as
+            free extension parameters via ``extra_params``. When provided,
+            the request emits the ``session.audio.input.format`` structure.
+        output_audio_config: AudioFormatConfig
+            output(downlink) audio format config. Supports separate format
+            type (pcm/wav) and sample rate (8000/16000/24000/48000), as well
+            as free extension parameters via ``extra_params``. When provided,
+            the request emits the ``session.audio.output.format`` structure.
+
+        Notes
+        -----
+        Once either ``input_audio_config`` or ``output_audio_config`` is
+        provided, the legacy top-level ``input_audio_format`` /
+        ``output_audio_format`` fields are not sent; an unspecified side
+        falls back to the server-side default. The top-level format fields
+        from ``transcription_params`` are also skipped in this case.
         """
         self.config = {
             "modalities": [m.value for m in output_modalities],
             "voice": voice,
-            "input_audio_format": input_audio_format.format_str,
-            "output_audio_format": output_audio_format.format_str,
         }
+        self._apply_audio_format(
+            input_audio_format,
+            output_audio_format,
+            input_audio_config,
+            output_audio_config,
+        )
         if enable_input_audio_transcription:
             self.config["input_audio_transcription"] = {
                 "model": input_audio_transcription_model,
@@ -315,23 +508,7 @@ def update_session(
                         "phrases": translation_params.corpus.phrases,
                     }
         if transcription_params is not None:
-            self.config["input_audio_transcription"] = {}
-            if transcription_params.language is not None:
-                self.config["input_audio_transcription"].update(
-                    {"language": transcription_params.language},
-                )
-            if transcription_params.corpus_text is not None:
-                transcription_params.corpus = {
-                    "text": transcription_params.corpus_text,
-                }
-            if transcription_params.corpus is not None:
-                self.config["input_audio_transcription"].update(
-                    {"corpus": transcription_params.corpus},
-                )
-            self.config[
-                "input_audio_format"
-            ] = transcription_params.input_audio_format
-            self.config["sample_rate"] = transcription_params.sample_rate
+            self._apply_transcription_params(transcription_params)
         self.config.update(kwargs)
         self.__send_str(
             json.dumps(