Add web support for pre-connect audio buffer

.changes/pre-connect-audio-web

@@ -0,0 +1 @@
+patch type="added" "Add web support for pre-connect audio buffer"

lib/src/preconnect/audio_frame_capture.dart

@@ -0,0 +1,60 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'dart:typed_data' show Uint8List;
+
+import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
+
+import 'audio_frame_capture_native.dart' if (dart.library.js_interop) 'audio_frame_capture_web.dart';
+
+/// A single frame of raw PCM audio data.
+class AudioFrame {
+  final int sampleRate;
+  final int channels;
+  final Uint8List data;
+  final String commonFormat;
+
+  const AudioFrame({
+    required this.sampleRate,
+    required this.channels,
+    required this.data,
+    required this.commonFormat,
+  });
+}
+
+/// Platform-agnostic interface for capturing raw PCM audio frames from a track.
+///
+/// On native (iOS/Android), this uses MethodChannel + EventChannel.
+/// On web, this uses Web Audio API with AudioWorklet.
+abstract class AudioFrameCapture {
+  /// Stream of raw PCM audio frames.
+  Stream<AudioFrame> get frameStream;
+
+  /// Start capturing audio from the given [track].
+  ///
+  /// Returns `true` if the renderer started successfully.
+  Future<bool> start({
+    required MediaStreamTrack track,
+    required String rendererId,
+    required int sampleRate,
+    required int channels,
+    required String commonFormat,
+  });
+
+  /// Stop capturing and release resources.
+  Future<void> stop();
+}
+
+/// Factory that returns the platform-appropriate implementation.
+AudioFrameCapture createAudioFrameCapture() => createAudioFrameCaptureImpl();

lib/src/preconnect/audio_frame_capture_native.dart

@@ -0,0 +1,93 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'dart:async';
+import 'dart:typed_data' show Uint8List;
+
+import 'package:flutter/services.dart' show EventChannel;
+
+import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
+
+import '../logger.dart';
+import '../support/native.dart';
+import 'audio_frame_capture.dart';
+
+/// Native (iOS/Android/macOS) implementation using MethodChannel + EventChannel.
+class AudioFrameCaptureNative implements AudioFrameCapture {
+  EventChannel? _eventChannel;
+  StreamSubscription? _streamSubscription;
+  final _controller = StreamController<AudioFrame>.broadcast();
+  String? _rendererId;
+
+  @override
+  Stream<AudioFrame> get frameStream => _controller.stream;
+
+  @override
+  Future<bool> start({
+    required MediaStreamTrack track,
+    required String rendererId,
+    required int sampleRate,
+    required int channels,
+    required String commonFormat,
+  }) async {
+    final result = await Native.startAudioRenderer(
+      trackId: track.id!,
+      rendererId: rendererId,
+      format: {
+        'commonFormat': commonFormat,
+        'sampleRate': sampleRate,
+        'channels': channels,
+      },
+    );
+
+    if (result != true) return false;
+
+    _rendererId = rendererId;
+
+    _eventChannel = EventChannel('io.livekit.audio.renderer/channel-$rendererId');
+    _streamSubscription = _eventChannel?.receiveBroadcastStream().listen((event) {
+      try {
+        _controller.add(AudioFrame(
+          sampleRate: event['sampleRate'] as int,
+          channels: event['channels'] as int,
+          data: event['data'] as Uint8List,
+          commonFormat: (event['commonFormat'] as String?) ?? commonFormat,
+        ));
+      } catch (e) {
+        logger.warning('[AudioFrameCapture] Error parsing native event: $e');
+      }
+    });
+
+    return true;
+  }
+
+  @override
+  Future<void> stop() async {
+    await _streamSubscription?.cancel();
+    _streamSubscription = null;
+    _eventChannel = null;
+
+    final rendererId = _rendererId;
+    if (rendererId != null) {
+      await Native.stopAudioRenderer(rendererId: rendererId);
+      _rendererId = null;
+    }
+
+    if (!_controller.isClosed) {
+      await _controller.close();
+    }
+  }
+}
+
+AudioFrameCapture createAudioFrameCaptureImpl() => AudioFrameCaptureNative();

lib/src/preconnect/audio_frame_capture_web.dart

@@ -0,0 +1,186 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'dart:async';
+import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
+import 'dart:typed_data' show Uint8List;
+
+import 'package:dart_webrtc/dart_webrtc.dart' show MediaStreamTrackWeb;
+import 'package:flutter_webrtc/flutter_webrtc.dart' show MediaStreamTrack;
+import 'package:web/web.dart' as web;
+
+import '../logger.dart';
+import '../support/audio_pcm_utils.dart';
+import 'audio_frame_capture.dart';
+
+/// JavaScript source for the AudioWorkletProcessor.
+///
+/// Runs on the audio rendering thread. Forwards raw float32 input samples
+/// to the main thread via MessagePort.
+const _workletProcessorJs = '''
+class AudioRendererProcessor extends AudioWorkletProcessor {
+  process(inputs, outputs, parameters) {
+    const input = inputs[0];
+    if (input && input.length > 0 && input[0].length > 0) {
+      const channels = input.length;
+      const frames = input[0].length;
+
+      // Interleave channels into a single Float32Array.
+      const interleaved = new Float32Array(frames * channels);
+      for (let frame = 0; frame < frames; frame++) {
+        for (let ch = 0; ch < channels; ch++) {
+          interleaved[frame * channels + ch] = input[ch][frame];
+        }
+      }
+
+      this.port.postMessage({
+        samples: interleaved.buffer,
+        channels: channels,
+        frames: frames,
+      }, [interleaved.buffer]);
+    }
+    return true;
+  }
+}
+registerProcessor('audio-renderer-processor', AudioRendererProcessor);
+''';
+
+/// Web implementation using AudioWorklet to capture raw PCM frames.
+class AudioFrameCaptureWeb implements AudioFrameCapture {
+  web.AudioContext? _audioContext;
+  web.AudioWorkletNode? _workletNode;
+  web.AudioNode? _sourceNode;
+  StreamController<AudioFrame>? _controller;
+  String _targetFormat = 'int16';
+  int _targetChannels = 1;
+
+  @override
+  Stream<AudioFrame> get frameStream => (_controller ??= StreamController<AudioFrame>.broadcast()).stream;
+
+  @override
+  Future<bool> start({
+    required MediaStreamTrack track,
+    required String rendererId,
+    required int sampleRate,
+    required int channels,
+    required String commonFormat,
+  }) async {
+    _targetFormat = commonFormat;
+    _targetChannels = channels;
+    _controller ??= StreamController<AudioFrame>.broadcast();
+
+    try {
+      // 1. Get the underlying JS MediaStreamTrack.
+      final jsTrack = (track as MediaStreamTrackWeb).jsTrack;
+      final mediaStream = web.MediaStream([jsTrack].toJS);
+
+      // 2. Create AudioContext.
+      _audioContext = web.AudioContext();
+      final ctx = _audioContext!;
+
+      // 3. Register worklet processor via Blob URL.
+      final blob = web.Blob(
+        [_workletProcessorJs.toJS].toJS,
+        web.BlobPropertyBag(type: 'application/javascript'),
+      );
+      final blobUrl = web.URL.createObjectURL(blob);
+      try {
+        await ctx.audioWorklet.addModule(blobUrl).toDart;
+      } finally {
+        web.URL.revokeObjectURL(blobUrl);
+      }
+
+      // 4. Create audio pipeline: source → worklet → destination.
+      _sourceNode = ctx.createMediaStreamSource(mediaStream);
+      _workletNode = web.AudioWorkletNode(ctx, 'audio-renderer-processor');
+
+      _sourceNode!.connect(_workletNode!);
+      // No destination connection needed — process() is called as long as
+      // there is input flowing and it returns true. Connecting to
+      // ctx.destination would route mic audio to speakers (echo/feedback).
+
+      // 5. Listen for PCM frames from the worklet thread.
+      _workletNode!.port.onmessage = _onWorkletMessage.toJS;
+
+      return true;
+    } catch (e) {
+      logger.warning('[AudioFrameCapture] Failed to start web capture: $e');
+      await stop();
+      return false;
+    }
+  }
+
+  void _onWorkletMessage(web.MessageEvent event) {
+    final controller = _controller;
+    if (controller == null || controller.isClosed) return;
+
+    try {
+      final data = event.data as JSObject;
+      final samplesBuffer = (data.getProperty('samples'.toJS) as JSArrayBuffer).toDart;
+      final channels = (data.getProperty('channels'.toJS) as JSNumber).toDartInt;
+      final frames = (data.getProperty('frames'.toJS) as JSNumber).toDartInt;
+
+      final outChannels = _targetChannels.clamp(1, channels);
+      final actualSampleRate = _audioContext?.sampleRate.toInt() ?? 48000;
+      final srcFloat32 = samplesBuffer.asFloat32List();
+
+      final Uint8List bytes;
+      if (_targetFormat == 'float32') {
+        bytes = float32ToFloat32Bytes(srcFloat32, channels, outChannels, frames);
+      } else {
+        bytes = float32ToInt16Bytes(srcFloat32, channels, outChannels, frames);
+      }
+
+      controller.add(AudioFrame(
+        sampleRate: actualSampleRate,
+        channels: outChannels,
+        data: bytes,
+        commonFormat: _targetFormat,
+      ));
+    } catch (e) {
+      logger.warning('[AudioFrameCapture] Error processing worklet frame: $e');
+    }
+  }
+
+  @override
+  Future<void> stop() async {
+    _workletNode?.port.onmessage = null;
+
+    try {
+      _workletNode?.disconnect();
+    } catch (_) {}
+    _workletNode = null;
+
+    try {
+      _sourceNode?.disconnect();
+    } catch (_) {}
+    _sourceNode = null;
+
+    try {
+      if (_audioContext?.state != 'closed') {
+        await _audioContext?.close().toDart;
+      }
+    } catch (_) {}
+    _audioContext = null;
+
+    final controller = _controller;
+    _controller = null;
+    if (controller != null && !controller.isClosed) {
+      await controller.close();
+    }
+  }
+}
+
+AudioFrameCapture createAudioFrameCaptureImpl() => AudioFrameCaptureWeb();