feat: add TrackProcessor support. (#657)

* feat: add TrackProcessor support.

* update.

* wip.

* bump version for flutter-webrtc

* import sorter.

* fix analyze.

* fix analyzer.

* update.

* add (T options) to restart.

lib/livekit_client.dart

@@ -44,6 +44,7 @@ export 'src/track/remote/audio.dart';
 export 'src/track/remote/remote.dart';
 export 'src/track/remote/video.dart';
 export 'src/track/track.dart';
+export 'src/track/processor.dart';
 export 'src/types/other.dart';
 export 'src/types/participant_permissions.dart';
 export 'src/types/video_dimensions.dart';

lib/src/core/engine.dart

@@ -133,6 +133,10 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
 
   RegionUrlProvider? _regionUrlProvider;
 
+  lk_models.ServerInfo? _serverInfo;
+
+  lk_models.ServerInfo? get serverInfo => _serverInfo;
+
   void clearReconnectTimeout() {
     if (reconnectTimeout != null) {
       reconnectTimeout?.cancel();
@@ -893,6 +897,7 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
     ..on<SignalJoinResponseEvent>((event) async {
       // create peer connections
       _subscriberPrimary = event.response.subscriberPrimary;
+      _serverInfo = event.response.serverInfo;
       var iceServersFromServer =
           event.response.iceServers.map((e) => e.toSDKType()).toList();
 

lib/src/events.dart

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import 'package:livekit_client/src/track/processor.dart';
 import 'core/engine.dart';
 import 'core/room.dart';
 import 'core/signal_client.dart';
@@ -578,3 +579,16 @@ class AudioVisualizerEvent with TrackEvent {
   String toString() => '${runtimeType}'
       'track: ${track})';
 }
+
+class TrackProcessorUpdateEvent with TrackEvent {
+  final Track track;
+  final TrackProcessor? processor;
+  const TrackProcessorUpdateEvent({
+    required this.track,
+    this.processor,
+  });
+
+  @override
+  String toString() => '${runtimeType}'
+      'track: ${track})';
+}

lib/src/participant/local.dart

@@ -115,6 +115,8 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
 
     // did publish
     await track.onPublish();
+    await track.processor?.onPublish(room);
+
     await room.applyAudioSpeakerSettings();
 
     var listener = track.createListener();
@@ -330,6 +332,7 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
 
     // did publish
     await track.onPublish();
+    await track.processor?.onPublish(room);
 
     var listener = track.createListener();
     listener.on((TrackEndedEvent event) {
@@ -384,6 +387,12 @@ class LocalParticipant extends Participant<LocalTrackPublication> {
 
       // did unpublish
       await track.onUnpublish();
+
+      if (track.processor != null) {
+        await track.processor?.onUnpublish();
+        await track.stopProcessor();
+      }
+
       await room.applyAudioSpeakerSettings();
     }
 

lib/src/track/local/audio.dart

@@ -136,12 +136,18 @@ class LocalAudioTrack extends LocalTrack
     options ??= const AudioCaptureOptions();
     final stream = await LocalTrack.createStream(options);
 
-    return LocalAudioTrack(
+    var track = LocalAudioTrack(
       TrackSource.microphone,
       stream,
       stream.getAudioTracks().first,
       options,
       enableVisualizer: enableVisualizer,
     );
+
+    if (options.processor != null) {
+      await track.setProcessor(options.processor);
+    }
+
+    return track;
   }
 }

lib/src/track/local/local.dart

@@ -31,6 +31,7 @@ import '../../support/native.dart';
 import '../../support/platform.dart';
 import '../../types/other.dart';
 import '../options.dart';
+import '../processor.dart';
 import '../remote/audio.dart';
 import '../remote/video.dart';
 import '../track.dart';
@@ -119,6 +120,10 @@ abstract class LocalTrack extends Track {
 
   bool _stopped = false;
 
+  TrackProcessor? _processor;
+
+  TrackProcessor? get processor => _processor;
+
   LocalTrack(
     TrackType kind,
     TrackSource source,
@@ -253,6 +258,10 @@ abstract class LocalTrack extends Track {
     final newStream = await LocalTrack.createStream(currentOptions);
     final newTrack = newStream.getTracks().first;
 
+    var processor = _processor;
+
+    await stopProcessor();
+
     // replace track on sender
     try {
       await sender?.replaceTrack(newTrack);
@@ -267,6 +276,10 @@ abstract class LocalTrack extends Track {
     // set new stream & track to this object
     updateMediaStreamAndTrack(newStream, newTrack);
 
+    if (processor != null) {
+      await setProcessor(processor);
+    }
+
     // mark as started
     await start();
 
@@ -277,6 +290,50 @@ abstract class LocalTrack extends Track {
     ));
   }
 
+  Future<void> setProcessor(TrackProcessor? processor) async {
+    if (processor == null) {
+      return;
+    }
+
+    if (_processor != null) {
+      await stopProcessor();
+    }
+
+    _processor = processor;
+
+    var processorOptions = ProcessorOptions(
+      kind: kind,
+      track: mediaStreamTrack,
+    );
+
+    await _processor!.init(processorOptions);
+
+    logger.fine('processor initialized');
+
+    events.emit(TrackProcessorUpdateEvent(track: this, processor: _processor));
+  }
+
+  @internal
+  Future<void> stopProcessor({bool keepElement = false}) async {
+    if (_processor == null) return;
+
+    logger.fine('stopping processor');
+    await _processor?.destroy();
+    _processor = null;
+
+    if (!keepElement) {
+      // processorElement?.remove();
+      // processorElement = null;
+    }
+
+    // apply original track constraints in case the processor changed them
+    //await this._mediaStreamTrack.applyConstraints(this._constraints);
+    // force re-setting of the mediaStreamTrack on the sender
+    //await this.setMediaStreamTrack(this._mediaStreamTrack, true);
+
+    events.emit(TrackProcessorUpdateEvent(track: this));
+  }
+
   @internal
   @mustCallSuper
   Future<bool> onPublish() async {

lib/src/track/local/video.dart

@@ -162,12 +162,9 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
   }
 
   // Private constructor
-  LocalVideoTrack._(
-    TrackSource source,
-    rtc.MediaStream stream,
-    rtc.MediaStreamTrack track,
-    this.currentOptions,
-  ) : super(
+  LocalVideoTrack._(TrackSource source, rtc.MediaStream stream,
+      rtc.MediaStreamTrack track, this.currentOptions)
+      : super(
           TrackType.VIDEO,
           source,
           stream,
@@ -181,12 +178,18 @@ class LocalVideoTrack extends LocalTrack with VideoTrack {
     options ??= const CameraCaptureOptions();
 
     final stream = await LocalTrack.createStream(options);
-    return LocalVideoTrack._(
+    var track = LocalVideoTrack._(
       TrackSource.camera,
       stream,
       stream.getVideoTracks().first,
       options,
     );
+
+    if (options.processor != null) {
+      await track.setProcessor(options.processor);
+    }
+
+    return track;
   }
 
   /// Creates a LocalVideoTrack from the display.

lib/src/track/options.dart

@@ -19,6 +19,7 @@ import '../support/platform.dart';
 import '../track/local/audio.dart';
 import '../track/local/video.dart';
 import '../types/video_parameters.dart';
+import 'processor.dart';
 
 /// A type that represents front or back of the camera.
 enum CameraPosition {
@@ -60,10 +61,12 @@ class CameraCaptureOptions extends VideoCaptureOptions {
     double? maxFrameRate,
     VideoParameters params = VideoParametersPresets.h720_169,
     this.stopCameraCaptureOnMute = true,
+    TrackProcessor<VideoProcessorOptions>? processor,
   }) : super(
           params: params,
           deviceId: deviceId,
           maxFrameRate: maxFrameRate,
+          processor: processor,
         );
 
   CameraCaptureOptions.from({required VideoCaptureOptions captureOptions})
@@ -217,10 +220,14 @@ abstract class VideoCaptureOptions extends LocalTrackOptions {
   // Limit the maximum frameRate of the capture device.
   final double? maxFrameRate;
 
+  /// A processor to apply to the video track.
+  final TrackProcessor<VideoProcessorOptions>? processor;
+
   const VideoCaptureOptions({
     this.params = VideoParametersPresets.h540_169,
     this.deviceId,
     this.maxFrameRate,
+    this.processor,
   });
 
   @override
@@ -269,6 +276,9 @@ class AudioCaptureOptions extends LocalTrackOptions {
   /// set to false to only toggle enabled instead of stop/replaceTrack for muting
   final bool stopAudioCaptureOnMute;
 
+  /// A processor to apply to the audio track.
+  final TrackProcessor<AudioProcessorOptions>? processor;
+
   const AudioCaptureOptions({
     this.deviceId,
     this.noiseSuppression = true,
@@ -278,6 +288,7 @@ class AudioCaptureOptions extends LocalTrackOptions {
     this.voiceIsolation = true,
     this.typingNoiseDetection = true,
     this.stopAudioCaptureOnMute = true,
+    this.processor,
   });
 
   @override

lib/src/track/processor.dart

@@ -0,0 +1,39 @@
+import 'package:flutter_webrtc/flutter_webrtc.dart';
+
+import '../core/room.dart';
+import '../types/other.dart';
+
+class ProcessorOptions<T extends TrackType> {
+  T kind;
+  MediaStreamTrack track;
+  ProcessorOptions({
+    required this.kind,
+    required this.track,
+  });
+}
+
+class AudioProcessorOptions extends ProcessorOptions {
+  AudioProcessorOptions({
+    required MediaStreamTrack track,
+  }) : super(kind: TrackType.AUDIO, track: track);
+}
+
+class VideoProcessorOptions extends ProcessorOptions {
+  VideoProcessorOptions({
+    required MediaStreamTrack track,
+  }) : super(kind: TrackType.VIDEO, track: track);
+}
+
+abstract class TrackProcessor<T extends ProcessorOptions> {
+  String get name;
+
+  Future<void> init(T options);
+
+  Future<void> restart(T options);
+
+  Future<void> destroy();
+
+  Future<void> onPublish(Room room);
+
+  Future<void> onUnpublish();
+}

lib/src/track/processor_web.dart

@@ -0,0 +1,14 @@
+import 'package:web/web.dart';
+
+import 'processor.dart';
+
+class AudioProcessorOptionsWeb extends AudioProcessorOptions {
+  AudioProcessorOptionsWeb({
+    this.audioElement,
+    this.audioContext,
+    required super.track,
+  });
+
+  HTMLAudioElement? audioElement;
+  AudioContext? audioContext;
+}

pubspec.yaml

@@ -37,11 +37,11 @@ dependencies:
   uuid: '>=3.0.6'
   synchronized: ^3.0.0+3
   protobuf: ^3.0.0
-  flutter_webrtc: ^0.12.5+hotfix.2
+  flutter_webrtc: ^0.12.4
   device_info_plus: ^11.1.1
   js: '>=0.6.4'
   platform_detect: ^2.0.7
-  dart_webrtc: ^1.4.9
+  dart_webrtc: ^1.4.10
   sdp_transform: ^0.3.2
   web: ^1.0.0