From f211282c1f3e72af7538bb21edb38a5fc15b5a4d Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 15 May 2026 21:27:02 -0400 Subject: [PATCH 01/18] Update example colors (#1086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I noticed the example's color scheme was out of date. So, I pulled in a bunch of colors from https://github.com/livekit-examples/agent-starter-react, made some small design updates to the button + text field, and swapped in the more modern LiveKit logo. Definitely a lot more that could be done here, I was aiming for "better" not "perfect". ## Screenshots Screenshot 2026-05-15 at 12 46 11 PM Screenshot 2026-05-15 at 12 46 39 PM --- example/images/logo-dark.svg | 33 ++-- example/lib/pages/connect.dart | 4 +- example/lib/pages/prejoin.dart | 2 +- example/lib/theme.dart | 260 ++++++++++++++++++++++++++-- example/lib/widgets/text_field.dart | 29 ++-- 5 files changed, 280 insertions(+), 48 deletions(-) diff --git a/example/images/logo-dark.svg b/example/images/logo-dark.svg index adfea0e6d..97fcbe7b3 100644 --- a/example/images/logo-dark.svg +++ b/example/images/logo-dark.svg @@ -1,16 +1,19 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/example/lib/pages/connect.dart b/example/lib/pages/connect.dart index f1ef90fc9..9ffe51331 100644 --- a/example/lib/pages/connect.dart +++ b/example/lib/pages/connect.dart @@ -211,7 +211,7 @@ class _ConnectPageState extends State { crossAxisAlignment: CrossAxisAlignment.stretch, children: [ Padding( - padding: const EdgeInsets.only(bottom: 70), + padding: const EdgeInsets.fromLTRB(64, 0, 64, 70), child: SvgPicture.asset( 'images/logo-dark.svg', ), @@ -352,7 +352,7 @@ class _ConnectPageState extends State { ), ), ), - const Text('CONNECT'), + const Text('Connect'), ], ), ), diff --git a/example/lib/pages/prejoin.dart b/example/lib/pages/prejoin.dart index fa078c86a..8579db86a 100644 --- a/example/lib/pages/prejoin.dart +++ b/example/lib/pages/prejoin.dart @@ -520,7 +520,7 @@ class _PreJoinPageState extends State { ), ), ), - const Text('JOIN'), + const Text('Join'), ], ), ), diff --git a/example/lib/theme.dart b/example/lib/theme.dart index 1e291235a..2b2b475fd 100644 --- a/example/lib/theme.dart +++ b/example/lib/theme.dart @@ -6,19 +6,231 @@ import 'package:google_fonts/google_fonts.dart'; // on Apple devices. // https://github.com/flutter/flutter/issues/55092 // https://github.com/flutter/flutter/issues/39113 -// extension LKColors on Colors { - static const lkBlue = Color(0xFF5A8BFF); - static const lkDarkBlue = Color(0xFF00153C); + // ===== LiveKit brand ===== + static const lkAccent = Color(0xFF002CF2); // light-mode accent + static const lkAccentDark = Color(0xFF1FD5F9); // dark-mode accent + + // Legacy aliases. Existing widgets reference these; values were refreshed to + // match the new dark-mode palette so the rest of the UI keeps building. + static const lkBlue = lkAccentDark; // was Color(0xFF5A8BFF) + static const lkDarkBlue = neutral900; // was Color(0xFF00153C) + + // ===== Semantic theme tokens — dark mode ===== + static const bgDark = neutral950; // --background + static const fgDark = neutral50; // --foreground + static const cardDark = neutral900; // --card + static const cardFgDark = neutral50; // --card-foreground + static const popoverDark = neutral800; // --popover + static const popoverFgDark = neutral50; // --popover-foreground + static const primaryDark = neutral200; // --primary + static const primaryFgDark = neutral900; // --primary-foreground + static const secondaryDark = neutral800; // --secondary + static const secondaryFgDark = neutral50; // --secondary-foreground + static const mutedDark = neutral800; // --muted + static const mutedFgDark = Color(0xFFA3A3A3); // --muted-foreground (~neutral-400) + static const accentMutedDark = Color(0xFF5C5C5C); // --accent (oklch(0.371)) + static const accentFgDark = neutral50; // --accent-foreground + static const destructiveDark = Color(0xFFF87171); // --destructive (~red-400) + static const borderDark = Color(0x1AFFFFFF); // --border (white/10) + static const inputDark = Color(0x26FFFFFF); // --input (white/15) + static const inputFillDark = Color(0x0DFFFFFF); // dark:bg-input/30 (~white/5) + static const ringDark = Color(0xFF8E8E8E); // --ring (oklch(0.556)) + static const sidebarDark = neutral900; + static const sidebarFgDark = neutral50; + static const sidebarBorderDark = Color(0x1AFFFFFF); + static const sidebarRingDark = Color(0xFF6F6F6F); // oklch(0.439) + + // ===== Semantic theme tokens — light mode ===== + static const bgLight = Color(0xFFFFFFFF); + static const fgLight = neutral950; + static const cardLight = Color(0xFFFFFFFF); + static const cardFgLight = neutral950; + static const popoverLight = Color(0xFFFFFFFF); + static const popoverFgLight = neutral950; + static const primaryLight = neutral900; + static const primaryFgLight = neutral50; + static const secondaryLight = neutral100; + static const secondaryFgLight = neutral900; + static const mutedLight = neutral100; + static const mutedFgLight = Color(0xFF737373); // oklch(0.556) ~ neutral-500 + static const accentMutedLight = neutral100; + static const accentFgLight = neutral900; + static const destructiveLight = Color(0xFFDC2626); // ~red-600 + static const borderLight = neutral200; + static const inputLight = neutral200; + static const ringLight = Color(0xFFB4B4B4); // oklch(0.708) + + // ===== Chart tokens ===== + // Light-mode charts (warm/cool mix; shadcn defaults). + static const chart1Light = orange500; + static const chart2Light = teal600; + static const chart3Light = blue800; + static const chart4Light = amber300; + static const chart5Light = amber400; + + // Dark-mode charts (vibrant against the dark surface). + static const chart1Dark = indigo500; + static const chart2Dark = emerald400; + static const chart3Dark = amber400; + static const chart4Dark = violet500; + static const chart5Dark = rose500; + + // ===== Tailwind color scales ===== + // Neutral (the backbone of the dark/light surfaces). + static const neutral50 = Color(0xFFFAFAFA); + static const neutral100 = Color(0xFFF5F5F5); + static const neutral200 = Color(0xFFE5E5E5); + static const neutral300 = Color(0xFFD4D4D4); + static const neutral400 = Color(0xFFA3A3A3); + static const neutral500 = Color(0xFF737373); + static const neutral600 = Color(0xFF525252); + static const neutral700 = Color(0xFF404040); + static const neutral800 = Color(0xFF262626); + static const neutral900 = Color(0xFF171717); + static const neutral950 = Color(0xFF0A0A0A); + + // Red (destructive family). + static const red50 = Color(0xFFFEF2F2); + static const red100 = Color(0xFFFEE2E2); + static const red200 = Color(0xFFFECACA); + static const red300 = Color(0xFFFCA5A5); + static const red400 = Color(0xFFF87171); + static const red500 = Color(0xFFEF4444); + static const red600 = Color(0xFFDC2626); + static const red700 = Color(0xFFB91C1C); + static const red800 = Color(0xFF991B1B); + static const red900 = Color(0xFF7F1D1D); + static const red950 = Color(0xFF450A0A); + + // Orange (chart-1 light family). + static const orange50 = Color(0xFFFFF7ED); + static const orange100 = Color(0xFFFFEDD5); + static const orange200 = Color(0xFFFED7AA); + static const orange300 = Color(0xFFFDBA74); + static const orange400 = Color(0xFFFB923C); + static const orange500 = Color(0xFFF97316); + static const orange600 = Color(0xFFEA580C); + static const orange700 = Color(0xFFC2410C); + static const orange800 = Color(0xFF9A3412); + static const orange900 = Color(0xFF7C2D12); + static const orange950 = Color(0xFF431407); + + // Amber (chart-4/-5 light, chart-3 dark). + static const amber50 = Color(0xFFFFFBEB); + static const amber100 = Color(0xFFFEF3C7); + static const amber200 = Color(0xFFFDE68A); + static const amber300 = Color(0xFFFCD34D); + static const amber400 = Color(0xFFFBBF24); + static const amber500 = Color(0xFFF59E0B); + static const amber600 = Color(0xFFD97706); + static const amber700 = Color(0xFFB45309); + static const amber800 = Color(0xFF92400E); + static const amber900 = Color(0xFF78350F); + static const amber950 = Color(0xFF451A03); + + // Teal (chart-2 light family). + static const teal50 = Color(0xFFF0FDFA); + static const teal100 = Color(0xFFCCFBF1); + static const teal200 = Color(0xFF99F6E4); + static const teal300 = Color(0xFF5EEAD4); + static const teal400 = Color(0xFF2DD4BF); + static const teal500 = Color(0xFF14B8A6); + static const teal600 = Color(0xFF0D9488); + static const teal700 = Color(0xFF0F766E); + static const teal800 = Color(0xFF115E59); + static const teal900 = Color(0xFF134E4A); + static const teal950 = Color(0xFF042F2E); + + // Blue (brand-blue family; chart-3 light leans into the deep end). + static const blue50 = Color(0xFFEFF6FF); + static const blue100 = Color(0xFFDBEAFE); + static const blue200 = Color(0xFFBFDBFE); + static const blue300 = Color(0xFF93C5FD); + static const blue400 = Color(0xFF60A5FA); + static const blue500 = Color(0xFF3B82F6); + static const blue600 = Color(0xFF2563EB); + static const blue700 = Color(0xFF1D4ED8); + static const blue800 = Color(0xFF1E40AF); + static const blue900 = Color(0xFF1E3A8A); + static const blue950 = Color(0xFF172554); + + // Cyan (dark-mode accent family; #1FD5F9 sits between cyan-300 and cyan-400). + static const cyan50 = Color(0xFFECFEFF); + static const cyan100 = Color(0xFFCFFAFE); + static const cyan200 = Color(0xFFA5F3FC); + static const cyan300 = Color(0xFF67E8F9); + static const cyan400 = Color(0xFF22D3EE); + static const cyan500 = Color(0xFF06B6D4); + static const cyan600 = Color(0xFF0891B2); + static const cyan700 = Color(0xFF0E7490); + static const cyan800 = Color(0xFF155E75); + static const cyan900 = Color(0xFF164E63); + static const cyan950 = Color(0xFF083344); + + // Indigo (chart-1 dark family). + static const indigo50 = Color(0xFFEEF2FF); + static const indigo100 = Color(0xFFE0E7FF); + static const indigo200 = Color(0xFFC7D2FE); + static const indigo300 = Color(0xFFA5B4FC); + static const indigo400 = Color(0xFF818CF8); + static const indigo500 = Color(0xFF6366F1); + static const indigo600 = Color(0xFF4F46E5); + static const indigo700 = Color(0xFF4338CA); + static const indigo800 = Color(0xFF3730A3); + static const indigo900 = Color(0xFF312E81); + static const indigo950 = Color(0xFF1E1B4B); + + // Emerald (chart-2 dark family). + static const emerald50 = Color(0xFFECFDF5); + static const emerald100 = Color(0xFFD1FAE5); + static const emerald200 = Color(0xFFA7F3D0); + static const emerald300 = Color(0xFF6EE7B7); + static const emerald400 = Color(0xFF34D399); + static const emerald500 = Color(0xFF10B981); + static const emerald600 = Color(0xFF059669); + static const emerald700 = Color(0xFF047857); + static const emerald800 = Color(0xFF065F46); + static const emerald900 = Color(0xFF064E3B); + static const emerald950 = Color(0xFF022C22); + + // Violet (chart-4 dark family). + static const violet50 = Color(0xFFF5F3FF); + static const violet100 = Color(0xFFEDE9FE); + static const violet200 = Color(0xFFDDD6FE); + static const violet300 = Color(0xFFC4B5FD); + static const violet400 = Color(0xFFA78BFA); + static const violet500 = Color(0xFF8B5CF6); + static const violet600 = Color(0xFF7C3AED); + static const violet700 = Color(0xFF6D28D9); + static const violet800 = Color(0xFF5B21B6); + static const violet900 = Color(0xFF4C1D95); + static const violet950 = Color(0xFF2E1065); + + // Rose (chart-5 dark family). + static const rose50 = Color(0xFFFFF1F2); + static const rose100 = Color(0xFFFFE4E6); + static const rose200 = Color(0xFFFECDD3); + static const rose300 = Color(0xFFFDA4AF); + static const rose400 = Color(0xFFFB7185); + static const rose500 = Color(0xFFF43F5E); + static const rose600 = Color(0xFFE11D48); + static const rose700 = Color(0xFFBE123C); + static const rose800 = Color(0xFF9F1239); + static const rose900 = Color(0xFF881337); + static const rose950 = Color(0xFF4C0519); } class LiveKitTheme { // - final bgColor = Colors.black; - final textColor = Colors.white; - final cardColor = LKColors.lkDarkBlue; - final accentColor = LKColors.lkBlue; + final bgColor = LKColors.bgDark; + final textColor = LKColors.fgDark; + final cardColor = LKColors.cardDark; + final accentColor = LKColors.lkAccentDark; + final mutedColor = LKColors.mutedFgDark; + final borderColor = LKColors.borderDark; + final destructiveColor = LKColors.destructiveDark; ThemeData buildThemeData(BuildContext ctx) => ThemeData( appBarTheme: AppBarTheme( @@ -34,22 +246,34 @@ class LiveKitTheme { style: ButtonStyle( textStyle: WidgetStateProperty.all(GoogleFonts.montserrat( fontSize: 15, + fontWeight: FontWeight.w500, )), padding: WidgetStateProperty.all(const EdgeInsets.symmetric(vertical: 20, horizontal: 25)), shape: WidgetStateProperty.all(RoundedRectangleBorder(borderRadius: BorderRadius.circular(8))), - foregroundColor: WidgetStateProperty.all(Colors.white), - // backgroundColor: WidgetStateProperty.all(accentColor), + foregroundColor: WidgetStateProperty.resolveWith((states) { + if (states.contains(WidgetState.disabled)) { + return LKColors.primaryFgDark.withValues(alpha: 0.5); + } + return LKColors.primaryFgDark; + }), backgroundColor: WidgetStateProperty.resolveWith((states) { if (states.contains(WidgetState.disabled)) { return accentColor.withValues(alpha: 0.5); } + if (states.contains(WidgetState.pressed) || + states.contains(WidgetState.hovered) || + states.contains(WidgetState.focused)) { + return accentColor.withValues(alpha: 0.9); + } return accentColor; }), + overlayColor: WidgetStateProperty.all(Colors.transparent), + animationDuration: const Duration(milliseconds: 150), ), ), checkboxTheme: CheckboxThemeData( - checkColor: WidgetStateProperty.all(Colors.white), + checkColor: WidgetStateProperty.all(LKColors.primaryFgDark), fillColor: WidgetStateProperty.all(accentColor), ), switchTheme: SwitchThemeData( @@ -61,9 +285,9 @@ class LiveKitTheme { }), thumbColor: WidgetStateProperty.resolveWith((states) { if (states.contains(WidgetState.selected)) { - return Colors.white; + return LKColors.fgDark; } - return Colors.white.withValues(alpha: 0.3); + return LKColors.fgDark.withValues(alpha: 0.3); }), ), dialogTheme: DialogThemeData( @@ -79,20 +303,24 @@ class LiveKitTheme { bodyColor: textColor, decorationColor: textColor, ), - hintColor: Colors.red, + hintColor: destructiveColor, inputDecorationTheme: InputDecorationTheme( labelStyle: const TextStyle( - color: LKColors.lkBlue, + color: LKColors.lkAccentDark, ), hintStyle: TextStyle( - color: LKColors.lkBlue.withValues(alpha: 5), + color: LKColors.lkAccentDark.withValues(alpha: 5), ), enabledBorder: InputBorder.none, focusedBorder: InputBorder.none, ), colorScheme: ColorScheme.fromSeed( - seedColor: Colors.white, + seedColor: accentColor, + brightness: Brightness.dark, surface: bgColor, + primary: accentColor, + secondary: LKColors.secondaryDark, + error: destructiveColor, ), ); } diff --git a/example/lib/widgets/text_field.dart b/example/lib/widgets/text_field.dart index f4173f3da..870f07fb2 100644 --- a/example/lib/widgets/text_field.dart +++ b/example/lib/widgets/text_field.dart @@ -1,5 +1,7 @@ import 'package:flutter/material.dart'; +import '../theme.dart'; + class LKTextField extends StatelessWidget { final String label; final TextEditingController? ctrl; @@ -14,32 +16,31 @@ class LKTextField extends StatelessWidget { crossAxisAlignment: CrossAxisAlignment.start, children: [ Padding( - padding: const EdgeInsets.only(bottom: 10), + padding: const EdgeInsets.only(bottom: 8), child: Text( label, style: const TextStyle( - fontSize: 16, - fontWeight: FontWeight.bold, + fontSize: 15, + fontWeight: FontWeight.w500, + color: LKColors.fgDark, ), ), ), Container( - padding: const EdgeInsets.symmetric( - vertical: 15, - horizontal: 15, - ), + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 12), decoration: BoxDecoration( - border: Border.all( - width: 1, - color: Colors.white.withValues(alpha: .3), - ), - borderRadius: BorderRadius.circular(8), + color: LKColors.inputFillDark, + border: Border.all(width: 1, color: LKColors.inputDark), + borderRadius: BorderRadius.circular(6), ), child: TextField( controller: ctrl, - decoration: const InputDecoration.collapsed( - hintText: '', + style: const TextStyle( + fontSize: 15, + color: LKColors.fgDark, ), + cursorColor: LKColors.fgDark, + decoration: const InputDecoration.collapsed(hintText: ''), keyboardType: TextInputType.url, autocorrect: false, ), From 3ec0288a5b239e6fd51ec9dc9422986943babbaa Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 19 May 2026 00:43:28 +0800 Subject: [PATCH 02/18] Update protocol to v1.45.8 (#1088) Bump PB for PacketTrailer related features --- .changes/update-protocol-v1-45-8 | 1 + lib/src/proto/livekit_metrics.pb.dart | 26 +++ lib/src/proto/livekit_metrics.pbjson.dart | 16 +- lib/src/proto/livekit_models.pb.dart | 92 ++++++++- lib/src/proto/livekit_models.pbenum.dart | 44 +++- lib/src/proto/livekit_models.pbjson.dart | 239 +++++++++++++--------- lib/src/proto/livekit_rtc.pb.dart | 9 + lib/src/proto/livekit_rtc.pbjson.dart | 105 +++++----- 8 files changed, 381 insertions(+), 151 deletions(-) create mode 100644 .changes/update-protocol-v1-45-8 diff --git a/.changes/update-protocol-v1-45-8 b/.changes/update-protocol-v1-45-8 new file mode 100644 index 000000000..6760a419c --- /dev/null +++ b/.changes/update-protocol-v1-45-8 @@ -0,0 +1 @@ +patch type="changed" "Update generated protocol definitions to v1.45.8" diff --git a/lib/src/proto/livekit_metrics.pb.dart b/lib/src/proto/livekit_metrics.pb.dart index 90a6d3a17..b87bce41b 100644 --- a/lib/src/proto/livekit_metrics.pb.dart +++ b/lib/src/proto/livekit_metrics.pb.dart @@ -432,12 +432,16 @@ class MetricsRecordingHeader extends $pb.GeneratedMessage { $fixnum.Int64? duration, $0.Timestamp? startTime, $core.Iterable<$core.MapEntry<$core.String, $core.String>>? roomTags, + $core.String? roomName, + $0.Timestamp? roomStartTime, }) { final result = create(); if (roomId != null) result.roomId = roomId; if (duration != null) result.duration = duration; if (startTime != null) result.startTime = startTime; if (roomTags != null) result.roomTags.addEntries(roomTags); + if (roomName != null) result.roomName = roomName; + if (roomStartTime != null) result.roomStartTime = roomStartTime; return result; } @@ -460,6 +464,8 @@ class MetricsRecordingHeader extends $pb.GeneratedMessage { keyFieldType: $pb.PbFieldType.OS, valueFieldType: $pb.PbFieldType.OS, packageName: const $pb.PackageName('livekit')) + ..aOS(6, _omitFieldNames ? '' : 'roomName') + ..aOM<$0.Timestamp>(7, _omitFieldNames ? '' : 'roomStartTime', subBuilder: $0.Timestamp.create) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -511,6 +517,26 @@ class MetricsRecordingHeader extends $pb.GeneratedMessage { @$pb.TagNumber(5) $pb.PbMap<$core.String, $core.String> get roomTags => $_getMap(3); + + @$pb.TagNumber(6) + $core.String get roomName => $_getSZ(4); + @$pb.TagNumber(6) + set roomName($core.String value) => $_setString(4, value); + @$pb.TagNumber(6) + $core.bool hasRoomName() => $_has(4); + @$pb.TagNumber(6) + void clearRoomName() => $_clearField(6); + + @$pb.TagNumber(7) + $0.Timestamp get roomStartTime => $_getN(5); + @$pb.TagNumber(7) + set roomStartTime($0.Timestamp value) => $_setField(7, value); + @$pb.TagNumber(7) + $core.bool hasRoomStartTime() => $_has(5); + @$pb.TagNumber(7) + void clearRoomStartTime() => $_clearField(7); + @$pb.TagNumber(7) + $0.Timestamp ensureRoomStartTime() => $_ensure(5); } const $core.bool _omitFieldNames = $core.bool.fromEnvironment('protobuf.omit_field_names'); diff --git a/lib/src/proto/livekit_metrics.pbjson.dart b/lib/src/proto/livekit_metrics.pbjson.dart index 14f4bcee0..310436b17 100644 --- a/lib/src/proto/livekit_metrics.pbjson.dart +++ b/lib/src/proto/livekit_metrics.pbjson.dart @@ -184,10 +184,12 @@ final $typed_data.Uint8List eventMetricDescriptor = const MetricsRecordingHeader$json = { '1': 'MetricsRecordingHeader', '2': [ - {'1': 'room_id', '3': 1, '4': 1, '5': 9, '10': 'roomId'}, + {'1': 'room_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'roomId'}, {'1': 'duration', '3': 3, '4': 1, '5': 4, '10': 'duration'}, {'1': 'start_time', '3': 4, '4': 1, '5': 11, '6': '.google.protobuf.Timestamp', '10': 'startTime'}, {'1': 'room_tags', '3': 5, '4': 3, '5': 11, '6': '.livekit.MetricsRecordingHeader.RoomTagsEntry', '10': 'roomTags'}, + {'1': 'room_name', '3': 6, '4': 1, '5': 9, '10': 'roomName'}, + {'1': 'room_start_time', '3': 7, '4': 1, '5': 11, '6': '.google.protobuf.Timestamp', '10': 'roomStartTime'}, ], '3': [MetricsRecordingHeader_RoomTagsEntry$json], }; @@ -204,8 +206,10 @@ const MetricsRecordingHeader_RoomTagsEntry$json = { /// Descriptor for `MetricsRecordingHeader`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List metricsRecordingHeaderDescriptor = - $convert.base64Decode('ChZNZXRyaWNzUmVjb3JkaW5nSGVhZGVyEhcKB3Jvb21faWQYASABKAlSBnJvb21JZBIaCghkdX' - 'JhdGlvbhgDIAEoBFIIZHVyYXRpb24SOQoKc3RhcnRfdGltZRgEIAEoCzIaLmdvb2dsZS5wcm90' - 'b2J1Zi5UaW1lc3RhbXBSCXN0YXJ0VGltZRJKCglyb29tX3RhZ3MYBSADKAsyLS5saXZla2l0Lk' - '1ldHJpY3NSZWNvcmRpbmdIZWFkZXIuUm9vbVRhZ3NFbnRyeVIIcm9vbVRhZ3MaOwoNUm9vbVRh' - 'Z3NFbnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6AjgB'); + $convert.base64Decode('ChZNZXRyaWNzUmVjb3JkaW5nSGVhZGVyEiIKB3Jvb21faWQYASABKAlCCbpQBnJvb21JRFIGcm' + '9vbUlkEhoKCGR1cmF0aW9uGAMgASgEUghkdXJhdGlvbhI5CgpzdGFydF90aW1lGAQgASgLMhou' + 'Z29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcFIJc3RhcnRUaW1lEkoKCXJvb21fdGFncxgFIAMoCz' + 'ItLmxpdmVraXQuTWV0cmljc1JlY29yZGluZ0hlYWRlci5Sb29tVGFnc0VudHJ5Ughyb29tVGFn' + 'cxIbCglyb29tX25hbWUYBiABKAlSCHJvb21OYW1lEkIKD3Jvb21fc3RhcnRfdGltZRgHIAEoCz' + 'IaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXBSDXJvb21TdGFydFRpbWUaOwoNUm9vbVRhZ3NF' + 'bnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6AjgB'); diff --git a/lib/src/proto/livekit_models.pb.dart b/lib/src/proto/livekit_models.pb.dart index 7bd4d1180..6ffa0300e 100644 --- a/lib/src/proto/livekit_models.pb.dart +++ b/lib/src/proto/livekit_models.pb.dart @@ -545,6 +545,7 @@ class ParticipantPermission extends $pb.GeneratedMessage { $core.bool? canUpdateMetadata, @$core.Deprecated('This field is deprecated.') $core.bool? agent, $core.bool? canSubscribeMetrics, + $core.bool? canManageAgentSession, }) { final result = create(); if (canSubscribe != null) result.canSubscribe = canSubscribe; @@ -556,6 +557,7 @@ class ParticipantPermission extends $pb.GeneratedMessage { if (canUpdateMetadata != null) result.canUpdateMetadata = canUpdateMetadata; if (agent != null) result.agent = agent; if (canSubscribeMetrics != null) result.canSubscribeMetrics = canSubscribeMetrics; + if (canManageAgentSession != null) result.canManageAgentSession = canManageAgentSession; return result; } @@ -580,6 +582,7 @@ class ParticipantPermission extends $pb.GeneratedMessage { ..aOB(10, _omitFieldNames ? '' : 'canUpdateMetadata') ..aOB(11, _omitFieldNames ? '' : 'agent') ..aOB(12, _omitFieldNames ? '' : 'canSubscribeMetrics') + ..aOB(13, _omitFieldNames ? '' : 'canManageAgentSession') ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -693,6 +696,16 @@ class ParticipantPermission extends $pb.GeneratedMessage { $core.bool hasCanSubscribeMetrics() => $_has(8); @$pb.TagNumber(12) void clearCanSubscribeMetrics() => $_clearField(12); + + /// if a participant can manage an agent session via RemoteSession (control and access state) + @$pb.TagNumber(13) + $core.bool get canManageAgentSession => $_getBF(9); + @$pb.TagNumber(13) + set canManageAgentSession($core.bool value) => $_setBool(9, value); + @$pb.TagNumber(13) + $core.bool hasCanManageAgentSession() => $_has(9); + @$pb.TagNumber(13) + void clearCanManageAgentSession() => $_clearField(13); } class ParticipantInfo extends $pb.GeneratedMessage { @@ -714,6 +727,7 @@ class ParticipantInfo extends $pb.GeneratedMessage { $fixnum.Int64? joinedAtMs, $core.Iterable? kindDetails, $core.Iterable? dataTracks, + $core.int? clientProtocol, }) { final result = create(); if (sid != null) result.sid = sid; @@ -733,6 +747,7 @@ class ParticipantInfo extends $pb.GeneratedMessage { if (joinedAtMs != null) result.joinedAtMs = joinedAtMs; if (kindDetails != null) result.kindDetails.addAll(kindDetails); if (dataTracks != null) result.dataTracks.addAll(dataTracks); + if (clientProtocol != null) result.clientProtocol = clientProtocol; return result; } @@ -770,6 +785,7 @@ class ParticipantInfo extends $pb.GeneratedMessage { enumValues: ParticipantInfo_KindDetail.values, defaultEnumValue: ParticipantInfo_KindDetail.CLOUD_AGENT) ..pPM(19, _omitFieldNames ? '' : 'dataTracks', subBuilder: DataTrackInfo.create) + ..aI(20, _omitFieldNames ? '' : 'clientProtocol') ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -924,6 +940,16 @@ class ParticipantInfo extends $pb.GeneratedMessage { @$pb.TagNumber(19) $pb.PbList get dataTracks => $_getList(16); + + /// protocol version used for client feature compatibility + @$pb.TagNumber(20) + $core.int get clientProtocol => $_getIZ(17); + @$pb.TagNumber(20) + set clientProtocol($core.int value) => $_setSignedInt32(17, value); + @$pb.TagNumber(20) + $core.bool hasClientProtocol() => $_has(17); + @$pb.TagNumber(20) + void clearClientProtocol() => $_clearField(20); } class Encryption extends $pb.GeneratedMessage { @@ -1090,6 +1116,7 @@ class TrackInfo extends $pb.GeneratedMessage { TimedVersion? version, $core.Iterable? audioFeatures, BackupCodecPolicy? backupCodecPolicy, + $core.Iterable? packetTrailerFeatures, }) { final result = create(); if (sid != null) result.sid = sid; @@ -1112,6 +1139,7 @@ class TrackInfo extends $pb.GeneratedMessage { if (version != null) result.version = version; if (audioFeatures != null) result.audioFeatures.addAll(audioFeatures); if (backupCodecPolicy != null) result.backupCodecPolicy = backupCodecPolicy; + if (packetTrailerFeatures != null) result.packetTrailerFeatures.addAll(packetTrailerFeatures); return result; } @@ -1148,6 +1176,10 @@ class TrackInfo extends $pb.GeneratedMessage { enumValues: AudioTrackFeature.values, defaultEnumValue: AudioTrackFeature.TF_STEREO) ..aE(20, _omitFieldNames ? '' : 'backupCodecPolicy', enumValues: BackupCodecPolicy.values) + ..pc(21, _omitFieldNames ? '' : 'packetTrailerFeatures', $pb.PbFieldType.KE, + valueOf: PacketTrailerFeature.valueOf, + enumValues: PacketTrailerFeature.values, + defaultEnumValue: PacketTrailerFeature.PTF_USER_TIMESTAMP) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -1351,6 +1383,9 @@ class TrackInfo extends $pb.GeneratedMessage { $core.bool hasBackupCodecPolicy() => $_has(19); @$pb.TagNumber(20) void clearBackupCodecPolicy() => $_clearField(20); + + @$pb.TagNumber(21) + $pb.PbList get packetTrailerFeatures => $_getList(20); } class DataTrackInfo extends $pb.GeneratedMessage { @@ -2965,6 +3000,7 @@ class RpcRequest extends $pb.GeneratedMessage { $core.String? payload, $core.int? responseTimeoutMs, $core.int? version, + $core.List<$core.int>? compressedPayload, }) { final result = create(); if (id != null) result.id = id; @@ -2972,6 +3008,7 @@ class RpcRequest extends $pb.GeneratedMessage { if (payload != null) result.payload = payload; if (responseTimeoutMs != null) result.responseTimeoutMs = responseTimeoutMs; if (version != null) result.version = version; + if (compressedPayload != null) result.compressedPayload = compressedPayload; return result; } @@ -2990,6 +3027,7 @@ class RpcRequest extends $pb.GeneratedMessage { ..aOS(3, _omitFieldNames ? '' : 'payload') ..aI(4, _omitFieldNames ? '' : 'responseTimeoutMs', fieldType: $pb.PbFieldType.OU3) ..aI(5, _omitFieldNames ? '' : 'version', fieldType: $pb.PbFieldType.OU3) + ..a<$core.List<$core.int>>(6, _omitFieldNames ? '' : 'compressedPayload', $pb.PbFieldType.OY) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -3053,6 +3091,16 @@ class RpcRequest extends $pb.GeneratedMessage { $core.bool hasVersion() => $_has(4); @$pb.TagNumber(5) void clearVersion() => $_clearField(5); + + /// Compressed payload data. When set, this field is used instead of `payload`. + @$pb.TagNumber(6) + $core.List<$core.int> get compressedPayload => $_getN(5); + @$pb.TagNumber(6) + set compressedPayload($core.List<$core.int> value) => $_setBytes(5, value); + @$pb.TagNumber(6) + $core.bool hasCompressedPayload() => $_has(5); + @$pb.TagNumber(6) + void clearCompressedPayload() => $_clearField(6); } class RpcAck extends $pb.GeneratedMessage { @@ -3103,18 +3151,20 @@ class RpcAck extends $pb.GeneratedMessage { void clearRequestId() => $_clearField(1); } -enum RpcResponse_Value { payload, error, notSet } +enum RpcResponse_Value { payload, error, compressedPayload, notSet } class RpcResponse extends $pb.GeneratedMessage { factory RpcResponse({ $core.String? requestId, $core.String? payload, RpcError? error, + $core.List<$core.int>? compressedPayload, }) { final result = create(); if (requestId != null) result.requestId = requestId; if (payload != null) result.payload = payload; if (error != null) result.error = error; + if (compressedPayload != null) result.compressedPayload = compressedPayload; return result; } @@ -3129,14 +3179,16 @@ class RpcResponse extends $pb.GeneratedMessage { static const $core.Map<$core.int, RpcResponse_Value> _RpcResponse_ValueByTag = { 2: RpcResponse_Value.payload, 3: RpcResponse_Value.error, + 4: RpcResponse_Value.compressedPayload, 0: RpcResponse_Value.notSet }; static final $pb.BuilderInfo _i = $pb.BuilderInfo(_omitMessageNames ? '' : 'RpcResponse', package: const $pb.PackageName(_omitMessageNames ? '' : 'livekit'), createEmptyInstance: create) - ..oo(0, [2, 3]) + ..oo(0, [2, 3, 4]) ..aOS(1, _omitFieldNames ? '' : 'requestId') ..aOS(2, _omitFieldNames ? '' : 'payload') ..aOM(3, _omitFieldNames ? '' : 'error', subBuilder: RpcError.create) + ..a<$core.List<$core.int>>(4, _omitFieldNames ? '' : 'compressedPayload', $pb.PbFieldType.OY) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -3158,9 +3210,11 @@ class RpcResponse extends $pb.GeneratedMessage { @$pb.TagNumber(2) @$pb.TagNumber(3) + @$pb.TagNumber(4) RpcResponse_Value whichValue() => _RpcResponse_ValueByTag[$_whichOneof(0)]!; @$pb.TagNumber(2) @$pb.TagNumber(3) + @$pb.TagNumber(4) void clearValue() => $_clearField($_whichOneof(0)); @$pb.TagNumber(1) @@ -3191,6 +3245,16 @@ class RpcResponse extends $pb.GeneratedMessage { void clearError() => $_clearField(3); @$pb.TagNumber(3) RpcError ensureError() => $_ensure(2); + + /// Compressed payload data. When set, this field is used instead of `payload`. + @$pb.TagNumber(4) + $core.List<$core.int> get compressedPayload => $_getN(3); + @$pb.TagNumber(4) + set compressedPayload($core.List<$core.int> value) => $_setBytes(3, value); + @$pb.TagNumber(4) + $core.bool hasCompressedPayload() => $_has(3); + @$pb.TagNumber(4) + void clearCompressedPayload() => $_clearField(4); } class RpcError extends $pb.GeneratedMessage { @@ -3461,6 +3525,8 @@ class ClientInfo extends $pb.GeneratedMessage { $core.String? address, $core.String? network, $core.String? otherSdks, + $core.int? clientProtocol, + $core.Iterable? capabilities, }) { final result = create(); if (sdk != null) result.sdk = sdk; @@ -3474,6 +3540,8 @@ class ClientInfo extends $pb.GeneratedMessage { if (address != null) result.address = address; if (network != null) result.network = network; if (otherSdks != null) result.otherSdks = otherSdks; + if (clientProtocol != null) result.clientProtocol = clientProtocol; + if (capabilities != null) result.capabilities.addAll(capabilities); return result; } @@ -3498,6 +3566,11 @@ class ClientInfo extends $pb.GeneratedMessage { ..aOS(9, _omitFieldNames ? '' : 'address') ..aOS(10, _omitFieldNames ? '' : 'network') ..aOS(11, _omitFieldNames ? '' : 'otherSdks') + ..aI(12, _omitFieldNames ? '' : 'clientProtocol') + ..pc(13, _omitFieldNames ? '' : 'capabilities', $pb.PbFieldType.KE, + valueOf: ClientInfo_Capability.valueOf, + enumValues: ClientInfo_Capability.values, + defaultEnumValue: ClientInfo_Capability.CAP_UNUSED) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -3618,6 +3691,21 @@ class ClientInfo extends $pb.GeneratedMessage { $core.bool hasOtherSdks() => $_has(10); @$pb.TagNumber(11) void clearOtherSdks() => $_clearField(11); + + /// client protocol version + @$pb.TagNumber(12) + $core.int get clientProtocol => $_getIZ(11); + @$pb.TagNumber(12) + set clientProtocol($core.int value) => $_setSignedInt32(11, value); + @$pb.TagNumber(12) + $core.bool hasClientProtocol() => $_has(11); + @$pb.TagNumber(12) + void clearClientProtocol() => $_clearField(12); + + /// capabilities the client advertises. Populated automatically by each SDK; + /// not a user-configurable setting. + @$pb.TagNumber(13) + $pb.PbList get capabilities => $_getList(12); } /// server provided client configuration diff --git a/lib/src/proto/livekit_models.pbenum.dart b/lib/src/proto/livekit_models.pbenum.dart index b8719fda9..35b6fad6a 100644 --- a/lib/src/proto/livekit_models.pbenum.dart +++ b/lib/src/proto/livekit_models.pbenum.dart @@ -253,6 +253,9 @@ class DisconnectReason extends $pb.ProtobufEnum { /// media stream failure or media timeout static const DisconnectReason MEDIA_FAILURE = DisconnectReason._(15, _omitEnumNames ? '' : 'MEDIA_FAILURE'); + /// agent encountered an error + static const DisconnectReason AGENT_ERROR = DisconnectReason._(16, _omitEnumNames ? '' : 'AGENT_ERROR'); + static const $core.List values = [ UNKNOWN_REASON, CLIENT_INITIATED, @@ -270,9 +273,10 @@ class DisconnectReason extends $pb.ProtobufEnum { SIP_TRUNK_FAILURE, CONNECTION_TIMEOUT, MEDIA_FAILURE, + AGENT_ERROR, ]; - static final $core.List _byValue = $pb.ProtobufEnum.$_initByValueList(values, 15); + static final $core.List _byValue = $pb.ProtobufEnum.$_initByValueList(values, 16); static DisconnectReason? valueOf($core.int value) => value < 0 || value >= _byValue.length ? null : _byValue[value]; const DisconnectReason._(super.value, super.name); @@ -349,6 +353,23 @@ class AudioTrackFeature extends $pb.ProtobufEnum { const AudioTrackFeature._(super.value, super.name); } +class PacketTrailerFeature extends $pb.ProtobufEnum { + static const PacketTrailerFeature PTF_USER_TIMESTAMP = + PacketTrailerFeature._(0, _omitEnumNames ? '' : 'PTF_USER_TIMESTAMP'); + static const PacketTrailerFeature PTF_FRAME_ID = PacketTrailerFeature._(1, _omitEnumNames ? '' : 'PTF_FRAME_ID'); + + static const $core.List values = [ + PTF_USER_TIMESTAMP, + PTF_FRAME_ID, + ]; + + static final $core.List _byValue = $pb.ProtobufEnum.$_initByValueList(values, 1); + static PacketTrailerFeature? valueOf($core.int value) => + value < 0 || value >= _byValue.length ? null : _byValue[value]; + + const PacketTrailerFeature._(super.value, super.name); +} + class ParticipantInfo_State extends $pb.ProtobufEnum { /// websocket' connected, but not offered yet static const ParticipantInfo_State JOINING = ParticipantInfo_State._(0, _omitEnumNames ? '' : 'JOINING'); @@ -552,6 +573,27 @@ class ClientInfo_SDK extends $pb.ProtobufEnum { const ClientInfo_SDK._(super.value, super.name); } +/// Optional capabilities advertised by the client at connect time. The SFU +/// uses these flags to decide whether to enable features that require +/// client-side support (e.g. passing RTP packet trailers through to the +/// subscriber instead of stripping them). +class ClientInfo_Capability extends $pb.ProtobufEnum { + static const ClientInfo_Capability CAP_UNUSED = ClientInfo_Capability._(0, _omitEnumNames ? '' : 'CAP_UNUSED'); + static const ClientInfo_Capability CAP_PACKET_TRAILER = + ClientInfo_Capability._(1, _omitEnumNames ? '' : 'CAP_PACKET_TRAILER'); + + static const $core.List values = [ + CAP_UNUSED, + CAP_PACKET_TRAILER, + ]; + + static final $core.List _byValue = $pb.ProtobufEnum.$_initByValueList(values, 1); + static ClientInfo_Capability? valueOf($core.int value) => + value < 0 || value >= _byValue.length ? null : _byValue[value]; + + const ClientInfo_Capability._(super.value, super.name); +} + /// enum for operation types (specific to TextHeader) class DataStream_OperationType extends $pb.ProtobufEnum { static const DataStream_OperationType CREATE = DataStream_OperationType._(0, _omitEnumNames ? '' : 'CREATE'); diff --git a/lib/src/proto/livekit_models.pbjson.dart b/lib/src/proto/livekit_models.pbjson.dart index 995047033..4f66de6bd 100644 --- a/lib/src/proto/livekit_models.pbjson.dart +++ b/lib/src/proto/livekit_models.pbjson.dart @@ -187,6 +187,7 @@ const DisconnectReason$json = { {'1': 'SIP_TRUNK_FAILURE', '2': 13}, {'1': 'CONNECTION_TIMEOUT', '2': 14}, {'1': 'MEDIA_FAILURE', '2': 15}, + {'1': 'AGENT_ERROR', '2': 16}, ], }; @@ -198,7 +199,7 @@ final $typed_data.Uint8List disconnectReasonDescriptor = 'AGEhAKDEpPSU5fRkFJTFVSRRAHEg0KCU1JR1JBVElPThAIEhAKDFNJR05BTF9DTE9TRRAJEg8K' 'C1JPT01fQ0xPU0VEEAoSFAoQVVNFUl9VTkFWQUlMQUJMRRALEhEKDVVTRVJfUkVKRUNURUQQDB' 'IVChFTSVBfVFJVTktfRkFJTFVSRRANEhYKEkNPTk5FQ1RJT05fVElNRU9VVBAOEhEKDU1FRElB' - 'X0ZBSUxVUkUQDw=='); + 'X0ZBSUxVUkUQDxIPCgtBR0VOVF9FUlJPUhAQ'); @$core.Deprecated('Use reconnectReasonDescriptor instead') const ReconnectReason$json = { @@ -254,19 +255,33 @@ final $typed_data.Uint8List audioTrackFeatureDescriptor = 'U0VfU1VQUFJFU1NJT04QBBIiCh5URl9FTkhBTkNFRF9OT0lTRV9DQU5DRUxMQVRJT04QBRIYCh' 'RURl9QUkVDT05ORUNUX0JVRkZFUhAG'); +@$core.Deprecated('Use packetTrailerFeatureDescriptor instead') +const PacketTrailerFeature$json = { + '1': 'PacketTrailerFeature', + '2': [ + {'1': 'PTF_USER_TIMESTAMP', '2': 0}, + {'1': 'PTF_FRAME_ID', '2': 1}, + ], +}; + +/// Descriptor for `PacketTrailerFeature`. Decode as a `google.protobuf.EnumDescriptorProto`. +final $typed_data.Uint8List packetTrailerFeatureDescriptor = + $convert.base64Decode('ChRQYWNrZXRUcmFpbGVyRmVhdHVyZRIWChJQVEZfVVNFUl9USU1FU1RBTVAQABIQCgxQVEZfRl' + 'JBTUVfSUQQAQ=='); + @$core.Deprecated('Use paginationDescriptor instead') const Pagination$json = { '1': 'Pagination', '2': [ - {'1': 'after_id', '3': 1, '4': 1, '5': 9, '10': 'afterId'}, + {'1': 'after_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'afterId'}, {'1': 'limit', '3': 2, '4': 1, '5': 5, '10': 'limit'}, ], }; /// Descriptor for `Pagination`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List paginationDescriptor = - $convert.base64Decode('CgpQYWdpbmF0aW9uEhkKCGFmdGVyX2lkGAEgASgJUgdhZnRlcklkEhQKBWxpbWl0GAIgASgFUg' - 'VsaW1pdA=='); + $convert.base64Decode('CgpQYWdpbmF0aW9uEiUKCGFmdGVyX2lkGAEgASgJQgq6UAdhZnRlcklEUgdhZnRlcklkEhQKBW' + 'xpbWl0GAIgASgFUgVsaW1pdA=='); @$core.Deprecated('Use tokenPaginationDescriptor instead') const TokenPagination$json = { @@ -325,11 +340,11 @@ final $typed_data.Uint8List roomDescriptor = '50cxIjCg1jcmVhdGlvbl90aW1lGAUgASgDUgxjcmVhdGlvblRpbWUSKAoQY3JlYXRpb25fdGlt' 'ZV9tcxgPIAEoA1IOY3JlYXRpb25UaW1lTXMSIwoNdHVybl9wYXNzd29yZBgGIAEoCVIMdHVybl' 'Bhc3N3b3JkEjUKDmVuYWJsZWRfY29kZWNzGAcgAygLMg4ubGl2ZWtpdC5Db2RlY1INZW5hYmxl' - 'ZENvZGVjcxJCCghtZXRhZGF0YRgIIAEoCUImiOwsAZLsLB48cmVkYWN0ZWQgKHt7IC5TaXplIH' - '19IGJ5dGVzKT5SCG1ldGFkYXRhEikKEG51bV9wYXJ0aWNpcGFudHMYCSABKA1SD251bVBhcnRp' - 'Y2lwYW50cxIlCg5udW1fcHVibGlzaGVycxgLIAEoDVINbnVtUHVibGlzaGVycxIpChBhY3Rpdm' - 'VfcmVjb3JkaW5nGAogASgIUg9hY3RpdmVSZWNvcmRpbmcSLwoHdmVyc2lvbhgNIAEoCzIVLmxp' - 'dmVraXQuVGltZWRWZXJzaW9uUgd2ZXJzaW9u'); + 'ZENvZGVjcxJACghtZXRhZGF0YRgIIAEoCUIkqFABslAePHJlZGFjdGVkICh7eyAuU2l6ZSB9fS' + 'BieXRlcyk+UghtZXRhZGF0YRIpChBudW1fcGFydGljaXBhbnRzGAkgASgNUg9udW1QYXJ0aWNp' + 'cGFudHMSJQoObnVtX3B1Ymxpc2hlcnMYCyABKA1SDW51bVB1Ymxpc2hlcnMSKQoQYWN0aXZlX3' + 'JlY29yZGluZxgKIAEoCFIPYWN0aXZlUmVjb3JkaW5nEi8KB3ZlcnNpb24YDSABKAsyFS5saXZl' + 'a2l0LlRpbWVkVmVyc2lvblIHdmVyc2lvbg=='); @$core.Deprecated('Use codecDescriptor instead') const Codec$json = { @@ -387,6 +402,7 @@ const ParticipantPermission$json = { '10': 'agent', }, {'1': 'can_subscribe_metrics', '3': 12, '4': 1, '5': 8, '10': 'canSubscribeMetrics'}, + {'1': 'can_manage_agent_session', '3': 13, '4': 1, '5': 8, '10': 'canManageAgentSession'}, ], }; @@ -398,7 +414,8 @@ final $typed_data.Uint8List participantPermissionDescriptor = '5saXZla2l0LlRyYWNrU291cmNlUhFjYW5QdWJsaXNoU291cmNlcxIWCgZoaWRkZW4YByABKAhS' 'BmhpZGRlbhIeCghyZWNvcmRlchgIIAEoCEICGAFSCHJlY29yZGVyEi4KE2Nhbl91cGRhdGVfbW' 'V0YWRhdGEYCiABKAhSEWNhblVwZGF0ZU1ldGFkYXRhEhgKBWFnZW50GAsgASgIQgIYAVIFYWdl' - 'bnQSMgoVY2FuX3N1YnNjcmliZV9tZXRyaWNzGAwgASgIUhNjYW5TdWJzY3JpYmVNZXRyaWNz'); + 'bnQSMgoVY2FuX3N1YnNjcmliZV9tZXRyaWNzGAwgASgIUhNjYW5TdWJzY3JpYmVNZXRyaWNzEj' + 'cKGGNhbl9tYW5hZ2VfYWdlbnRfc2Vzc2lvbhgNIAEoCFIVY2FuTWFuYWdlQWdlbnRTZXNzaW9u'); @$core.Deprecated('Use participantInfoDescriptor instead') const ParticipantInfo$json = { @@ -429,6 +446,7 @@ const ParticipantInfo$json = { {'1': 'disconnect_reason', '3': 16, '4': 1, '5': 14, '6': '.livekit.DisconnectReason', '10': 'disconnectReason'}, {'1': 'kind_details', '3': 18, '4': 3, '5': 14, '6': '.livekit.ParticipantInfo.KindDetail', '10': 'kindDetails'}, {'1': 'data_tracks', '3': 19, '4': 3, '5': 11, '6': '.livekit.DataTrackInfo', '10': 'dataTracks'}, + {'1': 'client_protocol', '3': 20, '4': 1, '5': 5, '10': 'clientProtocol'}, ], '3': [ParticipantInfo_AttributesEntry$json], '4': [ParticipantInfo_State$json, ParticipantInfo_Kind$json, ParticipantInfo_KindDetail$json], @@ -485,25 +503,26 @@ const ParticipantInfo_KindDetail$json = { final $typed_data.Uint8List participantInfoDescriptor = $convert.base64Decode('Cg9QYXJ0aWNpcGFudEluZm8SEAoDc2lkGAEgASgJUgNzaWQSGgoIaWRlbnRpdHkYAiABKAlSCG' 'lkZW50aXR5EjQKBXN0YXRlGAMgASgOMh4ubGl2ZWtpdC5QYXJ0aWNpcGFudEluZm8uU3RhdGVS' - 'BXN0YXRlEioKBnRyYWNrcxgEIAMoCzISLmxpdmVraXQuVHJhY2tJbmZvUgZ0cmFja3MSQgoIbW' - 'V0YWRhdGEYBSABKAlCJojsLAGS7CwePHJlZGFjdGVkICh7eyAuU2l6ZSB9fSBieXRlcyk+Ught' - 'ZXRhZGF0YRIbCglqb2luZWRfYXQYBiABKANSCGpvaW5lZEF0EiAKDGpvaW5lZF9hdF9tcxgRIA' - 'EoA1IKam9pbmVkQXRNcxIYCgRuYW1lGAkgASgJQgSI7CwBUgRuYW1lEhgKB3ZlcnNpb24YCiAB' - 'KA1SB3ZlcnNpb24SPgoKcGVybWlzc2lvbhgLIAEoCzIeLmxpdmVraXQuUGFydGljaXBhbnRQZX' - 'JtaXNzaW9uUgpwZXJtaXNzaW9uEhYKBnJlZ2lvbhgMIAEoCVIGcmVnaW9uEiEKDGlzX3B1Ymxp' - 'c2hlchgNIAEoCFILaXNQdWJsaXNoZXISMQoEa2luZBgOIAEoDjIdLmxpdmVraXQuUGFydGljaX' - 'BhbnRJbmZvLktpbmRSBGtpbmQScAoKYXR0cmlidXRlcxgPIAMoCzIoLmxpdmVraXQuUGFydGlj' - 'aXBhbnRJbmZvLkF0dHJpYnV0ZXNFbnRyeUImiOwsAZLsLB48cmVkYWN0ZWQgKHt7IC5TaXplIH' - '19IGJ5dGVzKT5SCmF0dHJpYnV0ZXMSRgoRZGlzY29ubmVjdF9yZWFzb24YECABKA4yGS5saXZl' - 'a2l0LkRpc2Nvbm5lY3RSZWFzb25SEGRpc2Nvbm5lY3RSZWFzb24SRgoMa2luZF9kZXRhaWxzGB' - 'IgAygOMiMubGl2ZWtpdC5QYXJ0aWNpcGFudEluZm8uS2luZERldGFpbFILa2luZERldGFpbHMS' - 'NwoLZGF0YV90cmFja3MYEyADKAsyFi5saXZla2l0LkRhdGFUcmFja0luZm9SCmRhdGFUcmFja3' - 'MaPQoPQXR0cmlidXRlc0VudHJ5EhAKA2tleRgBIAEoCVIDa2V5EhQKBXZhbHVlGAIgASgJUgV2' - 'YWx1ZToCOAEiPgoFU3RhdGUSCwoHSk9JTklORxAAEgoKBkpPSU5FRBABEgoKBkFDVElWRRACEh' - 'AKDERJU0NPTk5FQ1RFRBADIlwKBEtpbmQSDAoIU1RBTkRBUkQQABILCgdJTkdSRVNTEAESCgoG' - 'RUdSRVNTEAISBwoDU0lQEAMSCQoFQUdFTlQQBBINCglDT05ORUNUT1IQBxIKCgZCUklER0UQCC' - 'JrCgpLaW5kRGV0YWlsEg8KC0NMT1VEX0FHRU5UEAASDQoJRk9SV0FSREVEEAESFgoSQ09OTkVD' - 'VE9SX1dIQVRTQVBQEAISFAoQQ09OTkVDVE9SX1RXSUxJTxADEg8KC0JSSURHRV9SVFNQEAQ='); + 'BXN0YXRlEioKBnRyYWNrcxgEIAMoCzISLmxpdmVraXQuVHJhY2tJbmZvUgZ0cmFja3MSQAoIbW' + 'V0YWRhdGEYBSABKAlCJKhQAbJQHjxyZWRhY3RlZCAoe3sgLlNpemUgfX0gYnl0ZXMpPlIIbWV0' + 'YWRhdGESGwoJam9pbmVkX2F0GAYgASgDUghqb2luZWRBdBIgCgxqb2luZWRfYXRfbXMYESABKA' + 'NSCmpvaW5lZEF0TXMSFwoEbmFtZRgJIAEoCUIDqFABUgRuYW1lEhgKB3ZlcnNpb24YCiABKA1S' + 'B3ZlcnNpb24SPgoKcGVybWlzc2lvbhgLIAEoCzIeLmxpdmVraXQuUGFydGljaXBhbnRQZXJtaX' + 'NzaW9uUgpwZXJtaXNzaW9uEhYKBnJlZ2lvbhgMIAEoCVIGcmVnaW9uEiEKDGlzX3B1Ymxpc2hl' + 'chgNIAEoCFILaXNQdWJsaXNoZXISMQoEa2luZBgOIAEoDjIdLmxpdmVraXQuUGFydGljaXBhbn' + 'RJbmZvLktpbmRSBGtpbmQSbgoKYXR0cmlidXRlcxgPIAMoCzIoLmxpdmVraXQuUGFydGljaXBh' + 'bnRJbmZvLkF0dHJpYnV0ZXNFbnRyeUIkqFABslAePHJlZGFjdGVkICh7eyAuU2l6ZSB9fSBieX' + 'Rlcyk+UgphdHRyaWJ1dGVzEkYKEWRpc2Nvbm5lY3RfcmVhc29uGBAgASgOMhkubGl2ZWtpdC5E' + 'aXNjb25uZWN0UmVhc29uUhBkaXNjb25uZWN0UmVhc29uEkYKDGtpbmRfZGV0YWlscxgSIAMoDj' + 'IjLmxpdmVraXQuUGFydGljaXBhbnRJbmZvLktpbmREZXRhaWxSC2tpbmREZXRhaWxzEjcKC2Rh' + 'dGFfdHJhY2tzGBMgAygLMhYubGl2ZWtpdC5EYXRhVHJhY2tJbmZvUgpkYXRhVHJhY2tzEicKD2' + 'NsaWVudF9wcm90b2NvbBgUIAEoBVIOY2xpZW50UHJvdG9jb2waPQoPQXR0cmlidXRlc0VudHJ5' + 'EhAKA2tleRgBIAEoCVIDa2V5EhQKBXZhbHVlGAIgASgJUgV2YWx1ZToCOAEiPgoFU3RhdGUSCw' + 'oHSk9JTklORxAAEgoKBkpPSU5FRBABEgoKBkFDVElWRRACEhAKDERJU0NPTk5FQ1RFRBADIlwK' + 'BEtpbmQSDAoIU1RBTkRBUkQQABILCgdJTkdSRVNTEAESCgoGRUdSRVNTEAISBwoDU0lQEAMSCQ' + 'oFQUdFTlQQBBINCglDT05ORUNUT1IQBxIKCgZCUklER0UQCCJrCgpLaW5kRGV0YWlsEg8KC0NM' + 'T1VEX0FHRU5UEAASDQoJRk9SV0FSREVEEAESFgoSQ09OTkVDVE9SX1dIQVRTQVBQEAISFAoQQ0' + '9OTkVDVE9SX1RXSUxJTxADEg8KC0JSSURHRV9SVFNQEAQ='); @$core.Deprecated('Use encryptionDescriptor instead') const Encryption$json = { @@ -606,25 +625,35 @@ const TrackInfo$json = { '6': '.livekit.BackupCodecPolicy', '10': 'backupCodecPolicy' }, + { + '1': 'packet_trailer_features', + '3': 21, + '4': 3, + '5': 14, + '6': '.livekit.PacketTrailerFeature', + '10': 'packetTrailerFeatures' + }, ], }; /// Descriptor for `TrackInfo`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List trackInfoDescriptor = $convert.base64Decode('CglUcmFja0luZm8SEAoDc2lkGAEgASgJUgNzaWQSJgoEdHlwZRgCIAEoDjISLmxpdmVraXQuVH' - 'JhY2tUeXBlUgR0eXBlEhgKBG5hbWUYAyABKAlCBIjsLAFSBG5hbWUSFAoFbXV0ZWQYBCABKAhS' - 'BW11dGVkEhQKBXdpZHRoGAUgASgNUgV3aWR0aBIWCgZoZWlnaHQYBiABKA1SBmhlaWdodBIgCg' - 'lzaW11bGNhc3QYByABKAhCAhgBUglzaW11bGNhc3QSIwoLZGlzYWJsZV9kdHgYCCABKAhCAhgB' - 'UgpkaXNhYmxlRHR4EiwKBnNvdXJjZRgJIAEoDjIULmxpdmVraXQuVHJhY2tTb3VyY2VSBnNvdX' - 'JjZRIvCgZsYXllcnMYCiADKAsyEy5saXZla2l0LlZpZGVvTGF5ZXJCAhgBUgZsYXllcnMSGwoJ' - 'bWltZV90eXBlGAsgASgJUghtaW1lVHlwZRIQCgNtaWQYDCABKAlSA21pZBIzCgZjb2RlY3MYDS' - 'ADKAsyGy5saXZla2l0LlNpbXVsY2FzdENvZGVjSW5mb1IGY29kZWNzEhoKBnN0ZXJlbxgOIAEo' - 'CEICGAFSBnN0ZXJlbxIfCgtkaXNhYmxlX3JlZBgPIAEoCFIKZGlzYWJsZVJlZBI4CgplbmNyeX' - 'B0aW9uGBAgASgOMhgubGl2ZWtpdC5FbmNyeXB0aW9uLlR5cGVSCmVuY3J5cHRpb24SFgoGc3Ry' - 'ZWFtGBEgASgJUgZzdHJlYW0SLwoHdmVyc2lvbhgSIAEoCzIVLmxpdmVraXQuVGltZWRWZXJzaW' - '9uUgd2ZXJzaW9uEkEKDmF1ZGlvX2ZlYXR1cmVzGBMgAygOMhoubGl2ZWtpdC5BdWRpb1RyYWNr' - 'RmVhdHVyZVINYXVkaW9GZWF0dXJlcxJKChNiYWNrdXBfY29kZWNfcG9saWN5GBQgASgOMhoubG' - 'l2ZWtpdC5CYWNrdXBDb2RlY1BvbGljeVIRYmFja3VwQ29kZWNQb2xpY3k='); + 'JhY2tUeXBlUgR0eXBlEhcKBG5hbWUYAyABKAlCA6hQAVIEbmFtZRIUCgVtdXRlZBgEIAEoCFIF' + 'bXV0ZWQSFAoFd2lkdGgYBSABKA1SBXdpZHRoEhYKBmhlaWdodBgGIAEoDVIGaGVpZ2h0EiAKCX' + 'NpbXVsY2FzdBgHIAEoCEICGAFSCXNpbXVsY2FzdBIjCgtkaXNhYmxlX2R0eBgIIAEoCEICGAFS' + 'CmRpc2FibGVEdHgSLAoGc291cmNlGAkgASgOMhQubGl2ZWtpdC5UcmFja1NvdXJjZVIGc291cm' + 'NlEi8KBmxheWVycxgKIAMoCzITLmxpdmVraXQuVmlkZW9MYXllckICGAFSBmxheWVycxIbCglt' + 'aW1lX3R5cGUYCyABKAlSCG1pbWVUeXBlEhAKA21pZBgMIAEoCVIDbWlkEjMKBmNvZGVjcxgNIA' + 'MoCzIbLmxpdmVraXQuU2ltdWxjYXN0Q29kZWNJbmZvUgZjb2RlY3MSGgoGc3RlcmVvGA4gASgI' + 'QgIYAVIGc3RlcmVvEh8KC2Rpc2FibGVfcmVkGA8gASgIUgpkaXNhYmxlUmVkEjgKCmVuY3J5cH' + 'Rpb24YECABKA4yGC5saXZla2l0LkVuY3J5cHRpb24uVHlwZVIKZW5jcnlwdGlvbhIWCgZzdHJl' + 'YW0YESABKAlSBnN0cmVhbRIvCgd2ZXJzaW9uGBIgASgLMhUubGl2ZWtpdC5UaW1lZFZlcnNpb2' + '5SB3ZlcnNpb24SQQoOYXVkaW9fZmVhdHVyZXMYEyADKA4yGi5saXZla2l0LkF1ZGlvVHJhY2tG' + 'ZWF0dXJlUg1hdWRpb0ZlYXR1cmVzEkoKE2JhY2t1cF9jb2RlY19wb2xpY3kYFCABKA4yGi5saX' + 'Zla2l0LkJhY2t1cENvZGVjUG9saWN5UhFiYWNrdXBDb2RlY1BvbGljeRJVChdwYWNrZXRfdHJh' + 'aWxlcl9mZWF0dXJlcxgVIAMoDjIdLmxpdmVraXQuUGFja2V0VHJhaWxlckZlYXR1cmVSFXBhY2' + 'tldFRyYWlsZXJGZWF0dXJlcw=='); @$core.Deprecated('Use dataTrackInfoDescriptor instead') const DataTrackInfo$json = { @@ -962,7 +991,7 @@ const Transcription$json = { '1': 'Transcription', '2': [ {'1': 'transcribed_participant_identity', '3': 2, '4': 1, '5': 9, '10': 'transcribedParticipantIdentity'}, - {'1': 'track_id', '3': 3, '4': 1, '5': 9, '10': 'trackId'}, + {'1': 'track_id', '3': 3, '4': 1, '5': 9, '8': {}, '10': 'trackId'}, {'1': 'segments', '3': 4, '4': 3, '5': 11, '6': '.livekit.TranscriptionSegment', '10': 'segments'}, ], }; @@ -970,9 +999,9 @@ const Transcription$json = { /// Descriptor for `Transcription`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List transcriptionDescriptor = $convert.base64Decode('Cg1UcmFuc2NyaXB0aW9uEkgKIHRyYW5zY3JpYmVkX3BhcnRpY2lwYW50X2lkZW50aXR5GAIgAS' - 'gJUh50cmFuc2NyaWJlZFBhcnRpY2lwYW50SWRlbnRpdHkSGQoIdHJhY2tfaWQYAyABKAlSB3Ry' - 'YWNrSWQSOQoIc2VnbWVudHMYBCADKAsyHS5saXZla2l0LlRyYW5zY3JpcHRpb25TZWdtZW50Ug' - 'hzZWdtZW50cw=='); + 'gJUh50cmFuc2NyaWJlZFBhcnRpY2lwYW50SWRlbnRpdHkSJQoIdHJhY2tfaWQYAyABKAlCCrpQ' + 'B3RyYWNrSURSB3RyYWNrSWQSOQoIc2VnbWVudHMYBCADKAsyHS5saXZla2l0LlRyYW5zY3JpcH' + 'Rpb25TZWdtZW50UghzZWdtZW50cw=='); @$core.Deprecated('Use transcriptionSegmentDescriptor instead') const TranscriptionSegment$json = { @@ -1026,6 +1055,7 @@ const RpcRequest$json = { {'1': 'payload', '3': 3, '4': 1, '5': 9, '10': 'payload'}, {'1': 'response_timeout_ms', '3': 4, '4': 1, '5': 13, '10': 'responseTimeoutMs'}, {'1': 'version', '3': 5, '4': 1, '5': 13, '10': 'version'}, + {'1': 'compressed_payload', '3': 6, '4': 1, '5': 12, '10': 'compressedPayload'}, ], }; @@ -1033,27 +1063,29 @@ const RpcRequest$json = { final $typed_data.Uint8List rpcRequestDescriptor = $convert.base64Decode('CgpScGNSZXF1ZXN0Eg4KAmlkGAEgASgJUgJpZBIWCgZtZXRob2QYAiABKAlSBm1ldGhvZBIYCg' 'dwYXlsb2FkGAMgASgJUgdwYXlsb2FkEi4KE3Jlc3BvbnNlX3RpbWVvdXRfbXMYBCABKA1SEXJl' - 'c3BvbnNlVGltZW91dE1zEhgKB3ZlcnNpb24YBSABKA1SB3ZlcnNpb24='); + 'c3BvbnNlVGltZW91dE1zEhgKB3ZlcnNpb24YBSABKA1SB3ZlcnNpb24SLQoSY29tcHJlc3NlZF' + '9wYXlsb2FkGAYgASgMUhFjb21wcmVzc2VkUGF5bG9hZA=='); @$core.Deprecated('Use rpcAckDescriptor instead') const RpcAck$json = { '1': 'RpcAck', '2': [ - {'1': 'request_id', '3': 1, '4': 1, '5': 9, '10': 'requestId'}, + {'1': 'request_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'requestId'}, ], }; /// Descriptor for `RpcAck`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List rpcAckDescriptor = - $convert.base64Decode('CgZScGNBY2sSHQoKcmVxdWVzdF9pZBgBIAEoCVIJcmVxdWVzdElk'); + $convert.base64Decode('CgZScGNBY2sSKwoKcmVxdWVzdF9pZBgBIAEoCUIMulAJcmVxdWVzdElEUglyZXF1ZXN0SWQ='); @$core.Deprecated('Use rpcResponseDescriptor instead') const RpcResponse$json = { '1': 'RpcResponse', '2': [ - {'1': 'request_id', '3': 1, '4': 1, '5': 9, '10': 'requestId'}, + {'1': 'request_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'requestId'}, {'1': 'payload', '3': 2, '4': 1, '5': 9, '9': 0, '10': 'payload'}, {'1': 'error', '3': 3, '4': 1, '5': 11, '6': '.livekit.RpcError', '9': 0, '10': 'error'}, + {'1': 'compressed_payload', '3': 4, '4': 1, '5': 12, '9': 0, '10': 'compressedPayload'}, ], '8': [ {'1': 'value'}, @@ -1062,9 +1094,10 @@ const RpcResponse$json = { /// Descriptor for `RpcResponse`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List rpcResponseDescriptor = - $convert.base64Decode('CgtScGNSZXNwb25zZRIdCgpyZXF1ZXN0X2lkGAEgASgJUglyZXF1ZXN0SWQSGgoHcGF5bG9hZB' - 'gCIAEoCUgAUgdwYXlsb2FkEikKBWVycm9yGAMgASgLMhEubGl2ZWtpdC5ScGNFcnJvckgAUgVl' - 'cnJvckIHCgV2YWx1ZQ=='); + $convert.base64Decode('CgtScGNSZXNwb25zZRIrCgpyZXF1ZXN0X2lkGAEgASgJQgy6UAlyZXF1ZXN0SURSCXJlcXVlc3' + 'RJZBIaCgdwYXlsb2FkGAIgASgJSABSB3BheWxvYWQSKQoFZXJyb3IYAyABKAsyES5saXZla2l0' + 'LlJwY0Vycm9ySABSBWVycm9yEi8KEmNvbXByZXNzZWRfcGF5bG9hZBgEIAEoDEgAUhFjb21wcm' + 'Vzc2VkUGF5bG9hZEIHCgV2YWx1ZQ=='); @$core.Deprecated('Use rpcErrorDescriptor instead') const RpcError$json = { @@ -1103,7 +1136,7 @@ const ServerInfo$json = { {'1': 'version', '3': 2, '4': 1, '5': 9, '10': 'version'}, {'1': 'protocol', '3': 3, '4': 1, '5': 5, '10': 'protocol'}, {'1': 'region', '3': 4, '4': 1, '5': 9, '10': 'region'}, - {'1': 'node_id', '3': 5, '4': 1, '5': 9, '10': 'nodeId'}, + {'1': 'node_id', '3': 5, '4': 1, '5': 9, '8': {}, '10': 'nodeId'}, {'1': 'debug_info', '3': 6, '4': 1, '5': 9, '10': 'debugInfo'}, {'1': 'agent_protocol', '3': 7, '4': 1, '5': 5, '10': 'agentProtocol'}, ], @@ -1123,9 +1156,10 @@ const ServerInfo_Edition$json = { final $typed_data.Uint8List serverInfoDescriptor = $convert.base64Decode('CgpTZXJ2ZXJJbmZvEjUKB2VkaXRpb24YASABKA4yGy5saXZla2l0LlNlcnZlckluZm8uRWRpdG' 'lvblIHZWRpdGlvbhIYCgd2ZXJzaW9uGAIgASgJUgd2ZXJzaW9uEhoKCHByb3RvY29sGAMgASgF' - 'Ughwcm90b2NvbBIWCgZyZWdpb24YBCABKAlSBnJlZ2lvbhIXCgdub2RlX2lkGAUgASgJUgZub2' - 'RlSWQSHQoKZGVidWdfaW5mbxgGIAEoCVIJZGVidWdJbmZvEiUKDmFnZW50X3Byb3RvY29sGAcg' - 'ASgFUg1hZ2VudFByb3RvY29sIiIKB0VkaXRpb24SDAoIU3RhbmRhcmQQABIJCgVDbG91ZBAB'); + 'Ughwcm90b2NvbBIWCgZyZWdpb24YBCABKAlSBnJlZ2lvbhIiCgdub2RlX2lkGAUgASgJQgm6UA' + 'Zub2RlSURSBm5vZGVJZBIdCgpkZWJ1Z19pbmZvGAYgASgJUglkZWJ1Z0luZm8SJQoOYWdlbnRf' + 'cHJvdG9jb2wYByABKAVSDWFnZW50UHJvdG9jb2wiIgoHRWRpdGlvbhIMCghTdGFuZGFyZBAAEg' + 'kKBUNsb3VkEAE='); @$core.Deprecated('Use clientInfoDescriptor instead') const ClientInfo$json = { @@ -1142,8 +1176,10 @@ const ClientInfo$json = { {'1': 'address', '3': 9, '4': 1, '5': 9, '10': 'address'}, {'1': 'network', '3': 10, '4': 1, '5': 9, '10': 'network'}, {'1': 'other_sdks', '3': 11, '4': 1, '5': 9, '10': 'otherSdks'}, + {'1': 'client_protocol', '3': 12, '4': 1, '5': 5, '10': 'clientProtocol'}, + {'1': 'capabilities', '3': 13, '4': 3, '5': 14, '6': '.livekit.ClientInfo.Capability', '10': 'capabilities'}, ], - '4': [ClientInfo_SDK$json], + '4': [ClientInfo_SDK$json, ClientInfo_Capability$json], }; @$core.Deprecated('Use clientInfoDescriptor instead') @@ -1168,6 +1204,15 @@ const ClientInfo_SDK$json = { ], }; +@$core.Deprecated('Use clientInfoDescriptor instead') +const ClientInfo_Capability$json = { + '1': 'Capability', + '2': [ + {'1': 'CAP_UNUSED', '2': 0}, + {'1': 'CAP_PACKET_TRAILER', '2': 1}, + ], +}; + /// Descriptor for `ClientInfo`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List clientInfoDescriptor = $convert.base64Decode('CgpDbGllbnRJbmZvEikKA3NkaxgBIAEoDjIXLmxpdmVraXQuQ2xpZW50SW5mby5TREtSA3Nkax' @@ -1176,10 +1221,13 @@ final $typed_data.Uint8List clientInfoDescriptor = '9tb2RlbBgGIAEoCVILZGV2aWNlTW9kZWwSGAoHYnJvd3NlchgHIAEoCVIHYnJvd3NlchInCg9i' 'cm93c2VyX3ZlcnNpb24YCCABKAlSDmJyb3dzZXJWZXJzaW9uEhgKB2FkZHJlc3MYCSABKAlSB2' 'FkZHJlc3MSGAoHbmV0d29yaxgKIAEoCVIHbmV0d29yaxIdCgpvdGhlcl9zZGtzGAsgASgJUglv' - 'dGhlclNka3MiswEKA1NESxILCgdVTktOT1dOEAASBgoCSlMQARIJCgVTV0lGVBACEgsKB0FORF' - 'JPSUQQAxILCgdGTFVUVEVSEAQSBgoCR08QBRIJCgVVTklUWRAGEhAKDFJFQUNUX05BVElWRRAH' - 'EggKBFJVU1QQCBIKCgZQWVRIT04QCRIHCgNDUFAQChINCglVTklUWV9XRUIQCxIICgROT0RFEA' - 'wSCgoGVU5SRUFMEA0SCQoFRVNQMzIQDg=='); + 'dGhlclNka3MSJwoPY2xpZW50X3Byb3RvY29sGAwgASgFUg5jbGllbnRQcm90b2NvbBJCCgxjYX' + 'BhYmlsaXRpZXMYDSADKA4yHi5saXZla2l0LkNsaWVudEluZm8uQ2FwYWJpbGl0eVIMY2FwYWJp' + 'bGl0aWVzIrMBCgNTREsSCwoHVU5LTk9XThAAEgYKAkpTEAESCQoFU1dJRlQQAhILCgdBTkRST0' + 'lEEAMSCwoHRkxVVFRFUhAEEgYKAkdPEAUSCQoFVU5JVFkQBhIQCgxSRUFDVF9OQVRJVkUQBxII' + 'CgRSVVNUEAgSCgoGUFlUSE9OEAkSBwoDQ1BQEAoSDQoJVU5JVFlfV0VCEAsSCAoETk9ERRAMEg' + 'oKBlVOUkVBTBANEgkKBUVTUDMyEA4iNAoKQ2FwYWJpbGl0eRIOCgpDQVBfVU5VU0VEEAASFgoS' + 'Q0FQX1BBQ0tFVF9UUkFJTEVSEAE='); @$core.Deprecated('Use clientConfigurationDescriptor instead') const ClientConfiguration$json = { @@ -1442,7 +1490,7 @@ final $typed_data.Uint8List rTPMungerStateDescriptor = const VP8MungerState$json = { '1': 'VP8MungerState', '2': [ - {'1': 'ext_last_picture_id', '3': 1, '4': 1, '5': 5, '10': 'extLastPictureId'}, + {'1': 'ext_last_picture_id', '3': 1, '4': 1, '5': 5, '8': {}, '10': 'extLastPictureId'}, {'1': 'picture_id_used', '3': 2, '4': 1, '5': 8, '10': 'pictureIdUsed'}, {'1': 'last_tl0_pic_idx', '3': 3, '4': 1, '5': 13, '10': 'lastTl0PicIdx'}, {'1': 'tl0_pic_idx_used', '3': 4, '4': 1, '5': 8, '10': 'tl0PicIdxUsed'}, @@ -1454,12 +1502,12 @@ const VP8MungerState$json = { /// Descriptor for `VP8MungerState`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List vP8MungerStateDescriptor = - $convert.base64Decode('Cg5WUDhNdW5nZXJTdGF0ZRItChNleHRfbGFzdF9waWN0dXJlX2lkGAEgASgFUhBleHRMYXN0UG' - 'ljdHVyZUlkEiYKD3BpY3R1cmVfaWRfdXNlZBgCIAEoCFINcGljdHVyZUlkVXNlZBInChBsYXN0' - 'X3RsMF9waWNfaWR4GAMgASgNUg1sYXN0VGwwUGljSWR4EicKEHRsMF9waWNfaWR4X3VzZWQYBC' - 'ABKAhSDXRsMFBpY0lkeFVzZWQSGQoIdGlkX3VzZWQYBSABKAhSB3RpZFVzZWQSIAoMbGFzdF9r' - 'ZXlfaWR4GAYgASgNUgpsYXN0S2V5SWR4EiAKDGtleV9pZHhfdXNlZBgHIAEoCFIKa2V5SWR4VX' - 'NlZA=='); + $convert.base64Decode('Cg5WUDhNdW5nZXJTdGF0ZRJCChNleHRfbGFzdF9waWN0dXJlX2lkGAEgASgFQhO6UBBleHRMYX' + 'N0UGljdHVyZUlEUhBleHRMYXN0UGljdHVyZUlkEiYKD3BpY3R1cmVfaWRfdXNlZBgCIAEoCFIN' + 'cGljdHVyZUlkVXNlZBInChBsYXN0X3RsMF9waWNfaWR4GAMgASgNUg1sYXN0VGwwUGljSWR4Ei' + 'cKEHRsMF9waWNfaWR4X3VzZWQYBCABKAhSDXRsMFBpY0lkeFVzZWQSGQoIdGlkX3VzZWQYBSAB' + 'KAhSB3RpZFVzZWQSIAoMbGFzdF9rZXlfaWR4GAYgASgNUgpsYXN0S2V5SWR4EiAKDGtleV9pZH' + 'hfdXNlZBgHIAEoCFIKa2V5SWR4VXNlZA=='); @$core.Deprecated('Use timedVersionDescriptor instead') const TimedVersion$json = { @@ -1494,7 +1542,7 @@ const DataStream_TextHeader$json = { '2': [ {'1': 'operation_type', '3': 1, '4': 1, '5': 14, '6': '.livekit.DataStream.OperationType', '10': 'operationType'}, {'1': 'version', '3': 2, '4': 1, '5': 5, '10': 'version'}, - {'1': 'reply_to_stream_id', '3': 3, '4': 1, '5': 9, '10': 'replyToStreamId'}, + {'1': 'reply_to_stream_id', '3': 3, '4': 1, '5': 9, '8': {}, '10': 'replyToStreamId'}, {'1': 'attached_stream_ids', '3': 4, '4': 3, '5': 9, '10': 'attachedStreamIds'}, {'1': 'generated', '3': 5, '4': 1, '5': 8, '10': 'generated'}, ], @@ -1512,7 +1560,7 @@ const DataStream_ByteHeader$json = { const DataStream_Header$json = { '1': 'Header', '2': [ - {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '10': 'streamId'}, + {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'streamId'}, {'1': 'timestamp', '3': 2, '4': 1, '5': 3, '10': 'timestamp'}, {'1': 'topic', '3': 3, '4': 1, '5': 9, '10': 'topic'}, {'1': 'mime_type', '3': 4, '4': 1, '5': 9, '10': 'mimeType'}, @@ -1551,7 +1599,7 @@ const DataStream_Header_AttributesEntry$json = { const DataStream_Chunk$json = { '1': 'Chunk', '2': [ - {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '10': 'streamId'}, + {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'streamId'}, {'1': 'chunk_index', '3': 2, '4': 1, '5': 4, '10': 'chunkIndex'}, {'1': 'content', '3': 3, '4': 1, '5': 12, '10': 'content'}, {'1': 'version', '3': 4, '4': 1, '5': 5, '10': 'version'}, @@ -1575,7 +1623,7 @@ const DataStream_Chunk$json = { const DataStream_Trailer$json = { '1': 'Trailer', '2': [ - {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '10': 'streamId'}, + {'1': 'stream_id', '3': 1, '4': 1, '5': 9, '8': {}, '10': 'streamId'}, {'1': 'reason', '3': 2, '4': 1, '5': 9, '10': 'reason'}, { '1': 'attributes', @@ -1612,29 +1660,30 @@ const DataStream_OperationType$json = { /// Descriptor for `DataStream`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List dataStreamDescriptor = - $convert.base64Decode('CgpEYXRhU3RyZWFtGusBCgpUZXh0SGVhZGVyEkgKDm9wZXJhdGlvbl90eXBlGAEgASgOMiEubG' + $convert.base64Decode('CgpEYXRhU3RyZWFtGv8BCgpUZXh0SGVhZGVyEkgKDm9wZXJhdGlvbl90eXBlGAEgASgOMiEubG' 'l2ZWtpdC5EYXRhU3RyZWFtLk9wZXJhdGlvblR5cGVSDW9wZXJhdGlvblR5cGUSGAoHdmVyc2lv' - 'bhgCIAEoBVIHdmVyc2lvbhIrChJyZXBseV90b19zdHJlYW1faWQYAyABKAlSD3JlcGx5VG9TdH' - 'JlYW1JZBIuChNhdHRhY2hlZF9zdHJlYW1faWRzGAQgAygJUhFhdHRhY2hlZFN0cmVhbUlkcxIc' - 'CglnZW5lcmF0ZWQYBSABKAhSCWdlbmVyYXRlZBogCgpCeXRlSGVhZGVyEhIKBG5hbWUYASABKA' - 'lSBG5hbWUamQQKBkhlYWRlchIbCglzdHJlYW1faWQYASABKAlSCHN0cmVhbUlkEhwKCXRpbWVz' - 'dGFtcBgCIAEoA1IJdGltZXN0YW1wEhQKBXRvcGljGAMgASgJUgV0b3BpYxIbCgltaW1lX3R5cG' - 'UYBCABKAlSCG1pbWVUeXBlEiYKDHRvdGFsX2xlbmd0aBgFIAEoBEgBUgt0b3RhbExlbmd0aIgB' - 'ARJFCg9lbmNyeXB0aW9uX3R5cGUYByABKA4yGC5saXZla2l0LkVuY3J5cHRpb24uVHlwZUICGA' - 'FSDmVuY3J5cHRpb25UeXBlEkoKCmF0dHJpYnV0ZXMYCCADKAsyKi5saXZla2l0LkRhdGFTdHJl' - 'YW0uSGVhZGVyLkF0dHJpYnV0ZXNFbnRyeVIKYXR0cmlidXRlcxJBCgt0ZXh0X2hlYWRlchgJIA' - 'EoCzIeLmxpdmVraXQuRGF0YVN0cmVhbS5UZXh0SGVhZGVySABSCnRleHRIZWFkZXISQQoLYnl0' - 'ZV9oZWFkZXIYCiABKAsyHi5saXZla2l0LkRhdGFTdHJlYW0uQnl0ZUhlYWRlckgAUgpieXRlSG' - 'VhZGVyGj0KD0F0dHJpYnV0ZXNFbnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEo' - 'CVIFdmFsdWU6AjgBQhAKDmNvbnRlbnRfaGVhZGVyQg8KDV90b3RhbF9sZW5ndGgamQEKBUNodW' - '5rEhsKCXN0cmVhbV9pZBgBIAEoCVIIc3RyZWFtSWQSHwoLY2h1bmtfaW5kZXgYAiABKARSCmNo' - 'dW5rSW5kZXgSGAoHY29udGVudBgDIAEoDFIHY29udGVudBIYCgd2ZXJzaW9uGAQgASgFUgd2ZX' - 'JzaW9uEhcKAml2GAUgASgMQgIYAUgAUgJpdogBAUIFCgNfaXYaygEKB1RyYWlsZXISGwoJc3Ry' - 'ZWFtX2lkGAEgASgJUghzdHJlYW1JZBIWCgZyZWFzb24YAiABKAlSBnJlYXNvbhJLCgphdHRyaW' - 'J1dGVzGAMgAygLMisubGl2ZWtpdC5EYXRhU3RyZWFtLlRyYWlsZXIuQXR0cmlidXRlc0VudHJ5' - 'UgphdHRyaWJ1dGVzGj0KD0F0dHJpYnV0ZXNFbnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YW' - 'x1ZRgCIAEoCVIFdmFsdWU6AjgBIkEKDU9wZXJhdGlvblR5cGUSCgoGQ1JFQVRFEAASCgoGVVBE' - 'QVRFEAESCgoGREVMRVRFEAISDAoIUkVBQ1RJT04QAw=='); + 'bhgCIAEoBVIHdmVyc2lvbhI/ChJyZXBseV90b19zdHJlYW1faWQYAyABKAlCErpQD3JlcGx5VG' + '9TdHJlYW1JRFIPcmVwbHlUb1N0cmVhbUlkEi4KE2F0dGFjaGVkX3N0cmVhbV9pZHMYBCADKAlS' + 'EWF0dGFjaGVkU3RyZWFtSWRzEhwKCWdlbmVyYXRlZBgFIAEoCFIJZ2VuZXJhdGVkGiAKCkJ5dG' + 'VIZWFkZXISEgoEbmFtZRgBIAEoCVIEbmFtZRqmBAoGSGVhZGVyEigKCXN0cmVhbV9pZBgBIAEo' + 'CUILulAIc3RyZWFtSURSCHN0cmVhbUlkEhwKCXRpbWVzdGFtcBgCIAEoA1IJdGltZXN0YW1wEh' + 'QKBXRvcGljGAMgASgJUgV0b3BpYxIbCgltaW1lX3R5cGUYBCABKAlSCG1pbWVUeXBlEiYKDHRv' + 'dGFsX2xlbmd0aBgFIAEoBEgBUgt0b3RhbExlbmd0aIgBARJFCg9lbmNyeXB0aW9uX3R5cGUYBy' + 'ABKA4yGC5saXZla2l0LkVuY3J5cHRpb24uVHlwZUICGAFSDmVuY3J5cHRpb25UeXBlEkoKCmF0' + 'dHJpYnV0ZXMYCCADKAsyKi5saXZla2l0LkRhdGFTdHJlYW0uSGVhZGVyLkF0dHJpYnV0ZXNFbn' + 'RyeVIKYXR0cmlidXRlcxJBCgt0ZXh0X2hlYWRlchgJIAEoCzIeLmxpdmVraXQuRGF0YVN0cmVh' + 'bS5UZXh0SGVhZGVySABSCnRleHRIZWFkZXISQQoLYnl0ZV9oZWFkZXIYCiABKAsyHi5saXZla2' + 'l0LkRhdGFTdHJlYW0uQnl0ZUhlYWRlckgAUgpieXRlSGVhZGVyGj0KD0F0dHJpYnV0ZXNFbnRy' + 'eRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6AjgBQhAKDmNvbnRlbn' + 'RfaGVhZGVyQg8KDV90b3RhbF9sZW5ndGgapgEKBUNodW5rEigKCXN0cmVhbV9pZBgBIAEoCUIL' + 'ulAIc3RyZWFtSURSCHN0cmVhbUlkEh8KC2NodW5rX2luZGV4GAIgASgEUgpjaHVua0luZGV4Eh' + 'gKB2NvbnRlbnQYAyABKAxSB2NvbnRlbnQSGAoHdmVyc2lvbhgEIAEoBVIHdmVyc2lvbhIXCgJp' + 'dhgFIAEoDEICGAFIAFICaXaIAQFCBQoDX2l2GtcBCgdUcmFpbGVyEigKCXN0cmVhbV9pZBgBIA' + 'EoCUILulAIc3RyZWFtSURSCHN0cmVhbUlkEhYKBnJlYXNvbhgCIAEoCVIGcmVhc29uEksKCmF0' + 'dHJpYnV0ZXMYAyADKAsyKy5saXZla2l0LkRhdGFTdHJlYW0uVHJhaWxlci5BdHRyaWJ1dGVzRW' + '50cnlSCmF0dHJpYnV0ZXMaPQoPQXR0cmlidXRlc0VudHJ5EhAKA2tleRgBIAEoCVIDa2V5EhQK' + 'BXZhbHVlGAIgASgJUgV2YWx1ZToCOAEiQQoNT3BlcmF0aW9uVHlwZRIKCgZDUkVBVEUQABIKCg' + 'ZVUERBVEUQARIKCgZERUxFVEUQAhIMCghSRUFDVElPThAD'); @$core.Deprecated('Use filterParamsDescriptor instead') const FilterParams$json = { diff --git a/lib/src/proto/livekit_rtc.pb.dart b/lib/src/proto/livekit_rtc.pb.dart index 2277a2fd6..9b1ae3234 100644 --- a/lib/src/proto/livekit_rtc.pb.dart +++ b/lib/src/proto/livekit_rtc.pb.dart @@ -1143,6 +1143,7 @@ class AddTrackRequest extends $pb.GeneratedMessage { $core.String? stream, $0.BackupCodecPolicy? backupCodecPolicy, $core.Iterable<$0.AudioTrackFeature>? audioFeatures, + $core.Iterable<$0.PacketTrailerFeature>? packetTrailerFeatures, }) { final result = create(); if (cid != null) result.cid = cid; @@ -1162,6 +1163,7 @@ class AddTrackRequest extends $pb.GeneratedMessage { if (stream != null) result.stream = stream; if (backupCodecPolicy != null) result.backupCodecPolicy = backupCodecPolicy; if (audioFeatures != null) result.audioFeatures.addAll(audioFeatures); + if (packetTrailerFeatures != null) result.packetTrailerFeatures.addAll(packetTrailerFeatures); return result; } @@ -1195,6 +1197,10 @@ class AddTrackRequest extends $pb.GeneratedMessage { valueOf: $0.AudioTrackFeature.valueOf, enumValues: $0.AudioTrackFeature.values, defaultEnumValue: $0.AudioTrackFeature.TF_STEREO) + ..pc<$0.PacketTrailerFeature>(18, _omitFieldNames ? '' : 'packetTrailerFeatures', $pb.PbFieldType.KE, + valueOf: $0.PacketTrailerFeature.valueOf, + enumValues: $0.PacketTrailerFeature.values, + defaultEnumValue: $0.PacketTrailerFeature.PTF_USER_TIMESTAMP) ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -1364,6 +1370,9 @@ class AddTrackRequest extends $pb.GeneratedMessage { @$pb.TagNumber(17) $pb.PbList<$0.AudioTrackFeature> get audioFeatures => $_getList(16); + + @$pb.TagNumber(18) + $pb.PbList<$0.PacketTrailerFeature> get packetTrailerFeatures => $_getList(17); } class PublishDataTrackRequest extends $pb.GeneratedMessage { diff --git a/lib/src/proto/livekit_rtc.pbjson.dart b/lib/src/proto/livekit_rtc.pbjson.dart index 2df9359b7..f9eff6e6c 100644 --- a/lib/src/proto/livekit_rtc.pbjson.dart +++ b/lib/src/proto/livekit_rtc.pbjson.dart @@ -436,6 +436,14 @@ const AddTrackRequest$json = { '10': 'backupCodecPolicy' }, {'1': 'audio_features', '3': 17, '4': 3, '5': 14, '6': '.livekit.AudioTrackFeature', '10': 'audioFeatures'}, + { + '1': 'packet_trailer_features', + '3': 18, + '4': 3, + '5': 14, + '6': '.livekit.PacketTrailerFeature', + '10': 'packetTrailerFeatures' + }, ], }; @@ -452,7 +460,9 @@ final $typed_data.Uint8List addTrackRequestDescriptor = 'Y3J5cHRpb24YDiABKA4yGC5saXZla2l0LkVuY3J5cHRpb24uVHlwZVIKZW5jcnlwdGlvbhIWCg' 'ZzdHJlYW0YDyABKAlSBnN0cmVhbRJKChNiYWNrdXBfY29kZWNfcG9saWN5GBAgASgOMhoubGl2' 'ZWtpdC5CYWNrdXBDb2RlY1BvbGljeVIRYmFja3VwQ29kZWNQb2xpY3kSQQoOYXVkaW9fZmVhdH' - 'VyZXMYESADKA4yGi5saXZla2l0LkF1ZGlvVHJhY2tGZWF0dXJlUg1hdWRpb0ZlYXR1cmVz'); + 'VyZXMYESADKA4yGi5saXZla2l0LkF1ZGlvVHJhY2tGZWF0dXJlUg1hdWRpb0ZlYXR1cmVzElUK' + 'F3BhY2tldF90cmFpbGVyX2ZlYXR1cmVzGBIgAygOMh0ubGl2ZWtpdC5QYWNrZXRUcmFpbGVyRm' + 'VhdHVyZVIVcGFja2V0VHJhaWxlckZlYXR1cmVz'); @$core.Deprecated('Use publishDataTrackRequestDescriptor instead') const PublishDataTrackRequest$json = { @@ -704,6 +714,7 @@ const SessionDescription$json = { '4': 3, '5': 11, '6': '.livekit.SessionDescription.MidToTrackIdEntry', + '8': {}, '10': 'midToTrackId' }, ], @@ -723,10 +734,10 @@ const SessionDescription_MidToTrackIdEntry$json = { /// Descriptor for `SessionDescription`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List sessionDescriptionDescriptor = $convert.base64Decode('ChJTZXNzaW9uRGVzY3JpcHRpb24SEgoEdHlwZRgBIAEoCVIEdHlwZRIQCgNzZHAYAiABKAlSA3' - 'NkcBIOCgJpZBgDIAEoDVICaWQSVAoPbWlkX3RvX3RyYWNrX2lkGAQgAygLMi0ubGl2ZWtpdC5T' - 'ZXNzaW9uRGVzY3JpcHRpb24uTWlkVG9UcmFja0lkRW50cnlSDG1pZFRvVHJhY2tJZBo/ChFNaW' - 'RUb1RyYWNrSWRFbnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6' - 'AjgB'); + 'NkcBIOCgJpZBgDIAEoDVICaWQSZQoPbWlkX3RvX3RyYWNrX2lkGAQgAygLMi0ubGl2ZWtpdC5T' + 'ZXNzaW9uRGVzY3JpcHRpb24uTWlkVG9UcmFja0lkRW50cnlCD7pQDG1pZFRvVHJhY2tJRFIMbW' + 'lkVG9UcmFja0lkGj8KEU1pZFRvVHJhY2tJZEVudHJ5EhAKA2tleRgBIAEoCVIDa2V5EhQKBXZh' + 'bHVlGAIgASgJUgV2YWx1ZToCOAE='); @$core.Deprecated('Use participantUpdateDescriptor instead') const ParticipantUpdate$json = { @@ -894,7 +905,7 @@ const UpdateParticipantMetadata$json = { '8': {}, '10': 'attributes' }, - {'1': 'request_id', '3': 4, '4': 1, '5': 13, '10': 'requestId'}, + {'1': 'request_id', '3': 4, '4': 1, '5': 13, '8': {}, '10': 'requestId'}, ], '3': [UpdateParticipantMetadata_AttributesEntry$json], }; @@ -911,28 +922,28 @@ const UpdateParticipantMetadata_AttributesEntry$json = { /// Descriptor for `UpdateParticipantMetadata`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List updateParticipantMetadataDescriptor = - $convert.base64Decode('ChlVcGRhdGVQYXJ0aWNpcGFudE1ldGFkYXRhEkIKCG1ldGFkYXRhGAEgASgJQiaI7CwBkuwsHj' - 'xyZWRhY3RlZCAoe3sgLlNpemUgfX0gYnl0ZXMpPlIIbWV0YWRhdGESOgoEbmFtZRgCIAEoCUIm' - 'iOwsAZLsLB48cmVkYWN0ZWQgKHt7IC5TaXplIH19IGJ5dGVzKT5SBG5hbWUSegoKYXR0cmlidX' - 'RlcxgDIAMoCzIyLmxpdmVraXQuVXBkYXRlUGFydGljaXBhbnRNZXRhZGF0YS5BdHRyaWJ1dGVz' - 'RW50cnlCJojsLAGS7CwePHJlZGFjdGVkICh7eyAuU2l6ZSB9fSBieXRlcyk+UgphdHRyaWJ1dG' - 'VzEh0KCnJlcXVlc3RfaWQYBCABKA1SCXJlcXVlc3RJZBo9Cg9BdHRyaWJ1dGVzRW50cnkSEAoD' - 'a2V5GAEgASgJUgNrZXkSFAoFdmFsdWUYAiABKAlSBXZhbHVlOgI4AQ=='); + $convert.base64Decode('ChlVcGRhdGVQYXJ0aWNpcGFudE1ldGFkYXRhEkAKCG1ldGFkYXRhGAEgASgJQiSoUAGyUB48cm' + 'VkYWN0ZWQgKHt7IC5TaXplIH19IGJ5dGVzKT5SCG1ldGFkYXRhEjgKBG5hbWUYAiABKAlCJKhQ' + 'AbJQHjxyZWRhY3RlZCAoe3sgLlNpemUgfX0gYnl0ZXMpPlIEbmFtZRJ4CgphdHRyaWJ1dGVzGA' + 'MgAygLMjIubGl2ZWtpdC5VcGRhdGVQYXJ0aWNpcGFudE1ldGFkYXRhLkF0dHJpYnV0ZXNFbnRy' + 'eUIkqFABslAePHJlZGFjdGVkICh7eyAuU2l6ZSB9fSBieXRlcyk+UgphdHRyaWJ1dGVzEisKCn' + 'JlcXVlc3RfaWQYBCABKA1CDLpQCXJlcXVlc3RJRFIJcmVxdWVzdElkGj0KD0F0dHJpYnV0ZXNF' + 'bnRyeRIQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6AjgB'); @$core.Deprecated('Use iCEServerDescriptor instead') const ICEServer$json = { '1': 'ICEServer', '2': [ {'1': 'urls', '3': 1, '4': 3, '5': 9, '10': 'urls'}, - {'1': 'username', '3': 2, '4': 1, '5': 9, '10': 'username'}, - {'1': 'credential', '3': 3, '4': 1, '5': 9, '10': 'credential'}, + {'1': 'username', '3': 2, '4': 1, '5': 9, '8': {}, '10': 'username'}, + {'1': 'credential', '3': 3, '4': 1, '5': 9, '8': {}, '10': 'credential'}, ], }; /// Descriptor for `ICEServer`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List iCEServerDescriptor = - $convert.base64Decode('CglJQ0VTZXJ2ZXISEgoEdXJscxgBIAMoCVIEdXJscxIaCgh1c2VybmFtZRgCIAEoCVIIdXNlcm' - '5hbWUSHgoKY3JlZGVudGlhbBgDIAEoCVIKY3JlZGVudGlhbA=='); + $convert.base64Decode('CglJQ0VTZXJ2ZXISEgoEdXJscxgBIAMoCVIEdXJscxIfCgh1c2VybmFtZRgCIAEoCUIDqFABUg' + 'h1c2VybmFtZRIjCgpjcmVkZW50aWFsGAMgASgJQgOoUAFSCmNyZWRlbnRpYWw='); @$core.Deprecated('Use speakersChangedDescriptor instead') const SpeakersChanged$json = { @@ -1350,7 +1361,7 @@ final $typed_data.Uint8List subscriptionResponseDescriptor = const RequestResponse$json = { '1': 'RequestResponse', '2': [ - {'1': 'request_id', '3': 1, '4': 1, '5': 13, '10': 'requestId'}, + {'1': 'request_id', '3': 1, '4': 1, '5': 13, '8': {}, '10': 'requestId'}, {'1': 'reason', '3': 2, '4': 1, '5': 14, '6': '.livekit.RequestResponse.Reason', '10': 'reason'}, {'1': 'message', '3': 3, '4': 1, '5': 9, '10': 'message'}, {'1': 'trickle', '3': 4, '4': 1, '5': 11, '6': '.livekit.TrickleRequest', '9': 0, '10': 'trickle'}, @@ -1428,23 +1439,23 @@ const RequestResponse_Reason$json = { /// Descriptor for `RequestResponse`. Decode as a `google.protobuf.DescriptorProto`. final $typed_data.Uint8List requestResponseDescriptor = - $convert.base64Decode('Cg9SZXF1ZXN0UmVzcG9uc2USHQoKcmVxdWVzdF9pZBgBIAEoDVIJcmVxdWVzdElkEjcKBnJlYX' - 'NvbhgCIAEoDjIfLmxpdmVraXQuUmVxdWVzdFJlc3BvbnNlLlJlYXNvblIGcmVhc29uEhgKB21l' - 'c3NhZ2UYAyABKAlSB21lc3NhZ2USMwoHdHJpY2tsZRgEIAEoCzIXLmxpdmVraXQuVHJpY2tsZV' - 'JlcXVlc3RIAFIHdHJpY2tsZRI3CglhZGRfdHJhY2sYBSABKAsyGC5saXZla2l0LkFkZFRyYWNr' - 'UmVxdWVzdEgAUghhZGRUcmFjaxIvCgRtdXRlGAYgASgLMhkubGl2ZWtpdC5NdXRlVHJhY2tSZX' - 'F1ZXN0SABSBG11dGUSTQoPdXBkYXRlX21ldGFkYXRhGAcgASgLMiIubGl2ZWtpdC5VcGRhdGVQ' - 'YXJ0aWNpcGFudE1ldGFkYXRhSABSDnVwZGF0ZU1ldGFkYXRhEk4KEnVwZGF0ZV9hdWRpb190cm' - 'FjaxgIIAEoCzIeLmxpdmVraXQuVXBkYXRlTG9jYWxBdWRpb1RyYWNrSABSEHVwZGF0ZUF1ZGlv' - 'VHJhY2sSTgoSdXBkYXRlX3ZpZGVvX3RyYWNrGAkgASgLMh4ubGl2ZWtpdC5VcGRhdGVMb2NhbF' - 'ZpZGVvVHJhY2tIAFIQdXBkYXRlVmlkZW9UcmFjaxJQChJwdWJsaXNoX2RhdGFfdHJhY2sYCiAB' - 'KAsyIC5saXZla2l0LlB1Ymxpc2hEYXRhVHJhY2tSZXF1ZXN0SABSEHB1Ymxpc2hEYXRhVHJhY2' - 'sSVgoUdW5wdWJsaXNoX2RhdGFfdHJhY2sYCyABKAsyIi5saXZla2l0LlVucHVibGlzaERhdGFU' - 'cmFja1JlcXVlc3RIAFISdW5wdWJsaXNoRGF0YVRyYWNrIs4BCgZSZWFzb24SBgoCT0sQABINCg' - 'lOT1RfRk9VTkQQARIPCgtOT1RfQUxMT1dFRBACEhIKDkxJTUlUX0VYQ0VFREVEEAMSCgoGUVVF' - 'VUVEEAQSFAoQVU5TVVBQT1JURURfVFlQRRAFEhYKElVOQ0xBU1NJRklFRF9FUlJPUhAGEhIKDk' - 'lOVkFMSURfSEFORExFEAcSEAoMSU5WQUxJRF9OQU1FEAgSFAoQRFVQTElDQVRFX0hBTkRMRRAJ' - 'EhIKDkRVUExJQ0FURV9OQU1FEApCCQoHcmVxdWVzdA=='); + $convert.base64Decode('Cg9SZXF1ZXN0UmVzcG9uc2USKwoKcmVxdWVzdF9pZBgBIAEoDUIMulAJcmVxdWVzdElEUglyZX' + 'F1ZXN0SWQSNwoGcmVhc29uGAIgASgOMh8ubGl2ZWtpdC5SZXF1ZXN0UmVzcG9uc2UuUmVhc29u' + 'UgZyZWFzb24SGAoHbWVzc2FnZRgDIAEoCVIHbWVzc2FnZRIzCgd0cmlja2xlGAQgASgLMhcubG' + 'l2ZWtpdC5Ucmlja2xlUmVxdWVzdEgAUgd0cmlja2xlEjcKCWFkZF90cmFjaxgFIAEoCzIYLmxp' + 'dmVraXQuQWRkVHJhY2tSZXF1ZXN0SABSCGFkZFRyYWNrEi8KBG11dGUYBiABKAsyGS5saXZla2' + 'l0Lk11dGVUcmFja1JlcXVlc3RIAFIEbXV0ZRJNCg91cGRhdGVfbWV0YWRhdGEYByABKAsyIi5s' + 'aXZla2l0LlVwZGF0ZVBhcnRpY2lwYW50TWV0YWRhdGFIAFIOdXBkYXRlTWV0YWRhdGESTgoSdX' + 'BkYXRlX2F1ZGlvX3RyYWNrGAggASgLMh4ubGl2ZWtpdC5VcGRhdGVMb2NhbEF1ZGlvVHJhY2tI' + 'AFIQdXBkYXRlQXVkaW9UcmFjaxJOChJ1cGRhdGVfdmlkZW9fdHJhY2sYCSABKAsyHi5saXZla2' + 'l0LlVwZGF0ZUxvY2FsVmlkZW9UcmFja0gAUhB1cGRhdGVWaWRlb1RyYWNrElAKEnB1Ymxpc2hf' + 'ZGF0YV90cmFjaxgKIAEoCzIgLmxpdmVraXQuUHVibGlzaERhdGFUcmFja1JlcXVlc3RIAFIQcH' + 'VibGlzaERhdGFUcmFjaxJWChR1bnB1Ymxpc2hfZGF0YV90cmFjaxgLIAEoCzIiLmxpdmVraXQu' + 'VW5wdWJsaXNoRGF0YVRyYWNrUmVxdWVzdEgAUhJ1bnB1Ymxpc2hEYXRhVHJhY2sizgEKBlJlYX' + 'NvbhIGCgJPSxAAEg0KCU5PVF9GT1VORBABEg8KC05PVF9BTExPV0VEEAISEgoOTElNSVRfRVhD' + 'RUVERUQQAxIKCgZRVUVVRUQQBBIUChBVTlNVUFBPUlRFRF9UWVBFEAUSFgoSVU5DTEFTU0lGSU' + 'VEX0VSUk9SEAYSEgoOSU5WQUxJRF9IQU5ETEUQBxIQCgxJTlZBTElEX05BTUUQCBIUChBEVVBM' + 'SUNBVEVfSEFORExFEAkSEgoORFVQTElDQVRFX05BTUUQCkIJCgdyZXF1ZXN0'); @$core.Deprecated('Use trackSubscribedDescriptor instead') const TrackSubscribed$json = { @@ -1530,18 +1541,18 @@ const JoinRequest_ParticipantAttributesEntry$json = { final $typed_data.Uint8List joinRequestDescriptor = $convert.base64Decode('CgtKb2luUmVxdWVzdBI0CgtjbGllbnRfaW5mbxgBIAEoCzITLmxpdmVraXQuQ2xpZW50SW5mb1' 'IKY2xpZW50SW5mbxJMChNjb25uZWN0aW9uX3NldHRpbmdzGAIgASgLMhsubGl2ZWtpdC5Db25u' - 'ZWN0aW9uU2V0dGluZ3NSEmNvbm5lY3Rpb25TZXR0aW5ncxJCCghtZXRhZGF0YRgDIAEoCUImiO' - 'wsAZLsLB48cmVkYWN0ZWQgKHt7IC5TaXplIH19IGJ5dGVzKT5SCG1ldGFkYXRhEo4BChZwYXJ0' - 'aWNpcGFudF9hdHRyaWJ1dGVzGAQgAygLMi8ubGl2ZWtpdC5Kb2luUmVxdWVzdC5QYXJ0aWNpcG' - 'FudEF0dHJpYnV0ZXNFbnRyeUImiOwsAZLsLB48cmVkYWN0ZWQgKHt7IC5TaXplIH19IGJ5dGVz' - 'KT5SFXBhcnRpY2lwYW50QXR0cmlidXRlcxJGChJhZGRfdHJhY2tfcmVxdWVzdHMYBSADKAsyGC' - '5saXZla2l0LkFkZFRyYWNrUmVxdWVzdFIQYWRkVHJhY2tSZXF1ZXN0cxJECg9wdWJsaXNoZXJf' - 'b2ZmZXIYBiABKAsyGy5saXZla2l0LlNlc3Npb25EZXNjcmlwdGlvblIOcHVibGlzaGVyT2ZmZX' - 'ISHAoJcmVjb25uZWN0GAcgASgIUglyZWNvbm5lY3QSQwoQcmVjb25uZWN0X3JlYXNvbhgIIAEo' - 'DjIYLmxpdmVraXQuUmVjb25uZWN0UmVhc29uUg9yZWNvbm5lY3RSZWFzb24SJwoPcGFydGljaX' - 'BhbnRfc2lkGAkgASgJUg5wYXJ0aWNpcGFudFNpZBIxCgpzeW5jX3N0YXRlGAogASgLMhIubGl2' - 'ZWtpdC5TeW5jU3RhdGVSCXN5bmNTdGF0ZRpIChpQYXJ0aWNpcGFudEF0dHJpYnV0ZXNFbnRyeR' - 'IQCgNrZXkYASABKAlSA2tleRIUCgV2YWx1ZRgCIAEoCVIFdmFsdWU6AjgB'); + 'ZWN0aW9uU2V0dGluZ3NSEmNvbm5lY3Rpb25TZXR0aW5ncxJACghtZXRhZGF0YRgDIAEoCUIkqF' + 'ABslAePHJlZGFjdGVkICh7eyAuU2l6ZSB9fSBieXRlcyk+UghtZXRhZGF0YRKMAQoWcGFydGlj' + 'aXBhbnRfYXR0cmlidXRlcxgEIAMoCzIvLmxpdmVraXQuSm9pblJlcXVlc3QuUGFydGljaXBhbn' + 'RBdHRyaWJ1dGVzRW50cnlCJKhQAbJQHjxyZWRhY3RlZCAoe3sgLlNpemUgfX0gYnl0ZXMpPlIV' + 'cGFydGljaXBhbnRBdHRyaWJ1dGVzEkYKEmFkZF90cmFja19yZXF1ZXN0cxgFIAMoCzIYLmxpdm' + 'VraXQuQWRkVHJhY2tSZXF1ZXN0UhBhZGRUcmFja1JlcXVlc3RzEkQKD3B1Ymxpc2hlcl9vZmZl' + 'chgGIAEoCzIbLmxpdmVraXQuU2Vzc2lvbkRlc2NyaXB0aW9uUg5wdWJsaXNoZXJPZmZlchIcCg' + 'lyZWNvbm5lY3QYByABKAhSCXJlY29ubmVjdBJDChByZWNvbm5lY3RfcmVhc29uGAggASgOMhgu' + 'bGl2ZWtpdC5SZWNvbm5lY3RSZWFzb25SD3JlY29ubmVjdFJlYXNvbhInCg9wYXJ0aWNpcGFudF' + '9zaWQYCSABKAlSDnBhcnRpY2lwYW50U2lkEjEKCnN5bmNfc3RhdGUYCiABKAsyEi5saXZla2l0' + 'LlN5bmNTdGF0ZVIJc3luY1N0YXRlGkgKGlBhcnRpY2lwYW50QXR0cmlidXRlc0VudHJ5EhAKA2' + 'tleRgBIAEoCVIDa2V5EhQKBXZhbHVlGAIgASgJUgV2YWx1ZToCOAE='); @$core.Deprecated('Use wrappedJoinRequestDescriptor instead') const WrappedJoinRequest$json = { From c5103bf67ba101c259e300753377bdbb3d014d36 Mon Sep 17 00:00:00 2001 From: Aayush Saurabh <85521656+mechtech-mind@users.noreply.github.com> Date: Tue, 19 May 2026 07:46:29 +0530 Subject: [PATCH 03/18] fix(android): prevent screen from turning off during active video calls in example application (#1057) This PR fixes an issue in the Android example application where the device screen turns off automatically during an active video call due to screen timeout. ### **Problem** During an ongoing LiveKit call on Android, the screen is not kept awake, which leads to: - Poor user experience during video calls - Interrupted visibility when no user interaction occurs ### **Solution** This PR adds native Android window flag handling in MainActivity to ensure the screen remains active while a call is in progress: - Uses native flags to prevent screen timeout - Uses native flags as well to handle lock screen visibility ### **The flags are:** - **Enabled** when the room is active - **Cleared** when the room is disposed --------- Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> --- .../kotlin/io/livekit/example/MainActivity.kt | 46 ++++++++++++++++++- example/lib/pages/room.dart | 19 ++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/example/android/app/src/main/kotlin/io/livekit/example/MainActivity.kt b/example/android/app/src/main/kotlin/io/livekit/example/MainActivity.kt index 8096b9cae..9a7fa8afb 100644 --- a/example/android/app/src/main/kotlin/io/livekit/example/MainActivity.kt +++ b/example/android/app/src/main/kotlin/io/livekit/example/MainActivity.kt @@ -1,6 +1,48 @@ package io.livekit.example +import android.os.Build +import android.os.Bundle +import android.view.WindowManager import io.flutter.embedding.android.FlutterActivity +import io.flutter.embedding.engine.FlutterEngine +import io.flutter.plugin.common.MethodChannel -class MainActivity: FlutterActivity() { -} +class MainActivity : FlutterActivity() { + + private val CHANNEL = "livekit_incall" + + override fun configureFlutterEngine(flutterEngine: FlutterEngine) { + super.configureFlutterEngine(flutterEngine) + + MethodChannel( + flutterEngine.dartExecutor.binaryMessenger, + CHANNEL + ).setMethodCallHandler { call, result -> + when (call.method) { + "enableInCall" -> { + setInCallScreenFlags() + result.success(null) + } + "disableInCall" -> { + clearInCallScreenFlags() + result.success(null) + } + else -> result.notImplemented() + } + } + } + + private fun setInCallScreenFlags() { + val window = window + + // Keep screen on during call + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } + + private fun clearInCallScreenFlags() { + val window = window + + // Remove keep screen on + window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } +} \ No newline at end of file diff --git a/example/lib/pages/room.dart b/example/lib/pages/room.dart index 9551325ed..27aa394b8 100644 --- a/example/lib/pages/room.dart +++ b/example/lib/pages/room.dart @@ -3,6 +3,7 @@ import 'dart:convert'; import 'dart:math' as math; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:livekit_client/livekit_client.dart'; import '../exts.dart'; @@ -34,6 +35,11 @@ class _RoomPageState extends State { @override void initState() { super.initState(); + + if (lkPlatformIs(PlatformType.android)) { + unawaited(enableInCallFlags()); + } + // add callback for a `RoomEvent` as opposed to a `ParticipantEvent` widget.room.addListener(_onRoomDidUpdate); // add callbacks for finer grained events @@ -63,6 +69,9 @@ class _RoomPageState extends State { widget.room.removeListener(_onRoomDidUpdate); unawaited(_disposeRoomAsync()); onWindowShouldClose = null; + if (lkPlatformIs(PlatformType.android)) { + unawaited(disableInCallFlags()); + } super.dispose(); } @@ -71,6 +80,16 @@ class _RoomPageState extends State { await widget.room.dispose(); } + static const platform = MethodChannel('livekit_incall'); + + Future enableInCallFlags() async { + await platform.invokeMethod('enableInCall'); + } + + Future disableInCallFlags() async { + await platform.invokeMethod('disableInCall'); + } + /// for more information, see [event types](https://docs.livekit.io/client/events/#events) void _setUpListeners() => _listener ..on((event) async { From 3488ba805ceef1523bfbea64384a92eba2cdc543 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:58:34 +0900 Subject: [PATCH 04/18] Align screen-share simulcast default lower layer (#1089) ## Summary Align Flutter's generated default lower screen-share simulcast layer with the common SDK behavior. This updates the default generated lower layer to: - use half resolution - keep the top layer's frame rate - use `max(150kbps, topLayerBitrate / 4)` - carry through explicit bitrate/network priorities Also aligns the `screenShareH720FPS5` and `screenShareH1080FPS30` preset bitrates with the common SDK values. ## Notes This does not add the JS-only screen-share presets such as `h360fps15`, `h720fps30`, or `original`; that broader preset parity work should be handled separately. ## Tests - `dart format --set-exit-if-changed lib/src/types/video_parameters.dart lib/src/utils.dart test/utils_test.dart` - `flutter test test/utils_test.dart` - `flutter analyze --no-pub lib/src/types/video_parameters.dart lib/src/utils.dart test/utils_test.dart` --- .changes/align-screenshare-simulcast-defaults | 1 + lib/src/types/video_parameters.dart | 4 +- lib/src/utils.dart | 19 ++++--- test/utils_test.dart | 55 +++++++++++++++++++ 4 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 .changes/align-screenshare-simulcast-defaults diff --git a/.changes/align-screenshare-simulcast-defaults b/.changes/align-screenshare-simulcast-defaults new file mode 100644 index 000000000..5ef6b9d5a --- /dev/null +++ b/.changes/align-screenshare-simulcast-defaults @@ -0,0 +1 @@ +patch type="fixed" "Align screen share simulcast default low layer" diff --git a/lib/src/types/video_parameters.dart b/lib/src/types/video_parameters.dart index 0ce57f904..b982b64c6 100644 --- a/lib/src/types/video_parameters.dart +++ b/lib/src/types/video_parameters.dart @@ -274,7 +274,7 @@ extension VideoParametersPresets on VideoParameters { static const screenShareH720FPS5 = VideoParameters( dimensions: VideoDimensionsPresets.h720_169, encoding: VideoEncoding( - maxBitrate: 400 * 1000, + maxBitrate: 800 * 1000, maxFramerate: 5, ), ); @@ -298,7 +298,7 @@ extension VideoParametersPresets on VideoParameters { static const screenShareH1080FPS30 = VideoParameters( dimensions: VideoDimensionsPresets.h1080_169, encoding: VideoEncoding( - maxBitrate: 4000 * 1000, + maxBitrate: 5000 * 1000, maxFramerate: 30, ), ); diff --git a/lib/src/utils.dart b/lib/src/utils.dart index cb0297d05..0dc0a3ef2 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -229,23 +229,24 @@ class Utils { static List _computeDefaultScreenShareSimulcastParams({ required VideoParameters original, }) { - final layers = [rtc.RTCRtpEncoding(scaleResolutionDownBy: 2, maxFramerate: 3)]; - return layers.map((e) { - final scale = e.scaleResolutionDownBy ?? 1; - final fps = e.maxFramerate ?? 3; + final originalEncoding = original.encoding!; + const scale = 2.0; - return VideoParameters( + return [ + VideoParameters( dimensions: VideoDimensions((original.dimensions.width / scale).floor(), (original.dimensions.height / scale).floor()), encoding: VideoEncoding( maxBitrate: math.max( 150 * 1000, - (original.encoding!.maxBitrate / (math.pow(scale, 2) * (original.encoding!.maxFramerate / fps))).floor(), + (originalEncoding.maxBitrate / math.pow(scale, 2)).floor(), ), - maxFramerate: fps, + maxFramerate: originalEncoding.maxFramerate, + bitratePriority: originalEncoding.bitratePriority, + networkPriority: originalEncoding.networkPriority, ), - ); - }).toList(); + ), + ]; } static List _computeDefaultSimulcastParams({ diff --git a/test/utils_test.dart b/test/utils_test.dart index 0a1ef7a28..dd6212206 100644 --- a/test/utils_test.dart +++ b/test/utils_test.dart @@ -13,7 +13,9 @@ // limitations under the License. import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:livekit_client/livekit_client.dart' as lk; import 'package:livekit_client/src/utils.dart'; void main() { @@ -54,4 +56,57 @@ void main() { ), ); }); + + group('screen share simulcast encodings', () { + test('screen share preset bitrates match common SDK presets', () { + expect(lk.VideoParametersPresets.screenShareH720FPS5.encoding?.maxBitrate, 800000); + expect(lk.VideoParametersPresets.screenShareH1080FPS30.encoding?.maxBitrate, 5000000); + }); + + test('default lower layer follows top layer fps and priorities', () { + final encodings = Utils.computeVideoEncodings( + isScreenShare: true, + dimensions: const lk.VideoDimensions(1920, 1080), + options: const lk.VideoPublishOptions( + screenShareEncoding: lk.VideoEncoding( + maxBitrate: 2500000, + maxFramerate: 15, + bitratePriority: lk.Priority.high, + networkPriority: lk.Priority.high, + ), + ), + ); + + expect(encodings, hasLength(2)); + + final lowLayer = encodings![0]; + expect(lowLayer.rid, 'q'); + expect(lowLayer.scaleResolutionDownBy, 2); + expect(lowLayer.maxFramerate, 15); + expect(lowLayer.maxBitrate, 625000); + expect(lowLayer.priority, rtc.RTCPriorityType.high); + expect(lowLayer.networkPriority, rtc.RTCPriorityType.high); + }); + + test('default lower layer follows selected screen share preset', () { + final encodings = Utils.computeVideoEncodings( + isScreenShare: true, + dimensions: const lk.VideoDimensions(1920, 1080), + ); + + expect(encodings, hasLength(2)); + + final lowLayer = encodings![0]; + expect(lowLayer.rid, 'q'); + expect(lowLayer.scaleResolutionDownBy, 2); + expect(lowLayer.maxFramerate, 15); + expect(lowLayer.maxBitrate, 625000); + + final topLayer = encodings[1]; + expect(topLayer.rid, 'h'); + expect(topLayer.scaleResolutionDownBy, 1); + expect(topLayer.maxFramerate, 15); + expect(topLayer.maxBitrate, 2500000); + }); + }); } From 55295de356f973c2dd0697e74ed6b25d88f83c86 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:34:00 +0900 Subject: [PATCH 05/18] chore: fix Windows CI and tidy check names (#1099) ## Summary - Suppress the C++/WinRT experimental coroutine deprecation error for `permission_handler_windows_plugin`. - Keeps the example Windows build working on the `windows-2025-vs2026` GitHub Actions image. - Shorten Build workflow job names so checks render as `Build / Android`, `Build / Dart Analyze`, etc. ## Testing - `git diff --check` - Parsed `.github/workflows/build.yaml` with Ruby YAML. --- .github/workflows/build.yaml | 24 ++++++++++++------------ example/windows/CMakeLists.txt | 5 +++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 09343a033..99d70fc57 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,7 +24,7 @@ on: jobs: dart-format-check: - name: Dart Format Check + name: Dart Format runs-on: ubuntu-latest steps: @@ -35,7 +35,7 @@ jobs: run: dart format . --set-exit-if-changed import-sorter-check: - name: Import Sorter Check + name: Import Sorter runs-on: ubuntu-latest steps: @@ -46,7 +46,7 @@ jobs: run: dart run import_sorter:main --no-comments --exit-if-changed version-consistency-check: - name: Version Consistency Check + name: Version Consistency runs-on: ubuntu-latest steps: @@ -57,7 +57,7 @@ jobs: run: dart run scripts/check_version.dart dart-analyze-check: - name: Dart Analyze Check + name: Dart Analyze runs-on: ubuntu-latest steps: @@ -68,7 +68,7 @@ jobs: run: flutter analyze dart-test-check: - name: Dart Test Check + name: Dart Test runs-on: ubuntu-latest steps: @@ -79,7 +79,7 @@ jobs: run: flutter test build-for-android: - name: Build for Flutter Android + name: Android runs-on: ubuntu-latest steps: @@ -93,7 +93,7 @@ jobs: run: flutter build apk build-for-ios: - name: Build for Flutter iOS + name: iOS runs-on: macos-latest steps: @@ -112,7 +112,7 @@ jobs: run: flutter build ios --release --no-codesign build-for-windows: - name: Build for Flutter Windows + name: Windows runs-on: windows-latest steps: @@ -124,7 +124,7 @@ jobs: run: flutter build windows --release build-for-macos: - name: Build for Flutter macOS + name: macOS runs-on: macos-latest steps: @@ -136,7 +136,7 @@ jobs: run: flutter build macos --release build-for-linux: - name: Build for Flutter Linux + name: Linux runs-on: ubuntu-latest steps: @@ -154,7 +154,7 @@ jobs: run: flutter build linux build-for-web: - name: Build for Flutter Web + name: Web runs-on: ubuntu-latest steps: @@ -168,7 +168,7 @@ jobs: run: flutter build web build-for-web-wasm: - name: Build for Flutter Web WASM + name: Web WASM runs-on: ubuntu-latest steps: diff --git a/example/windows/CMakeLists.txt b/example/windows/CMakeLists.txt index 0e847c34f..c8614c3b2 100644 --- a/example/windows/CMakeLists.txt +++ b/example/windows/CMakeLists.txt @@ -50,6 +50,11 @@ add_subdirectory("runner") # them to the application. include(flutter/generated_plugins.cmake) +# VS 2026 treats C++/WinRT's experimental coroutine include as a hard error. +if(TARGET permission_handler_windows_plugin) + target_compile_definitions(permission_handler_windows_plugin PRIVATE + _SILENCE_EXPERIMENTAL_COROUTINE_DEPRECATION_WARNINGS) +endif() # === Installation === # Support files are copied into place next to the executable, so that it can From d83f2720bb306d4c19b5625545773f4ee9af3fcc Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:04:38 +0900 Subject: [PATCH 06/18] Clamp simulcast lower layers to top layer (#1084) ## Summary - Port Swift SDK simulcast lower-layer clamping behavior to Flutter. - Clamp lower-layer max framerate so it does not exceed the configured top layer. - Clamp lower-layer bitrate only when the layer does not actually downscale resolution. - Use the larger output dimension when computing simulcast scale-down values. - Add coverage for clamp behavior, ladder length, unchanged presets, priority preservation, and computed encodings. ## Testing - `dart format --output=none --set-exit-if-changed lib/src/types/video_parameters.dart lib/src/utils.dart test/utils_test.dart` - `flutter test test/utils_test.dart` - `flutter analyze --no-pub lib/src/types/video_parameters.dart lib/src/utils.dart test/utils_test.dart` --- .changes/clamp-simulcast-lower-layers | 1 + lib/src/utils.dart | 94 ++++++++++--- test/utils_test.dart | 183 ++++++++++++++++++++++++-- 3 files changed, 248 insertions(+), 30 deletions(-) create mode 100644 .changes/clamp-simulcast-lower-layers diff --git a/.changes/clamp-simulcast-lower-layers b/.changes/clamp-simulcast-lower-layers new file mode 100644 index 000000000..2426b13b7 --- /dev/null +++ b/.changes/clamp-simulcast-lower-layers @@ -0,0 +1 @@ +patch type="fixed" "Clamp simulcast lower layers to the top layer" diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 0dc0a3ef2..1da153515 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -312,18 +312,83 @@ class Utils { if (i >= videoRids.length) { return; } - final size = dimensions.min(); + final size = dimensions.max(); final rid = videoRids[i]; if (e.encoding != null) { result.add(e.encoding!.toRTCRtpEncoding( rid: rid, - scaleResolutionDownBy: math.max(1, size / e.dimensions.min()), + scaleResolutionDownBy: math.max(1, size / e.dimensions.max()), )); } }); return result; } + @internal + static List computeSimulcastPresets({ + required VideoDimensions dimensions, + required VideoParameters original, + required List requestedPresets, + required bool isScreenShare, + }) { + final params = (requestedPresets.isNotEmpty + ? requestedPresets + : _computeDefaultSimulcastParams(isScreenShare: isScreenShare, original: original)) + .sorted(); + + if (params.isEmpty) { + return [original]; + } + final lowPreset = params.first; + final midPreset = params.length > 1 ? params[1] : null; + + final size = dimensions.max(); + if (size >= 960 && midPreset != null) { + return [ + _clampSimulcastPreset(lowPreset, to: original, inDimensions: dimensions), + _clampSimulcastPreset(midPreset, to: original, inDimensions: dimensions), + original, + ]; + } + if (size >= 480) { + return [ + _clampSimulcastPreset(lowPreset, to: original, inDimensions: dimensions), + original, + ]; + } + return [original]; + } + + static VideoParameters _clampSimulcastPreset( + VideoParameters preset, { + required VideoParameters to, + required VideoDimensions inDimensions, + }) { + final presetEncoding = preset.encoding; + final topEncoding = to.encoding; + if (presetEncoding == null || topEncoding == null) { + return preset; + } + + final rawScaleDownBy = inDimensions.max() / preset.dimensions.max(); + final clampedFramerate = math.min(presetEncoding.maxFramerate, topEncoding.maxFramerate); + final clampedBitrate = + rawScaleDownBy <= 1.0 ? math.min(presetEncoding.maxBitrate, topEncoding.maxBitrate) : presetEncoding.maxBitrate; + + if (clampedFramerate == presetEncoding.maxFramerate && clampedBitrate == presetEncoding.maxBitrate) { + return preset; + } + + return VideoParameters( + description: preset.description, + dimensions: preset.dimensions, + encoding: presetEncoding.copyWith( + maxFramerate: clampedFramerate, + maxBitrate: clampedBitrate, + ), + ); + } + @internal static FutureOr getNetworkType() async { if (!kIsWeb && lkPlatformIsTest()) { @@ -451,25 +516,12 @@ class Utils { // compute simulcast encodings final userParams = isScreenShare ? options.screenShareSimulcastLayers : options.videoSimulcastLayers; - final params = (userParams.isNotEmpty - ? userParams - : _computeDefaultSimulcastParams(isScreenShare: isScreenShare, original: original)) - .sorted(); - - final VideoParameters lowPreset = params.first; - VideoParameters? midPreset; - if (params.length > 1) { - midPreset = params[1]; - } - - final size = dimensions.max(); - List computedParams = [original]; - - if (size >= 960 && midPreset != null) { - computedParams = [lowPreset, midPreset, original]; - } else if (size >= 480) { - computedParams = [lowPreset, original]; - } + final computedParams = computeSimulcastPresets( + dimensions: dimensions, + original: original, + requestedPresets: userParams, + isScreenShare: isScreenShare, + ); return encodingsFromPresets( dimensions, diff --git a/test/utils_test.dart b/test/utils_test.dart index dd6212206..e95a8304a 100644 --- a/test/utils_test.dart +++ b/test/utils_test.dart @@ -15,7 +15,7 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:livekit_client/livekit_client.dart' as lk; +import 'package:livekit_client/livekit_client.dart'; import 'package:livekit_client/src/utils.dart'; void main() { @@ -57,22 +57,187 @@ void main() { ); }); + group('simulcast encodings', () { + test('same-resolution lower layer is clamped to top encoding', () { + final presets = Utils.computeSimulcastPresets( + dimensions: const VideoDimensions(1280, 720), + original: const VideoParameters( + dimensions: VideoDimensions(1280, 720), + encoding: VideoEncoding(maxBitrate: 1500000, maxFramerate: 24), + ), + requestedPresets: const [ + VideoParametersPresets.h720_169, + VideoParametersPresets.h360_169, + ], + isScreenShare: false, + ); + + expect(presets, hasLength(3)); + + expect(presets[0], VideoParametersPresets.h360_169); + + expect(presets[1].dimensions, VideoDimensionsPresets.h720_169); + expect(presets[1].encoding?.maxFramerate, 24); + expect(presets[1].encoding?.maxBitrate, 1500000); + + expect(presets[2].dimensions, const VideoDimensions(1280, 720)); + expect(presets[2].encoding?.maxFramerate, 24); + expect(presets[2].encoding?.maxBitrate, 1500000); + }); + + test('lower-resolution layer clamps framerate but preserves preset bitrate', () { + final presets = Utils.computeSimulcastPresets( + dimensions: const VideoDimensions(1280, 720), + original: const VideoParameters( + dimensions: VideoDimensions(1280, 720), + encoding: VideoEncoding(maxBitrate: 500000, maxFramerate: 15), + ), + requestedPresets: const [], + isScreenShare: false, + ); + + expect(presets, hasLength(3)); + expect(presets[1].dimensions, VideoDimensionsPresets.h360_169); + expect(presets[1].encoding?.maxFramerate, 15); + expect(presets[1].encoding?.maxBitrate, 450000); + }); + + test('same-resolution full clamp', () { + final presets = Utils.computeSimulcastPresets( + dimensions: const VideoDimensions(854, 480), + original: const VideoParameters( + dimensions: VideoDimensions(854, 480), + encoding: VideoEncoding(maxBitrate: 600000, maxFramerate: 15), + ), + requestedPresets: const [ + VideoParameters( + dimensions: VideoDimensions(854, 480), + encoding: VideoEncoding(maxBitrate: 2000000, maxFramerate: 30), + ), + ], + isScreenShare: false, + ); + + expect(presets, hasLength(2)); + expect(presets[0].encoding?.maxFramerate, 15); + expect(presets[0].encoding?.maxBitrate, 600000); + expect(presets[1].dimensions, const VideoDimensions(854, 480)); + expect(presets[1].encoding?.maxFramerate, 15); + expect(presets[1].encoding?.maxBitrate, 600000); + }); + + test('ladder length follows the larger output dimension', () { + const cases = [ + (VideoDimensions(320, 240), 1), + (VideoDimensions(640, 480), 2), + (VideoDimensions(1280, 720), 3), + ]; + + for (final (dimensions, expectedCount) in cases) { + final presets = Utils.computeSimulcastPresets( + dimensions: dimensions, + original: VideoParameters( + dimensions: dimensions, + encoding: const VideoEncoding(maxBitrate: 1000000, maxFramerate: 30), + ), + requestedPresets: const [], + isScreenShare: false, + ); + + expect(presets, hasLength(expectedCount), reason: 'dimensions=$dimensions'); + } + }); + + test("presets that don't overshoot are passed through unchanged", () { + const original = VideoParameters( + dimensions: VideoDimensions(1920, 1080), + encoding: VideoEncoding(maxBitrate: 5000000, maxFramerate: 30), + ); + + final presets = Utils.computeSimulcastPresets( + dimensions: const VideoDimensions(1920, 1080), + original: original, + requestedPresets: const [ + VideoParametersPresets.h360_169, + VideoParametersPresets.h720_169, + ], + isScreenShare: false, + ); + + expect(presets, hasLength(3)); + expect(presets[0], VideoParametersPresets.h360_169); + expect(presets[1], VideoParametersPresets.h720_169); + expect(presets[2], original); + }); + + test('clamped layer carries forward per-layer priorities', () { + const prioritized = VideoParameters( + dimensions: VideoDimensions(1280, 720), + encoding: VideoEncoding( + maxBitrate: 1700000, + maxFramerate: 30, + bitratePriority: Priority.high, + networkPriority: Priority.high, + ), + ); + + final presets = Utils.computeSimulcastPresets( + dimensions: const VideoDimensions(1280, 720), + original: const VideoParameters( + dimensions: VideoDimensions(1280, 720), + encoding: VideoEncoding(maxBitrate: 1500000, maxFramerate: 24), + ), + requestedPresets: const [ + VideoParametersPresets.h360_169, + prioritized, + ], + isScreenShare: false, + ); + + expect(presets, hasLength(3)); + expect(presets[1].encoding?.maxFramerate, 24); + expect(presets[1].encoding?.maxBitrate, 1500000); + expect(presets[1].encoding?.bitratePriority, Priority.high); + expect(presets[1].encoding?.networkPriority, Priority.high); + }); + + test('computed encodings use clamped presets', () { + final encodings = Utils.computeVideoEncodings( + isScreenShare: false, + dimensions: const VideoDimensions(1280, 720), + options: const VideoPublishOptions( + videoEncoding: VideoEncoding(maxBitrate: 1500000, maxFramerate: 24), + videoSimulcastLayers: [ + VideoParametersPresets.h720_169, + VideoParametersPresets.h360_169, + ], + ), + ); + + expect(encodings, hasLength(3)); + expect(encodings![1].rid, 'h'); + expect(encodings[1].maxFramerate, 24); + expect(encodings[1].maxBitrate, 1500000); + expect(encodings[1].scaleResolutionDownBy, 1); + }); + }); + group('screen share simulcast encodings', () { test('screen share preset bitrates match common SDK presets', () { - expect(lk.VideoParametersPresets.screenShareH720FPS5.encoding?.maxBitrate, 800000); - expect(lk.VideoParametersPresets.screenShareH1080FPS30.encoding?.maxBitrate, 5000000); + expect(VideoParametersPresets.screenShareH720FPS5.encoding?.maxBitrate, 800000); + expect(VideoParametersPresets.screenShareH1080FPS30.encoding?.maxBitrate, 5000000); }); test('default lower layer follows top layer fps and priorities', () { final encodings = Utils.computeVideoEncodings( isScreenShare: true, - dimensions: const lk.VideoDimensions(1920, 1080), - options: const lk.VideoPublishOptions( - screenShareEncoding: lk.VideoEncoding( + dimensions: const VideoDimensions(1920, 1080), + options: const VideoPublishOptions( + screenShareEncoding: VideoEncoding( maxBitrate: 2500000, maxFramerate: 15, - bitratePriority: lk.Priority.high, - networkPriority: lk.Priority.high, + bitratePriority: Priority.high, + networkPriority: Priority.high, ), ), ); @@ -91,7 +256,7 @@ void main() { test('default lower layer follows selected screen share preset', () { final encodings = Utils.computeVideoEncodings( isScreenShare: true, - dimensions: const lk.VideoDimensions(1920, 1080), + dimensions: const VideoDimensions(1920, 1080), ); expect(encodings, hasLength(2)); From ead71ca97297e46a8745b433bd8c2737d2dedb25 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:26:29 +0900 Subject: [PATCH 07/18] Merge manual video quality/dimensions with adaptive stream (#1047) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Previously, `setVideoQuality` / `setVideoDimensions` on a remote track were **ignored** (with a warning) whenever `adaptiveStream` was enabled. This PR adopts the JS SDK's approach instead: the manual preference is **merged** client-side with the dimensions derived from the visible views, and the **smaller (more conservative)** of the two is sent to the server. Along the way it fixes a few related correctness issues in the adaptive-stream path. The logic mirrors `client-sdk-js`'s `RemoteTrackPublication` (`isEnabled`, `emitTrackUpdate`, `areDimensionsSmaller`, `layerDimensionsFor`). ## Changes **Manual quality + adaptive stream merge** - `setVideoQuality` / `setVideoDimensions` / `setVideoFPS` are no longer rejected when adaptive stream is on; the request is merged with the visibility-derived dimensions, smaller area wins (matching JS `areDimensionsSmaller`, strict `<`). - When only a quality is requested, it's compared against that quality's simulcast layer dimensions before deciding which to send. **Explicit enable/disable overrides visibility** - Enable/disable state is modeled as an internal tri-state (`TrackEnabledPreference`: unset / enabled / disabled), mirroring JS's `requestedDisabled`. - An explicit `enable()` / `disable()` now always wins over adaptive-stream visibility. Previously the visibility timer computed `disabled` independently and silently ignored the user's request. **Debounce correctness** - A manual update cancels any pending debounced visibility update, and the debounced send rebuilds settings from current state at fire time — so a stale snapshot can't clobber a newer manual update. **Misc** - `fps` is now preserved across adaptive-stream visibility updates (previously dropped). - Adaptive-stream visibility state is reset when the track changes, so stale dimensions can't leak into a later update. - Merge/disable/build logic extracted into pure functions (`resolveVideoSettings`, `resolveDisabled`, `buildUpdateTrackSettings`) in `track_settings.dart`. ## Tests - Unit tests for the pure resolution/build logic: merge precedence, equal-area tie-break, tri-state disable, and proto building. - Publication-level test asserting the `UpdateTrackSettings` actually sent to the signal client for `enable()` / `disable()`. --- .changes/adaptive-stream-manual-quality-merge | 1 + lib/src/publication/remote.dart | 181 ++++++--- lib/src/publication/track_settings.dart | 137 +++++++ .../remote_track_publication_test.dart | 342 ++++++++++++++++++ test/publication/track_settings_test.dart | 265 ++++++++++++++ 5 files changed, 867 insertions(+), 59 deletions(-) create mode 100644 .changes/adaptive-stream-manual-quality-merge create mode 100644 lib/src/publication/track_settings.dart create mode 100644 test/publication/remote_track_publication_test.dart create mode 100644 test/publication/track_settings_test.dart diff --git a/.changes/adaptive-stream-manual-quality-merge b/.changes/adaptive-stream-manual-quality-merge new file mode 100644 index 000000000..083a3def6 --- /dev/null +++ b/.changes/adaptive-stream-manual-quality-merge @@ -0,0 +1 @@ +patch type="improved" "Allow manual video quality selection with adaptive stream enabled" diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index f9e30bb5e..1ac92fbd6 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -33,6 +33,7 @@ import '../types/other.dart'; import '../types/video_dimensions.dart'; import '../utils.dart'; import 'track_publication.dart'; +import 'track_settings.dart'; /// Represents a track publication from a RemoteParticipant. Provides methods to /// control if we should subscribe to the track, and its quality (for video). @@ -41,18 +42,35 @@ class RemoteTrackPublication extends TrackPublication @override final RemoteParticipant participant; - bool get enabled => _enabled; - bool _enabled = true; + bool get enabled => !resolveDisabled( + enabledPreference: _enabledPreference, + adaptiveStreamActive: _adaptiveStreamActive, + adaptiveStreamVisible: _adaptiveStreamVisible, + ); + + /// The user's explicit enable/disable request via [enable] / [disable]. + /// [TrackEnabledPreference.unset] means no explicit request, in which case + /// adaptive-stream visibility decides. An explicit request takes precedence + /// over visibility. + TrackEnabledPreference _enabledPreference = TrackEnabledPreference.unset; /// The current desired FPS of the track. This is only available for video tracks that support SVC. int? _fps; int get fps => _fps ?? 0; - VideoQuality? _videoQuality = VideoQuality.HIGH; - VideoQuality get videoQuality => _videoQuality ?? VideoQuality.HIGH; + // Manual settings (set by user via setVideoQuality / setVideoDimensions) + VideoSettings? _userPreference; + + // Adaptive stream state (set automatically by visibility observer) + VideoDimensions? _adaptiveStreamDimensions; + // Whether adaptive stream is active for this publication (room option on + + // remote video track). When false, view visibility never gates `disabled`. + bool _adaptiveStreamActive = false; + // Whether at least one view of this track is currently visible/sized. + bool _adaptiveStreamVisible = true; - VideoDimensions? _videoDimensions; - VideoDimensions? get videoDimensions => _videoDimensions; + VideoQuality get videoQuality => _userPreference?.quality ?? VideoQuality.HIGH; + VideoDimensions? get videoDimensions => _userPreference?.dimensions; /// The server may pause the track when they are bandwidth limitations and resume /// when there is more capacity. This property will be updated when the track is @@ -144,11 +162,6 @@ class RemoteTrackPublication extends TrackPublication final videoTrack = track as VideoTrack; - final settings = lk_rtc.UpdateTrackSettings( - trackSids: [sid], - disabled: true, - ); - // filter visible build contexts final viewSizes = videoTrack.viewKeys .map((e) => e.currentContext) @@ -161,15 +174,19 @@ class RemoteTrackPublication extends TrackPublication logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); if (viewSizes.isNotEmpty) { - // compute largest size final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element)); - - settings - ..disabled = false - ..width = largestSize.width.ceil() - ..height = largestSize.height.ceil(); + _adaptiveStreamDimensions = VideoDimensions( + largestSize.width.ceil(), + largestSize.height.ceil(), + ); + _adaptiveStreamVisible = true; + } else { + _adaptiveStreamDimensions = null; + _adaptiveStreamVisible = false; } + final settings = _buildTrackSettings(); + // Only send new settings to server if it changed if (settings != _lastSentTrackSettings) { _lastSentTrackSettings = settings; @@ -182,7 +199,13 @@ class RemoteTrackPublication extends TrackPublication } } - void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings settings) { + void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings _) { + // Re-build from the current state at fire time instead of replaying the + // snapshot captured when the debounce was scheduled. Otherwise a stale + // snapshot could be sent after newer state (e.g. a manual setVideoQuality) + // has already been applied, clobbering it. + final settings = _buildTrackSettings(); + _lastSentTrackSettings = settings; logger.fine('[Visibility] Sending... ${settings.toProto3Json()}'); participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } @@ -198,8 +221,17 @@ class RemoteTrackPublication extends TrackPublication _cancelPendingTrackSettingsUpdateRequest?.call(); _visibilityTimer?.cancel(); + // The track changed, so any adaptive-stream visibility computed for the + // previous track is stale. Reset to the construction defaults so it can't + // leak into a later _buildTrackSettings (e.g. via enable() / disable(), + // which emit regardless of visibility). Repopulated by the visibility + // observer below while adaptive stream is active. + _adaptiveStreamDimensions = null; + _adaptiveStreamVisible = true; + final roomOptions = participant.room.roomOptions; if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) { + _adaptiveStreamActive = true; // Start monitoring visibility _visibilityTimer = Timer.periodic( const Duration(milliseconds: 300), @@ -214,6 +246,10 @@ class RemoteTrackPublication extends TrackPublication _computeVideoViewVisibility(quick: true); } }; + + _computeVideoViewVisibility(quick: true); + } else { + _adaptiveStreamActive = false; } if (newValue != null) { @@ -229,7 +265,7 @@ class RemoteTrackPublication extends TrackPublication return didUpdate; } - bool _canUpdateManualVideoSettings() { + bool _isManualOperationAllowed() { if (kind != TrackType.VIDEO) { logger.warning('Manual video setting updates are only supported for video tracks'); return false; @@ -240,55 +276,59 @@ class RemoteTrackPublication extends TrackPublication return false; } - if (participant.room.roomOptions.adaptiveStream) { - logger.warning('Manual video setting update ignored because adaptive stream is enabled'); - return false; - } - return true; } + /// For tracks that support simulcasting, adjust subscribed quality. + /// + /// This indicates the highest quality the client can accept. If network + /// bandwidth does not allow, the server will automatically reduce quality to + /// optimize for uninterrupted video. + /// + /// When adaptive stream is active, this preference is merged client-side with + /// the dimensions computed from the visible views, and the smaller (more + /// conservative) of the two is sent to the server. Future setVideoQuality(VideoQuality newValue) async { - if (newValue == _videoQuality) return; - if (!_canUpdateManualVideoSettings()) return; - _videoQuality = newValue; - _videoDimensions = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.quality) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.quality(newValue); + _emitTrackUpdate(); } /// Set preferred video dimensions for this track. /// /// Server will choose the appropriate layer based on these dimensions. /// Will override previous calls to [setVideoQuality]. + /// + /// When adaptive stream is active, this preference is merged client-side with + /// the dimensions computed from the visible views, and the smaller (more + /// conservative) of the two is sent to the server. Future setVideoDimensions(VideoDimensions newValue) async { - if (newValue.width == _videoDimensions?.width && newValue.height == _videoDimensions?.height) { - return; - } - if (!_canUpdateManualVideoSettings()) return; - _videoDimensions = newValue; - _videoQuality = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.dimensions) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.dimensions(newValue); + _emitTrackUpdate(); } /// Set desired FPS, server will do its best to return FPS close to this. /// It's only supported for video codecs that support SVC currently. Future setVideoFPS(int newValue) async { if (newValue == _fps) return; - if (!_canUpdateManualVideoSettings()) return; + if (!_isManualOperationAllowed()) return; _fps = newValue; - sendUpdateTrackSettings(); + _emitTrackUpdate(); } Future enable() async { - if (_enabled) return; - _enabled = true; - sendUpdateTrackSettings(); + if (_enabledPreference == TrackEnabledPreference.enabled) return; + _enabledPreference = TrackEnabledPreference.enabled; + _emitTrackUpdate(); } Future disable() async { - if (!_enabled) return; - _enabled = false; - sendUpdateTrackSettings(); + if (_enabledPreference == TrackEnabledPreference.disabled) return; + _enabledPreference = TrackEnabledPreference.disabled; + _emitTrackUpdate(); } Future subscribe() async { @@ -333,26 +373,49 @@ class RemoteTrackPublication extends TrackPublication participant.room.engine.signalClient.sendUpdateSubscription(subscription); } - @internal - void sendUpdateTrackSettings() { - final settings = lk_rtc.UpdateTrackSettings( - trackSids: [sid], - disabled: !_enabled, + lk_rtc.UpdateTrackSettings _buildTrackSettings() { + final isDisabled = resolveDisabled( + enabledPreference: _enabledPreference, + adaptiveStreamActive: _adaptiveStreamActive, + adaptiveStreamVisible: _adaptiveStreamVisible, ); - if (kind == TrackType.VIDEO) { - if (_videoDimensions != null) { - settings.width = _videoDimensions!.width; - settings.height = _videoDimensions!.height; - } else if (_videoQuality != null) { - settings.quality = _videoQuality!.toPBType(); - } else { - settings.quality = VideoQuality.HIGH.toPBType(); - } - if (_fps != null) settings.fps = _fps!; + + if (kind != TrackType.VIDEO) { + return buildUpdateTrackSettings(sid: sid, disabled: isDisabled); } + + final resolved = resolveVideoSettings( + adaptiveStreamDimensions: _adaptiveStreamDimensions, + userPreference: _userPreference, + layerDimensionsForQuality: (quality) { + final pbQuality = quality.toPBType(); + final layer = latestInfo?.layers.where((l) => l.quality == pbQuality).firstOrNull; + if (layer == null) return null; + return VideoDimensions(layer.width, layer.height); + }, + ); + + return buildUpdateTrackSettings( + sid: sid, + disabled: isDisabled, + dimensions: resolved.dimensions, + quality: resolved.quality?.toPBType(), + fps: _fps, + ); + } + + void _emitTrackUpdate() { + // Cancel any pending debounced visibility update so its (now potentially + // stale) snapshot cannot fire after — and clobber — this immediate update. + _cancelPendingTrackSettingsUpdateRequest?.call(); + final settings = _buildTrackSettings(); + _lastSentTrackSettings = settings; participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } + @internal + void sendUpdateTrackSettings() => _emitTrackUpdate(); + @internal // Update internal var and return true if changed Future updateSubscriptionAllowed(bool allowed) async { diff --git a/lib/src/publication/track_settings.dart b/lib/src/publication/track_settings.dart new file mode 100644 index 000000000..9f8d6b352 --- /dev/null +++ b/lib/src/publication/track_settings.dart @@ -0,0 +1,137 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart' show immutable, internal; + +import '../proto/livekit_models.pb.dart' as lk_models; +import '../proto/livekit_rtc.pb.dart' as lk_rtc; +import '../types/other.dart'; +import '../types/video_dimensions.dart'; + +/// Represents a video quality setting — either explicit dimensions or a +/// quality level (LOW/MEDIUM/HIGH), never both. +/// +/// Used for both user-requested settings and the resolved merge result. +@internal +@immutable +class VideoSettings { + final VideoDimensions? dimensions; + final VideoQuality? quality; + + const VideoSettings.dimensions(VideoDimensions this.dimensions) : quality = null; + + const VideoSettings.quality(VideoQuality this.quality) : dimensions = null; + + static const high = VideoSettings.quality(VideoQuality.HIGH); +} + +/// Merges adaptive stream dimensions with manual [VideoSettings], +/// always picking the more conservative (smaller) of the two. +/// +/// This matches the JS SDK's merge behavior in `emitTrackUpdate()`. +@internal +VideoSettings resolveVideoSettings({ + VideoDimensions? adaptiveStreamDimensions, + VideoSettings? userPreference, + VideoDimensions? Function(VideoQuality quality)? layerDimensionsForQuality, +}) { + VideoDimensions? minDimensions = userPreference?.dimensions; + + if (adaptiveStreamDimensions != null) { + if (minDimensions != null) { + // Use the smaller of adaptive vs manually requested dimensions + if (adaptiveStreamDimensions.area() < minDimensions.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else if (userPreference?.quality != null) { + // Compare adaptive dimensions with the dimensions implied by the requested quality. + final requestedQualityLayerDimensions = layerDimensionsForQuality?.call(userPreference!.quality!); + if (requestedQualityLayerDimensions != null && + adaptiveStreamDimensions.area() < requestedQualityLayerDimensions.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else { + minDimensions = adaptiveStreamDimensions; + } + } + + if (minDimensions != null) { + return VideoSettings.dimensions(minDimensions); + } else if (userPreference?.quality != null) { + return VideoSettings.quality(userPreference!.quality!); + } + return VideoSettings.high; +} + +/// The user's explicit enable/disable request for a track, used to decide +/// whether visibility may gate the track. Equivalent to the JS SDK's +/// `requestedDisabled` tri-state (`undefined` / `false` / `true`). +@internal +enum TrackEnabledPreference { + /// No explicit request; adaptive-stream visibility decides. + unset, + + /// User explicitly enabled; overrides visibility (track keeps streaming). + enabled, + + /// User explicitly disabled; overrides visibility (track stays off). + disabled, +} + +/// Resolves whether a subscribed track should be sent as `disabled`. +/// +/// Mirrors the JS SDK's `isEnabled` precedence: an explicit user +/// enable/disable always wins; otherwise, when adaptive stream is active for +/// the track, view visibility decides; otherwise the track is enabled. +@internal +bool resolveDisabled({ + required TrackEnabledPreference enabledPreference, + required bool adaptiveStreamActive, + required bool adaptiveStreamVisible, +}) { + switch (enabledPreference) { + case TrackEnabledPreference.enabled: + return false; + case TrackEnabledPreference.disabled: + return true; + case TrackEnabledPreference.unset: + return adaptiveStreamActive ? !adaptiveStreamVisible : false; + } +} + +/// Builds the [lk_rtc.UpdateTrackSettings] request sent to the server from the +/// already-resolved [disabled] flag and, for video, the resolved [dimensions] +/// or [quality] plus an optional [fps]. [dimensions] takes precedence over +/// [quality]; pass neither for non-video tracks. +@internal +lk_rtc.UpdateTrackSettings buildUpdateTrackSettings({ + required String sid, + required bool disabled, + VideoDimensions? dimensions, + lk_models.VideoQuality? quality, + int? fps, +}) { + final settings = lk_rtc.UpdateTrackSettings( + trackSids: [sid], + disabled: disabled, + ); + if (dimensions != null) { + settings.width = dimensions.width; + settings.height = dimensions.height; + } else if (quality != null) { + settings.quality = quality; + } + if (fps != null) settings.fps = fps; + return settings; +} diff --git a/test/publication/remote_track_publication_test.dart b/test/publication/remote_track_publication_test.dart new file mode 100644 index 000000000..b77b0d533 --- /dev/null +++ b/test/publication/remote_track_publication_test.dart @@ -0,0 +1,342 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +@Timeout(Duration(seconds: 10)) +library; + +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; + +import 'package:livekit_client/livekit_client.dart'; +import 'package:livekit_client/src/core/engine.dart'; +import 'package:livekit_client/src/core/signal_client.dart'; +import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; +import 'package:livekit_client/src/proto/livekit_rtc.pb.dart' as lk_rtc; +import 'package:livekit_client/src/support/websocket.dart'; +import '../core/signal_client_test.dart'; +import '../mock/peerconnection_mock.dart'; + +/// A websocket that records every outgoing [lk_rtc.SignalRequest] so tests can +/// assert on what the SDK actually sends to the server. +class _CapturingWebSocket extends LiveKitWebSocket { + final List sent = []; + + @override + void send(List data) => sent.add(lk_rtc.SignalRequest.fromBuffer(data)); +} + +class _CapturingConnector { + final socket = _CapturingWebSocket(); + WebSocketEventHandlers? handlers; + + WebSocketOnData get onData => handlers!.onData!; + + Future connect( + Uri uri, { + WebSocketEventHandlers? options, + Map? headers, + }) async { + handlers = options; + return socket; + } +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late _CapturingConnector connector; + late Room room; + + Future connectTestRoom({ + RoomOptions roomOptions = const RoomOptions(), + }) async { + connector = _CapturingConnector(); + final client = SignalClient(connector.connect); + final engine = Engine( + connectOptions: const ConnectOptions(), + roomOptions: roomOptions, + signalClient: client, + peerConnectionCreate: MockPeerConnection.create, + ); + room = Room(engine: engine); + + final connectFuture = room.connect(exampleUri, token); + Future.delayed(const Duration(milliseconds: 1), () { + connector.onData(joinResponse.writeToBuffer()); + connector.onData(offerResponse.writeToBuffer()); + }); + await connectFuture; + + connector.onData(participantJoinResponse.writeToBuffer()); + await room.events.waitFor(duration: const Duration(seconds: 1)); + } + + setUp(() async { + // adaptiveStream defaults to false, so no visibility timer runs and the + // emitted settings depend only on the explicit enable/disable preference. + await connectTestRoom(); + }); + + tearDown(() async { + await room.dispose(); + }); + + /// The most recent [lk_rtc.UpdateTrackSettings] the SDK sent for [sid], if any. + lk_rtc.UpdateTrackSettings? lastSettingsFor(String sid) { + final matches = + connector.socket.sent.where((r) => r.hasTrackSetting() && r.trackSetting.trackSids.contains(sid)).toList(); + return matches.isEmpty ? null : matches.last.trackSetting; + } + + group('enable/disable wiring', () { + test('disable() then enable() emit the disabled flag through the real publication path', () async { + final participant = room.remoteParticipants.values.first; + const sid = 'TR_remote_pub_test'; + final pub = RemoteTrackPublication( + participant: participant, + info: lk_models.TrackInfo( + sid: sid, + name: 'video', + type: lk_models.TrackType.VIDEO, + ), + ); + addTearDown(() async => await pub.dispose()); + + // No explicit preference yet: enabled by default, nothing sent. + expect(pub.enabled, isTrue); + expect(lastSettingsFor(sid), isNull); + + await pub.disable(); + final disabled = lastSettingsFor(sid); + expect(disabled, isNotNull, reason: 'disable() should send an UpdateTrackSettings'); + expect(disabled!.disabled, isTrue); + expect(disabled.trackSids, [sid]); + // The default video path resolves to HIGH quality. + expect(disabled.quality, lk_models.VideoQuality.HIGH); + expect(pub.enabled, isFalse); + + await pub.enable(); + final enabled = lastSettingsFor(sid); + expect(enabled!.disabled, isFalse); + expect(pub.enabled, isTrue); + }); + + test('repeated disable() is a no-op (no duplicate send)', () async { + final participant = room.remoteParticipants.values.first; + const sid = 'TR_remote_pub_dedup'; + final pub = RemoteTrackPublication( + participant: participant, + info: lk_models.TrackInfo( + sid: sid, + name: 'video', + type: lk_models.TrackType.VIDEO, + ), + ); + addTearDown(() async => await pub.dispose()); + + await pub.disable(); + final countAfterFirst = + connector.socket.sent.where((r) => r.hasTrackSetting() && r.trackSetting.trackSids.contains(sid)).length; + + await pub.disable(); + final countAfterSecond = + connector.socket.sent.where((r) => r.hasTrackSetting() && r.trackSetting.trackSids.contains(sid)).length; + + expect(countAfterFirst, 1); + expect(countAfterSecond, 1, reason: 'a second disable() with no state change should not re-send'); + }); + + test('enabled follows adaptive visibility when there is no explicit preference', () async { + await room.dispose(); + await connectTestRoom( + roomOptions: const RoomOptions(adaptiveStream: true), + ); + + final participant = room.remoteParticipants.values.first; + const sid = 'TR_remote_pub_adaptive_hidden'; + final pub = RemoteTrackPublication( + participant: participant, + info: lk_models.TrackInfo( + sid: sid, + name: 'video', + type: lk_models.TrackType.VIDEO, + ), + ); + addTearDown(() async => await pub.dispose()); + + final stream = _FakeMediaStream('stream-$sid'); + final track = _FakeMediaStreamTrack(id: sid, kind: 'video'); + await stream.addTrack(track); + + expect(pub.enabled, isTrue); + + await pub.updateTrack(RemoteVideoTrack( + TrackSource.camera, + stream, + track, + )); + + await Future.delayed(const Duration(milliseconds: 350)); + + expect(pub.enabled, isFalse); + + await pub.enable(); + expect(pub.enabled, isTrue, reason: 'an explicit enable still overrides adaptive visibility'); + }); + + test('adaptive visibility is computed immediately when track is attached', () async { + await room.dispose(); + await connectTestRoom( + roomOptions: const RoomOptions(adaptiveStream: true), + ); + + final participant = room.remoteParticipants.values.first; + const sid = 'TR_remote_pub_adaptive_initial'; + final pub = RemoteTrackPublication( + participant: participant, + info: lk_models.TrackInfo( + sid: sid, + name: 'video', + type: lk_models.TrackType.VIDEO, + ), + ); + addTearDown(() async => await pub.dispose()); + + final stream = _FakeMediaStream('stream-$sid'); + final track = _FakeMediaStreamTrack(id: sid, kind: 'video'); + await stream.addTrack(track); + + await pub.updateTrack(RemoteVideoTrack( + TrackSource.camera, + stream, + track, + )); + + final initialSettings = lastSettingsFor(sid); + expect(initialSettings, isNotNull); + expect(initialSettings!.disabled, isTrue); + + await pub.setVideoQuality(VideoQuality.LOW); + final manualSettings = lastSettingsFor(sid); + expect(manualSettings!.disabled, isTrue, reason: 'manual video settings should keep hidden tracks disabled'); + expect(manualSettings.quality, lk_models.VideoQuality.LOW); + }); + }); +} + +class _FakeMediaStream extends rtc.MediaStream { + final List _tracks = []; + + _FakeMediaStream(String id) : super(id, 'fake-owner'); + + @override + bool? get active => true; + + @override + Future addTrack(rtc.MediaStreamTrack track, {bool addToNative = true}) async { + _tracks.add(track); + } + + @override + Future clone() async => _FakeMediaStream('${id}_clone'); + + @override + List getAudioTracks() => _tracks.where((t) => t.kind == 'audio').toList(); + + @override + Future getMediaTracks() async {} + + @override + List getTracks() => List.from(_tracks); + + @override + List getVideoTracks() => _tracks.where((t) => t.kind == 'video').toList(); + + @override + Future removeTrack(rtc.MediaStreamTrack track, {bool removeFromNative = true}) async { + _tracks.remove(track); + } +} + +class _FakeMediaStreamTrack implements rtc.MediaStreamTrack { + @override + rtc.StreamTrackCallback? onEnded; + + @override + rtc.StreamTrackCallback? onMute; + + @override + rtc.StreamTrackCallback? onUnMute; + + @override + bool enabled; + + @override + final String id; + + @override + final String kind; + + @override + String? get label => '$kind-track'; + + @override + bool? get muted => false; + + _FakeMediaStreamTrack({ + required this.id, + required this.kind, + this.enabled = true, + }); + + @override + Future adaptRes(int width, int height) async {} + + @override + Future applyConstraints([Map? constraints]) async {} + + @override + Future captureFrame() { + throw UnimplementedError(); + } + + @override + Future clone() async => _FakeMediaStreamTrack(id: id, kind: kind, enabled: enabled); + + @override + Future dispose() async {} + + @override + Map getConstraints() => const {}; + + @override + Map getSettings() => const {}; + + @override + Future hasTorch() async => false; + + @override + void enableSpeakerphone(bool enable) {} + + @override + Future setTorch(bool torch) async {} + + @override + Future stop() async {} + + @override + Future switchCamera() async => false; +} diff --git a/test/publication/track_settings_test.dart b/test/publication/track_settings_test.dart new file mode 100644 index 000000000..e63f6a581 --- /dev/null +++ b/test/publication/track_settings_test.dart @@ -0,0 +1,265 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/proto/livekit_models.pb.dart' as lk_models; +import 'package:livekit_client/src/publication/track_settings.dart'; +import 'package:livekit_client/src/types/other.dart'; +import 'package:livekit_client/src/types/video_dimensions.dart'; + +/// Test helper: returns layer dimensions for a standard 3-layer SVC/simulcast track. +VideoDimensions? _testLayerDimensions(VideoQuality quality) { + return { + VideoQuality.LOW: VideoDimensions(320, 180), + VideoQuality.MEDIUM: VideoDimensions(640, 360), + VideoQuality.HIGH: VideoDimensions(1280, 720), + }[quality]; +} + +void main() { + group('resolveVideoSettings', () { + group('no adaptive stream', () { + test('defaults to HIGH quality when nothing set', () { + final r = resolveVideoSettings(); + expect(r.quality, VideoQuality.HIGH); + expect(r.dimensions, isNull); + }); + + test('uses preferred quality', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('uses preferred dimensions', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.dimensions(VideoDimensions(800, 600)), + ); + expect(r.dimensions, VideoDimensions(800, 600)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream only', () { + test('uses adaptive stream dimensions', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(480, 270), + ); + expect(r.dimensions, VideoDimensions(480, 270)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream + preferred dimensions', () { + test('adaptive wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.dimensions(VideoDimensions(1280, 720)), + ); + expect(r.dimensions, VideoDimensions(320, 180)); + }); + + test('preferred wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + + test('equal areas keep preferred', () { + // 720*320 == 640*360 == 230400. Distinct dimensions with equal area + // so the assertion can actually distinguish strict `<` (keep preferred) + // from `<=` (switch to adaptive), matching JS areDimensionsSmaller. + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(720, 320), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + + test('adaptive wins when area is one smaller', () { + // 639*360 = 230040 < 640*360 = 230400, so adaptive is strictly smaller. + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(639, 360), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(639, 360)); + }); + }); + + group('adaptive stream + preferred quality', () { + test('adaptive wins when smaller than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.HIGH), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 320*180 < HIGH 1280*720 → sends adaptive dimensions + expect(r.dimensions, VideoDimensions(320, 180)); + expect(r.quality, isNull); + }); + + test('quality wins when adaptive is larger than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.quality(VideoQuality.LOW), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 1920*1080 > LOW 320*180 → sends quality directly + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent directly when no layer info available', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent when layer lookup returns null', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.MEDIUM), + layerDimensionsForQuality: (_) => null, + ); + expect(r.quality, VideoQuality.MEDIUM); + expect(r.dimensions, isNull); + }); + }); + }); + + group('resolveDisabled', () { + test('not disabled by default (unset preference, adaptive inactive)', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: false, + adaptiveStreamVisible: true, + ), + isFalse, + ); + }); + + test('explicit disable wins even when visible', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.disabled, + adaptiveStreamActive: true, + adaptiveStreamVisible: true, + ), + isTrue, + ); + }); + + test('explicit enable wins even when not visible (JS tri-state parity)', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.enabled, + adaptiveStreamActive: true, + adaptiveStreamVisible: false, + ), + isFalse, + ); + }); + + test('adaptive visibility decides when preference is unset', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: true, + adaptiveStreamVisible: true, + ), + isFalse, + ); + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: true, + adaptiveStreamVisible: false, + ), + isTrue, + ); + }); + + test('visibility is ignored when adaptive stream is inactive', () { + expect( + resolveDisabled( + enabledPreference: TrackEnabledPreference.unset, + adaptiveStreamActive: false, + adaptiveStreamVisible: false, + ), + isFalse, + ); + }); + }); + + group('buildUpdateTrackSettings', () { + test('sets sid and disabled flag', () { + final s = buildUpdateTrackSettings(sid: 'TR_abc', disabled: true); + expect(s.trackSids, ['TR_abc']); + expect(s.disabled, isTrue); + expect(s.hasWidth(), isFalse); + expect(s.hasQuality(), isFalse); + expect(s.hasFps(), isFalse); + }); + + test('dimensions are written, quality is not', () { + final s = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + dimensions: VideoDimensions(640, 360), + quality: lk_models.VideoQuality.LOW, + ); + expect(s.width, 640); + expect(s.height, 360); + expect(s.hasQuality(), isFalse); + }); + + test('quality is written when no dimensions', () { + final s = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + ); + expect(s.quality, lk_models.VideoQuality.HIGH); + expect(s.hasWidth(), isFalse); + expect(s.hasHeight(), isFalse); + }); + + test('fps is forwarded when set and omitted when null', () { + final withFps = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + fps: 30, + ); + expect(withFps.hasFps(), isTrue); + expect(withFps.fps, 30); + + final withoutFps = buildUpdateTrackSettings( + sid: 'TR_abc', + disabled: false, + quality: lk_models.VideoQuality.HIGH, + ); + expect(withoutFps.hasFps(), isFalse); + }); + }); +} From a05a782de15f4405a6ac2c31031b0cba6a9d7825 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:48:48 +0900 Subject: [PATCH 08/18] Fix pixel density in AdaptiveStream (#1098) The adaptive-stream visibility observer fed each view's logical (DIP) size straight to the server, so on retina/HiDPI displays it under-requested by the device pixel ratio and the server returned an upscaled/soft layer. Server layers are sized in physical pixels. - Add `AdaptiveStreamPixelDensity` (`auto` | `fixed(double)`) mirroring the JS SDK's `pixelDensity` option, plus `VideoTrackRenderer.adaptiveStreamPixelDensity` (default: `auto`), set per view rather than room-wide. Fractional densities are supported and the resolved value is capped at 3x to bound bandwidth. - Track each rendered view with a `VideoTrackViewRegistration` that owns both the `GlobalKey` and pixel density, so density updates stay tied to the renderer lifecycle. - Resolve auto mode from each view's `devicePixelRatio` via `MediaQuery.maybeDevicePixelRatioOf`; fixed modes use a constant multiplier. - `_computeVideoViewVisibility` scales each view's logical size by the resolved density, materializes the visible sizes once per tick, and requests the largest scaled dimensions across all views. --- .changes/adaptive-stream-pixel-density | 1 + lib/src/publication/remote.dart | 29 ++++--- lib/src/track/local/local.dart | 21 +++-- .../track/video_track_view_registration.dart | 33 ++++++++ lib/src/types/other.dart | 47 ++++++++++++ lib/src/widgets/video_track_renderer.dart | 28 ++++--- .../adaptive_stream_pixel_density_test.dart | 76 +++++++++++++++++++ 7 files changed, 209 insertions(+), 26 deletions(-) create mode 100644 .changes/adaptive-stream-pixel-density create mode 100644 lib/src/track/video_track_view_registration.dart create mode 100644 test/options/adaptive_stream_pixel_density_test.dart diff --git a/.changes/adaptive-stream-pixel-density b/.changes/adaptive-stream-pixel-density new file mode 100644 index 000000000..94de7146f --- /dev/null +++ b/.changes/adaptive-stream-pixel-density @@ -0,0 +1 @@ +patch type="fixed" "Fix adaptive stream dimensions on high-density displays" diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index 1ac92fbd6..c47ba6938 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -162,19 +162,30 @@ class RemoteTrackPublication extends TrackPublication final videoTrack = track as VideoTrack; - // filter visible build contexts - final viewSizes = videoTrack.viewKeys - .map((e) => e.currentContext) + // Filter visible build contexts and scale each view's logical size by its + // own pixel density, so the server is asked for physical-pixel dimensions + // (retina-aware). Each view's density is configured on its VideoTrackRenderer + // and resolved per-view; with AdaptiveStreamPixelDensity.auto the actual + // device pixel ratio is read from that view via MediaQuery. The largest + // resulting size across all of the track's views is requested. + final viewSizes = videoTrack.viewRegistrations + .map((registration) { + final context = registration.key.currentContext; + if (context == null) return null; + final renderBox = context.findRenderObject() as RenderBox?; + if (renderBox == null || !renderBox.hasSize) return null; + final density = registration.pixelDensity.resolve( + MediaQuery.maybeDevicePixelRatioOf(context) ?? 1.0, + ); + return renderBox.size * density; + }) .nonNulls - .map((e) => e.findRenderObject() as RenderBox?) - .nonNulls - .where((e) => e.hasSize) - .map((e) => e.size); + .toList(); logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); if (viewSizes.isNotEmpty) { - final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element)); + final largestSize = viewSizes.reduce(maxOfSizes); _adaptiveStreamDimensions = VideoDimensions( largestSize.width.ceil(), largestSize.height.ceil(), @@ -238,7 +249,7 @@ class RemoteTrackPublication extends TrackPublication (_) => _computeVideoViewVisibility(), ); - newValue.onVideoViewBuild = (_) { + newValue.onVideoViewBuild = () { logger.finer('[Visibility] VideoView did build'); if (_lastSentTrackSettings?.disabled == true) { // quick enable diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index e520b8712..1b197c50f 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -35,27 +35,32 @@ import '../processor_native.dart' if (dart.library.js_interop) '../processor_web import '../remote/audio.dart'; import '../remote/video.dart'; import '../track.dart'; +import '../video_track_view_registration.dart'; import 'audio.dart'; import 'video.dart'; /// Used to group [LocalVideoTrack] and [RemoteVideoTrack]. mixin VideoTrack on Track { + /// The views attached to this track. Set by [VideoTrackRenderer] and read by + /// the visibility observer to compute adaptive-stream dimensions. @internal - final List viewKeys = []; + final List viewRegistrations = []; @internal - Function(Key)? onVideoViewBuild; + VoidCallback? onVideoViewBuild; @internal - GlobalKey addViewKey() { - final key = GlobalKey(); - viewKeys.add(key); - return key; + VideoTrackViewRegistration addViewRegistration({ + AdaptiveStreamPixelDensity pixelDensity = AdaptiveStreamPixelDensity.auto, + }) { + final registration = VideoTrackViewRegistration(pixelDensity: pixelDensity); + viewRegistrations.add(registration); + return registration; } @internal - void removeViewKey(GlobalKey key) { - viewKeys.remove(key); + void removeViewRegistration(VideoTrackViewRegistration registration) { + viewRegistrations.remove(registration); } } diff --git a/lib/src/track/video_track_view_registration.dart b/lib/src/track/video_track_view_registration.dart new file mode 100644 index 000000000..3fc548c1f --- /dev/null +++ b/lib/src/track/video_track_view_registration.dart @@ -0,0 +1,33 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter/material.dart'; + +import 'package:meta/meta.dart'; + +import '../types/other.dart'; + +@internal +class VideoTrackViewRegistration { + /// The widget key used by adaptive stream to find this view's render context. + final GlobalKey key = GlobalKey(); + + /// The pixel density used to convert this view's logical size to physical + /// pixels when computing adaptive-stream dimensions. + AdaptiveStreamPixelDensity pixelDensity; + + VideoTrackViewRegistration({ + this.pixelDensity = AdaptiveStreamPixelDensity.auto, + }); +} diff --git a/lib/src/types/other.dart b/lib/src/types/other.dart index d6d99d77a..08d2494ff 100644 --- a/lib/src/types/other.dart +++ b/lib/src/types/other.dart @@ -233,3 +233,50 @@ class ParticipantTrackPermission { this.allowedTrackSids, ); } + +/// Controls how a video view's logical size is scaled to physical pixels when +/// computing adaptive-stream dimensions. Mirrors the JS SDK's `pixelDensity` +/// option (`number | 'screen'`). +/// +/// Server layers are sized in physical pixels, so on high-density (retina) +/// displays the logical view size under-represents the pixels needed. Set on a +/// view via [VideoTrackRenderer]; the largest result is requested across all +/// views attached to the track. +class AdaptiveStreamPixelDensity { + /// Upper bound applied to the resolved density to keep bandwidth in check. + static const maxDensity = 3.0; + + /// Fixed multiplier, or `null` to use the view's device pixel ratio ([auto]). + final double? value; + + const AdaptiveStreamPixelDensity._(this.value); + + /// Use the view's actual device pixel ratio, read via `MediaQuery`. + /// Equivalent to the JS SDK's `'screen'` setting. Capped at [maxDensity]. + static const auto = AdaptiveStreamPixelDensity._(null); + + /// A positive fixed pixel-density multiplier (fractional allowed, e.g. `1.5`, + /// `2.0`, `2.75`). The effective value is capped at [maxDensity] (3x) when + /// resolved. + const AdaptiveStreamPixelDensity.fixed(double density) + : assert(density > 0, 'density must be positive'), + value = density; + + /// Resolves the effective multiplier, capped at [maxDensity]. For [auto], + /// falls back to the supplied [devicePixelRatio]. + double resolve(double devicePixelRatio) { + final density = value ?? devicePixelRatio; + if (density.isNaN || density <= 0) return 1.0; + return density > maxDensity ? maxDensity : density; + } + + @override + bool operator ==(Object other) => + identical(this, other) || (other is AdaptiveStreamPixelDensity && other.value == value); + + @override + int get hashCode => value.hashCode; + + @override + String toString() => value == null ? 'AdaptiveStreamPixelDensity.auto' : 'AdaptiveStreamPixelDensity.fixed($value)'; +} diff --git a/lib/src/widgets/video_track_renderer.dart b/lib/src/widgets/video_track_renderer.dart index 95b683bce..2457129e4 100644 --- a/lib/src/widgets/video_track_renderer.dart +++ b/lib/src/widgets/video_track_renderer.dart @@ -29,6 +29,7 @@ import '../support/platform.dart'; import '../track/local/local.dart'; import '../track/local/video.dart'; import '../track/options.dart'; +import '../track/video_track_view_registration.dart'; import '../types/other.dart'; enum VideoViewMirrorMode { @@ -69,6 +70,12 @@ class VideoTrackRenderer extends StatefulWidget { /// wrap the video view in a Center widget (if [fit] is [VideoViewFit.contain]) final bool autoCenter; + /// Controls how this view's logical size is converted to the physical-pixel + /// dimensions requested from the server when adaptive stream is enabled. + /// Defaults to [AdaptiveStreamPixelDensity.auto] (the view's own device pixel + /// ratio), avoiding an under-sized layer on retina / high-density displays. + final AdaptiveStreamPixelDensity adaptiveStreamPixelDensity; + const VideoTrackRenderer( this.track, { this.fit = VideoViewFit.contain, @@ -77,6 +84,7 @@ class VideoTrackRenderer extends StatefulWidget { this.autoDisposeRenderer = true, this.cachedRenderer, this.autoCenter = true, + this.adaptiveStreamPixelDensity = AdaptiveStreamPixelDensity.auto, Key? key, }) : super(key: key); @@ -91,7 +99,7 @@ class _VideoTrackRendererState extends State { double? _aspectRatio; EventsListener? _listener; // Used to compute visibility information - late GlobalKey _internalKey; + late VideoTrackViewRegistration _viewRegistration; Future _initializeRenderer() async { if (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView) { @@ -142,7 +150,7 @@ class _VideoTrackRendererState extends State { if (widget.cachedRenderer != null) { _renderer = widget.cachedRenderer; } - _internalKey = widget.track.addViewKey(); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); if (kIsWeb) { unawaited(() async { await _initializeRenderer(); @@ -154,7 +162,7 @@ class _VideoTrackRendererState extends State { @override void dispose() { - widget.track.removeViewKey(_internalKey); + widget.track.removeViewRegistration(_viewRegistration); unawaited(_listener?.dispose()); if (widget.autoDisposeRenderer) { disposeRenderer(); @@ -188,11 +196,13 @@ class _VideoTrackRendererState extends State { void didUpdateWidget(covariant VideoTrackRenderer oldWidget) { super.didUpdateWidget(oldWidget); if (widget.track != oldWidget.track) { - oldWidget.track.removeViewKey(_internalKey); - _internalKey = widget.track.addViewKey(); + oldWidget.track.removeViewRegistration(_viewRegistration); + _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity); unawaited(() async { await _attach(); }()); + } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) { + _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity; } if ([BrowserType.safari, BrowserType.firefox].contains(lkBrowser()) && oldWidget.key != widget.key) { @@ -203,11 +213,11 @@ class _VideoTrackRendererState extends State { Widget _videoViewForWeb() => !_rendererReadyForWeb ? Container() : Builder( - key: _internalKey, + key: _viewRegistration.key, builder: (ctx) { // let it render before notifying build WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(_internalKey); + widget.track.onVideoViewBuild?.call(); }); return rtc.RTCVideoView( _renderer! as rtc.RTCVideoRenderer, @@ -244,11 +254,11 @@ class _VideoTrackRendererState extends State { if ((snapshot.hasData && _renderer != null) || (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView)) { return Builder( - key: _internalKey, + key: _viewRegistration.key, builder: (ctx) { // let it render before notifying build WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) { - widget.track.onVideoViewBuild?.call(_internalKey); + widget.track.onVideoViewBuild?.call(); }); if (!lkPlatformIsMobile() || widget.track is! LocalVideoTrack) { diff --git a/test/options/adaptive_stream_pixel_density_test.dart b/test/options/adaptive_stream_pixel_density_test.dart new file mode 100644 index 000000000..d2c248795 --- /dev/null +++ b/test/options/adaptive_stream_pixel_density_test.dart @@ -0,0 +1,76 @@ +// Copyright 2025 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/types/other.dart'; + +void main() { + group('AdaptiveStreamPixelDensity.resolve', () { + test('fixed densities ignore the device pixel ratio', () { + expect(const AdaptiveStreamPixelDensity.fixed(1.0).resolve(3.0), 1.0); + expect(const AdaptiveStreamPixelDensity.fixed(2.0).resolve(1.0), 2.0); + }); + + test('fractional fixed densities are supported', () { + expect(const AdaptiveStreamPixelDensity.fixed(1.5).resolve(3.0), 1.5); + expect(const AdaptiveStreamPixelDensity.fixed(2.75).resolve(1.0), 2.75); + }); + + test('auto falls back to the supplied device pixel ratio', () { + expect(AdaptiveStreamPixelDensity.auto.resolve(1.0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(2.0), 2.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(2.625), 2.625); + }); + + test('caps at 3x for both fixed and auto', () { + expect(const AdaptiveStreamPixelDensity.fixed(4.0).resolve(1.0), 3.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(4.0), 3.0); + expect(AdaptiveStreamPixelDensity.maxDensity, 3.0); + }); + + test('falls back for invalid auto device pixel ratios', () { + expect(AdaptiveStreamPixelDensity.auto.resolve(0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(-2.0), 1.0); + expect(AdaptiveStreamPixelDensity.auto.resolve(double.nan), 1.0); + }); + + test('fixed densities must be positive', () { + expect( + () => AdaptiveStreamPixelDensity.fixed(0), + throwsA(isA()), + ); + expect( + () => AdaptiveStreamPixelDensity.fixed(-1.0), + throwsA(isA()), + ); + }); + + test('value is null only for auto', () { + expect(AdaptiveStreamPixelDensity.auto.value, isNull); + expect(const AdaptiveStreamPixelDensity.fixed(1.5).value, 1.5); + }); + + test('equality is by value', () { + expect( + const AdaptiveStreamPixelDensity.fixed(2.0), + const AdaptiveStreamPixelDensity.fixed(2.0), + ); + expect( + const AdaptiveStreamPixelDensity.fixed(2.0) == AdaptiveStreamPixelDensity.auto, + isFalse, + ); + }); + }); +} From 707792b434cf45c91e72e62aeba7417ad2b87db8 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:23:17 +0900 Subject: [PATCH 09/18] v2.8.0 (#1101) * Added: Session API support for simpler E2EE setup * Changed: Manual video quality selection can be used with adaptive stream enabled * Changed: Generated protocol definitions for LiveKit protocol v1.45.8 * Fixed: waitForBufferStatusLow busy-waiting after engine close * Fixed: Simulcast lower layers exceeding the top layer * Fixed: forceRelay log message interpolation * Fixed: sendSyncState error handling so sync-state preparation failures are not swallowed * Fixed: Screen share simulcast default low layer alignment * Fixed: Region failover null-provider dereference * Fixed: Android builds with dependencies that require compileSdk 36 * Fixed: Deferred track listener leaks across reconnects * Fixed: Adaptive stream dimensions on high-density displays * Fixed: Session.start() reentrancy during concurrent calls * Fixed: Connected server address resolving from the wrong peer connection * Fixed: Reconnect counter null assertion on the first reconnect attempt * Fixed: Premature publication disposal during unpublish --- .changes/adaptive-stream-manual-quality-merge | 1 - .changes/adaptive-stream-pixel-density | 1 - .changes/align-screenshare-simulcast-defaults | 1 - .changes/clamp-simulcast-lower-layers | 1 - .changes/fix-buffer-status-busy-wait | 1 - .changes/fix-connected-server-address | 1 - .changes/fix-deferred-track-listener-leak | 1 - .changes/fix-log-interpolation | 1 - .changes/fix-premature-publication-dispose | 1 - .changes/fix-reconnect-counter | 1 - .changes/fix-region-failover-condition | 1 - .changes/fix-send-sync-state-return-type | 1 - .changes/fix-session-start-reentrance | 1 - .changes/simplify-session-e2ee | 1 - .changes/update-protocol-v1-45-8 | 1 - .version | 2 +- CHANGELOG.md | 19 ++++++++ README.md | 2 +- ios/livekit_client.podspec | 2 +- lib/src/livekit.dart | 2 +- macos/livekit_client.podspec | 2 +- pubspec.lock | 8 ++-- pubspec.yaml | 2 +- scripts/create_version.dart | 48 +++++++++++++------ 24 files changed, 63 insertions(+), 39 deletions(-) delete mode 100644 .changes/adaptive-stream-manual-quality-merge delete mode 100644 .changes/adaptive-stream-pixel-density delete mode 100644 .changes/align-screenshare-simulcast-defaults delete mode 100644 .changes/clamp-simulcast-lower-layers delete mode 100644 .changes/fix-buffer-status-busy-wait delete mode 100644 .changes/fix-connected-server-address delete mode 100644 .changes/fix-deferred-track-listener-leak delete mode 100644 .changes/fix-log-interpolation delete mode 100644 .changes/fix-premature-publication-dispose delete mode 100644 .changes/fix-reconnect-counter delete mode 100644 .changes/fix-region-failover-condition delete mode 100644 .changes/fix-send-sync-state-return-type delete mode 100644 .changes/fix-session-start-reentrance delete mode 100644 .changes/simplify-session-e2ee delete mode 100644 .changes/update-protocol-v1-45-8 diff --git a/.changes/adaptive-stream-manual-quality-merge b/.changes/adaptive-stream-manual-quality-merge deleted file mode 100644 index 083a3def6..000000000 --- a/.changes/adaptive-stream-manual-quality-merge +++ /dev/null @@ -1 +0,0 @@ -patch type="improved" "Allow manual video quality selection with adaptive stream enabled" diff --git a/.changes/adaptive-stream-pixel-density b/.changes/adaptive-stream-pixel-density deleted file mode 100644 index 94de7146f..000000000 --- a/.changes/adaptive-stream-pixel-density +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix adaptive stream dimensions on high-density displays" diff --git a/.changes/align-screenshare-simulcast-defaults b/.changes/align-screenshare-simulcast-defaults deleted file mode 100644 index 5ef6b9d5a..000000000 --- a/.changes/align-screenshare-simulcast-defaults +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Align screen share simulcast default low layer" diff --git a/.changes/clamp-simulcast-lower-layers b/.changes/clamp-simulcast-lower-layers deleted file mode 100644 index 2426b13b7..000000000 --- a/.changes/clamp-simulcast-lower-layers +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Clamp simulcast lower layers to the top layer" diff --git a/.changes/fix-buffer-status-busy-wait b/.changes/fix-buffer-status-busy-wait deleted file mode 100644 index 9f048b1d1..000000000 --- a/.changes/fix-buffer-status-busy-wait +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix waitForBufferStatusLow busy-wait after engine close" diff --git a/.changes/fix-connected-server-address b/.changes/fix-connected-server-address deleted file mode 100644 index e55d6ccea..000000000 --- a/.changes/fix-connected-server-address +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix connected server address using wrong peer connection" diff --git a/.changes/fix-deferred-track-listener-leak b/.changes/fix-deferred-track-listener-leak deleted file mode 100644 index fa6955d75..000000000 --- a/.changes/fix-deferred-track-listener-leak +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix deferred track listener leak across reconnects" diff --git a/.changes/fix-log-interpolation b/.changes/fix-log-interpolation deleted file mode 100644 index 8919c3a8e..000000000 --- a/.changes/fix-log-interpolation +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix string interpolation in forceRelay log messages" diff --git a/.changes/fix-premature-publication-dispose b/.changes/fix-premature-publication-dispose deleted file mode 100644 index e97a95a32..000000000 --- a/.changes/fix-premature-publication-dispose +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix premature publication dispose during unpublish" diff --git a/.changes/fix-reconnect-counter b/.changes/fix-reconnect-counter deleted file mode 100644 index 7291fd827..000000000 --- a/.changes/fix-reconnect-counter +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix reconnect counter null assertion on first reconnect attempt" diff --git a/.changes/fix-region-failover-condition b/.changes/fix-region-failover-condition deleted file mode 100644 index 722a10933..000000000 --- a/.changes/fix-region-failover-condition +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix region failover condition allowing null provider dereference" diff --git a/.changes/fix-send-sync-state-return-type b/.changes/fix-send-sync-state-return-type deleted file mode 100644 index ed91cbb36..000000000 --- a/.changes/fix-send-sync-state-return-type +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Fix sendSyncState using async void and swallowing sync-state preparation errors" diff --git a/.changes/fix-session-start-reentrance b/.changes/fix-session-start-reentrance deleted file mode 100644 index 286024ba9..000000000 --- a/.changes/fix-session-start-reentrance +++ /dev/null @@ -1 +0,0 @@ -patch type="fixed" "Guard Session.start() against concurrent calls" diff --git a/.changes/simplify-session-e2ee b/.changes/simplify-session-e2ee deleted file mode 100644 index d63c8d54c..000000000 --- a/.changes/simplify-session-e2ee +++ /dev/null @@ -1 +0,0 @@ -minor type="added" "Simplify enabling E2EE with Session API" diff --git a/.changes/update-protocol-v1-45-8 b/.changes/update-protocol-v1-45-8 deleted file mode 100644 index 6760a419c..000000000 --- a/.changes/update-protocol-v1-45-8 +++ /dev/null @@ -1 +0,0 @@ -patch type="changed" "Update generated protocol definitions to v1.45.8" diff --git a/.version b/.version index 24ba9a38d..834f26295 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.7.0 +2.8.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2b2620c..0f1778128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # CHANGELOG +## 2.8.0 + +* Added: Session API support for simpler E2EE setup +* Changed: Manual video quality selection can be used with adaptive stream enabled +* Changed: Generated protocol definitions for LiveKit protocol v1.45.8 +* Fixed: waitForBufferStatusLow busy-waiting after engine close +* Fixed: Simulcast lower layers exceeding the top layer +* Fixed: forceRelay log message interpolation +* Fixed: sendSyncState error handling so sync-state preparation failures are not swallowed +* Fixed: Screen share simulcast default low layer alignment +* Fixed: Region failover null-provider dereference +* Fixed: Android builds with dependencies that require compileSdk 36 +* Fixed: Deferred track listener leaks across reconnects +* Fixed: Adaptive stream dimensions on high-density displays +* Fixed: Session.start() reentrancy during concurrent calls +* Fixed: Connected server address resolving from the wrong peer connection +* Fixed: Reconnect counter null assertion on the first reconnect attempt +* Fixed: Premature publication disposal during unpublish + ## 2.7.0 * Added: Add setVideoDimensions for remote track publications diff --git a/README.md b/README.md index 1bbbce27f..9a80d3dfb 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Include this package to your `pubspec.yaml` ```yaml --- dependencies: - livekit_client: ^2.7.0 + livekit_client: ^2.8.0 ``` ### iOS diff --git a/ios/livekit_client.podspec b/ios/livekit_client.podspec index ff758ac38..e9b3032eb 100644 --- a/ios/livekit_client.podspec +++ b/ios/livekit_client.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'livekit_client' - s.version = '2.7.0' + s.version = '2.8.0' s.summary = 'Open source platform for real-time audio and video.' s.description = 'Open source platform for real-time audio and video.' s.homepage = 'https://livekit.io/' diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 413774285..6cdc2cba8 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -20,7 +20,7 @@ import 'support/platform.dart' show lkPlatformIsMobile; /// Main entry point to connect to a room. /// {@category Room} class LiveKitClient { - static const version = '2.7.0'; + static const version = '2.8.0'; /// Initialize the WebRTC plugin. If this is not manually called, will be /// initialized with default settings. diff --git a/macos/livekit_client.podspec b/macos/livekit_client.podspec index 10cbd676c..16c318b85 100644 --- a/macos/livekit_client.podspec +++ b/macos/livekit_client.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'livekit_client' - s.version = '2.7.0' + s.version = '2.8.0' s.summary = 'Open source platform for real-time audio and video.' s.description = 'Open source platform for real-time audio and video.' s.homepage = 'https://livekit.io/' diff --git a/pubspec.lock b/pubspec.lock index 9a7a4b124..39350a0a9 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -452,10 +452,10 @@ packages: dependency: "direct main" description: name: meta - sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" + sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349" url: "https://pub.dev" source: hosted - version: "1.17.0" + version: "1.18.0" mime: dependency: transitive description: @@ -737,10 +737,10 @@ packages: dependency: transitive description: name: test_api - sha256: "8161c84903fd860b26bfdefb7963b3f0b68fee7adea0f59ef805ecca346f0c7a" + sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e" url: "https://pub.dev" source: hosted - version: "0.7.10" + version: "0.7.11" timing: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index a72f7b8eb..2c8783473 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -15,7 +15,7 @@ name: livekit_client description: Flutter Client SDK for LiveKit. Build real-time video and audio into your apps. Supports iOS, Android, and Web. -version: 2.7.0 +version: 2.8.0 homepage: https://github.com/livekit/client-sdk-flutter environment: diff --git a/scripts/create_version.dart b/scripts/create_version.dart index ec4d96f0d..26e176d2b 100755 --- a/scripts/create_version.dart +++ b/scripts/create_version.dart @@ -23,7 +23,7 @@ import 'dart:io'; /// /// Where: /// - level: One of [patch, minor, major] indicating the version bump level -/// - kind: One of [added, changed, fixed, refactor, performance, security, deprecated, removed, docs, chore] +/// - kind: One of [added, changed, fixed, refactor, performance, security, deprecated, removed, docs] /// - description: A detailed description of the change /// /// Examples: @@ -103,6 +103,10 @@ class Change { }); } +String _allowedChangeKinds() => ChangeKind.values.map((e) => e.name).join(', '); + +String _allowedChangeLevels() => ChangeLevel.values.map((e) => e.name).join(', '); + class SemanticVersion { final int major; final int minor; @@ -154,39 +158,55 @@ List parseChanges() { } final changes = []; + final errors = []; final files = changesDir.listSync().whereType().where((f) => !f.path.split('/').last.startsWith('.')); for (final file in files) { final content = file.readAsStringSync(); final lines = content.split('\n'); - for (final line in lines) { + for (var i = 0; i < lines.length; i++) { + final rawLine = lines[i]; + final line = rawLine.trim(); + final location = '${file.path}:${i + 1}'; + // Skip empty lines - if (line.trim().isEmpty) continue; + if (line.isEmpty) continue; // Parse format: level type="kind" "description" - final parts = line.split(RegExp(r'\s+')); - if (parts.length < 3) continue; + final match = RegExp(r'^(\w+)\s+type="([^"]+)"\s+"([^"]+)"$').firstMatch(line); + if (match == null) { + errors.add('$location: expected `level type="kind" "description"`, found `$rawLine`'); + continue; + } // Extract level - final level = ChangeLevel.fromString(parts[0]); - if (level == null) continue; + final levelName = match.group(1)!; + final level = ChangeLevel.fromString(levelName); + if (level == null) { + errors.add('$location: unsupported level `$levelName`; expected one of: ${_allowedChangeLevels()}'); + continue; + } // Extract type from type="kind" format - final typeMatch = RegExp(r'type="(\w+)"').firstMatch(parts[1]); - if (typeMatch == null) continue; - final kind = ChangeKind.fromString(typeMatch.group(1)!); - if (kind == null) continue; + final kindName = match.group(2)!; + final kind = ChangeKind.fromString(kindName); + if (kind == null) { + errors.add('$location: unsupported type `$kindName`; expected one of: ${_allowedChangeKinds()}'); + continue; + } // Extract description from the last quoted string - final descMatch = RegExp(r'"([^"]+)"$').firstMatch(line); - if (descMatch == null) continue; - final description = descMatch.group(1)!; + final description = match.group(3)!; changes.add(Change(level: level, kind: kind, description: description)); } } + if (errors.isNotEmpty) { + throw Exception('Invalid change entries:\n${errors.map((e) => ' - $e').join('\n')}'); + } + if (changes.isEmpty) { throw Exception('No changes found in ${_Path.changes}'); } From b6cc65dbdf9f58d18dbe43e1573928078efacfdf Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 6 Jun 2026 16:45:56 +0900 Subject: [PATCH 10/18] Use maintain-resolution as the default video degradation preference (#1106) ## Summary - Align Flutter default local-video degradation behavior with the Swift SDK. - Default unset `VideoPublishOptions.degradationPreference` to `maintainResolution` for camera and screen-share publishing. - Keep explicit degradation preferences overrideable by apps. ## Context Related to #1097, which explores preserving video quality through a live-streaming option. This PR takes the smaller SDK-default approach instead: use maintain-resolution by default, matching Swift, without adding a separate app-facing toggle for this behavior. ## Testing - `dart analyze` - `flutter test test/core/room_e2e_test.dart` --- .changes/video-degradation-default | 1 + lib/src/participant/local.dart | 21 ++------------------- 2 files changed, 3 insertions(+), 19 deletions(-) create mode 100644 .changes/video-degradation-default diff --git a/.changes/video-degradation-default b/.changes/video-degradation-default new file mode 100644 index 000000000..f3edc2c76 --- /dev/null +++ b/.changes/video-degradation-default @@ -0,0 +1 @@ +patch type="fixed" "Use maintain-resolution as the default video degradation preference for local video publishing" diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index e785b72fa..1d2781d91 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -386,10 +386,7 @@ class LocalParticipant extends Participant { } if ([TrackSource.camera, TrackSource.screenShareVideo].contains(track.source)) { - final degradationPreference = publishOptions.degradationPreference ?? - getDefaultDegradationPreference( - track, - ); + final degradationPreference = publishOptions.degradationPreference ?? DegradationPreference.maintainResolution; await track.setDegradationPreference(degradationPreference); } @@ -487,10 +484,7 @@ class LocalParticipant extends Participant { } if ([TrackSource.camera, TrackSource.screenShareVideo].contains(track.source)) { - final degradationPreference = publishOptions.degradationPreference ?? - getDefaultDegradationPreference( - track, - ); + final degradationPreference = publishOptions.degradationPreference ?? DegradationPreference.maintainResolution; await track.setDegradationPreference(degradationPreference); } @@ -593,17 +587,6 @@ class LocalParticipant extends Participant { await pub.dispose(); } - DegradationPreference getDefaultDegradationPreference(LocalVideoTrack track) { - // a few of reasons we have different default paths: - // 1. without this, Chrome seems to aggressively resize the SVC video stating `quality-limitation: bandwidth` even when BW isn't an issue - // 2. since we are overriding contentHint to motion (to workaround L1T3 publishing), it overrides the default degradationPreference to `balanced` - final VideoDimensions dimensions = track.currentOptions.params.dimensions; - if (track.source == TrackSource.screenShareVideo || dimensions.height >= 1080) { - return DegradationPreference.maintainResolution; - } - return DegradationPreference.balanced; - } - /// Convenience method to unpublish all tracks. Future unpublishAllTracks({bool notify = true, bool? stopOnUnpublish}) async { final trackSids = trackPublications.keys.toSet(); From d031c3f3da3646bc357e2c402ddc63fd8fb3bf9c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 12 Jun 2026 02:47:29 +0900 Subject: [PATCH 11/18] chore(android): support AGP 9 built-in Kotlin (#1102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Migrates the Android plugin to AGP 9's built-in Kotlin while keeping older toolchains building (same pattern as flutter-webrtc/flutter-webrtc#2075): - Apply the **Kotlin Gradle Plugin only when built-in Kotlin is inactive** — AGP < 9, or AGP 9 with `android.builtInKotlin=false` (the configuration Flutter currently ships by default while the ecosystem migrates). When AGP 9's built-in Kotlin is active it registers the `kotlin` extension itself and rejects KGP, so applying it is skipped. - Set the JVM target through the `kotlin { compilerOptions {} }` DSL **when the extension supports it** (KGP 1.9+ / AGP 9 built-in Kotlin), falling back to the legacy `kotlinOptions` DSL for apps still on KGP 1.8.x. - Bump the standalone buildscript fallback KGP to 2.1.0 so it is self-consistent. - Add a changeset entry for the generated release notes. ## Context AGP 9 uses built-in Kotlin support and rejects Android plugins that still apply KGP directly. This follows the Flutter compatibility migration path instead of raising the minimum supported toolchain. ## Verification - Example app builds (`flutter build apk --debug`) on the current stable toolchain (AGP 8.x + modern KGP path). - The AGP 9 built-in path mirrors the reviewed and merged flutter-webrtc implementation. --- .changes/agp9-built-in-kotlin | 1 + android/build.gradle | 35 +++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 .changes/agp9-built-in-kotlin diff --git a/.changes/agp9-built-in-kotlin b/.changes/agp9-built-in-kotlin new file mode 100644 index 000000000..bb0857676 --- /dev/null +++ b/.changes/agp9-built-in-kotlin @@ -0,0 +1 @@ +patch type="fixed" "Android plugin compatibility with AGP 9 built-in Kotlin" diff --git a/android/build.gradle b/android/build.gradle index dbc0cb96b..17fa1f448 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -2,7 +2,7 @@ group = "io.livekit.plugin" version = "1.0-SNAPSHOT" buildscript { - ext.kotlin_version = "1.8.22" + ext.kotlin_version = "2.1.0" repositories { google() mavenCentral() @@ -22,7 +22,18 @@ allprojects { } apply plugin: "com.android.library" -apply plugin: "kotlin-android" + +// AGP 9's built-in Kotlin compiles Kotlin itself and rejects the Kotlin Gradle +// Plugin. Apply KGP only when built-in Kotlin is NOT active: that means AGP < 9, +// or AGP 9 with android.builtInKotlin=false (the configuration Flutter currently +// ships by default while the ecosystem migrates). +def agpMajor = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION.tokenize(".")[0] as int +def builtInKotlinActive = agpMajor >= 9 && + (!project.hasProperty("android.builtInKotlin") || + Boolean.parseBoolean(project.property("android.builtInKotlin").toString())) +if (!builtInKotlinActive) { + apply plugin: "kotlin-android" +} android { if (project.android.hasProperty("namespace")) { @@ -36,10 +47,6 @@ android { targetCompatibility = JavaVersion.VERSION_1_8 } - kotlinOptions { - jvmTarget = JavaVersion.VERSION_1_8 - } - sourceSets { main.java.srcDirs += "src/main/kotlin" test.java.srcDirs += "src/test/kotlin" @@ -68,3 +75,19 @@ android { } } } + +// Configure the Kotlin JVM target. The compilerOptions DSL requires KGP 1.9+ or +// AGP 9 built-in Kotlin; older Flutter app templates ship KGP 1.8.x, which only +// supports the legacy kotlinOptions DSL. +def kotlinExt = project.extensions.findByName("kotlin") +if (kotlinExt?.hasProperty("compilerOptions")) { + kotlin { + compilerOptions { + jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8 + } + } +} else { + android.kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8.toString() + } +} From 1c03b5526e86cd9358f3bd56eae037b2887e51a1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 12 Jun 2026 19:02:56 +0900 Subject: [PATCH 12/18] feat(audio): runtime audio processing options and engine-wide state read-back (#1107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Runtime control of audio processing (AEC / NS / AGC / HPF) for local audio tracks, plus an engine-wide diagnostic read-back, built on the WebRTC-SDK audio processing options API (webrtc-sdk/webrtc#247 + webrtc-sdk/webrtc#254). ## API **Set** — `AudioProcessingOptions` with per-component enabled flags and modes (`automatic` / `platform` / `software`), applied either at capture time via `AudioCaptureOptions` or at runtime: ```dart final result = await localAudioTrack.setAudioProcessingOptions(options); ``` Caller bugs (invalid combination, remote track) throw `AudioProcessingException`; legitimate outcomes return a typed `AudioProcessingApplyResult` (`applied` / `stored` / rejections). **Read** — the audio processing module is owned by the native peer connection factory and shared engine-wide, so the snapshot lives on `AudioManager`: ```dart final state = await AudioManager.instance.getAudioProcessingState(); ``` Per component: `requested` (nullable — null means nothing was ever applied), `isSoftwareResolved` / `isSoftwareActive`, `isPlatformAvailable` / `isPlatformResolved` / `isPlatformActive`, and `effective` as the merged verdict. Same requested → resolved → active → effective vocabulary as the native SDKs. ## Commits Bottom-up, each builds standalone: 1. `chore(deps)`: WebRTC-SDK pin bump 2. Dart `AudioProcessingOptions` for `LocalAudioTrack` 3. Routing through the LiveKit native plugin (iOS + Android handlers) 4. Typed apply results 5. Engine-wide v2 state read-back on `AudioManager` ## Dependencies / not yet done - **Lib pin is still `144.7559.08`** — will bump to `144.7559.09` (which carries the state v2 API from webrtc-sdk/webrtc#254) once published. The state read-back native code requires `.09` to compile. - **Android requires `FlutterWebRTCPlugin.getPeerConnectionFactory()`** — flutter-webrtc/flutter-webrtc#2077. - Device smoke test on iOS + Android pending the `.09` artifacts. --- .changes/runtime-audio-options | 1 + android/build.gradle | 2 +- .../kotlin/io/livekit/plugin/LiveKitPlugin.kt | 122 ++++++++++++ ios/livekit_client.podspec | 2 +- lib/livekit_client.dart | 2 + lib/src/audio/audio_manager.dart | 40 ++++ lib/src/audio/audio_processing_state.dart | 145 ++++++++++++++ lib/src/support/native.dart | 43 ++++ lib/src/track/local/audio.dart | 52 ++++- lib/src/track/options.dart | 183 +++++++++++++++++- macos/livekit_client.podspec | 2 +- pubspec.lock | 4 +- pubspec.yaml | 2 +- shared_swift/LiveKitPlugin.swift | 119 ++++++++++++ 14 files changed, 704 insertions(+), 15 deletions(-) create mode 100644 .changes/runtime-audio-options create mode 100644 lib/src/audio/audio_manager.dart create mode 100644 lib/src/audio/audio_processing_state.dart diff --git a/.changes/runtime-audio-options b/.changes/runtime-audio-options new file mode 100644 index 000000000..7cb3ff1ec --- /dev/null +++ b/.changes/runtime-audio-options @@ -0,0 +1 @@ +minor type="added" "Runtime audio processing options for local audio tracks and engine-wide audio processing state read-back on AudioManager" diff --git a/android/build.gradle b/android/build.gradle index 17fa1f448..82e5dd8d7 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -59,7 +59,7 @@ android { dependencies { testImplementation("org.jetbrains.kotlin:kotlin-test") testImplementation("org.mockito:mockito-core:5.0.0") - implementation 'io.github.webrtc-sdk:android:144.7559.01' + implementation 'io.github.webrtc-sdk:android:144.7559.09' implementation 'io.livekit:noise:2.0.0' } diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt index 88c73c658..95a242497 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt @@ -29,6 +29,13 @@ import com.cloudwebrtc.webrtc.FlutterWebRTCPlugin import com.cloudwebrtc.webrtc.audio.LocalAudioTrack import io.flutter.plugin.common.BinaryMessenger import org.webrtc.AudioTrack +import org.webrtc.audio.AudioProcessingComponentOptions +import org.webrtc.audio.AudioProcessingComponentState +import org.webrtc.audio.AudioProcessingImplementation +import org.webrtc.audio.AudioProcessingMode +import org.webrtc.audio.AudioProcessingOptions +import org.webrtc.audio.AudioProcessingOptionsResult +import org.webrtc.audio.AudioProcessingState /** LiveKitPlugin */ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { @@ -210,6 +217,113 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { result.success(true) } + private fun handleSetAudioProcessingOptions(call: MethodCall, result: Result) { + val trackId = call.argument("trackId") + if (trackId == null) { + result.error("INVALID_ARGUMENT", "trackId is required", null) + return + } + + val mediaTrack = (flutterWebRTCPlugin.getLocalTrack(trackId) as? LocalAudioTrack)?.track + if (mediaTrack !is AudioTrack) { + result.error("INVALID_ARGUMENT", "track is not a local audio track", null) + return + } + + val options = AudioProcessingOptions( + AudioProcessingComponentOptions( + call.argument("echoCancellation") ?: true, + audioProcessingMode(call.argument("echoCancellationMode")), + ), + AudioProcessingComponentOptions( + call.argument("noiseSuppression") ?: true, + audioProcessingMode(call.argument("noiseSuppressionMode")), + ), + AudioProcessingComponentOptions( + call.argument("autoGainControl") ?: true, + audioProcessingMode(call.argument("autoGainControlMode")), + ), + AudioProcessingComponentOptions( + call.argument("highPassFilter") ?: false, + audioProcessingMode(call.argument("highPassFilterMode")), + ), + ) + + val processingResult = mediaTrack.setAudioProcessingOptions(options) + result.success( + mapOf( + "result" to processingResult.isSuccess, + "code" to audioProcessingResultCodeString(processingResult.code), + "message" to processingResult.message, + ), + ) + } + + private fun audioProcessingMode(value: String?): AudioProcessingMode = when (value) { + "platform" -> AudioProcessingMode.PLATFORM + "software" -> AudioProcessingMode.SOFTWARE + else -> AudioProcessingMode.AUTOMATIC + } + + private fun audioProcessingResultCodeString(code: AudioProcessingOptionsResult.Code): String = when (code) { + AudioProcessingOptionsResult.Code.APPLIED -> "applied" + AudioProcessingOptionsResult.Code.STORED -> "stored" + AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "rejectedRemoteTrack" + AudioProcessingOptionsResult.Code.REJECTED_INVALID_COMBINATION -> "rejectedInvalidCombination" + AudioProcessingOptionsResult.Code.REJECTED_PLATFORM_UNAVAILABLE -> "rejectedPlatformUnavailable" + AudioProcessingOptionsResult.Code.APPLY_FAILED -> "applyFailed" + } + + private fun handleGetAudioProcessingState(result: Result) { + val factory = flutterWebRTCPlugin.getPeerConnectionFactory() + if (factory == null) { + result.success(null) + return + } + result.success(audioProcessingStateToMap(factory.audioProcessingState)) + } + + private fun audioProcessingModeString(mode: AudioProcessingMode): String = when (mode) { + AudioProcessingMode.PLATFORM -> "platform" + AudioProcessingMode.SOFTWARE -> "software" + AudioProcessingMode.AUTOMATIC -> "auto" + } + + private fun audioProcessingImplementationString(implementation: AudioProcessingImplementation): String = + when (implementation) { + AudioProcessingImplementation.UNKNOWN -> "unknown" + AudioProcessingImplementation.DISABLED -> "disabled" + AudioProcessingImplementation.SOFTWARE -> "software" + AudioProcessingImplementation.PLATFORM -> "platform" + AudioProcessingImplementation.SOFTWARE_AND_PLATFORM -> "softwareAndPlatform" + } + + private fun requestedToMap(requested: AudioProcessingComponentOptions?): Map? = + requested?.let { + mapOf( + "enabled" to it.isEnabled, + "mode" to audioProcessingModeString(it.mode), + ) + } + + private fun componentToMap(state: AudioProcessingComponentState): Map = mapOf( + "requested" to requestedToMap(state.requested), + "isSoftwareResolved" to state.isSoftwareResolved, + "isSoftwareActive" to state.isSoftwareActive, + "isPlatformAvailable" to state.isPlatformAvailable, + "isPlatformResolved" to state.isPlatformResolved, + "isPlatformActive" to state.isPlatformActive, + "effective" to audioProcessingImplementationString(state.effective), + ) + + private fun audioProcessingStateToMap(state: AudioProcessingState): Map = mapOf( + "hasAudioProcessingModule" to state.hasAudioProcessingModule, + "echoCancellation" to componentToMap(state.echoCancellation), + "noiseSuppression" to componentToMap(state.noiseSuppression), + "autoGainControl" to componentToMap(state.autoGainControl), + "highPassFilter" to componentToMap(state.highPassFilter), + ) + override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { when (call.method) { "startVisualizer" -> { @@ -228,6 +342,14 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { handleStopAudioRenderer(call, result) } + "setAudioProcessingOptions" -> { + handleSetAudioProcessingOptions(call, result) + } + + "getAudioProcessingState" -> { + handleGetAudioProcessingState(result) + } + else -> { result.notImplemented() } diff --git a/ios/livekit_client.podspec b/ios/livekit_client.podspec index e9b3032eb..004152ca5 100644 --- a/ios/livekit_client.podspec +++ b/ios/livekit_client.podspec @@ -16,6 +16,6 @@ Pod::Spec.new do |s| s.static_framework = true s.dependency 'Flutter' - s.dependency 'WebRTC-SDK', '144.7559.01' + s.dependency 'WebRTC-SDK', '144.7559.09' s.dependency 'flutter_webrtc' end diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart index cb0408d67..9756d5a07 100644 --- a/lib/livekit_client.dart +++ b/lib/livekit_client.dart @@ -41,12 +41,14 @@ export 'src/agent/room_agent.dart'; export 'src/participant/local.dart'; export 'src/participant/participant.dart'; export 'src/participant/remote.dart' hide ParticipantCreationResult; +export 'src/audio/audio_manager.dart'; export 'src/audio/audio_frame_capture.dart' show AudioFormat, AudioFrame, AudioFrameCallback, AudioRendererOptions; export 'src/preconnect/pre_connect_audio_buffer.dart'; export 'src/publication/local.dart'; export 'src/publication/remote.dart'; export 'src/publication/track_publication.dart'; export 'src/support/platform.dart'; +export 'src/audio/audio_processing_state.dart'; export 'src/track/audio_visualizer.dart'; export 'src/track/local/audio.dart'; export 'src/track/local/local.dart'; diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart new file mode 100644 index 000000000..3034a3706 --- /dev/null +++ b/lib/src/audio/audio_manager.dart @@ -0,0 +1,40 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import '../support/native.dart'; +import 'audio_processing_state.dart'; + +/// Controls LiveKit's process-wide platform audio behavior. +/// +/// The platform audio engine and its audio processing module are global to the +/// app process, so engine-scoped audio state lives here rather than on a `Room` +/// or an individual track. +class AudioManager { + AudioManager._(); + + static final AudioManager instance = AudioManager._(); + + /// Diagnostic snapshot of the resolved audio processing state. + /// + /// The audio processing module is owned by the native peer connection factory + /// and shared engine-wide, so this reflects what is actually applied across + /// the engine rather than any single track — use it to verify what a + /// `LocalAudioTrack.setAudioProcessingOptions` request resolved to. Returns + /// `null` when the native side cannot provide it. + Future getAudioProcessingState() async { + final response = await Native.getAudioProcessingState(); + if (response == null) return null; + return AudioProcessingState.fromMap(response); + } +} diff --git a/lib/src/audio/audio_processing_state.dart b/lib/src/audio/audio_processing_state.dart new file mode 100644 index 000000000..a7d21e12c --- /dev/null +++ b/lib/src/audio/audio_processing_state.dart @@ -0,0 +1,145 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import '../track/options.dart'; + +/// The implementation in effect for an audio processing component. +enum AudioProcessingImplementation { + unknown('unknown'), + disabled('disabled'), + software('software'), + platform('platform'), + softwareAndPlatform('softwareAndPlatform'); + + const AudioProcessingImplementation(this.value); + + final String value; + + static AudioProcessingImplementation fromValue(String? value) => AudioProcessingImplementation.values.firstWhere( + (e) => e.value == value, + orElse: () => AudioProcessingImplementation.unknown, + ); +} + +AudioProcessingMode _modeFromValue(String? value) { + for (final mode in AudioProcessingMode.values) { + if (mode.constraintValue == value) return mode; + } + return AudioProcessingMode.automatic; +} + +/// The caller's request for one audio processing component: enabled flag plus +/// implementation mode. +class AudioProcessingComponentRequest { + const AudioProcessingComponentRequest({ + required this.enabled, + required this.mode, + }); + + factory AudioProcessingComponentRequest.fromMap(Map map) => AudioProcessingComponentRequest( + enabled: (map['enabled'] as bool?) ?? false, + mode: _modeFromValue(map['mode'] as String?), + ); + + final bool enabled; + final AudioProcessingMode mode; +} + +/// Diagnostic state of one audio processing component (echo cancellation, +/// noise suppression, auto gain control or high-pass filter), observed at +/// three stages of one pipeline: requested (caller intent) -> resolved (the +/// engine's per-path decision) -> active (live truth), with [effective] as +/// the merged verdict. +class AudioProcessingComponentState { + const AudioProcessingComponentState({ + this.requested, + required this.isSoftwareResolved, + required this.isSoftwareActive, + required this.isPlatformAvailable, + required this.isPlatformResolved, + required this.isPlatformActive, + required this.effective, + }); + + factory AudioProcessingComponentState.fromMap(Map map) => AudioProcessingComponentState( + requested: map['requested'] is Map + ? AudioProcessingComponentRequest.fromMap(Map.from(map['requested'] as Map)) + : null, + isSoftwareResolved: (map['isSoftwareResolved'] as bool?) ?? false, + isSoftwareActive: (map['isSoftwareActive'] as bool?) ?? false, + isPlatformAvailable: (map['isPlatformAvailable'] as bool?) ?? false, + isPlatformResolved: (map['isPlatformResolved'] as bool?) ?? false, + isPlatformActive: (map['isPlatformActive'] as bool?) ?? false, + effective: AudioProcessingImplementation.fromValue(map['effective'] as String?), + ); + + /// What the caller most recently requested for this component. Null when no + /// audio processing options have ever been applied — "nobody asked". + final AudioProcessingComponentRequest? requested; + + /// Whether the resolver decided the WebRTC software (APM) implementation + /// should run, after weighing the requested mode against platform + /// availability, coupling, and policy. + final bool isSoftwareResolved; + + /// Whether APM's live configuration currently has this component enabled. + final bool isSoftwareActive; + + /// Whether this device/OS offers a built-in implementation at all. + final bool isPlatformAvailable; + + /// Whether the engine asked the OS to run the platform implementation. The + /// OS owns the outcome: it can decline, defer, or couple components. + final bool isPlatformResolved; + + /// Whether the device reports the platform implementation actually running. + final bool isPlatformActive; + + /// The verdict: which implementation is in effect right now. + final AudioProcessingImplementation effective; +} + +/// Diagnostic snapshot of the resolved audio processing state for the shared +/// audio processing module. +/// +/// The module is owned by the native peer connection factory and shared +/// engine-wide, so this reflects what is actually applied (per-component +/// [AudioProcessingComponentState.effective]) versus what was requested — for +/// the whole engine, not a single track. +class AudioProcessingState { + const AudioProcessingState({ + required this.hasAudioProcessingModule, + required this.echoCancellation, + required this.noiseSuppression, + required this.autoGainControl, + required this.highPassFilter, + }); + + factory AudioProcessingState.fromMap(Map map) => AudioProcessingState( + hasAudioProcessingModule: (map['hasAudioProcessingModule'] as bool?) ?? false, + echoCancellation: + AudioProcessingComponentState.fromMap(Map.from(map['echoCancellation'] as Map)), + noiseSuppression: + AudioProcessingComponentState.fromMap(Map.from(map['noiseSuppression'] as Map)), + autoGainControl: + AudioProcessingComponentState.fromMap(Map.from(map['autoGainControl'] as Map)), + highPassFilter: AudioProcessingComponentState.fromMap(Map.from(map['highPassFilter'] as Map)), + ); + + final bool hasAudioProcessingModule; + final AudioProcessingComponentState echoCancellation; + final AudioProcessingComponentState noiseSuppression; + final AudioProcessingComponentState autoGainControl; + final AudioProcessingComponentState highPassFilter; +} diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index 2f3b3e24d..9cdfe928a 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -50,6 +50,49 @@ class Native { } } + /// Applies runtime audio processing options to a local audio track. + /// + /// Resolved natively against the underlying WebRTC audio track owned by + /// flutter_webrtc; [options] is the serialized [AudioProcessingOptions] map. + /// Returns the native result map (`result`/`code`/`message`) so the caller + /// can surface typed rejections. Channel errors propagate to the caller. + @internal + static Future> setAudioProcessingOptions( + String trackId, + Map options, + ) async { + final response = await channel.invokeMethod( + 'setAudioProcessingOptions', + { + 'trackId': trackId, + ...options, + }, + ); + if (response is Map) { + return response.map((key, value) => MapEntry(key.toString(), value)); + } + return {}; + } + + /// Reads the engine-wide audio processing state from the native peer + /// connection factory. Returns `null` when unavailable (e.g. the factory + /// does not exist yet, or the platform cannot provide it). + @internal + static Future?> getAudioProcessingState() async { + try { + final response = await channel.invokeMethod( + 'getAudioProcessingState', + {}, + ); + if (response is Map) { + return response.map((key, value) => MapEntry(key.toString(), value)); + } + } catch (error) { + logger.warning('getAudioProcessingState did throw $error'); + } + return null; + } + @internal static Future startVisualizer( String trackId, { diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index e3675a340..48468b8cc 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -19,19 +19,21 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; import '../../events.dart'; +import '../../internal/events.dart'; import '../../logger.dart'; import '../../options.dart'; import '../../stats/audio_source_stats.dart'; import '../../stats/stats.dart'; +import '../../support/native.dart'; import '../../types/other.dart'; import '../audio_management.dart'; -import '../options.dart'; +import '../options.dart' as track_options; import 'local.dart'; class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMixin { // Options used for this track @override - covariant AudioCaptureOptions currentOptions; + covariant track_options.AudioCaptureOptions currentOptions; AudioPublishOptions? lastPublishOptions; @@ -45,6 +47,38 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi } } + Future setAudioProcessingOptions( + track_options.AudioProcessingOptions options) async { + final nextOptions = currentOptions.copyWith(processing: options); + final response = await Native.setAudioProcessingOptions( + mediaStreamTrack.id!, + options.toMap(), + ); + + final code = track_options.AudioProcessingOptionsResultCode.fromValue(response['code'] as String?); + final message = (response['message'] as String?) ?? ''; + + // Malformed requests (incompatible modes, or a non-local track) are caller + // bugs — surface them loudly rather than as a silently-unsuccessful result. + if (code == track_options.AudioProcessingOptionsResultCode.rejectedInvalidCombination || + code == track_options.AudioProcessingOptionsResultCode.rejectedRemoteTrack) { + throw track_options.AudioProcessingException( + code, + message.isNotEmpty ? message : 'Unable to apply audio processing options', + ); + } + + final result = track_options.AudioProcessingApplyResult(code, message); + if (result.isSuccess) { + currentOptions = nextOptions; + events.emit(LocalTrackOptionsUpdatedEvent( + track: this, + options: currentOptions, + )); + } + return result; + } + num? _currentBitrate; num? get currentBitrate => _currentBitrate; @@ -126,9 +160,9 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi /// Creates a new audio track from the default audio input device. static Future create([ - AudioCaptureOptions? options, + track_options.AudioCaptureOptions? options, ]) async { - options ??= const AudioCaptureOptions(); + options ??= const track_options.AudioCaptureOptions(); final stream = await LocalTrack.createStream(options); final track = LocalAudioTrack( @@ -142,6 +176,16 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi await track.setProcessor(options.processor); } + // Per-component processing modes are not part of standard capture + // constraints; apply them through the native audio processing path. + final processing = options.processing; + if (processing.echoCancellationMode != track_options.AudioProcessingMode.automatic || + processing.noiseSuppressionMode != track_options.AudioProcessingMode.automatic || + processing.autoGainControlMode != track_options.AudioProcessingMode.automatic || + processing.highPassFilterMode != track_options.AudioProcessingMode.automatic) { + await track.setAudioProcessingOptions(processing); + } + return track; } } diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index d5bfc5bbf..aa0ba9f62 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -242,8 +242,79 @@ abstract class VideoCaptureOptions extends LocalTrackOptions { Map toMediaConstraintsMap() => params.toMediaConstraintsMap(); } +/// Selects whether a voice-processing component uses platform or software processing. +enum AudioProcessingMode { + automatic('auto'), + platform('platform'), + software('software'); + + const AudioProcessingMode(this.constraintValue); + + final String constraintValue; +} + +/// Runtime voice-processing options for a [LocalAudioTrack]. +/// +/// These values update the native local audio source without restarting +/// capture. When the track is being sent, the native WebRTC sender reapplies +/// the updated processing config. The effective audio processing module config +/// is shared by the native voice engine/channel, so conflicting updates from +/// multiple local tracks are not isolated per track. +class AudioProcessingOptions { + const AudioProcessingOptions({ + required this.echoCancellation, + required this.noiseSuppression, + required this.autoGainControl, + required this.highPassFilter, + this.echoCancellationMode = AudioProcessingMode.automatic, + this.noiseSuppressionMode = AudioProcessingMode.automatic, + this.autoGainControlMode = AudioProcessingMode.automatic, + this.highPassFilterMode = AudioProcessingMode.automatic, + }); + + const AudioProcessingOptions.communication() + : echoCancellation = true, + noiseSuppression = true, + autoGainControl = true, + highPassFilter = true, + echoCancellationMode = AudioProcessingMode.automatic, + noiseSuppressionMode = AudioProcessingMode.automatic, + autoGainControlMode = AudioProcessingMode.automatic, + highPassFilterMode = AudioProcessingMode.automatic; + + const AudioProcessingOptions.raw() + : echoCancellation = false, + noiseSuppression = false, + autoGainControl = false, + highPassFilter = false, + echoCancellationMode = AudioProcessingMode.automatic, + noiseSuppressionMode = AudioProcessingMode.automatic, + autoGainControlMode = AudioProcessingMode.automatic, + highPassFilterMode = AudioProcessingMode.automatic; + + final bool echoCancellation; + final bool noiseSuppression; + final bool autoGainControl; + final bool highPassFilter; + final AudioProcessingMode echoCancellationMode; + final AudioProcessingMode noiseSuppressionMode; + final AudioProcessingMode autoGainControlMode; + final AudioProcessingMode highPassFilterMode; + + Map toMap() => { + 'echoCancellation': echoCancellation, + 'noiseSuppression': noiseSuppression, + 'autoGainControl': autoGainControl, + 'highPassFilter': highPassFilter, + 'echoCancellationMode': echoCancellationMode.constraintValue, + 'noiseSuppressionMode': noiseSuppressionMode.constraintValue, + 'autoGainControlMode': autoGainControlMode.constraintValue, + 'highPassFilterMode': highPassFilterMode.constraintValue, + }; +} + /// Options used when creating a [LocalAudioTrack]. -class AudioCaptureOptions extends LocalTrackOptions { +class AudioCaptureOptions extends LocalTrackOptions implements AudioProcessingOptions { /// The deviceId of the capture device to use. /// Available deviceIds can be obtained through `flutter_webrtc`: /// ``` @@ -256,22 +327,38 @@ class AudioCaptureOptions extends LocalTrackOptions { /// Attempt to use noiseSuppression option (if supported by the platform) /// See https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings/noiseSuppression /// Defaults to true. + @override final bool noiseSuppression; /// Attempt to use echoCancellation option (if supported by the platform) /// See https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings/echoCancellation /// Defaults to true. + @override final bool echoCancellation; /// Attempt to use autoGainControl option (if supported by the platform) /// See https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints/autoGainControl /// Defaults to true. + @override final bool autoGainControl; /// Attempt to use highPassFilter options (if supported by the platform) /// Defaults to false. + @override final bool highPassFilter; + @override + final AudioProcessingMode echoCancellationMode; + + @override + final AudioProcessingMode noiseSuppressionMode; + + @override + final AudioProcessingMode autoGainControlMode; + + @override + final AudioProcessingMode highPassFilterMode; + /// Attempt to use typingNoiseDetection option (if supported by the platform) /// Defaults to true. final bool typingNoiseDetection; @@ -292,12 +379,30 @@ class AudioCaptureOptions extends LocalTrackOptions { this.echoCancellation = true, this.autoGainControl = true, this.highPassFilter = false, + this.echoCancellationMode = AudioProcessingMode.automatic, + this.noiseSuppressionMode = AudioProcessingMode.automatic, + this.autoGainControlMode = AudioProcessingMode.automatic, + this.highPassFilterMode = AudioProcessingMode.automatic, this.voiceIsolation = true, this.typingNoiseDetection = true, this.stopAudioCaptureOnMute = true, this.processor, }); + AudioProcessingOptions get processing => AudioProcessingOptions( + echoCancellation: echoCancellation, + noiseSuppression: noiseSuppression, + autoGainControl: autoGainControl, + highPassFilter: highPassFilter, + echoCancellationMode: echoCancellationMode, + noiseSuppressionMode: noiseSuppressionMode, + autoGainControlMode: autoGainControlMode, + highPassFilterMode: highPassFilterMode, + ); + + @override + Map toMap() => processing.toMap(); + @override Map toMediaConstraintsMap() { final constraints = {}; @@ -358,15 +463,30 @@ class AudioCaptureOptions extends LocalTrackOptions { bool? echoCancellation, bool? autoGainControl, bool? highPassFilter, + AudioProcessingMode? echoCancellationMode, + AudioProcessingMode? noiseSuppressionMode, + AudioProcessingMode? autoGainControlMode, + AudioProcessingMode? highPassFilterMode, + AudioProcessingOptions? processing, + bool? voiceIsolation, bool? typingNoiseDetection, + bool? stopAudioCaptureOnMute, + TrackProcessor? processor, }) { return AudioCaptureOptions( deviceId: deviceId ?? this.deviceId, - noiseSuppression: noiseSuppression ?? this.noiseSuppression, - echoCancellation: echoCancellation ?? this.echoCancellation, - autoGainControl: autoGainControl ?? this.autoGainControl, - highPassFilter: highPassFilter ?? this.highPassFilter, + noiseSuppression: processing?.noiseSuppression ?? noiseSuppression ?? this.noiseSuppression, + echoCancellation: processing?.echoCancellation ?? echoCancellation ?? this.echoCancellation, + autoGainControl: processing?.autoGainControl ?? autoGainControl ?? this.autoGainControl, + highPassFilter: processing?.highPassFilter ?? highPassFilter ?? this.highPassFilter, + echoCancellationMode: processing?.echoCancellationMode ?? echoCancellationMode ?? this.echoCancellationMode, + noiseSuppressionMode: processing?.noiseSuppressionMode ?? noiseSuppressionMode ?? this.noiseSuppressionMode, + autoGainControlMode: processing?.autoGainControlMode ?? autoGainControlMode ?? this.autoGainControlMode, + highPassFilterMode: processing?.highPassFilterMode ?? highPassFilterMode ?? this.highPassFilterMode, + voiceIsolation: voiceIsolation ?? this.voiceIsolation, typingNoiseDetection: typingNoiseDetection ?? this.typingNoiseDetection, + stopAudioCaptureOnMute: stopAudioCaptureOnMute ?? this.stopAudioCaptureOnMute, + processor: processor ?? this.processor, ); } } @@ -388,3 +508,56 @@ class AudioOutputOptions { ); } } + +/// Result code from applying [AudioProcessingOptions], mirroring the native +/// `AudioProcessingOptionsResult`. `applied`/`stored` are success; the rest are +/// rejections. +enum AudioProcessingOptionsResultCode { + applied('applied'), + stored('stored'), + rejectedRemoteTrack('rejectedRemoteTrack'), + rejectedInvalidCombination('rejectedInvalidCombination'), + rejectedPlatformUnavailable('rejectedPlatformUnavailable'), + applyFailed('applyFailed'); + + const AudioProcessingOptionsResultCode(this.value); + + final String value; + + static AudioProcessingOptionsResultCode fromValue(String? value) => + AudioProcessingOptionsResultCode.values.firstWhere( + (e) => e.value == value, + orElse: () => AudioProcessingOptionsResultCode.applyFailed, + ); + + bool get isSuccess => + this == AudioProcessingOptionsResultCode.applied || this == AudioProcessingOptionsResultCode.stored; +} + +/// Thrown when the native layer rejects requested [AudioProcessingOptions] +/// (e.g. an invalid platform/software combination, or platform processing that +/// is unavailable on the device). +class AudioProcessingException implements Exception { + AudioProcessingException(this.code, this.message); + + final AudioProcessingOptionsResultCode code; + final String message; + + @override + String toString() => 'AudioProcessingException(${code.value}): $message'; +} + +/// Outcome of applying [AudioProcessingOptions]. +/// +/// Returned for operational outcomes — `applied`/`stored` (success), and the +/// device-capability rejections `rejectedPlatformUnavailable`/`applyFailed` +/// (inspect [isSuccess]). A malformed request (incompatible modes, or a +/// non-local track) throws [AudioProcessingException] instead. +class AudioProcessingApplyResult { + AudioProcessingApplyResult(this.code, this.message); + + final AudioProcessingOptionsResultCode code; + final String message; + + bool get isSuccess => code.isSuccess; +} diff --git a/macos/livekit_client.podspec b/macos/livekit_client.podspec index 16c318b85..d96841f45 100644 --- a/macos/livekit_client.podspec +++ b/macos/livekit_client.podspec @@ -16,6 +16,6 @@ Pod::Spec.new do |s| s.static_framework = true s.dependency 'FlutterMacOS' - s.dependency 'WebRTC-SDK', '144.7559.01' + s.dependency 'WebRTC-SDK', '144.7559.09' s.dependency 'flutter_webrtc' end diff --git a/pubspec.lock b/pubspec.lock index 39350a0a9..3aefe0abc 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -284,10 +284,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: "8b220dc006c4891266735e516f7679bd08b7caaf7c36b1a93fb9357cec555f92" + sha256: d8c89028d29e5693742190285b2e3c8a117531b0960ae0693d84273a53968d28 url: "https://pub.dev" source: hosted - version: "1.4.0" + version: "1.5.0" frontend_server_client: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 2c8783473..2c768472e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -46,7 +46,7 @@ dependencies: json_annotation: ^4.9.0 # Fix version to avoid version conflicts between WebRTC-SDK pods, which both this package and flutter_webrtc depend on. - flutter_webrtc: 1.4.0 + flutter_webrtc: 1.5.0 dart_webrtc: ^1.8.0 dev_dependencies: diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index a2d8d8639..fa2367bbe 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -388,6 +388,121 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { return versions.map { String($0) }.joined(separator: ".") } + public func handleSetAudioProcessingOptions(args: [String: Any?], result: @escaping FlutterResult) { + guard let trackId = args["trackId"] as? String else { + result(FlutterError(code: "setAudioProcessingOptions", message: "trackId is required", details: nil)) + return + } + + let webrtc = FlutterWebRTCPlugin.sharedSingleton() + guard let localTrack = webrtc?.localTracks?[trackId] as? LocalAudioTrack, + let audioTrack = localTrack.track() as? RTCAudioTrack + else { + result(FlutterError(code: "setAudioProcessingOptions", message: "track is not a local audio track", details: nil)) + return + } + + let options = RTCAudioProcessingOptions( + echoCancellationOptions: RTCAudioProcessingComponentOptions( + enabled: (args["echoCancellation"] as? Bool) ?? true, + mode: LiveKitPlugin.audioProcessingMode(from: args["echoCancellationMode"] as? String) + ), + noiseSuppressionOptions: RTCAudioProcessingComponentOptions( + enabled: (args["noiseSuppression"] as? Bool) ?? true, + mode: LiveKitPlugin.audioProcessingMode(from: args["noiseSuppressionMode"] as? String) + ), + autoGainControlOptions: RTCAudioProcessingComponentOptions( + enabled: (args["autoGainControl"] as? Bool) ?? true, + mode: LiveKitPlugin.audioProcessingMode(from: args["autoGainControlMode"] as? String) + ), + highPassFilterOptions: RTCAudioProcessingComponentOptions( + enabled: (args["highPassFilter"] as? Bool) ?? false, + mode: LiveKitPlugin.audioProcessingMode(from: args["highPassFilterMode"] as? String) + ) + ) + + let processingResult = audioTrack.setAudioProcessingOptions(options) + result([ + "result": processingResult.isSuccess, + "code": LiveKitPlugin.audioProcessingResultCodeString(processingResult.code), + "message": processingResult.message, + ]) + } + + static func audioProcessingMode(from string: String?) -> RTCAudioProcessingMode { + switch string { + case "platform": return .platform + case "software": return .software + default: return .automatic + } + } + + static func audioProcessingResultCodeString(_ code: RTCAudioProcessingOptionsResultCode) -> String { + switch code { + case .applied: return "applied" + case .stored: return "stored" + case .rejectedRemoteTrack: return "rejectedRemoteTrack" + case .rejectedInvalidCombination: return "rejectedInvalidCombination" + case .rejectedPlatformUnavailable: return "rejectedPlatformUnavailable" + case .applyFailed: return "applyFailed" + @unknown default: return "applyFailed" + } + } + + public func handleGetAudioProcessingState(result: @escaping FlutterResult) { + guard let factory = FlutterWebRTCPlugin.sharedSingleton()?.peerConnectionFactory else { + result(nil) + return + } + result(LiveKitPlugin.toMap(state: factory.audioProcessingState)) + } + + static func audioProcessingModeString(_ mode: RTCAudioProcessingMode) -> String { + switch mode { + case .platform: return "platform" + case .software: return "software" + default: return "auto" + } + } + + static func audioProcessingImplementationString(_ implementation: RTCAudioProcessingImplementation) -> String { + switch implementation { + case .disabled: return "disabled" + case .software: return "software" + case .platform: return "platform" + case .softwareAndPlatform: return "softwareAndPlatform" + default: return "unknown" + } + } + + static func toMap(component state: RTCAudioProcessingComponentState) -> [String: Any] { + var map: [String: Any] = [ + "isSoftwareResolved": state.isSoftwareResolved, + "isSoftwareActive": state.isSoftwareActive, + "isPlatformAvailable": state.isPlatformAvailable, + "isPlatformResolved": state.isPlatformResolved, + "isPlatformActive": state.isPlatformActive, + "effective": audioProcessingImplementationString(state.effective), + ] + if let requested = state.requested { + map["requested"] = [ + "enabled": requested.isEnabled, + "mode": audioProcessingModeString(requested.mode), + ] + } + return map + } + + static func toMap(state: RTCAudioProcessingState) -> [String: Any] { + [ + "hasAudioProcessingModule": state.hasAudioProcessingModule, + "echoCancellation": toMap(component: state.echoCancellation), + "noiseSuppression": toMap(component: state.noiseSuppression), + "autoGainControl": toMap(component: state.autoGainControl), + "highPassFilter": toMap(component: state.highPassFilter), + ] + } + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { guard let args = call.arguments as? [String: Any?] else { print("[LiveKit] arguments must be a dictionary") @@ -406,6 +521,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { handleStartAudioRenderer(args: args, result: result) case "stopAudioRenderer": handleStopAudioRenderer(args: args, result: result) + case "setAudioProcessingOptions": + handleSetAudioProcessingOptions(args: args, result: result) + case "getAudioProcessingState": + handleGetAudioProcessingState(result: result) case "osVersionString": result(LiveKitPlugin.osVersionString()) #if os(iOS) From a1934012f9636d2f72224171e868df6e139f8df9 Mon Sep 17 00:00:00 2001 From: xianshijing-lk Date: Thu, 18 Jun 2026 07:32:03 +0800 Subject: [PATCH 13/18] feat: add deployment field to agent dispatch (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add `deployment` field to `RoomAgentDispatch` for targeting specific agent deployments - Add `agentDeployment` to `TokenRequestOptions` to pass deployment through token requests - Update generated JSON serialization code The `deployment` field allows targeting a specific agent deployment (e.g., "staging"). Leave empty to target the production deployment. Related PRs: - node-sdks: https://github.com/livekit/node-sdks/pull/675 - python-sdks: https://github.com/livekit/python-sdks/pull/722 - rust-sdks: https://github.com/livekit/rust-sdks/pull/1176 - client-sdk-swift: https://github.com/livekit/client-sdk-swift/pull/1043 - client-sdk-js: https://github.com/livekit/client-sdk-js/pull/1971 ## Usage ```dart final options = TokenRequestOptions( roomName: 'my-room', agentName: 'my-agent', agentDeployment: 'staging', // Optional: target specific deployment ); ``` Or directly via `RoomAgentDispatch`: ```dart final dispatch = RoomAgentDispatch( agentName: 'my-agent', metadata: 'my-metadata', deployment: 'staging', ); ``` ## Test plan ### Unit Tests ```bash flutter test flutter test test/token/token_source_test.dart -v ``` ### Manual Verification **1. Verify JSON serialization includes deployment:** ```dart final dispatch = RoomAgentDispatch( agentName: 'my-agent', deployment: 'staging', ); final json = dispatch.toJson(); print(json); // Should include 'deployment': 'staging' ``` **2. Verify TokenRequestOptions converts to request correctly:** ```dart final options = TokenRequestOptions( roomName: 'test-room', agentName: 'my-agent', agentDeployment: 'staging', ); final request = options.toRequest(); print(request.roomConfiguration?.agents?.first?.deployment); // Should print 'staging' ``` **3. Verify JSON round-trip:** ```dart final original = RoomAgentDispatch( agentName: 'my-agent', deployment: 'staging', ); final json = original.toJson(); final restored = RoomAgentDispatch.fromJson(json); assert(restored.deployment == 'staging'); ``` ### End-to-End Verification 1. Use TokenSource to get credentials with agentDeployment set 2. Connect to room - agent with matching deployment should join 3. Verify only staging agent receives the dispatch 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 --- lib/src/token_source/room_configuration.dart | 4 ++++ lib/src/token_source/room_configuration.g.dart | 2 ++ lib/src/token_source/token_source.dart | 10 ++++++++-- lib/src/token_source/token_source.g.dart | 2 ++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/src/token_source/room_configuration.dart b/lib/src/token_source/room_configuration.dart index 24ce8af8b..82a3898b1 100644 --- a/lib/src/token_source/room_configuration.dart +++ b/lib/src/token_source/room_configuration.dart @@ -26,9 +26,13 @@ class RoomAgentDispatch { /// Metadata for the agent. final String? metadata; + /// Optional deployment to target. Leave empty to target the production deployment. + final String? deployment; + const RoomAgentDispatch({ this.agentName, this.metadata, + this.deployment, }); factory RoomAgentDispatch.fromJson(Map json) => _$RoomAgentDispatchFromJson(json); diff --git a/lib/src/token_source/room_configuration.g.dart b/lib/src/token_source/room_configuration.g.dart index c7517a575..9bfaee4d2 100644 --- a/lib/src/token_source/room_configuration.g.dart +++ b/lib/src/token_source/room_configuration.g.dart @@ -9,11 +9,13 @@ part of 'room_configuration.dart'; RoomAgentDispatch _$RoomAgentDispatchFromJson(Map json) => RoomAgentDispatch( agentName: json['agent_name'] as String?, metadata: json['metadata'] as String?, + deployment: json['deployment'] as String?, ); Map _$RoomAgentDispatchToJson(RoomAgentDispatch instance) => { if (instance.agentName case final value?) 'agent_name': value, if (instance.metadata case final value?) 'metadata': value, + if (instance.deployment case final value?) 'deployment': value, }; RoomConfiguration _$RoomConfigurationFromJson(Map json) => RoomConfiguration( diff --git a/lib/src/token_source/token_source.dart b/lib/src/token_source/token_source.dart index f2f4e8304..392c62557 100644 --- a/lib/src/token_source/token_source.dart +++ b/lib/src/token_source/token_source.dart @@ -43,6 +43,9 @@ class TokenRequestOptions { /// Metadata passed to the agent job. final String? agentMetadata; + /// Optional deployment to target. Leave empty to target the production deployment. + final String? agentDeployment; + const TokenRequestOptions({ this.roomName, this.participantName, @@ -51,6 +54,7 @@ class TokenRequestOptions { this.participantAttributes, this.agentName, this.agentMetadata, + this.agentDeployment, }); factory TokenRequestOptions.fromJson(Map json) => _$TokenRequestOptionsFromJson(json); @@ -58,8 +62,8 @@ class TokenRequestOptions { /// Converts this options object to a wire-format request. TokenSourceRequest toRequest() { - final List? agents = (agentName != null || agentMetadata != null) - ? [RoomAgentDispatch(agentName: agentName, metadata: agentMetadata)] + final List? agents = (agentName != null || agentMetadata != null || agentDeployment != null) + ? [RoomAgentDispatch(agentName: agentName, metadata: agentMetadata, deployment: agentDeployment)] : null; return TokenSourceRequest( @@ -83,6 +87,7 @@ class TokenRequestOptions { other.participantMetadata == participantMetadata && other.agentName == agentName && other.agentMetadata == agentMetadata && + other.agentDeployment == agentDeployment && const MapEquality().equals(other.participantAttributes, participantAttributes); } @@ -95,6 +100,7 @@ class TokenRequestOptions { participantMetadata, agentName, agentMetadata, + agentDeployment, const MapEquality().hash(participantAttributes), ); } diff --git a/lib/src/token_source/token_source.g.dart b/lib/src/token_source/token_source.g.dart index a5ac7308e..ec930e18c 100644 --- a/lib/src/token_source/token_source.g.dart +++ b/lib/src/token_source/token_source.g.dart @@ -16,6 +16,7 @@ TokenRequestOptions _$TokenRequestOptionsFromJson(Map json) => ), agentName: json['agentName'] as String?, agentMetadata: json['agentMetadata'] as String?, + agentDeployment: json['agentDeployment'] as String?, ); Map _$TokenRequestOptionsToJson(TokenRequestOptions instance) => { @@ -26,6 +27,7 @@ Map _$TokenRequestOptionsToJson(TokenRequestOptions instance) = if (instance.participantAttributes case final value?) 'participantAttributes': value, if (instance.agentName case final value?) 'agentName': value, if (instance.agentMetadata case final value?) 'agentMetadata': value, + if (instance.agentDeployment case final value?) 'agentDeployment': value, }; TokenSourceRequest _$TokenSourceRequestFromJson(Map json) => TokenSourceRequest( From 55af814ab6c837aaee01ec4e3cabbba0854b90a1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 19 Jun 2026 08:31:26 +0900 Subject: [PATCH 14/18] feat(audio): add AudioManager session and routing APIs (#1108) ## What Adds first-class, process-wide audio session and routing control through `AudioManager` on iOS and Android. LiveKit owns the platform audio session by default, while apps that need exact platform behavior can switch to manual mode and apply typed session configs. ## API and behavior - **Automatic by default**: calls need no setup. LiveKit applies a managed communication policy. - **iOS automatic mode**: the native WebRTC audio-engine delegate drives `AVAudioSession` from engine lifecycle events. Listen-only playout uses `playback`; recording uses `playAndRecord`. - **Android automatic mode**: LiveKit uses a communication session through the new AudioSwitch-backed `LKAudioSwitchManager`. - **Manual mode**: `setAudioSessionOptions(...)` and `deactivateAudioSession()` switch `AudioManager` to manual mode. `setAudioSessionManagementMode(AudioSessionManagementMode.automatic)` hands lifecycle control back to LiveKit. - **Typed options**: `AudioSessionOptions.communication()` and `AudioSessionOptions.media()` pre-fill Apple and Android configs, with per-platform overrides applied verbatim in manual mode. - **Speaker routing**: `AudioManager.instance.setSpeakerOutputPreferred(...)` owns speaker preference and forced speaker routing. Wired and Bluetooth devices still win unless `force: true`. ## Compatibility - Existing calls keep working without audio-session setup. - `Hardware` audio members and `Room.setSpeakerOn(...)` are deprecated forwarders to `AudioManager`. - `flutter_webrtc` native audio-session management is disabled so LiveKit has one owner for the session. - `bypassVoiceProcessing` now only controls WebRTC voice processing; it no longer changes the session intent. ## Docs and tests - Adds `docs/audio.md` and updates the README audio sections. - Adds coverage for session options, automatic/manual mode transitions, Apple/Android policy resolution, routing serialization, and engine-state observation. - Verified locally with `dart analyze`, `flutter test test/audio/audio_session_test.dart`, and `flutter test --reporter compact`. --- .changes/audio-manager-api | 1 + README.md | 33 +- android/build.gradle | 4 + .../io/livekit/plugin/LKAudioSwitchManager.kt | 322 ++++++++++ .../kotlin/io/livekit/plugin/LiveKitPlugin.kt | 29 + docs/audio.md | 226 +++++++ example/lib/pages/room.dart | 2 +- example/lib/widgets/controls.dart | 4 +- lib/livekit_client.dart | 1 + .../audio/android_audio_session_adapter.dart | 38 ++ lib/src/audio/audio_manager.dart | 268 +++++++- lib/src/audio/audio_session.dart | 275 +++++++++ lib/src/audio/audio_session_policy.dart | 61 ++ lib/src/core/room.dart | 49 +- lib/src/hardware/hardware.dart | 90 +-- lib/src/livekit.dart | 25 +- lib/src/participant/local.dart | 4 - lib/src/support/native.dart | 88 ++- lib/src/support/native_audio.dart | 85 +-- lib/src/support/value_or_absent.dart | 63 ++ lib/src/track/audio_management.dart | 162 +---- lib/src/track/options.dart | 2 +- pubspec.lock | 4 +- pubspec.yaml | 2 +- shared_swift/LiveKitPlugin.swift | 378 ++++++++++-- test/audio/audio_session_test.dart | 579 ++++++++++++++++++ 26 files changed, 2408 insertions(+), 387 deletions(-) create mode 100644 .changes/audio-manager-api create mode 100644 android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt create mode 100644 docs/audio.md create mode 100644 lib/src/audio/android_audio_session_adapter.dart create mode 100644 lib/src/audio/audio_session.dart create mode 100644 lib/src/audio/audio_session_policy.dart create mode 100644 lib/src/support/value_or_absent.dart create mode 100644 test/audio/audio_session_test.dart diff --git a/.changes/audio-manager-api b/.changes/audio-manager-api new file mode 100644 index 000000000..2fd918ba1 --- /dev/null +++ b/.changes/audio-manager-api @@ -0,0 +1 @@ +minor type="added" "AudioManager audio session options with engine-driven native lifecycle and platform routing controls" diff --git a/README.md b/README.md index 9a80d3dfb..31a36b056 100644 --- a/README.md +++ b/README.md @@ -150,30 +150,18 @@ void main() async { #### Audio Modes -By default, we use the `communication` audio mode on Android which works best for two-way voice communication. +By default LiveKit uses the `communication` audio mode on Android, which works best for two-way voice communication. -If your app is media playback oriented and does not need the use of the device's microphone, you can use the `media` -audio mode which will provide better audio quality. +If your app is media playback oriented and does not need the device's microphone, apply the `media` session yourself. This +switches `AudioManager` to manual mode, where your app owns the session. ```dart -import 'package:flutter_webrtc/flutter_webrtc.dart' as webrtc; - -Future _initializeAndroidAudioSettings() async { - await webrtc.WebRTC.initialize(options: { - 'androidAudioConfiguration': webrtc.AndroidAudioConfiguration.media.toMap() - }); - webrtc.Helper.setAndroidAudioConfiguration( - webrtc.AndroidAudioConfiguration.media); -} - -void main() async { - await _initializeAudioSettings(); - runApp(const MyApp()); -} +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.media(), +); ``` -Note: the audio routing will become controlled by the system and cannot be manually changed with functions like -`Hardware.selectAudioOutput`. +See the [audio session guide](https://github.com/livekit/client-sdk-flutter/blob/main/docs/audio.md) for more. ### Desktop support @@ -322,6 +310,13 @@ Widget build(BuildContext context) { Audio tracks are played automatically as long as you are subscribed to them. +LiveKit owns the platform audio session through `AudioManager`. A call is managed automatically with no setup. Speaker routing and, when you need it, manual session control go through the same object. See the [audio session guide](https://github.com/livekit/client-sdk-flutter/blob/main/docs/audio.md) for examples covering the automatic and manual modes, speaker routing, per platform overrides, and migration from the older `Hardware` APIs. + +```dart +// A call is managed automatically. Route to the speaker when you want. +await AudioManager.instance.setSpeakerOutputPreferred(true); +``` + ### Handling changes LiveKit client makes it simple to build declarative UI that reacts to state changes. It notifies changes in two ways diff --git a/android/build.gradle b/android/build.gradle index 82e5dd8d7..5a671d109 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -18,6 +18,7 @@ allprojects { repositories { google() mavenCentral() + maven { url 'https://jitpack.io' } } } @@ -61,6 +62,9 @@ android { testImplementation("org.mockito:mockito-core:5.0.0") implementation 'io.github.webrtc-sdk:android:144.7559.09' implementation 'io.livekit:noise:2.0.0' + // Audio device/focus/mode routing. Pinned to the same revision used by + // the LiveKit Android SDK (AudioSwitchHandler). + implementation 'com.github.davidliu:audioswitch:039a35aefab7747c557242fa216c9ea11743b604' } testOptions { diff --git a/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt new file mode 100644 index 000000000..463d5ceec --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt @@ -0,0 +1,322 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.os.HandlerThread +import com.twilio.audioswitch.AbstractAudioSwitch +import com.twilio.audioswitch.AudioDevice +import com.twilio.audioswitch.AudioSwitch +import com.twilio.audioswitch.CommDeviceAudioSwitch +import com.twilio.audioswitch.LegacyAudioSwitch + +/** + * Manages the Android platform audio session (audio mode, audio focus, and + * output routing) for the LiveKit Flutter SDK, built on top of [AudioSwitch]. + * + * This is LiveKit's own port of the audio-handling best practices from the + * LiveKit Android SDK (`AudioSwitchHandler`) and flutter_webrtc + * (`AudioSwitchManager`), so the Flutter SDK can own the platform audio session + * directly instead of delegating to flutter_webrtc's native audio management. + * + * [AudioSwitch] is not thread-safe, so every interaction with it runs on a + * single dedicated [HandlerThread]. + */ +internal class LKAudioSwitchManager(private val context: Context) { + // AudioSwitch is not threadsafe, so confine all access to a single long-lived + // thread. The AudioSwitch instance is recreated per active session, while + // queued lifecycle work stays serialized on this thread. + private val thread = HandlerThread("LKAudioSwitchThread").also { it.start() } + private val handler = Handler(thread.looper) + + private var audioSwitch: AbstractAudioSwitch? = null + private var isActive = false + + // Configuration. Defaults mirror a communication/VoIP session and match the + // AudioSwitchHandler defaults in the LiveKit Android SDK. + private var manageAudioFocus = true + private var audioMode = AudioManager.MODE_IN_COMMUNICATION + private var focusMode = AudioManager.AUDIOFOCUS_GAIN + private var audioStreamType = AudioManager.STREAM_VOICE_CALL + private var audioAttributeUsageType = AudioAttributes.USAGE_VOICE_COMMUNICATION + private var audioAttributeContentType = AudioAttributes.CONTENT_TYPE_SPEECH + private var forceHandleAudioRouting = false + + private var speakerOutputPreferred = true + private var speakerOutputForced = false + + /** + * Apply an audio session configuration. Unspecified keys keep their current + * value. When the session is already active, changes that only take effect at + * activate() time trigger a deactivate and activate cycle so they apply live. + */ + @Synchronized + fun configure(configuration: Map) { + val previous = sessionConfigSnapshot() + (configuration["manageAudioFocus"] as? Boolean)?.let { manageAudioFocus = it } + audioModeForName(configuration["androidAudioMode"] as? String)?.let { audioMode = it } + focusModeForName(configuration["androidAudioFocusMode"] as? String)?.let { focusMode = it } + streamTypeForName(configuration["androidAudioStreamType"] as? String)?.let { audioStreamType = it } + usageTypeForName(configuration["androidAudioAttributesUsageType"] as? String)?.let { audioAttributeUsageType = it } + contentTypeForName(configuration["androidAudioAttributesContentType"] as? String)?.let { audioAttributeContentType = it } + (configuration["forceHandleAudioRouting"] as? Boolean)?.let { forceHandleAudioRouting = it } + val sessionConfig = sessionConfigSnapshot() + val sessionConfigChanged = sessionConfig != previous + val speakerRouting = speakerRoutingSnapshot() + + handler.post { + val switch = audioSwitch ?: return@post + applyConfiguration(switch, sessionConfig) + // AudioSwitch applies the audio mode, focus, and attributes at activate() + // time, so a live reconfiguration (e.g. communication to media) needs a + // deactivate and activate cycle to take effect on an already active + // session. Reassert speaker routing afterward. + if (isActive && sessionConfigChanged) { + switch.deactivate() + switch.activate() + applySpeakerRouting(switch, speakerRouting) + } + } + } + + // Snapshot of the AudioSwitch properties applied only at activate() time, used + // to detect when a live session needs a deactivate and activate cycle to pick + // up a configuration change. + private fun sessionConfigSnapshot() = SessionConfig( + manageAudioFocus = manageAudioFocus, + audioMode = audioMode, + focusMode = focusMode, + audioStreamType = audioStreamType, + audioAttributeUsageType = audioAttributeUsageType, + audioAttributeContentType = audioAttributeContentType, + forceHandleAudioRouting = forceHandleAudioRouting, + ) + + /** Create (if needed) and activate the audio session: acquire focus, set mode and routing. */ + @Synchronized + fun start() { + val sessionConfig = sessionConfigSnapshot() + val speakerRouting = speakerRoutingSnapshot() + handler.post { + val switch = audioSwitch ?: createSwitch(sessionConfig, speakerRouting).also { audioSwitch = it } + if (!isActive) { + switch.activate() + applySpeakerRouting(switch, speakerRouting) + isActive = true + } + } + } + + /** Deactivate and tear down the audio session: release focus and restore the previous mode. */ + @Synchronized + fun stop() { + handler.post { + audioSwitch?.stop() + audioSwitch = null + isActive = false + } + } + + /** Final cleanup when the plugin detaches. The manager must not be used after this. */ + @Synchronized + fun dispose() { + handler.post { + audioSwitch?.stop() + audioSwitch = null + isActive = false + thread.quitSafely() + } + } + + /** + * Prefer routing to/from the speaker, letting a connected headset keep priority + * unless [force] is true. + */ + @Synchronized + fun setSpeakerphoneOn(enable: Boolean, force: Boolean) { + speakerOutputPreferred = enable + speakerOutputForced = enable && force + val speakerRouting = speakerRoutingSnapshot() + handler.post { + val switch = audioSwitch ?: return@post + applySpeakerRouting(switch, speakerRouting) + } + } + + private fun createSwitch( + sessionConfig: SessionConfig, + speakerRouting: SpeakerRouting, + ): AbstractAudioSwitch { + val focusListener = AudioManager.OnAudioFocusChangeListener { } + // API-aware switch selection, matching the LiveKit Android SDK's + // AudioSwitchHandler: CommDeviceAudioSwitch uses the modern + // AudioManager.setCommunicationDevice routing on API 31+. + val switch = when { + Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> + CommDeviceAudioSwitch(context, false, focusListener, speakerRouting.preferredDeviceList) + + Build.VERSION.SDK_INT >= Build.VERSION_CODES.M -> + AudioSwitch(context, false, focusListener, speakerRouting.preferredDeviceList) + + else -> + LegacyAudioSwitch(context, false, focusListener, speakerRouting.preferredDeviceList) + } + applyConfiguration(switch, sessionConfig) + switch.start { _, _ -> } + return switch + } + + private fun applyConfiguration(switch: AbstractAudioSwitch, sessionConfig: SessionConfig) { + switch.manageAudioFocus = sessionConfig.manageAudioFocus + switch.audioMode = sessionConfig.audioMode + switch.focusMode = sessionConfig.focusMode + switch.audioStreamType = sessionConfig.audioStreamType + switch.audioAttributeUsageType = sessionConfig.audioAttributeUsageType + switch.audioAttributeContentType = sessionConfig.audioAttributeContentType + switch.forceHandleAudioRouting = sessionConfig.forceHandleAudioRouting + } + + private fun applySpeakerRouting(switch: AbstractAudioSwitch, speakerRouting: SpeakerRouting) { + switch.setPreferredDeviceList(speakerRouting.preferredDeviceList) + val forcedSpeaker = if (speakerRouting.speakerOutputForced) { + switch.availableAudioDevices.firstOrNull { it is AudioDevice.Speakerphone } + } else { + null + } + // AudioSwitch selections are sticky. Use them only for forced speaker output. + // Clearing the selection lets the preferred-device list handle normal routing + // and headset hot-plug priority. + switch.selectDevice(forcedSpeaker) + } + + private fun speakerRoutingSnapshot() = SpeakerRouting( + speakerOutputForced = speakerOutputForced, + preferredDeviceList = preferredDeviceList( + speakerOutputPreferred = speakerOutputPreferred, + speakerOutputForced = speakerOutputForced, + ), + ) + + private fun preferredDeviceList( + speakerOutputPreferred: Boolean, + speakerOutputForced: Boolean, + ): List> = + when { + speakerOutputForced -> listOf( + AudioDevice.Speakerphone::class.java, + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Earpiece::class.java, + ) + + speakerOutputPreferred -> listOf( + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Speakerphone::class.java, + AudioDevice.Earpiece::class.java, + ) + + else -> listOf( + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Earpiece::class.java, + AudioDevice.Speakerphone::class.java, + ) + } + + private data class SessionConfig( + val manageAudioFocus: Boolean, + val audioMode: Int, + val focusMode: Int, + val audioStreamType: Int, + val audioAttributeUsageType: Int, + val audioAttributeContentType: Int, + val forceHandleAudioRouting: Boolean, + ) + + private data class SpeakerRouting( + val speakerOutputForced: Boolean, + val preferredDeviceList: List>, + ) +} + +// Map the Flutter-side enum names (see android_audio_session_adapter.dart) to +// Android framework constants. Ported from flutter_webrtc's AudioUtils. + +private fun audioModeForName(name: String?): Int? = when (name) { + null -> null + "normal" -> AudioManager.MODE_NORMAL + "callScreening" -> AudioManager.MODE_CALL_SCREENING + "inCall" -> AudioManager.MODE_IN_CALL + "inCommunication" -> AudioManager.MODE_IN_COMMUNICATION + "ringtone" -> AudioManager.MODE_RINGTONE + else -> null +} + +private fun focusModeForName(name: String?): Int? = when (name) { + null -> null + "gain" -> AudioManager.AUDIOFOCUS_GAIN + "gainTransient" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + "gainTransientExclusive" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + "gainTransientMayDuck" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + else -> null +} + +private fun streamTypeForName(name: String?): Int? = when (name) { + null -> null + "accessibility" -> AudioManager.STREAM_ACCESSIBILITY + "alarm" -> AudioManager.STREAM_ALARM + "dtmf" -> AudioManager.STREAM_DTMF + "music" -> AudioManager.STREAM_MUSIC + "notification" -> AudioManager.STREAM_NOTIFICATION + "ring" -> AudioManager.STREAM_RING + "system" -> AudioManager.STREAM_SYSTEM + "voiceCall" -> AudioManager.STREAM_VOICE_CALL + else -> null +} + +private fun usageTypeForName(name: String?): Int? = when (name) { + null -> null + "alarm" -> AudioAttributes.USAGE_ALARM + "assistanceAccessibility" -> AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY + "assistanceNavigationGuidance" -> AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE + "assistanceSonification" -> AudioAttributes.USAGE_ASSISTANCE_SONIFICATION + "assistant" -> AudioAttributes.USAGE_ASSISTANT + "game" -> AudioAttributes.USAGE_GAME + "media" -> AudioAttributes.USAGE_MEDIA + "notification" -> AudioAttributes.USAGE_NOTIFICATION + "notificationEvent" -> AudioAttributes.USAGE_NOTIFICATION_EVENT + "notificationRingtone" -> AudioAttributes.USAGE_NOTIFICATION_RINGTONE + "unknown" -> AudioAttributes.USAGE_UNKNOWN + "voiceCommunication" -> AudioAttributes.USAGE_VOICE_COMMUNICATION + "voiceCommunicationSignalling" -> AudioAttributes.USAGE_VOICE_COMMUNICATION_SIGNALLING + else -> null +} + +private fun contentTypeForName(name: String?): Int? = when (name) { + null -> null + "movie" -> AudioAttributes.CONTENT_TYPE_MOVIE + "music" -> AudioAttributes.CONTENT_TYPE_MUSIC + "sonification" -> AudioAttributes.CONTENT_TYPE_SONIFICATION + "speech" -> AudioAttributes.CONTENT_TYPE_SPEECH + "unknown" -> AudioAttributes.CONTENT_TYPE_UNKNOWN + else -> null +} diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt index 95a242497..925890628 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt @@ -26,6 +26,7 @@ import io.flutter.plugin.common.MethodChannel.MethodCallHandler import io.flutter.plugin.common.MethodChannel.Result import com.cloudwebrtc.webrtc.FlutterWebRTCPlugin +import com.cloudwebrtc.webrtc.audio.AudioSwitchManager import com.cloudwebrtc.webrtc.audio.LocalAudioTrack import io.flutter.plugin.common.BinaryMessenger import org.webrtc.AudioTrack @@ -42,6 +43,7 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private var audioProcessors = mutableMapOf() private var flutterWebRTCPlugin = FlutterWebRTCPlugin.sharedSingleton private var binaryMessenger: BinaryMessenger? = null + private var audioSwitchManager: LKAudioSwitchManager? = null /// The MethodChannel that will the communication between Flutter and native Android /// @@ -50,9 +52,13 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private lateinit var channel: MethodChannel override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + // LiveKit owns the platform audio session, so disable flutter_webrtc's own + // native audio management. Set at registration, before any audio op. + AudioSwitchManager.setAudioSessionManagementEnabled(false) channel = MethodChannel(flutterPluginBinding.binaryMessenger, "livekit_client") channel.setMethodCallHandler(this) binaryMessenger = flutterPluginBinding.binaryMessenger + audioSwitchManager = LKAudioSwitchManager(flutterPluginBinding.applicationContext) } @SuppressLint("SuspiciousIndentation") @@ -350,6 +356,26 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { handleGetAudioProcessingState(result) } + "configureAndroidAudioSession" -> { + @Suppress("UNCHECKED_CAST") + val configuration = call.arguments as? Map ?: emptyMap() + audioSwitchManager?.configure(configuration) + audioSwitchManager?.start() + result.success(null) + } + + "stopAndroidAudioSession" -> { + audioSwitchManager?.stop() + result.success(null) + } + + "setAndroidSpeakerphoneOn" -> { + val enable = call.argument("enable") ?: false + val force = call.argument("force") ?: false + audioSwitchManager?.setSpeakerphoneOn(enable, force) + result.success(null) + } + else -> { result.notImplemented() } @@ -359,6 +385,9 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { channel.setMethodCallHandler(null) + audioSwitchManager?.dispose() + audioSwitchManager = null + // Cleanup all processors audioProcessors.values.forEach { it.cleanup() } audioProcessors.clear() diff --git a/docs/audio.md b/docs/audio.md new file mode 100644 index 000000000..0f11b5c5e --- /dev/null +++ b/docs/audio.md @@ -0,0 +1,226 @@ +# Audio session management + +LiveKit owns the platform audio session on iOS and Android through a single process-wide entry point, `AudioManager`. By default it manages a communication session for you, choosing the right native category, mode, focus, and routing. When you need more control you switch to manual mode and apply typed options yourself. On macOS, `AudioManager` reports native audio-engine state but does not configure a platform audio session. + +`AudioManager` is a singleton, reached through `AudioManager.instance`. It is also where you read back the engine-wide audio processing state, so one object covers both the audio session and the signal processing applied to your audio. + +## Defaults if you do nothing + +With no configuration, LiveKit manages the session automatically with the `communication` intent, which is meant for calls. A call needs no setup. On iOS this resolves to a `playAndRecord` session while the microphone is active and a `playback` session for listen only playout. On Android it resolves to communication mode with voice call routing and audio focus. + +LiveKit disables flutter_webrtc's own native audio management automatically when the plugin loads, so it owns the session without any setup from you. + +Speaker output is preferred by default, but a wired or Bluetooth headset still wins over the speaker. Forced speaker output is off, so the speaker is never forced over a connected headset unless you ask for it. + +The default audio capture options apply standard voice processing, so echo cancellation, noise suppression, and auto gain control are on and the high pass filter is off. You can change this per track with `AudioProcessingOptions`. + +On macOS the audio engine state is reported but no `AVAudioSession` is configured. On web, Windows, and Linux the session APIs do not configure native audio. Speaker switching is available only on iOS and Android, where `AudioManager.instance.canSwitchSpeakerphone` is true. + +## Quick start + +For a call you do not need to configure anything. LiveKit manages a communication session automatically. + +To take control of the session yourself, apply options. This switches `AudioManager` to manual mode, where your app owns the session and LiveKit stops managing it from room and engine lifecycle. + +```dart +import 'package:livekit_client/livekit_client.dart'; + +// Take manual control and apply a media playback session. +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.media(), +); +``` + +See the next section for the full rule. Apply options before connecting when you can. + +## Automatic vs manual mode + +The two modes differ in who owns the session lifecycle. + +In automatic mode (the default) LiveKit manages the session from room, connect, and engine lifecycle and chooses the configuration for you. It does not take session options in this mode. + +In manual mode LiveKit does not touch the session on its own, and your app owns it. Enter manual mode when you need to apply a fixed platform configuration or deactivate the session yourself. + +```dart +// Apply a fixed config. This enters manual mode. +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.media(), +); + +// Later, hand control back to LiveKit. +await AudioManager.instance.setAudioSessionManagementMode( + AudioSessionManagementMode.automatic, +); +``` + +You can also enter manual mode without applying a new config: + +```dart +await AudioManager.instance.setAudioSessionManagementMode( + AudioSessionManagementMode.manual, +); +``` + +To release the active session, call `deactivateAudioSession`. It also enters manual mode if needed, so LiveKit does not immediately reactivate the session from engine lifecycle. + +```dart +// Release the active session when your manual lifecycle no longer needs it. +await AudioManager.instance.deactivateAudioSession(); +``` + +Prefer setting the mode before connecting to a room. + +## Speaker routing + +```dart +// Prefer the speaker. A wired or Bluetooth headset still takes priority. +await AudioManager.instance.setSpeakerOutputPreferred(true); + +// Force the speaker even when a headset is connected. +await AudioManager.instance.setSpeakerOutputPreferred(true, force: true); + +// Route back to the earpiece or the connected headset when supported by the +// active platform session. +await AudioManager.instance.setSpeakerOutputPreferred(false); +``` + +Speaker routing is independent of the management mode and does not switch it. On Android and in iOS automatic mode, LiveKit applies the preference through its managed route policy. In iOS manual mode, the fixed Apple config you apply owns non-forced receiver vs speaker behavior. `force: true` still uses Apple's speaker override when the active category is `playAndRecord`. Read the current preference through `AudioManager.instance.isSpeakerOutputPreferred` and `AudioManager.instance.isSpeakerOutputForced`. `AudioManager.instance.canSwitchSpeakerphone` is true on iOS and Android. + +`Room.setSpeakerOn(...)` is deprecated and forwards to `AudioManager.instance.setSpeakerOutputPreferred`. You can also set an initial preference through `RoomOptions` (`defaultAudioOutputOptions.speakerOn`) before connecting, which LiveKit applies when the session starts. + +## Observing audio engine state + +On iOS and macOS the native audio engine reports when playout and recording turn on and off. This is the source of truth for audio activity. + +```dart +final sub = AudioManager.instance.audioEngineStateStream.listen((state) { + print('playout ${state.isPlayoutEnabled} recording ${state.isRecordingEnabled}'); + if (state.isIdle) { + print('engine is idle'); + } +}); + +// Current snapshot without listening. +final now = AudioManager.instance.audioEngineState; +``` + +## Audio processing + +Session management is one half of the audio stack. The other half is audio processing, the signal processing applied to captured audio such as echo cancellation, noise suppression, auto gain control, and the high pass filter. `AudioManager` is the home for both, and they compose cleanly. + +The session intent decides how the platform treats audio. Processing options decide what happens to captured local microphone audio: + +- A call usually uses the automatic `communication` session. The default `AudioCaptureOptions` enable echo cancellation, noise suppression, and auto gain control, while leaving the high pass filter off. +- Use `AudioProcessingOptions.communication()` when you want all four voice filters on for an existing local audio track. +- Use `AudioProcessingOptions.noProcessing()` for local capture where you want minimal processing, such as high quality recording or app-managed audio effects. + +Processing is applied per local audio track. A malformed request (an incompatible mode, or a remote track) throws, while a request the platform could not honor comes back as an unsuccessful result: + +```dart +try { + final result = await localAudioTrack.setAudioProcessingOptions( + const AudioProcessingOptions.noProcessing(), + ); + if (!result.isSuccess) { + print('audio processing not applied: ${result.code.value} ${result.message}'); + } +} on AudioProcessingException catch (error) { + print('invalid audio processing request: ${error.code.value} ${error.message}'); +} +``` + +The processing module is owned by the native peer connection factory and shared across the whole engine, so the resolved state is read back through `AudioManager` rather than from a single track: + +```dart +final state = await AudioManager.instance.getAudioProcessingState(); +print('echo cancellation in effect: ${state?.echoCancellation.effective}'); +``` + +## Per platform overrides + +When the preset constructors are not enough you can pin exact platform values. Supplying options through `setAudioSessionOptions` switches to manual mode, so these configs are a manual-mode tool. `AudioSessionOptions.communication()` and `AudioSessionOptions.media()` pre-fill Apple and Android configs. Passing `apple` or `android` replaces that platform config rather than merging with the preset. + +```dart +await AudioManager.instance.setAudioSessionOptions( + AudioSessionOptions.communication( + apple: const AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.mixWithOthers, + }, + mode: AppleAudioMode.voiceChat, + ), + android: AndroidAudioSessionConfiguration.communication, + ), +); +``` + +For an `apple` config, your exact category, options, and mode are applied as written. For an `android` config, any field you leave null is omitted so the native manager keeps its current value for that field. + +### Updating options with copyWith + +`AudioSessionOptions.copyWith` uses `ValueOrAbsent` to replace the Apple or Android config as a whole. A bare `copyWith()` keeps the existing config, and `ValueOrAbsent.value(x)` sets a new config. The Apple and Android config objects have their own `copyWith` methods for clearing nullable native fields with `ValueOrAbsent.value(null)`. + +```dart +const base = AudioSessionOptions.communication(); + +// Use the media Android config, keep the Apple config. +final updated = base.copyWith( + android: const ValueOrAbsent.value(AndroidAudioSessionConfiguration.media), +); + +// Clear just the Apple mode field inside the Apple config. +final clearedMode = updated.copyWith( + apple: ValueOrAbsent.value( + updated.apple.copyWith(mode: const ValueOrAbsent.value(null)), + ), +); +``` + +Create a new `AudioSessionOptions.communication()` or `AudioSessionOptions.media()` when you want to start from a different preset config. + +## Platform support + +| Platform | Audio session | Speaker routing | Engine state | +| --- | --- | --- | --- | +| iOS | Automatic mode follows live engine state. Manual mode applies your Apple config verbatim. | Yes. Normal preference respects wired and Bluetooth devices. Forced speaker uses Apple's speaker override while the active category is `playAndRecord`. | Yes, from native WebRTC engine events. | +| macOS | Not configured. There is no `AVAudioSession`. | No. `canSwitchSpeakerphone` is false. | Yes, the same engine events are reported. | +| Android | Automatic mode uses the communication session (in-communication mode, voice call stream). A media session is available in manual mode. Managed through LiveKit's AudioSwitch handler. | Yes. Normal preference orders headsets before the speaker. Forced speaker selects the speaker device. | Not reported, the Dart state stays idle. | +| Web, Windows, Linux | Not configured. | No. `canSwitchSpeakerphone` is false. | Not reported. | + +On iOS automatic mode, listen only playout uses `playback`. When recording starts, LiveKit reapplies the session as `playAndRecord`. In manual mode, non-forced receiver vs speaker behavior comes from the Apple config you applied. + +Dart track counting no longer drives the session. The native audio engine delegate drives it from real lifecycle events, which removes the timing races and missed deactivations of the older counting approach. + +## Migrating from the old APIs + +The legacy `Hardware` audio members still work but are deprecated and forward to `AudioManager`. + +| Old | New | +| --- | --- | +| `Hardware.instance.setSpeakerphoneOn(true)` | `AudioManager.instance.setSpeakerOutputPreferred(true)` | +| `room.setSpeakerOn(true)` | `AudioManager.instance.setSpeakerOutputPreferred(true)` | +| `Hardware.instance.speakerOn` | `AudioManager.instance.isSpeakerOutputPreferred` | +| `Hardware.instance.preferSpeakerOutput` | `AudioManager.instance.isSpeakerOutputPreferred` | +| `Hardware.instance.forceSpeakerOutput` | `AudioManager.instance.isSpeakerOutputForced` | +| `Hardware.instance.setAutomaticConfigurationEnabled(enable: false)` | `AudioManager.instance.setAudioSessionManagementMode(AudioSessionManagementMode.manual)` | + +The old `onConfigureNativeAudio` hook (a deep src import) is removed. Replace a custom configuration function with explicit options, which run in manual mode. + +```dart +// Before: assigning onConfigureNativeAudio with a custom function. +// After: +await AudioManager.instance.setAudioSessionOptions( + AudioSessionOptions.communication( + apple: const AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + mode: AppleAudioMode.videoChat, + ), + ), +); +``` + +## API reference + +The full generated API reference for these types lives at [pub.dev](https://pub.dev/documentation/livekit_client/latest/). Start from `AudioManager`, `AudioSessionOptions`, `AppleAudioSessionConfiguration`, `AndroidAudioSessionConfiguration`, and `AudioProcessingOptions`. diff --git a/example/lib/pages/room.dart b/example/lib/pages/room.dart index 27aa394b8..42295242e 100644 --- a/example/lib/pages/room.dart +++ b/example/lib/pages/room.dart @@ -52,7 +52,7 @@ class _RoomPageState extends State { }); if (lkPlatformIs(PlatformType.android)) { - unawaited(Hardware.instance.setSpeakerphoneOn(true)); + unawaited(AudioManager.instance.setSpeakerOutputPreferred(true)); } if (lkPlatformIsDesktop()) { diff --git a/example/lib/widgets/controls.dart b/example/lib/widgets/controls.dart index 3c96b4ebc..50cf2fee7 100644 --- a/example/lib/widgets/controls.dart +++ b/example/lib/widgets/controls.dart @@ -36,7 +36,7 @@ class _ControlsWidgetState extends State { StreamSubscription? _subscription; - bool _speakerphoneOn = Hardware.instance.speakerOn ?? false; + bool _speakerphoneOn = AudioManager.instance.isSpeakerOutputPreferred; @override void initState() { @@ -109,7 +109,7 @@ class _ControlsWidgetState extends State { void _setSpeakerphoneOn() async { _speakerphoneOn = !_speakerphoneOn; - await widget.room.setSpeakerOn(_speakerphoneOn, forceSpeakerOutput: false); + await AudioManager.instance.setSpeakerOutputPreferred(_speakerphoneOn, force: false); setState(() {}); } diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart index 9756d5a07..20fb5e289 100644 --- a/lib/livekit_client.dart +++ b/lib/livekit_client.dart @@ -43,6 +43,7 @@ export 'src/participant/participant.dart'; export 'src/participant/remote.dart' hide ParticipantCreationResult; export 'src/audio/audio_manager.dart'; export 'src/audio/audio_frame_capture.dart' show AudioFormat, AudioFrame, AudioFrameCallback, AudioRendererOptions; +export 'src/audio/audio_session.dart'; export 'src/preconnect/pre_connect_audio_buffer.dart'; export 'src/publication/local.dart'; export 'src/publication/remote.dart'; diff --git a/lib/src/audio/android_audio_session_adapter.dart b/lib/src/audio/android_audio_session_adapter.dart new file mode 100644 index 000000000..fe7434b04 --- /dev/null +++ b/lib/src/audio/android_audio_session_adapter.dart @@ -0,0 +1,38 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +import '../support/native.dart'; +import 'audio_session.dart'; + +/// Serializes an [AndroidAudioSessionConfiguration] into the map consumed by +/// LiveKit's native Android audio session manager. Unset fields are omitted so +/// the native side keeps its current value. +@internal +Map androidAudioSessionConfigurationToMap(AndroidAudioSessionConfiguration config) => + { + if (config.manageAudioFocus != null) 'manageAudioFocus': config.manageAudioFocus!, + if (config.audioMode != null) 'androidAudioMode': config.audioMode!.name, + if (config.focusMode != null) 'androidAudioFocusMode': config.focusMode!.name, + if (config.streamType != null) 'androidAudioStreamType': config.streamType!.name, + if (config.usageType != null) 'androidAudioAttributesUsageType': config.usageType!.name, + if (config.contentType != null) 'androidAudioAttributesContentType': config.contentType!.name, + if (config.forceAudioRouting != null) 'forceHandleAudioRouting': config.forceAudioRouting!, + }; + +@internal +Future setAndroidAudioSessionConfiguration(AndroidAudioSessionConfiguration config) async { + await Native.configureAndroidAudioSession(androidAudioSessionConfigurationToMap(config)); +} diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart index 3034a3706..116b2d25f 100644 --- a/lib/src/audio/audio_manager.dart +++ b/lib/src/audio/audio_manager.dart @@ -12,19 +12,281 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'dart:async'; + +import 'package:meta/meta.dart'; + +import '../logger.dart'; import '../support/native.dart'; +import '../support/platform.dart'; +import 'android_audio_session_adapter.dart'; import 'audio_processing_state.dart'; +import 'audio_session.dart'; +import 'audio_session_policy.dart'; + +/// Snapshot of the WebRTC audio engine's playout/recording state. +/// +/// Surfaced by [AudioManager] from real audio-engine lifecycle events on the +/// native side (iOS and macOS). This is the source of truth for audio activity, +/// replacing the legacy track-counting state. +class AudioEngineState { + /// Whether the engine has playout (output / remote audio) enabled. + final bool isPlayoutEnabled; + + /// Whether the engine has recording (input / local mic) enabled. + final bool isRecordingEnabled; + + const AudioEngineState({ + required this.isPlayoutEnabled, + required this.isRecordingEnabled, + }); + + /// Whether the engine is neither playing out nor recording. + bool get isIdle => !isPlayoutEnabled && !isRecordingEnabled; + + @override + bool operator ==(Object other) => + other is AudioEngineState && + other.isPlayoutEnabled == isPlayoutEnabled && + other.isRecordingEnabled == isRecordingEnabled; + + @override + int get hashCode => Object.hash(isPlayoutEnabled, isRecordingEnabled); + + @override + String toString() => 'AudioEngineState(isPlayoutEnabled: $isPlayoutEnabled, isRecordingEnabled: $isRecordingEnabled)'; +} /// Controls LiveKit's process-wide platform audio behavior. /// -/// The platform audio engine and its audio processing module are global to the -/// app process, so engine-scoped audio state lives here rather than on a `Room` -/// or an individual track. +/// Platform audio sessions and the audio processing module are global to the +/// app process, so session options and engine-scoped audio state live here +/// rather than on a `Room` or an individual track. class AudioManager { AudioManager._(); static final AudioManager instance = AudioManager._(); + AudioSessionOptions _options = const AudioSessionOptions.communication(); + AudioSessionManagementMode _managementMode = AudioSessionManagementMode.automatic; + bool _preferSpeakerOutput = true; + bool _forceSpeakerOutput = false; + bool _isPlayoutEnabled = false; + bool _isRecordingEnabled = false; + final StreamController _audioEngineStateController = StreamController.broadcast(); + + AudioSessionOptions get options => _options; + AudioSessionManagementMode get managementMode => _managementMode; + + /// Whether the speaker is the preferred audio output. + bool get isSpeakerOutputPreferred => _preferSpeakerOutput; + + /// Whether speaker output is forced even when a headset/Bluetooth device is + /// connected. + bool get isSpeakerOutputForced => _forceSpeakerOutput && _preferSpeakerOutput; + + /// Whether the platform supports switching the speaker output (iOS/Android). + bool get canSwitchSpeakerphone => lkPlatformIsMobile(); + + /// The current audio engine state, derived from native engine lifecycle + /// events (iOS/macOS). On platforms without engine events this stays idle. + AudioEngineState get audioEngineState => + AudioEngineState(isPlayoutEnabled: _isPlayoutEnabled, isRecordingEnabled: _isRecordingEnabled); + + /// A broadcast stream of audio engine state changes (native engine lifecycle). + Stream get audioEngineStateStream => _audioEngineStateController.stream; + + bool get _isAutomaticConfigurationEnabled => _managementMode == AudioSessionManagementMode.automatic; + + @visibleForTesting + void resetForTest() { + _options = const AudioSessionOptions.communication(); + _managementMode = AudioSessionManagementMode.automatic; + _preferSpeakerOutput = true; + _forceSpeakerOutput = false; + _isPlayoutEnabled = false; + _isRecordingEnabled = false; + } + + /// Invoked from native when the WebRTC audio engine's playout/recording state + /// changes. Audio-engine lifecycle events are the single source of truth for + /// audio activity. This replaces the legacy track-counting path, which had + /// timing races and could miss session deactivation. + /// + /// On iOS the native engine delegate also owns audio-session activation + /// timing (configure + activate on enable, deactivate on disable). This Dart + /// hop is non-blocking and only keeps the observable state in sync. macOS + /// emits the same events (no `AVAudioSession` to configure) so engine state + /// stays authoritative there too. + @internal + void handleAudioEngineState({ + required bool isPlayoutEnabled, + required bool isRecordingEnabled, + }) { + final nextState = AudioEngineState( + isPlayoutEnabled: isPlayoutEnabled, + isRecordingEnabled: isRecordingEnabled, + ); + if (nextState == audioEngineState) { + return; + } + + _isPlayoutEnabled = isPlayoutEnabled; + _isRecordingEnabled = isRecordingEnabled; + _audioEngineStateController.add(nextState); + } + + /// Applies an explicit audio session configuration and switches to manual mode. + /// + /// Calling this puts [AudioManager] in [AudioSessionManagementMode.manual]: + /// LiveKit stops managing the session from room, connect, and engine + /// lifecycle, and the app owns it from here. Hand control back to LiveKit with + /// [setAudioSessionManagementMode] and [AudioSessionManagementMode.automatic]. + /// + /// The speaker preference and force flag are owned by setSpeakerOutputPreferred + /// and are preserved across this call. + Future setAudioSessionOptions(AudioSessionOptions options) async { + await _enterManualMode(); + _options = options; + await _applyCurrentAudioSessionPolicy(); + } + + /// Selects whether LiveKit manages the platform audio session automatically. + /// + /// In [AudioSessionManagementMode.manual], LiveKit does not update the audio + /// session from room, connect, or track lifecycle. The app can still apply a + /// configuration explicitly with [setAudioSessionOptions] and release it with + /// [deactivateAudioSession]. + /// + /// Prefer setting this before connecting to a room. flutter_webrtc's own + /// native audio management is always disabled (LiveKit owns the session). + /// Switching back to automatic mode reapplies LiveKit's managed policy. + Future setAudioSessionManagementMode(AudioSessionManagementMode mode) async { + final previousMode = _managementMode; + _managementMode = mode; + await _syncAppleAudioSessionManagementMode(); + if (previousMode != AudioSessionManagementMode.automatic && mode == AudioSessionManagementMode.automatic) { + await _applyCurrentAudioSessionPolicy(); + } + } + + /// Switches to manual mode if not already, syncing the native side once. + Future _enterManualMode() async { + if (_managementMode == AudioSessionManagementMode.manual) return; + _managementMode = AudioSessionManagementMode.manual; + await _syncAppleAudioSessionManagementMode(); + } + + /// Deactivates the current platform audio session and switches to manual mode. + /// + /// Like [setAudioSessionOptions], calling this puts [AudioManager] in + /// [AudioSessionManagementMode.manual] so LiveKit does not re-activate the + /// session on its own. Re-apply a configuration with [setAudioSessionOptions], + /// or hand control back with [setAudioSessionManagementMode]. + Future deactivateAudioSession() async { + await _enterManualMode(); + if (lkPlatformIs(PlatformType.iOS)) { + await Native.deactivateAppleAudioSession(); + } else if (lkPlatformIs(PlatformType.android)) { + await Native.stopAndroidAudioSession(); + } + } + + /// Prefers routing audio output to/from the speaker. + /// + /// By default a connected wired/Bluetooth headset still takes priority even + /// when [preferred] is true. Set [force] to force the speaker even when a + /// headset is connected. + /// + /// LiveKit owns this routing on both platforms (Android via its own + /// audioswitch handler and iOS via its audio session), so it does not depend + /// on flutter_webrtc. + Future setSpeakerOutputPreferred(bool preferred, {bool force = false}) async { + if (!canSwitchSpeakerphone) { + logger.warning('setSpeakerOutputPreferred is only supported on iOS/Android'); + return; + } + _preferSpeakerOutput = preferred; + _forceSpeakerOutput = preferred && force; + + if (lkPlatformIs(PlatformType.iOS)) { + if (_isAutomaticConfigurationEnabled) { + final policy = _resolvedAudioSessionPolicy(_options); + // Automatic mode: the native audio-engine delegate owns activation + // timing, so this caches the policy and applies now only if the engine + // is already running. Category is resolved natively from engine state. + await Native.configureAudio( + policy.appleConfiguration, + automatic: true, + selectCategoryByEngineState: true, + forceSpeakerOutput: policy.forceSpeakerOutput, + ); + } else { + // Manual mode: re-apply the fixed Apple config. Non-forced receiver vs + // speaker behavior comes from that config. Force is carried separately + // to native for playAndRecord sessions. + await _configureAppleAudioSession(_options); + } + } else if (lkPlatformIs(PlatformType.android)) { + await Native.setAndroidSpeakerphoneOn(preferred, force: _forceSpeakerOutput); + } + } + + Future _applyCurrentAudioSessionPolicy() async { + if (lkPlatformIs(PlatformType.iOS)) { + await _configureAppleAudioSession(_options); + } else if (lkPlatformIs(PlatformType.android)) { + await _configureAndroidAudioSession(_options); + } + } + + @internal + Future applyOptionsForConnect() async { + await _syncAppleAudioSessionManagementMode(); + if (_isAutomaticConfigurationEnabled) { + await _applyCurrentAudioSessionPolicy(); + } + } + + Future _syncAppleAudioSessionManagementMode() async { + if (lkPlatformIs(PlatformType.iOS)) { + await Native.setAppleAudioSessionAutomaticManagementEnabled(_isAutomaticConfigurationEnabled); + } + } + + Future _configureAppleAudioSession(AudioSessionOptions options) async { + final policy = _resolvedAudioSessionPolicy(options); + final config = policy.appleConfiguration; + logger.fine('configuring Apple audio session using $config...'); + // In automatic mode the native audio-engine delegate owns activation timing, + // so this caches the policy and applies now only if the engine is already + // running. Automatic mode resolves the category from engine state. Manual + // mode applies the resolved config immediately and verbatim. + await Native.configureAudio( + config, + automatic: _isAutomaticConfigurationEnabled, + selectCategoryByEngineState: _isAutomaticConfigurationEnabled, + forceSpeakerOutput: policy.forceSpeakerOutput, + ); + } + + Future _configureAndroidAudioSession(AudioSessionOptions options) async { + final policy = _resolvedAudioSessionPolicy(options); + final config = policy.androidConfiguration; + logger.fine( + 'configuring Android audio session using ${androidAudioSessionConfigurationToMap(config)}...', + ); + await setAndroidAudioSessionConfiguration(config); + await Native.setAndroidSpeakerphoneOn(policy.preferSpeakerOutput, force: policy.forceSpeakerOutput); + } + + ResolvedAudioSessionPolicy _resolvedAudioSessionPolicy(AudioSessionOptions options) => ResolvedAudioSessionPolicy( + options: options, + preferSpeakerOutput: _preferSpeakerOutput, + forceSpeakerOutput: _forceSpeakerOutput && _preferSpeakerOutput, + automatic: _isAutomaticConfigurationEnabled, + ); + /// Diagnostic snapshot of the resolved audio processing state. /// /// The audio processing module is owned by the native peer connection factory diff --git a/lib/src/audio/audio_session.dart b/lib/src/audio/audio_session.dart new file mode 100644 index 000000000..506488931 --- /dev/null +++ b/lib/src/audio/audio_session.dart @@ -0,0 +1,275 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export '../support/value_or_absent.dart'; + +import 'package:meta/meta.dart'; + +import '../support/value_or_absent.dart'; + +enum AudioSessionManagementMode { + /// LiveKit updates the platform audio session based on room/track lifecycle. + automatic, + + /// LiveKit does not update the platform audio session automatically. + /// + /// The app must call AudioManager APIs when it wants to apply a session + /// configuration. + manual, +} + +@immutable +class AudioSessionOptions { + /// Exact Apple session configuration for manual mode. + final AppleAudioSessionConfiguration apple; + + /// Exact Android session configuration for manual mode. + final AndroidAudioSessionConfiguration android; + + const AudioSessionOptions._({ + required this.apple, + required this.android, + }); + + /// Two-way audio preset for calls, rooms, and microphone capture. + /// + /// This pre-fills communication-oriented platform policies. Speaker + /// routing is a runtime preference set with + /// `AudioManager.setSpeakerOutputPreferred`. Override [apple] or [android] + /// for exact platform behavior. + const AudioSessionOptions.communication({ + AppleAudioSessionConfiguration apple = AppleAudioSessionConfiguration.communication, + AndroidAudioSessionConfiguration android = AndroidAudioSessionConfiguration.communication, + }) : this._(apple: apple, android: android); + + /// One-way media playback preset. + /// + /// This pre-fills playback-oriented platform policies. Apple playback policy + /// leaves routing to the platform, while Android speaker routing remains a + /// runtime preference. Override [apple] or [android] for exact platform + /// behavior. + const AudioSessionOptions.media({ + AppleAudioSessionConfiguration apple = AppleAudioSessionConfiguration.media, + AndroidAudioSessionConfiguration android = AndroidAudioSessionConfiguration.media, + }) : this._(apple: apple, android: android); + + /// Returns a copy with selected fields replaced. + AudioSessionOptions copyWith({ + ValueOrAbsent apple = const ValueOrAbsent.absent(), + ValueOrAbsent android = const ValueOrAbsent.absent(), + }) => + AudioSessionOptions._( + apple: apple.valueOr(this.apple), + android: android.valueOr(this.android), + ); +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/category +enum AppleAudioCategory { + soloAmbient, + playback, + record, + playAndRecord, + multiRoute, +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/categoryoptions +enum AppleAudioCategoryOption { + mixWithOthers, // Only playAndRecord, playback, or multiRoute. + duckOthers, // Only playAndRecord, playback, or multiRoute. + interruptSpokenAudioAndMixWithOthers, + allowBluetooth, // Only playAndRecord or record. + allowBluetoothA2DP, + allowAirPlay, + defaultToSpeaker, +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/mode +enum AppleAudioMode { + default_, + gameChat, + measurement, + moviePlayback, + spokenAudio, + videoChat, + videoRecording, + voiceChat, + voicePrompt, +} + +@immutable +class AppleAudioSessionConfiguration { + /// AVAudioSession category. + final AppleAudioCategory? category; + + /// AVAudioSession category options. + final Set? categoryOptions; + + /// AVAudioSession mode. + final AppleAudioMode? mode; + + const AppleAudioSessionConfiguration({ + this.category, + this.categoryOptions, + this.mode, + }); + + static const communication = AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + mode: AppleAudioMode.videoChat, + ); + + static const media = AppleAudioSessionConfiguration( + category: AppleAudioCategory.playback, + categoryOptions: {AppleAudioCategoryOption.mixWithOthers}, + mode: AppleAudioMode.spokenAudio, + ); + + AppleAudioSessionConfiguration copyWith({ + ValueOrAbsent category = const ValueOrAbsent.absent(), + ValueOrAbsent?> categoryOptions = const ValueOrAbsent.absent(), + ValueOrAbsent mode = const ValueOrAbsent.absent(), + }) => + AppleAudioSessionConfiguration( + category: category.valueOr(this.category), + categoryOptions: categoryOptions.valueOr(this.categoryOptions), + mode: mode.valueOr(this.mode), + ); +} + +enum AndroidAudioMode { + normal, + callScreening, + inCall, + inCommunication, + ringtone, +} + +enum AndroidAudioFocusMode { + gain, + gainTransient, + gainTransientExclusive, + gainTransientMayDuck, +} + +enum AndroidAudioStreamType { + accessibility, + alarm, + dtmf, + music, + notification, + ring, + system, + voiceCall, +} + +enum AndroidAudioAttributesUsageType { + alarm, + assistanceAccessibility, + assistanceNavigationGuidance, + assistanceSonification, + assistant, + game, + media, + notification, + notificationEvent, + notificationRingtone, + unknown, + voiceCommunication, + voiceCommunicationSignalling, +} + +enum AndroidAudioAttributesContentType { + movie, + music, + sonification, + speech, + unknown, +} + +@immutable +class AndroidAudioSessionConfiguration { + /// Android AudioManager mode. + final AndroidAudioMode? audioMode; + + /// Whether LiveKit should manage Android audio focus. + final bool? manageAudioFocus; + + /// Requested Android audio focus gain type. + final AndroidAudioFocusMode? focusMode; + + /// Legacy Android stream type. + final AndroidAudioStreamType? streamType; + + /// Android AudioAttributes usage. + final AndroidAudioAttributesUsageType? usageType; + + /// Android AudioAttributes content type. + final AndroidAudioAttributesContentType? contentType; + + /// Forces LiveKit audio routing even outside communication/call modes. + final bool? forceAudioRouting; + + const AndroidAudioSessionConfiguration({ + this.audioMode, + this.manageAudioFocus, + this.focusMode, + this.streamType, + this.usageType, + this.contentType, + this.forceAudioRouting, + }); + + static const communication = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.inCommunication, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.voiceCall, + usageType: AndroidAudioAttributesUsageType.voiceCommunication, + contentType: AndroidAudioAttributesContentType.speech, + ); + + static const media = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.music, + usageType: AndroidAudioAttributesUsageType.media, + contentType: AndroidAudioAttributesContentType.unknown, + ); + + AndroidAudioSessionConfiguration copyWith({ + ValueOrAbsent audioMode = const ValueOrAbsent.absent(), + ValueOrAbsent manageAudioFocus = const ValueOrAbsent.absent(), + ValueOrAbsent focusMode = const ValueOrAbsent.absent(), + ValueOrAbsent streamType = const ValueOrAbsent.absent(), + ValueOrAbsent usageType = const ValueOrAbsent.absent(), + ValueOrAbsent contentType = const ValueOrAbsent.absent(), + ValueOrAbsent forceAudioRouting = const ValueOrAbsent.absent(), + }) => + AndroidAudioSessionConfiguration( + audioMode: audioMode.valueOr(this.audioMode), + manageAudioFocus: manageAudioFocus.valueOr(this.manageAudioFocus), + focusMode: focusMode.valueOr(this.focusMode), + streamType: streamType.valueOr(this.streamType), + usageType: usageType.valueOr(this.usageType), + contentType: contentType.valueOr(this.contentType), + forceAudioRouting: forceAudioRouting.valueOr(this.forceAudioRouting), + ); +} diff --git a/lib/src/audio/audio_session_policy.dart b/lib/src/audio/audio_session_policy.dart new file mode 100644 index 000000000..bc2abe410 --- /dev/null +++ b/lib/src/audio/audio_session_policy.dart @@ -0,0 +1,61 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +import '../support/native_audio.dart'; +import 'audio_session.dart'; + +@internal +class ResolvedAudioSessionPolicy { + const ResolvedAudioSessionPolicy({ + required this.options, + required this.preferSpeakerOutput, + required this.forceSpeakerOutput, + required this.automatic, + }); + + final AudioSessionOptions options; + final bool preferSpeakerOutput; + final bool forceSpeakerOutput; + final bool automatic; + + NativeAudioConfiguration get appleConfiguration { + if (automatic) { + return NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + appleAudioMode: preferSpeakerOutput ? AppleAudioMode.videoChat : AppleAudioMode.voiceChat, + ); + } + + final apple = options.apple; + return NativeAudioConfiguration( + appleAudioCategory: apple.category, + appleAudioCategoryOptions: apple.categoryOptions, + appleAudioMode: apple.mode, + ); + } + + AndroidAudioSessionConfiguration get androidConfiguration { + if (automatic) { + return AndroidAudioSessionConfiguration.communication; + } + return options.android; + } +} diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index f274a0289..9550bcbef 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -19,6 +19,7 @@ import 'package:collection/collection.dart'; import 'package:http/http.dart' as http; import 'package:meta/meta.dart'; +import '../audio/audio_manager.dart'; import '../core/signal_client.dart'; import '../data_stream/errors.dart'; import '../data_stream/stream_reader.dart'; @@ -103,6 +104,9 @@ class Room extends DisposableChangeNotifier with EventsEmittable { bool _isRecording = false; bool _audioEnabled = true; + // Whether the one-time RoomOptions speaker preference bridge has run. + bool _legacySpeakerBridged = false; + lk_models.Room? _roomInfo; /// a list of participants that are actively speaking, including local participant. @@ -288,9 +292,20 @@ class Room extends DisposableChangeNotifier with EventsEmittable { })); } + // Bridge a legacy RoomOptions speaker preference into the process-wide + // AudioManager once, on the first connect. Skipping it on a later manual + // connect of the same Room keeps a runtime speaker change from being + // reverted. New code should call setSpeakerOutputPreferred directly. + final legacySpeakerOn = roomOptions.defaultAudioOutputOptions.speakerOn; + if (legacySpeakerOn != null && !_legacySpeakerBridged && lkPlatformIsMobile()) { + _legacySpeakerBridged = true; + await AudioManager.instance.setSpeakerOutputPreferred(legacySpeakerOn); + } + // configure audio for native platform await NativeAudioManagement.start(); + var didConnect = false; try { await engine.connect( _regionUrl ?? url, @@ -300,6 +315,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { fastConnectOptions: fastConnectOptions, regionUrlProvider: _regionUrlProvider, ); + didConnect = true; } catch (e) { logger.warning('could not connect to $url $e'); if (_regionUrlProvider != null && @@ -322,12 +338,17 @@ class Room extends DisposableChangeNotifier with EventsEmittable { fastConnectOptions: fastConnectOptions, regionUrlProvider: _regionUrlProvider, ); + didConnect = true; } else { rethrow; } } else { rethrow; } + } finally { + if (!didConnect) { + await NativeAudioManagement.stop(); + } } } @@ -1121,7 +1142,8 @@ extension RoomHardwareManagementMethods on Room { roomOptions.defaultCameraCaptureOptions.deviceId ?? Hardware.instance.selectedVideoInput?.deviceId; /// Get mobile device's speaker status. - bool? get speakerOn => roomOptions.defaultAudioOutputOptions.speakerOn; + @Deprecated('Use AudioManager.instance.isSpeakerOutputPreferred instead') + bool? get speakerOn => AudioManager.instance.isSpeakerOutputPreferred; /// Set audio output device. Future setAudioOutputDevice(MediaDevice device) async { @@ -1186,27 +1208,10 @@ extension RoomHardwareManagementMethods on Room { /// [speakerOn] set speakerphone on or off, by default wired/bluetooth headsets will still /// be prioritized even if set to true. /// [forceSpeakerOutput] if true, will force speaker output even if headphones - /// or bluetooth is connected, only supported on iOS for now - Future setSpeakerOn(bool speakerOn, {bool forceSpeakerOutput = false}) async { - if (lkPlatformIsMobile()) { - await Hardware.instance.setSpeakerphoneOn(speakerOn, forceSpeakerOutput: forceSpeakerOutput); - engine.roomOptions = engine.roomOptions.copyWith( - defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith( - speakerOn: speakerOn, - ), - ); - } - } - - /// Apply audio output device settings. - @internal - Future applyAudioSpeakerSettings() async { - if (roomOptions.defaultAudioOutputOptions.speakerOn != null) { - if (lkPlatformIsMobile()) { - await Hardware.instance.setSpeakerphoneOn(roomOptions.defaultAudioOutputOptions.speakerOn!); - } - } - } + /// or bluetooth is connected. + @Deprecated('Use AudioManager.instance.setSpeakerOutputPreferred instead') + Future setSpeakerOn(bool speakerOn, {bool forceSpeakerOutput = false}) => + AudioManager.instance.setSpeakerOutputPreferred(speakerOn, force: forceSpeakerOutput); Future startAudio() async { try { diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index 065d4f361..6f75f94d1 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -17,11 +17,10 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import '../audio/audio_manager.dart'; +import '../audio/audio_session.dart'; import '../logger.dart'; -import '../support/native.dart'; -import '../support/native_audio.dart'; import '../support/platform.dart'; -import '../track/audio_management.dart'; class MediaDevice { const MediaDevice(this.deviceId, this.label, this.kind, this.groupId); @@ -69,27 +68,33 @@ class Hardware { MediaDevice? selectedVideoInput; - bool? get speakerOn => _preferSpeakerOutput; + @Deprecated('Use AudioManager.instance.isSpeakerOutputPreferred instead') + bool? get speakerOn => AudioManager.instance.isSpeakerOutputPreferred; - bool _preferSpeakerOutput = true; - - bool get preferSpeakerOutput => _preferSpeakerOutput; - - bool _forceSpeakerOutput = false; + @Deprecated('Use AudioManager.instance.isSpeakerOutputPreferred instead') + bool get preferSpeakerOutput => AudioManager.instance.isSpeakerOutputPreferred; /// if true, will force speaker output even if headphones or bluetooth is connected - /// only supported on iOS for now - bool get forceSpeakerOutput => _forceSpeakerOutput && _preferSpeakerOutput; - - // This flag is used to determine if automatic native configuration - // of audio is enabled. If set to false Natvive.configureAudio - // will not be called, and the user is responsible for configuring - // the native audio configuration manually. - bool _isAutomaticConfigurationEnabled = true; - bool get isAutomaticConfigurationEnabled => _isAutomaticConfigurationEnabled; - + @Deprecated('Use AudioManager.instance.isSpeakerOutputForced instead') + bool get forceSpeakerOutput => AudioManager.instance.isSpeakerOutputForced; + + // Whether automatic native audio configuration is enabled. If disabled, + // Native.configureAudio is not called and the app is responsible for + // configuring the native audio session manually. + // + // Backed by [AudioManager] so there is a single source of truth for the + // management mode. See [AudioManager.setAudioSessionManagementMode]. + @Deprecated('Use AudioManager.instance.managementMode instead') + bool get isAutomaticConfigurationEnabled => + AudioManager.instance.managementMode == AudioSessionManagementMode.automatic; + + @Deprecated('Use AudioManager.instance.setAudioSessionManagementMode instead') void setAutomaticConfigurationEnabled({required bool enable}) { - _isAutomaticConfigurationEnabled = enable; + unawaited( + AudioManager.instance.setAudioSessionManagementMode( + enable ? AudioSessionManagementMode.automatic : AudioSessionManagementMode.manual, + ), + ); } Future> enumerateDevices({String? type}) async { @@ -131,48 +136,19 @@ class Hardware { await rtc.Helper.selectAudioInput(device.deviceId); } - @Deprecated('use setSpeakerphoneOn') - Future setPreferSpeakerOutput(bool enable) => setSpeakerphoneOn(enable); + @Deprecated('Use AudioManager.instance.setSpeakerOutputPreferred instead') + Future setPreferSpeakerOutput(bool enable) => AudioManager.instance.setSpeakerOutputPreferred(enable); - bool get canSwitchSpeakerphone => lkPlatformIsMobile(); + @Deprecated('Use AudioManager.instance.canSwitchSpeakerphone instead') + bool get canSwitchSpeakerphone => AudioManager.instance.canSwitchSpeakerphone; /// [enable] set speakerphone on or off, by default wired/bluetooth headsets will still /// be prioritized even if set to true. /// [forceSpeakerOutput] if true, will force speaker output even if headphones - /// or bluetooth is connected, only supported on iOS for now - Future setSpeakerphoneOn(bool enable, {bool forceSpeakerOutput = false}) async { - if (canSwitchSpeakerphone) { - _preferSpeakerOutput = enable; - _forceSpeakerOutput = forceSpeakerOutput; - if (lkPlatformIs(PlatformType.iOS)) { - NativeAudioConfiguration? config; - if (lkPlatformIs(PlatformType.iOS)) { - // Only iOS for now... - config = await onConfigureNativeAudio.call(audioTrackState); - if (_preferSpeakerOutput && _forceSpeakerOutput) { - config = config.copyWith( - appleAudioCategoryOptions: { - ...?config.appleAudioCategoryOptions, - AppleAudioCategoryOption.defaultToSpeaker, - }, - ); - } - logger.fine('configuring for ${audioTrackState} using ${config}...'); - try { - if (_isAutomaticConfigurationEnabled) { - await Native.configureAudio(config); - } - } catch (error) { - logger.warning('failed to configure ${error}'); - } - } - } else { - await rtc.Helper.setSpeakerphoneOn(enable); - } - } else { - logger.warning('setSpeakerphoneOn only support on iOS/Android'); - } - } + /// or bluetooth is connected. + @Deprecated('Use AudioManager.instance.setSpeakerOutputPreferred instead') + Future setSpeakerphoneOn(bool enable, {bool forceSpeakerOutput = false}) => + AudioManager.instance.setSpeakerOutputPreferred(enable, force: forceSpeakerOutput); Future openCamera({MediaDevice? device, bool? facingMode}) async { final constraints = { diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 6cdc2cba8..498091bae 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -14,6 +14,7 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'audio/audio_manager.dart'; import 'support/native.dart'; import 'support/platform.dart' show lkPlatformIsMobile; @@ -22,16 +23,28 @@ import 'support/platform.dart' show lkPlatformIsMobile; class LiveKitClient { static const version = '2.8.0'; - /// Initialize the WebRTC plugin. If this is not manually called, will be - /// initialized with default settings. - /// This method must be called before calling any LiveKit SDK API. - static Future initialize({bool bypassVoiceProcessing = false}) async { + /// Initialize the WebRTC plugin. + /// + /// Optional: call once at startup to enable [bypassVoiceProcessing] before + /// connecting. Otherwise WebRTC initializes lazily with defaults. + /// + /// LiveKit owns the platform audio session, and flutter_webrtc's own native + /// audio management is disabled automatically when the LiveKit plugin loads + /// (done natively at registration), so that does not depend on this call. + /// + /// Configure audio-session behavior through [AudioManager] before connecting, + /// e.g. `await AudioManager.instance.setAudioSessionManagementMode(...)` and + /// `await AudioManager.instance.setAudioSessionOptions(...)`. + static Future initialize({ + bool bypassVoiceProcessing = false, + }) async { if (lkPlatformIsMobile()) { + // bypassVoiceProcessing controls only WebRTC voice processing, not the + // session intent. The audio session is owned by AudioManager. + Native.bypassVoiceProcessing = bypassVoiceProcessing; await rtc.WebRTC.initialize(options: { if (bypassVoiceProcessing) 'bypassVoiceProcessing': bypassVoiceProcessing, }); - - Native.bypassVoiceProcessing = bypassVoiceProcessing; } } } diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 1d2781d91..c5b5f86c1 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -239,8 +239,6 @@ class LocalParticipant extends Participant { await track.onPublish(); await track.processor?.onPublish(room); - await room.applyAudioSpeakerSettings(); - final listener = track.createListener(); listener.on((TrackEndedEvent event) async { logger.fine('TrackEndedEvent: ${event.track}'); @@ -573,8 +571,6 @@ class LocalParticipant extends Participant { await track.processor?.onUnpublish(); await track.stopProcessor(); } - - await room.applyAudioSpeakerSettings(); } if (notify) { diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index 9cdfe928a..b71dd18ab 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -18,6 +18,7 @@ import 'package:flutter/services.dart' show MethodChannel, MethodCall; import 'package:meta/meta.dart'; +import '../audio/audio_manager.dart'; import '../logger.dart'; import '../managers/broadcast_manager.dart'; import 'native_audio.dart'; @@ -36,12 +37,29 @@ class Native { @internal static bool bypassVoiceProcessing = false; + /// Configures (and caches) the Apple audio session. + /// + /// When [automatic] is true, the native audio-engine delegate owns activation + /// timing: the configuration is cached and (re)applied on engine lifecycle + /// events, and only applied immediately here if the engine is already + /// running. When false (manual mode / explicit apply) it is applied + /// immediately. @internal - static Future configureAudio(NativeAudioConfiguration configuration) async { + static Future configureAudio( + NativeAudioConfiguration configuration, { + bool automatic = false, + bool selectCategoryByEngineState = false, + bool forceSpeakerOutput = false, + }) async { try { final result = await channel.invokeMethod( 'configureNativeAudio', - configuration.toMap(), + { + ...configuration.toMap(), + 'automatic': automatic, + 'selectCategoryByEngineState': selectCategoryByEngineState, + 'forceSpeakerOutput': forceSpeakerOutput, + }, ); return result == true; } catch (error) { @@ -93,6 +111,63 @@ class Native { return null; } + /// Configure and activate LiveKit's Android audio session (mode/focus/routing). + @internal + static Future configureAndroidAudioSession(Map configuration) async { + try { + await channel.invokeMethod('configureAndroidAudioSession', configuration); + } catch (error) { + logger.warning('configureAndroidAudioSession did throw $error'); + } + } + + /// Deactivate LiveKit's Android audio session (release focus, restore mode). + @internal + static Future stopAndroidAudioSession() async { + try { + await channel.invokeMethod('stopAndroidAudioSession'); + } catch (error) { + logger.warning('stopAndroidAudioSession did throw $error'); + } + } + + /// Deactivate LiveKit's Apple audio session. + @internal + static Future deactivateAppleAudioSession() async { + try { + await channel.invokeMethod('deactivateAppleAudioSession', {}); + } catch (error) { + logger.warning('deactivateAppleAudioSession did throw $error'); + } + } + + /// Route Android audio output to/from the speakerphone. + @internal + static Future setAndroidSpeakerphoneOn(bool enable, {bool force = false}) async { + try { + await channel.invokeMethod( + 'setAndroidSpeakerphoneOn', + {'enable': enable, 'force': force}, + ); + } catch (error) { + logger.warning('setAndroidSpeakerphoneOn did throw $error'); + } + } + + /// Enable or disable LiveKit's automatic iOS audio-session management from + /// native WebRTC audio-engine lifecycle callbacks. + @internal + static Future setAppleAudioSessionAutomaticManagementEnabled(bool enabled) async { + try { + await channel.invokeMethod( + 'setAppleAudioSessionAutomaticManagementEnabled', + {'enabled': enabled}, + ); + } catch (error) { + logger.warning('setAppleAudioSessionAutomaticManagementEnabled did throw $error'); + } + } + @internal static Future startVisualizer( String trackId, { @@ -196,6 +271,15 @@ class Native { } _broadcastStateChanged(call.arguments as bool); return null; + case 'onAudioEngineState': + final args = call.arguments; + if (args is Map) { + AudioManager.instance.handleAudioEngineState( + isPlayoutEnabled: args['isPlayoutEnabled'] == true, + isRecordingEnabled: args['isRecordingEnabled'] == true, + ); + } + return null; default: logger.warning('Method ${call.method} is not implemented.'); return null; diff --git a/lib/src/support/native_audio.dart b/lib/src/support/native_audio.dart index 70bc59dce..0cce554f0 100644 --- a/lib/src/support/native_audio.dart +++ b/lib/src/support/native_audio.dart @@ -12,38 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// https://developer.apple.com/documentation/avfaudio/avaudiosession/category -enum AppleAudioCategory { - soloAmbient, - playback, - record, - playAndRecord, - multiRoute, -} - -// https://developer.apple.com/documentation/avfaudio/avaudiosession/categoryoptions -enum AppleAudioCategoryOption { - mixWithOthers, // Only playAndRecord, playback, or multiRoute. - duckOthers, // Only playAndRecord, playback, or multiRoute. - interruptSpokenAudioAndMixWithOthers, - allowBluetooth, // Only playAndRecord or record. - allowBluetoothA2DP, - allowAirPlay, - defaultToSpeaker, -} - -// https://developer.apple.com/documentation/avfaudio/avaudiosession/mode -enum AppleAudioMode { - default_, - gameChat, - measurement, - moviePlayback, - spokenAudio, - videoChat, - videoRecording, - voiceChat, - voicePrompt, -} +import '../audio/audio_session.dart' show AppleAudioCategory, AppleAudioCategoryOption, AppleAudioMode; +import 'value_or_absent.dart'; extension AppleAudioCategoryExt on AppleAudioCategory { String toStringValue() => { @@ -85,47 +55,13 @@ class NativeAudioConfiguration { final AppleAudioCategory? appleAudioCategory; final Set? appleAudioCategoryOptions; final AppleAudioMode? appleAudioMode; - final bool? preferSpeakerOutput; - - static final soloAmbient = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.soloAmbient, - appleAudioCategoryOptions: {}, - appleAudioMode: AppleAudioMode.default_, - ); - - static final playback = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playback, - appleAudioCategoryOptions: {AppleAudioCategoryOption.mixWithOthers}, - appleAudioMode: AppleAudioMode.spokenAudio, - ); - - static final playAndRecordSpeaker = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playAndRecord, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.allowBluetooth, - AppleAudioCategoryOption.allowBluetoothA2DP, - AppleAudioCategoryOption.allowAirPlay, - }, - appleAudioMode: AppleAudioMode.videoChat, - ); - - static final playAndRecordReceiver = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playAndRecord, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.allowBluetooth, - AppleAudioCategoryOption.allowBluetoothA2DP, - AppleAudioCategoryOption.allowAirPlay, - }, - appleAudioMode: AppleAudioMode.voiceChat, - ); NativeAudioConfiguration( { // for iOS / Mac this.appleAudioCategory, this.appleAudioCategoryOptions, - this.appleAudioMode, - this.preferSpeakerOutput + this.appleAudioMode // Android options // ... }); @@ -135,19 +71,16 @@ class NativeAudioConfiguration { if (appleAudioCategoryOptions != null) 'appleAudioCategoryOptions': appleAudioCategoryOptions!.map((e) => e.toStringValue()).toList(), if (appleAudioMode != null) 'appleAudioMode': appleAudioMode!.toStringValue(), - if (preferSpeakerOutput != null) 'preferSpeakerOutput': preferSpeakerOutput, }; NativeAudioConfiguration copyWith({ - AppleAudioCategory? appleAudioCategory, - Set? appleAudioCategoryOptions, - AppleAudioMode? appleAudioMode, - bool? preferSpeakerOutput, + ValueOrAbsent appleAudioCategory = const ValueOrAbsent.absent(), + ValueOrAbsent?> appleAudioCategoryOptions = const ValueOrAbsent.absent(), + ValueOrAbsent appleAudioMode = const ValueOrAbsent.absent(), }) => NativeAudioConfiguration( - appleAudioCategory: appleAudioCategory ?? this.appleAudioCategory, - appleAudioCategoryOptions: appleAudioCategoryOptions ?? this.appleAudioCategoryOptions, - appleAudioMode: appleAudioMode ?? this.appleAudioMode, - preferSpeakerOutput: preferSpeakerOutput ?? this.preferSpeakerOutput, + appleAudioCategory: appleAudioCategory.valueOr(this.appleAudioCategory), + appleAudioCategoryOptions: appleAudioCategoryOptions.valueOr(this.appleAudioCategoryOptions), + appleAudioMode: appleAudioMode.valueOr(this.appleAudioMode), ); } diff --git a/lib/src/support/value_or_absent.dart b/lib/src/support/value_or_absent.dart new file mode 100644 index 000000000..ab1abd1ef --- /dev/null +++ b/lib/src/support/value_or_absent.dart @@ -0,0 +1,63 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Distinguishes an omitted copy value from an explicit replacement value. +/// +/// This is useful for `copyWith` methods where a nullable field must be able to +/// keep its current value, change to a non-null value, or change to `null`. +/// +/// ```dart +/// class Example { +/// const Example({this.name}); +/// +/// final String? name; +/// +/// Example copyWith({ +/// ValueOrAbsent name = const ValueOrAbsent.absent(), +/// }) => +/// Example(name: name.valueOr(this.name)); +/// } +/// +/// example.copyWith(); // keep existing name +/// example.copyWith(name: ValueOrAbsent.value('room')); // set name +/// example.copyWith(name: ValueOrAbsent.value(null)); // clear name +/// ``` +sealed class ValueOrAbsent { + const ValueOrAbsent(); + + /// Creates an explicit replacement value. + const factory ValueOrAbsent.value(T value) = _Value; + + /// Creates an omitted value that preserves the current field. + const factory ValueOrAbsent.absent() = _Absent; + + /// Returns the explicit value, or [other] when this value is absent. + T valueOr(T other); +} + +final class _Value extends ValueOrAbsent { + const _Value(this.value); + + final T value; + + @override + T valueOr(T other) => value; +} + +final class _Absent extends ValueOrAbsent { + const _Absent(); + + @override + T valueOr(T other) => other; +} diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index f287ca03b..085c63781 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -12,170 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:synchronized/synchronized.dart' as sync; - -import '../hardware/hardware.dart'; -import '../logger.dart'; +import '../audio/audio_manager.dart'; +import '../audio/audio_session.dart'; import '../support/native.dart'; -import '../support/native_audio.dart'; import '../support/platform.dart'; import 'local/local.dart'; import 'remote/remote.dart'; -enum AudioTrackState { - none, - remoteOnly, - localOnly, - localAndRemote, -} - -typedef ConfigureNativeAudioFunc = Future Function(AudioTrackState state); - -// it's possible to set custom function here to customize audio session configuration -ConfigureNativeAudioFunc onConfigureNativeAudio = defaultNativeAudioConfigurationFunc; - -final _trackCounterLock = sync.Lock(); -AudioTrackState _audioTrackState = AudioTrackState.none; - -AudioTrackState get audioTrackState => _audioTrackState; - -int _localTrackCount = 0; -int _remoteTrackCount = 0; - -mixin LocalAudioManagementMixin on LocalTrack, AudioTrack { - @override - Future onPublish() async { - final didUpdate = await super.onPublish(); - if (didUpdate) { - // update counter - await _trackCounterLock.synchronized(() async { - _localTrackCount++; - await _onAudioTrackCountDidChange(); - }); - } - return didUpdate; - } +@Deprecated('Audio session lifecycle is managed by AudioManager instead') +mixin LocalAudioManagementMixin on LocalTrack, AudioTrack {} - @override - Future onUnpublish() async { - final didUpdate = await super.onUnpublish(); - if (didUpdate) { - // update counter - await _trackCounterLock.synchronized(() async { - _localTrackCount--; - await _onAudioTrackCountDidChange(); - }); - } - return didUpdate; - } -} -mixin RemoteAudioManagementMixin on RemoteTrack, AudioTrack { - /// Start playing audio track. On web platform, create an audio element and - /// start playback - @override - Future start() async { - final didStart = await super.start(); - if (didStart) { - await _trackCounterLock.synchronized(() async { - _remoteTrackCount++; - await _onAudioTrackCountDidChange(); - }); - } - return didStart; - } - - @override - Future stop() async { - final didStop = await super.stop(); - if (didStop) { - await _trackCounterLock.synchronized(() async { - _remoteTrackCount--; - await _onAudioTrackCountDidChange(); - }); - } - return didStop; - } -} - -Future _onAudioTrackCountDidChange() async { - logger.fine('onAudioTrackCountDidChange: ' - 'local: $_localTrackCount, remote: $_remoteTrackCount'); - - final newState = _computeAudioTrackState(); - - if (_audioTrackState != newState) { - _audioTrackState = newState; - logger.fine('didUpdateSate: $_audioTrackState'); - - NativeAudioConfiguration? config; - if (lkPlatformIs(PlatformType.iOS)) { - // Only iOS for now... - config = await onConfigureNativeAudio.call(_audioTrackState); - - if (Hardware.instance.forceSpeakerOutput) { - config = config.copyWith( - appleAudioCategoryOptions: { - ...?config.appleAudioCategoryOptions, - AppleAudioCategoryOption.defaultToSpeaker, - }, - ); - } - } - - if (config != null) { - logger.fine('configuring for ${_audioTrackState} using ${config}...'); - try { - if (Hardware.instance.isAutomaticConfigurationEnabled) { - logger.fine('configuring native audio...'); - await Native.configureAudio(config); - } - } catch (error) { - logger.warning('failed to configure ${error}'); - } - } - } -} - -AudioTrackState _computeAudioTrackState() { - if (_localTrackCount > 0 && _remoteTrackCount == 0) { - return AudioTrackState.localOnly; - } else if (_localTrackCount == 0 && _remoteTrackCount > 0) { - return AudioTrackState.remoteOnly; - } else if (_localTrackCount > 0 && _remoteTrackCount > 0) { - return AudioTrackState.localAndRemote; - } - // Default - return AudioTrackState.none; -} - -Future defaultNativeAudioConfigurationFunc(AudioTrackState state) async { - if (state == AudioTrackState.none) { - return NativeAudioConfiguration.soloAmbient; - } else if (state == AudioTrackState.remoteOnly && Hardware.instance.preferSpeakerOutput) { - return NativeAudioConfiguration.playback; - } - - return Hardware.instance.preferSpeakerOutput - ? NativeAudioConfiguration.playAndRecordSpeaker - : NativeAudioConfiguration.playAndRecordReceiver; -} +@Deprecated('Audio session lifecycle is managed by AudioManager instead') +mixin RemoteAudioManagementMixin on RemoteTrack, AudioTrack {} class NativeAudioManagement { static Future start() async { - // Audio configuration for Android. - if (lkPlatformIs(PlatformType.android)) { - if (Native.bypassVoiceProcessing) { - await rtc.Helper.setAndroidAudioConfiguration(rtc.AndroidAudioConfiguration.media); - } else { - await rtc.Helper.setAndroidAudioConfiguration(rtc.AndroidAudioConfiguration.communication); - } - } + await AudioManager.instance.applyOptionsForConnect(); } static Future stop() async { - if (lkPlatformIs(PlatformType.android)) { - await rtc.Helper.clearAndroidCommunicationDevice(); + if (lkPlatformIs(PlatformType.android) && + AudioManager.instance.managementMode == AudioSessionManagementMode.automatic) { + await Native.stopAndroidAudioSession(); } } } diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index aa0ba9f62..cc717a957 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -282,7 +282,7 @@ class AudioProcessingOptions { autoGainControlMode = AudioProcessingMode.automatic, highPassFilterMode = AudioProcessingMode.automatic; - const AudioProcessingOptions.raw() + const AudioProcessingOptions.noProcessing() : echoCancellation = false, noiseSuppression = false, autoGainControl = false, diff --git a/pubspec.lock b/pubspec.lock index 3aefe0abc..c78edefdc 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -284,10 +284,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: d8c89028d29e5693742190285b2e3c8a117531b0960ae0693d84273a53968d28 + sha256: c4e8db6ed337b8c30d76cd7cd8c91693f5495e1aeb556cbcb73f1e5d5bdcf020 url: "https://pub.dev" source: hosted - version: "1.5.0" + version: "1.5.1" frontend_server_client: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 2c768472e..0665dc206 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -46,7 +46,7 @@ dependencies: json_annotation: ^4.9.0 # Fix version to avoid version conflicts between WebRTC-SDK pods, which both this package and flutter_webrtc depend on. - flutter_webrtc: 1.5.0 + flutter_webrtc: 1.5.1 dart_webrtc: ^1.8.0 dev_dependencies: diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index fa2367bbe..8a93f8431 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import AVFoundation import flutter_webrtc import WebRTC @@ -52,6 +53,11 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { var binaryMessenger: FlutterBinaryMessenger? + // Retained strong reference to the audio-engine delegate. flutter_webrtc and + // the audio device module both hold it weakly, so LiveKit must keep it alive. + var channel: FlutterMethodChannel? + var audioEngineObserver: LKAudioEngineObserver? + #if os(iOS) var cancellable = Set() #endif @@ -68,6 +74,21 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { instance.binaryMessenger = messenger registrar.addMethodCallDelegate(instance, channel: channel) + // LiveKit owns the platform audio session, so disable flutter_webrtc's + // own native audio management. Set at registration, before any audio op. + FlutterWebRTCPlugin.setAudioSessionManagementEnabled(false) + + // Own the audio device module's engine-lifecycle delegate so LiveKit + // drives the audio session from real engine events (configure + activate + // on enable, deactivate on disable) instead of track counting. The + // engine emits these events on both iOS and macOS. macOS has no + // AVAudioSession to configure, so there it only surfaces engine state. + // Set before the peer connection factory is created. + instance.channel = channel + let audioEngineObserver = LKAudioEngineObserver(channel: channel) + instance.audioEngineObserver = audioEngineObserver + FlutterWebRTCPlugin.setAudioDeviceModuleObserver(audioEngineObserver) + #if os(iOS) BroadcastManager.shared.isBroadcastingPublisher .sink { isBroadcasting in @@ -310,13 +331,11 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let category = categoryMap[string] { configuration.category = category.rawValue - print("[LiveKit] Configuring category: ", configuration.category) } // CategoryOptions if let strings = args["appleAudioCategoryOptions"] as? [String] { configuration.categoryOptions = categoryOptions(fromFlutter: strings) - print("[LiveKit] Configuring categoryOptions: ", strings) } // Mode @@ -324,50 +343,58 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let mode = modeMap[string] { configuration.mode = mode.rawValue - print("[LiveKit] Configuring mode: ", configuration.mode) } - // get `RTCAudioSession` and lock - let rtcSession = RTCAudioSession.sharedInstance() - rtcSession.lockForConfiguration() - - var isLocked = true - let unlock = { - guard isLocked else { - print("[LiveKit] not locked, ignoring unlock") - return - } - rtcSession.unlockForConfiguration() - isLocked = false + // Cache the policy so the audio-engine delegate can (re)apply it on + // engine lifecycle events. In automatic mode the delegate owns + // activation timing (configure + activate on engine enable), so here we + // only apply immediately if the engine is already running. Manual mode + // (or no `automatic` flag) applies immediately. + let automatic = args["automatic"] as? Bool ?? false + let selectCategoryByEngineState = args["selectCategoryByEngineState"] as? Bool ?? false + let forceSpeakerOutput = args["forceSpeakerOutput"] as? Bool ?? false + audioEngineObserver?.updatePolicy(configuration, + automaticManagementEnabled: automatic, + selectCategoryByEngineState: selectCategoryByEngineState, + forceSpeakerOutput: forceSpeakerOutput) + + let shouldApplyNow = !automatic || (audioEngineObserver?.isSessionActive ?? false) + guard shouldApplyNow else { + result(true) + return } - // always `unlock()` when exiting scope, calling multiple times has no side-effect - defer { - unlock() + // Apply through the observer so the category is resolved from the live + // engine state (when category selection is enabled), matching what the + // engine-lifecycle callbacks would apply. + if let error = audioEngineObserver?.applyCachedConfiguration() { + print("[LiveKit] Configure audio error: ", error) + result(FlutterError(code: "configure", message: error.localizedDescription, details: nil)) + } else { + result(true) } + #endif + } - do { - try rtcSession.setConfiguration(configuration, active: true) - // unlock here before configuring `AVAudioSession` - // unlock() - print("[LiveKit] RTCAudioSession Configure success") - - // also configure longFormAudio - // let avSession = AVAudioSession.sharedInstance() - // try avSession.setCategory(AVAudioSession.Category(rawValue: configuration.category), - // mode: AVAudioSession.Mode(rawValue: configuration.mode), - // policy: .default, - // options: configuration.categoryOptions) - // print("[LiveKit] AVAudioSession Configure success") - - // preferSpeakerOutput - if let preferSpeakerOutput = args["preferSpeakerOutput"] as? Bool { - try rtcSession.overrideOutputAudioPort(preferSpeakerOutput ? .speaker : .none) - } + public func handleSetAppleAudioSessionAutomaticManagementEnabled(args: [String: Any?], result: @escaping FlutterResult) { + #if os(macOS) + result(FlutterMethodNotImplemented) + #else + let enabled = (args["enabled"] as? Bool) ?? true + audioEngineObserver?.setAutomaticManagementEnabled(enabled) + result(true) + #endif + } + + public func handleDeactivateAppleAudioSession(result: @escaping FlutterResult) { + #if os(macOS) + result(FlutterMethodNotImplemented) + #else + if let error = LiveKitPlugin.deactivateAudioSession() { + print("[LiveKit] Deactivate audio session error: ", error) + result(FlutterError(code: "deactivateAudioSession", message: error.localizedDescription, details: nil)) + } else { result(true) - } catch { - print("[LiveKit] Configure audio error: ", error) - result(FlutterError(code: "configure", message: error.localizedDescription, details: nil)) } #endif } @@ -513,6 +540,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { switch call.method { case "configureNativeAudio": handleConfigureNativeAudio(args: args, result: result) + case "setAppleAudioSessionAutomaticManagementEnabled": + handleSetAppleAudioSessionAutomaticManagementEnabled(args: args, result: result) + case "deactivateAppleAudioSession": + handleDeactivateAppleAudioSession(result: result) case "startVisualizer": handleStartAudioVisualizer(args: args, result: result) case "stopVisualizer": @@ -541,3 +572,272 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { } } } + +#if !os(macOS) +@available(iOS 13.0, *) +extension LiveKitPlugin { + /// SDK-side audio engine error code (mirrors client-sdk-swift): returned + /// from a delegate callback to make WebRTC abort / roll back the engine + /// operation when the audio session cannot be configured. + static let kAudioEngineErrorFailedToConfigureAudioSession = -4100 + + /// Applies an `RTCAudioSessionConfiguration` to the shared `RTCAudioSession`. + /// Returns `nil` on success or the thrown error. Safe to call on any thread. + static func applyAudioSessionConfiguration(_ configuration: RTCAudioSessionConfiguration, + forceSpeakerOutput: Bool, + active: Bool) -> Error? { + let rtcSession = RTCAudioSession.sharedInstance() + rtcSession.lockForConfiguration() + defer { rtcSession.unlockForConfiguration() } + do { + try rtcSession.setConfiguration(configuration, active: active) + // overrideOutputAudioPort hard-routes to the speaker even over a + // connected headset. Plain speaker preference is expressed by the + // selected audio mode/category options, so clear any stale hard + // override unless the app explicitly forced speaker output. + // Only valid for the playAndRecord category. + if active, configuration.category == AVAudioSession.Category.playAndRecord.rawValue { + try rtcSession.overrideOutputAudioPort(forceSpeakerOutput ? .speaker : .none) + } + return nil + } catch { + return error + } + } + + /// Deactivates the shared `RTCAudioSession`. Returns `nil` on success. + static func deactivateAudioSession() -> Error? { + let rtcSession = RTCAudioSession.sharedInstance() + rtcSession.lockForConfiguration() + defer { rtcSession.unlockForConfiguration() } + do { + try rtcSession.setActive(false) + return nil + } catch { + return error + } + } +} +#endif + +/// Receives the WebRTC audio device module's engine-lifecycle callbacks and, +/// on iOS, drives the audio session: configure + activate when the engine +/// enables, deactivate when it disables. Replaces the old track-counting +/// trigger. On macOS there is no `AVAudioSession`, so it only surfaces engine +/// state to Dart (keeping engine state the single source of truth there too). +/// +/// The engine-lifecycle methods are invoked synchronously on WebRTC's worker +/// thread. The engine blocks on the return value (`0` = proceed, non-zero = +/// abort / roll back), so the session work here is synchronous and never calls +/// back into the audio device module. The Dart notification is dispatched +/// asynchronously and is purely informational. It never blocks the engine. +@available(iOS 13.0, macOS 10.15, *) +class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { + private let lock = NSLock() + private weak var channel: FlutterMethodChannel? + + #if !os(macOS) + private var cachedConfiguration: RTCAudioSessionConfiguration? + // When true, the category is chosen from the live engine state at apply time + // (playAndRecord while recording, playback for playout-only) rather than + // taken from the pushed config. This is what keeps the category correct as + // recording/playout come and go. The pushed config still supplies the mode, + // options and speaker preference. False for an explicit per-platform + // override or manual mode, where the config is applied verbatim. + private var selectCategoryByEngineState = false + private var forceSpeakerOutput = false + private var isAutomaticManagementEnabled = true + // True when cached policy changes should apply immediately. This includes + // an engine already running under manual mode, because switching back to + // automatic should configure the live session without waiting for another + // engine lifecycle event. + private var sessionActive = false + // Last recording state seen, so an immediate re-apply (e.g. speaker toggle + // while the engine is running) can resolve the category from current state. + private var lastIsRecordingEnabled = false + #endif + + init(channel: FlutterMethodChannel) { + self.channel = channel + super.init() + } + + #if !os(macOS) + var isSessionActive: Bool { + lock.lock(); defer { lock.unlock() } + return sessionActive + } + + func setAutomaticManagementEnabled(_ enabled: Bool) { + lock.lock() + isAutomaticManagementEnabled = enabled + lock.unlock() + } + + /// Stores the audio session policy pushed from Dart. Pure cache, where the + /// delegate callbacks apply it. Callers decide whether to apply immediately. + func updatePolicy(_ configuration: RTCAudioSessionConfiguration, + automaticManagementEnabled: Bool, + selectCategoryByEngineState: Bool, + forceSpeakerOutput: Bool) { + let cachedConfiguration = copyConfiguration(configuration) + lock.lock() + self.cachedConfiguration = cachedConfiguration + self.isAutomaticManagementEnabled = automaticManagementEnabled + self.selectCategoryByEngineState = selectCategoryByEngineState + self.forceSpeakerOutput = forceSpeakerOutput + lock.unlock() + } + + /// Applies the cached configuration immediately, resolving the category from + /// the last known engine state when category selection is enabled. Used for + /// manual mode and for re-applying while the engine is already running. + func applyCachedConfiguration() -> Error? { + lock.lock() + let configuration = effectiveConfigurationLocked(isRecordingEnabled: lastIsRecordingEnabled) + let forceSpeakerOutput = self.forceSpeakerOutput + lock.unlock() + guard let configuration else { return nil } + return LiveKitPlugin.applyAudioSessionConfiguration(configuration, + forceSpeakerOutput: forceSpeakerOutput, + active: true) + } + + private func applyManagedConfiguration(isRecordingEnabled: Bool) -> Error? { + lock.lock() + let shouldManageSession = isAutomaticManagementEnabled + let configuration = effectiveConfigurationLocked(isRecordingEnabled: isRecordingEnabled) + let forceSpeakerOutput = self.forceSpeakerOutput + lock.unlock() + + guard shouldManageSession, let configuration else { return nil } + return LiveKitPlugin.applyAudioSessionConfiguration(configuration, + forceSpeakerOutput: forceSpeakerOutput, + active: true) + } + + private func recordEngineState(isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + lock.lock() + lastIsRecordingEnabled = isRecordingEnabled + sessionActive = isPlayoutEnabled || isRecordingEnabled + lock.unlock() + } + + /// Resolves the configuration to apply for a given engine state. Must be + /// called with `lock` held. + /// + /// When category selection is disabled (explicit Apple override or manual + /// mode) the pushed config is applied verbatim. When enabled, recording + /// uses the pushed config (resolved as a playAndRecord policy by Dart), + /// while playout-only uses a coherent playback policy: flipping only the + /// category would leave playAndRecord-only mode/options (e.g. videoChat, + /// allowBluetooth) that are invalid for the playback category. Mirrors the + /// Swift SDK's `.playback` preset (playback + spokenAudio + mixWithOthers). + private func effectiveConfigurationLocked(isRecordingEnabled: Bool) -> RTCAudioSessionConfiguration? { + guard let configuration = cachedConfiguration else { return nil } + guard selectCategoryByEngineState, !isRecordingEnabled else { return configuration } + + let playback = copyConfiguration(configuration) + playback.category = AVAudioSession.Category.playback.rawValue + playback.categoryOptions = [.mixWithOthers] + playback.mode = AVAudioSession.Mode.spokenAudio.rawValue + return playback + } + + private func copyConfiguration(_ configuration: RTCAudioSessionConfiguration) -> RTCAudioSessionConfiguration { + let copy = RTCAudioSessionConfiguration() + copy.category = configuration.category + copy.categoryOptions = configuration.categoryOptions + copy.mode = configuration.mode + copy.sampleRate = configuration.sampleRate + copy.ioBufferDuration = configuration.ioBufferDuration + copy.inputNumberOfChannels = configuration.inputNumberOfChannels + copy.outputNumberOfChannels = configuration.outputNumberOfChannels + return copy + } + #endif + + // MARK: RTCAudioDeviceModuleDelegate, engine lifecycle + + func audioDeviceModule(_: RTCAudioDeviceModule, + willEnableEngine _: AVAudioEngine, + isPlayoutEnabled: Bool, + isRecordingEnabled: Bool) -> Int { + var resultCode = 0 + #if !os(macOS) + if isPlayoutEnabled || isRecordingEnabled { + if let error = applyManagedConfiguration(isRecordingEnabled: isRecordingEnabled) { + print("[LiveKit] AudioEngine willEnable: failed to configure audio session: \(error)") + resultCode = LiveKitPlugin.kAudioEngineErrorFailedToConfigureAudioSession + } + if resultCode == 0 { + recordEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + } + #endif + if resultCode == 0 { + notifyEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + return resultCode + } + + func audioDeviceModule(_: RTCAudioDeviceModule, + didDisableEngine _: AVAudioEngine, + isPlayoutEnabled: Bool, + isRecordingEnabled: Bool) -> Int { + var resultCode = 0 + #if !os(macOS) + if isPlayoutEnabled || isRecordingEnabled { + // A disable event can leave one side of the engine running (for + // example, mic off while remote playout continues). Re-apply so + // dynamic category selection follows the new engine state. + if let error = applyManagedConfiguration(isRecordingEnabled: isRecordingEnabled) { + print("[LiveKit] AudioEngine didDisable: failed to configure audio session: \(error)") + resultCode = LiveKitPlugin.kAudioEngineErrorFailedToConfigureAudioSession + } + } else { + lock.lock() + let shouldManageSession = isAutomaticManagementEnabled + lock.unlock() + + if shouldManageSession, let error = LiveKitPlugin.deactivateAudioSession() { + // Leave sessionActive unchanged (still true) so cached state + // keeps reflecting the live session. Flipping it to false here + // would make a later configureNativeAudio(automatic:) cache-only + // while the session is in fact still active. + print("[LiveKit] AudioEngine didDisable: failed to deactivate audio session: \(error)") + resultCode = LiveKitPlugin.kAudioEngineErrorFailedToConfigureAudioSession + } + } + if resultCode == 0 { + recordEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + #endif + if resultCode == 0 { + notifyEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + return resultCode + } + + // Remaining callbacks are not needed for session management (proceed / no-op). + func audioDeviceModule(_: RTCAudioDeviceModule, didCreateEngine _: AVAudioEngine) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, willStartEngine _: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, didStopEngine _: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, willReleaseEngine _: AVAudioEngine) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, engine _: AVAudioEngine, configureInputFromSource _: AVAudioNode?, toDestination _: AVAudioNode, format _: AVAudioFormat, context _: [AnyHashable: Any]) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, engine _: AVAudioEngine, configureOutputFromSource _: AVAudioNode, toDestination _: AVAudioNode?, format _: AVAudioFormat, context _: [AnyHashable: Any]) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, didReceiveSpeechActivityEvent _: RTCSpeechActivityEvent) {} + func audioDeviceModuleDidUpdateDevices(_ audioDeviceModule: RTCAudioDeviceModule) { + FlutterWebRTCPlugin.sharedSingleton()?.audioDeviceModuleDidUpdateDevices(audioDeviceModule) + } + + private func notifyEngineState(isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let channel = channel else { return } + DispatchQueue.main.async { + channel.invokeMethod("onAudioEngineState", arguments: [ + "isPlayoutEnabled": isPlayoutEnabled, + "isRecordingEnabled": isRecordingEnabled, + ]) + } + } +} diff --git a/test/audio/audio_session_test.dart b/test/audio/audio_session_test.dart new file mode 100644 index 000000000..0705d57c6 --- /dev/null +++ b/test/audio/audio_session_test.dart @@ -0,0 +1,579 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter/services.dart'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/audio/android_audio_session_adapter.dart'; +import 'package:livekit_client/src/audio/audio_manager.dart'; +import 'package:livekit_client/src/audio/audio_session.dart'; +import 'package:livekit_client/src/audio/audio_session_policy.dart'; +import 'package:livekit_client/src/support/native.dart'; +import 'package:livekit_client/src/support/native_audio.dart' as native_audio; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + setUp(() { + AudioManager.instance.resetForTest(); + Native.bypassVoiceProcessing = false; + }); + + native_audio.NativeAudioConfiguration resolveApplePolicy( + AudioSessionOptions options, { + bool preferSpeakerOutput = true, + bool forceSpeakerOutput = false, + bool automatic = true, + }) => + ResolvedAudioSessionPolicy( + options: options, + preferSpeakerOutput: preferSpeakerOutput, + forceSpeakerOutput: forceSpeakerOutput && preferSpeakerOutput, + automatic: automatic, + ).appleConfiguration; + + AndroidAudioSessionConfiguration resolveAndroidPolicy( + AudioSessionOptions options, { + bool automatic = true, + }) => + ResolvedAudioSessionPolicy( + options: options, + preferSpeakerOutput: AudioManager.instance.isSpeakerOutputPreferred, + forceSpeakerOutput: AudioManager.instance.isSpeakerOutputForced, + automatic: automatic, + ).androidConfiguration; + + group('AudioSessionManagementMode', () { + test('supports automatic and manual management', () { + expect( + AudioSessionManagementMode.values, + [ + AudioSessionManagementMode.automatic, + AudioSessionManagementMode.manual, + ], + ); + }); + }); + + group('AudioSessionOptions', () { + test('communication constructor pre-fills platform configs', () { + const options = AudioSessionOptions.communication(); + + expect(options.apple.category, AppleAudioCategory.playAndRecord); + expect( + options.apple.categoryOptions, + { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + expect(options.apple.mode, AppleAudioMode.videoChat); + expect(options.android.audioMode, AndroidAudioMode.inCommunication); + expect(options.android.streamType, AndroidAudioStreamType.voiceCall); + }); + + test('media constructor pre-fills platform configs', () { + const options = AudioSessionOptions.media(); + + expect(options.apple.category, AppleAudioCategory.playback); + expect(options.apple.categoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + expect(options.apple.mode, AppleAudioMode.spokenAudio); + expect(options.android.audioMode, AndroidAudioMode.normal); + expect(options.android.streamType, AndroidAudioStreamType.music); + }); + + test('copyWith replaces platform configs', () { + const options = AudioSessionOptions.communication(); + + final updated = options.copyWith( + apple: const ValueOrAbsent.value( + AppleAudioSessionConfiguration( + mode: AppleAudioMode.voiceChat, + ), + ), + ); + + expect(updated.apple.category, isNull); + expect(updated.apple.mode, AppleAudioMode.voiceChat); + expect(updated.android.audioMode, AndroidAudioMode.inCommunication); + + final restored = updated.copyWith( + apple: const ValueOrAbsent.value(AppleAudioSessionConfiguration.communication), + android: const ValueOrAbsent.value(AndroidAudioSessionConfiguration.communication), + ); + + expect(restored.apple.category, AppleAudioCategory.playAndRecord); + expect(restored.apple.mode, AppleAudioMode.videoChat); + expect(restored.android.audioMode, AndroidAudioMode.inCommunication); + }); + + test('Apple configuration copyWith updates and clears nullable fields', () { + const config = AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: {AppleAudioCategoryOption.allowBluetooth}, + mode: AppleAudioMode.voiceChat, + ); + + final updated = config.copyWith( + category: const ValueOrAbsent.value(AppleAudioCategory.playback), + categoryOptions: const ValueOrAbsent.value({AppleAudioCategoryOption.mixWithOthers}), + mode: const ValueOrAbsent.value(AppleAudioMode.spokenAudio), + ); + + expect(updated.category, AppleAudioCategory.playback); + expect(updated.categoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + expect(updated.mode, AppleAudioMode.spokenAudio); + + final cleared = updated.copyWith( + category: const ValueOrAbsent.value(null), + categoryOptions: const ValueOrAbsent.value(null), + mode: const ValueOrAbsent.value(null), + ); + + expect(cleared.category, isNull); + expect(cleared.categoryOptions, isNull); + expect(cleared.mode, isNull); + }); + + test('Android configuration copyWith updates and clears nullable fields', () { + const config = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.inCommunication, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.voiceCall, + usageType: AndroidAudioAttributesUsageType.voiceCommunication, + contentType: AndroidAudioAttributesContentType.speech, + forceAudioRouting: true, + ); + + final updated = config.copyWith( + audioMode: const ValueOrAbsent.value(AndroidAudioMode.normal), + manageAudioFocus: const ValueOrAbsent.value(false), + focusMode: const ValueOrAbsent.value(AndroidAudioFocusMode.gainTransient), + streamType: const ValueOrAbsent.value(AndroidAudioStreamType.music), + usageType: const ValueOrAbsent.value(AndroidAudioAttributesUsageType.media), + contentType: const ValueOrAbsent.value(AndroidAudioAttributesContentType.unknown), + forceAudioRouting: const ValueOrAbsent.value(false), + ); + + expect(updated.audioMode, AndroidAudioMode.normal); + expect(updated.manageAudioFocus, isFalse); + expect(updated.focusMode, AndroidAudioFocusMode.gainTransient); + expect(updated.streamType, AndroidAudioStreamType.music); + expect(updated.usageType, AndroidAudioAttributesUsageType.media); + expect(updated.contentType, AndroidAudioAttributesContentType.unknown); + expect(updated.forceAudioRouting, isFalse); + + final cleared = updated.copyWith( + audioMode: const ValueOrAbsent.value(null), + manageAudioFocus: const ValueOrAbsent.value(null), + focusMode: const ValueOrAbsent.value(null), + streamType: const ValueOrAbsent.value(null), + usageType: const ValueOrAbsent.value(null), + contentType: const ValueOrAbsent.value(null), + forceAudioRouting: const ValueOrAbsent.value(null), + ); + + expect(cleared.audioMode, isNull); + expect(cleared.manageAudioFocus, isNull); + expect(cleared.focusMode, isNull); + expect(cleared.streamType, isNull); + expect(cleared.usageType, isNull); + expect(cleared.contentType, isNull); + expect(cleared.forceAudioRouting, isNull); + }); + }); + + group('AudioManager', () { + test('management mode can be set independently from options', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionManagementMode(AudioSessionManagementMode.manual); + + expect(manager.managementMode, AudioSessionManagementMode.manual); + expect(manager.options.android.audioMode, AndroidAudioMode.inCommunication); + + await manager.setAudioSessionManagementMode(AudioSessionManagementMode.automatic); + }); + + test('setAudioSessionOptions switches management to manual', () async { + final manager = AudioManager.instance; + expect(manager.managementMode, AudioSessionManagementMode.automatic); + + await manager.setAudioSessionOptions(const AudioSessionOptions.media()); + + expect(manager.managementMode, AudioSessionManagementMode.manual); + expect(manager.options.apple.category, AppleAudioCategory.playback); + expect(manager.options.android.streamType, AndroidAudioStreamType.music); + }); + + test('deactivateAudioSession switches management to manual', () async { + final manager = AudioManager.instance; + expect(manager.managementMode, AudioSessionManagementMode.automatic); + + await manager.deactivateAudioSession(); + + expect(manager.managementMode, AudioSessionManagementMode.manual); + }); + + test('setAudioSessionOptions preserves the runtime speaker preference', () async { + final manager = AudioManager.instance; + + // The speaker preference is runtime state owned by + // setSpeakerOutputPreferred, so changing the session intent must not reset + // it. + expect(manager.isSpeakerOutputPreferred, isTrue); + + await manager.setAudioSessionOptions( + const AudioSessionOptions.media(), + ); + expect(manager.isSpeakerOutputPreferred, isTrue); + + await manager.setAudioSessionOptions( + const AudioSessionOptions.communication(), + ); + expect(manager.isSpeakerOutputPreferred, isTrue); + }); + + test('resolves communication Apple session policy from speaker preference', () { + final speaker = resolveApplePolicy( + const AudioSessionOptions.communication(), + preferSpeakerOutput: true, + ); + + expect(speaker.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(speaker.appleAudioMode, AppleAudioMode.videoChat); + expect( + speaker.appleAudioCategoryOptions, + { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + + final receiver = resolveApplePolicy( + const AudioSessionOptions.communication(), + preferSpeakerOutput: false, + ); + + expect(receiver.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(receiver.appleAudioMode, AppleAudioMode.voiceChat); + }); + + test('automatic Apple policy ignores manual media options', () { + final config = resolveApplePolicy( + const AudioSessionOptions.media(), + ); + + expect(config.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(config.appleAudioMode, AppleAudioMode.videoChat); + expect( + config.appleAudioCategoryOptions, + { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + }); + + test('resolves manual media Apple session policy as fixed playback', () { + final config = resolveApplePolicy( + const AudioSessionOptions.media(), + automatic: false, + ); + + expect(config.appleAudioCategory, AppleAudioCategory.playback); + expect(config.appleAudioMode, AppleAudioMode.spokenAudio); + expect(config.appleAudioCategoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + }); + + test('forced speaker does not mutate Apple category options', () { + final playback = resolveApplePolicy( + const AudioSessionOptions.media( + apple: AppleAudioSessionConfiguration( + category: AppleAudioCategory.playback, + categoryOptions: {AppleAudioCategoryOption.mixWithOthers}, + ), + ), + automatic: false, + forceSpeakerOutput: true, + ); + + expect(playback.appleAudioCategory, AppleAudioCategory.playback); + expect(playback.appleAudioCategoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + + final playAndRecord = resolveApplePolicy( + const AudioSessionOptions.communication( + apple: AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: {AppleAudioCategoryOption.allowBluetooth}, + ), + ), + automatic: false, + forceSpeakerOutput: true, + ); + + expect(playAndRecord.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect( + playAndRecord.appleAudioCategoryOptions, + {AppleAudioCategoryOption.allowBluetooth}, + ); + }); + + test('resolves Android session policy from automatic mode or manual options', () { + final automaticMedia = resolveAndroidPolicy( + const AudioSessionOptions.media(), + ); + + expect(automaticMedia.audioMode, AndroidAudioMode.inCommunication); + expect(automaticMedia.streamType, AndroidAudioStreamType.voiceCall); + + final media = resolveAndroidPolicy( + const AudioSessionOptions.media(), + automatic: false, + ); + + expect(media.audioMode, AndroidAudioMode.normal); + expect(media.streamType, AndroidAudioStreamType.music); + + final explicit = resolveAndroidPolicy( + const AudioSessionOptions.communication( + android: AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + forceAudioRouting: true, + ), + ), + automatic: false, + ); + + expect(explicit.audioMode, AndroidAudioMode.normal); + expect(explicit.forceAudioRouting, isTrue); + }); + + test('automatic mode ignores stored manual options after switching back', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionOptions(const AudioSessionOptions.media()); + await manager.setAudioSessionManagementMode(AudioSessionManagementMode.automatic); + + final isAutomatic = manager.managementMode == AudioSessionManagementMode.automatic; + final apple = resolveApplePolicy( + manager.options, + automatic: isAutomatic, + preferSpeakerOutput: false, + ); + final android = resolveAndroidPolicy( + manager.options, + automatic: isAutomatic, + ); + + expect(apple.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(apple.appleAudioMode, AppleAudioMode.voiceChat); + expect( + apple.appleAudioCategoryOptions, + { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + expect(android.audioMode, AndroidAudioMode.inCommunication); + expect(android.streamType, AndroidAudioStreamType.voiceCall); + }); + + test('handleAudioEngineState updates snapshot and stream', () async { + final manager = AudioManager.instance; + final states = []; + final subscription = manager.audioEngineStateStream.listen(states.add); + + manager.handleAudioEngineState( + isPlayoutEnabled: true, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect( + manager.audioEngineState, + const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false), + ); + expect(states, [const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false)]); + + manager.handleAudioEngineState( + isPlayoutEnabled: true, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect(states, [const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false)]); + + manager.handleAudioEngineState( + isPlayoutEnabled: false, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect(manager.audioEngineState.isIdle, isTrue); + expect( + states, + [ + const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false), + const AudioEngineState(isPlayoutEnabled: false, isRecordingEnabled: false), + ], + ); + + await subscription.cancel(); + }); + }); + + group('AndroidAudioSessionConfiguration', () { + test('communication preset uses voice communication values', () { + final config = AndroidAudioSessionConfiguration.communication; + + expect(config.manageAudioFocus, isTrue); + expect(config.audioMode, AndroidAudioMode.inCommunication); + expect(config.focusMode, AndroidAudioFocusMode.gain); + expect(config.streamType, AndroidAudioStreamType.voiceCall); + expect(config.usageType, AndroidAudioAttributesUsageType.voiceCommunication); + expect(config.contentType, AndroidAudioAttributesContentType.speech); + }); + + test('media preset uses non-communication media values', () { + final config = AndroidAudioSessionConfiguration.media; + + expect(config.manageAudioFocus, isTrue); + expect(config.audioMode, AndroidAudioMode.normal); + expect(config.focusMode, AndroidAudioFocusMode.gain); + expect(config.streamType, AndroidAudioStreamType.music); + expect(config.usageType, AndroidAudioAttributesUsageType.media); + expect(config.contentType, AndroidAudioAttributesContentType.unknown); + }); + }); + + group('NativeAudioConfiguration', () { + test('serializes Apple audio wire format', () { + final map = native_audio.NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.defaultToSpeaker, + }, + appleAudioMode: AppleAudioMode.default_, + ).toMap(); + + expect(map['appleAudioCategory'], 'playAndRecord'); + expect( + map['appleAudioCategoryOptions'], + unorderedEquals([ + 'allowBluetooth', + 'defaultToSpeaker', + ]), + ); + expect(map['appleAudioMode'], 'default'); + expect(map.containsKey('preferSpeakerOutput'), isFalse); + }); + }); + + group('Native audio channel', () { + late List calls; + + setUp(() { + calls = []; + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + calls.add(call); + return call.method == 'configureNativeAudio' ? true : null; + }, + ); + }); + + tearDown(() { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + null, + ); + }); + + test('passes forced speaker routing to Android platform method', () async { + await Native.setAndroidSpeakerphoneOn(true, force: true); + + expect(calls.single.method, 'setAndroidSpeakerphoneOn'); + expect(calls.single.arguments, {'enable': true, 'force': true}); + }); + + test('passes audio session deactivation to platform methods', () async { + await Native.stopAndroidAudioSession(); + await Native.deactivateAppleAudioSession(); + + expect(calls[0].method, 'stopAndroidAudioSession'); + expect(calls[0].arguments, isNull); + expect(calls[1].method, 'deactivateAppleAudioSession'); + expect(calls[1].arguments, {}); + }); + + test('passes forced speaker routing to automatic Apple configuration', () async { + final result = await Native.configureAudio( + native_audio.NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioMode: AppleAudioMode.videoChat, + ), + automatic: true, + selectCategoryByEngineState: true, + forceSpeakerOutput: true, + ); + + expect(result, isTrue); + expect(calls.single.method, 'configureNativeAudio'); + expect( + calls.single.arguments, + containsPair('forceSpeakerOutput', true), + ); + }); + }); + + group('androidAudioSessionConfigurationToMap', () { + test('serializes communication preset for the native session manager', () { + expect( + androidAudioSessionConfigurationToMap(AndroidAudioSessionConfiguration.communication), + { + 'manageAudioFocus': true, + 'androidAudioMode': 'inCommunication', + 'androidAudioFocusMode': 'gain', + 'androidAudioStreamType': 'voiceCall', + 'androidAudioAttributesUsageType': 'voiceCommunication', + 'androidAudioAttributesContentType': 'speech', + }, + ); + }); + + test('omits unset Android fields', () { + final config = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + forceAudioRouting: true, + ); + + expect( + androidAudioSessionConfigurationToMap(config), + { + 'androidAudioMode': 'normal', + 'forceHandleAudioRouting': true, + }, + ); + }); + }); +} From d1fe342c17be4340011248d72f53f2937c8e645e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 20 Jun 2026 03:17:14 +0900 Subject: [PATCH 15/18] Apply audio processing options when local capture starts (#1115) ## Summary - move create-time audio processing setup into the local capture start path so publish/preconnect prepares platform audio processing before WebRTC opens the microphone, matching the Swift SDK flow - keep `LocalAudioTrack.setAudioProcessingOptions` as a command-style runtime API that returns on success and throws `AudioProcessingException` on failure - expose structured failure reasons for invalid combinations, unavailable platform support, native apply failures, and unknown failures - clean up local audio tracks when capture/publish startup fails and keep cleanup failures from masking the original error - bump `flutter_webrtc` to `1.5.2` for Android audio device module access ## Behavior - `LocalAudioTrack.create(...)` stores requested audio processing options; it no longer attempts to apply them immediately. - `track.start()` starts local capture and applies stored processing options before the microphone opens on supported platforms. - Publish and preconnect paths fail during capture start if the exposed native platform API reports audio processing setup failure. - Runtime `setAudioProcessingOptions(...)` still applies immediately for active local audio tracks and throws on failed native apply/store. - Android uses `JavaAudioDeviceModule.prewarmRecording(options)`. That WebRTC API returns `void`, so this PR can surface thrown failures, but clean internal `false` returns from `initRecordingIfNeeded()` / `prewarmRecordingIfNeeded()` are not observable until WebRTC/flutter_webrtc exposes a resultful API. ## Testing - `flutter pub get` - `flutter pub get` in `example` - `flutter analyze --no-pub` - `flutter test --no-pub` - `flutter build apk --debug --no-pub` in `example` - Android emulator smoke test: example joined a room, published local microphone audio, muted/unpublished, and showed no ADM/platform-unavailable errors --- .changes/audio-processing-create-cleanup | 1 + .changes/audio-processing-platform-gate | 1 + .changes/audio-processing-start-flow | 1 + .changes/runtime-audio-options | 2 +- .../kotlin/io/livekit/plugin/LiveKitPlugin.kt | 111 +++++++++-- docs/audio.md | 24 ++- lib/src/audio/audio_manager.dart | 4 +- lib/src/participant/local.dart | 136 ++++++++------ .../preconnect/pre_connect_audio_buffer.dart | 20 +- lib/src/support/native.dart | 89 +++++++-- lib/src/support/platform.dart | 3 + lib/src/track/local/audio.dart | 129 +++++++++---- lib/src/track/local/local.dart | 8 +- lib/src/track/options.dart | 78 ++++---- lib/src/track/track.dart | 14 +- pubspec.lock | 4 +- pubspec.yaml | 2 +- shared_swift/LiveKitPlugin.swift | 72 +++++++- test/audio/audio_session_test.dart | 174 ++++++++++++++++++ 19 files changed, 670 insertions(+), 203 deletions(-) create mode 100644 .changes/audio-processing-create-cleanup create mode 100644 .changes/audio-processing-platform-gate create mode 100644 .changes/audio-processing-start-flow diff --git a/.changes/audio-processing-create-cleanup b/.changes/audio-processing-create-cleanup new file mode 100644 index 000000000..780bd0711 --- /dev/null +++ b/.changes/audio-processing-create-cleanup @@ -0,0 +1 @@ +patch type="fixed" "Clean up local audio tracks when recording start fails" diff --git a/.changes/audio-processing-platform-gate b/.changes/audio-processing-platform-gate new file mode 100644 index 000000000..82d744616 --- /dev/null +++ b/.changes/audio-processing-platform-gate @@ -0,0 +1 @@ +patch type="fixed" "Throw platformUnavailable when runtime audio processing is unsupported" diff --git a/.changes/audio-processing-start-flow b/.changes/audio-processing-start-flow new file mode 100644 index 000000000..0ad50dd92 --- /dev/null +++ b/.changes/audio-processing-start-flow @@ -0,0 +1 @@ +patch type="fixed" "Apply create-time audio processing when local recording is prepared" diff --git a/.changes/runtime-audio-options b/.changes/runtime-audio-options index 7cb3ff1ec..f4a43122d 100644 --- a/.changes/runtime-audio-options +++ b/.changes/runtime-audio-options @@ -1 +1 @@ -minor type="added" "Runtime audio processing options for local audio tracks and engine-wide audio processing state read-back on AudioManager" +minor type="added" "Runtime audio processing controls for local audio tracks" diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt index 925890628..11a04536e 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt @@ -17,6 +17,8 @@ package io.livekit.plugin import android.annotation.SuppressLint +import android.os.Handler +import android.os.Looper import androidx.annotation.NonNull import io.flutter.embedding.engine.plugins.FlutterPlugin @@ -37,6 +39,9 @@ import org.webrtc.audio.AudioProcessingMode import org.webrtc.audio.AudioProcessingOptions import org.webrtc.audio.AudioProcessingOptionsResult import org.webrtc.audio.AudioProcessingState +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.RejectedExecutionException /** LiveKitPlugin */ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { @@ -44,6 +49,8 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private var flutterWebRTCPlugin = FlutterWebRTCPlugin.sharedSingleton private var binaryMessenger: BinaryMessenger? = null private var audioSwitchManager: LKAudioSwitchManager? = null + private var audioDeviceModuleExecutor: ExecutorService? = null + private val mainHandler = Handler(Looper.getMainLooper()) /// The MethodChannel that will the communication between Flutter and native Android /// @@ -59,6 +66,8 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { channel.setMethodCallHandler(this) binaryMessenger = flutterPluginBinding.binaryMessenger audioSwitchManager = LKAudioSwitchManager(flutterPluginBinding.applicationContext) + audioDeviceModuleExecutor?.shutdown() + audioDeviceModuleExecutor = Executors.newSingleThreadExecutor() } @SuppressLint("SuspiciousIndentation") @@ -236,7 +245,84 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { return } - val options = AudioProcessingOptions( + val options = audioProcessingOptions(call) + val processingResult = mediaTrack.setAudioProcessingOptions(options) + result.success( + mapOf( + "result" to processingResult.isSuccess, + "code" to audioProcessingResultCodeString(processingResult.code), + "message" to processingResult.message, + ), + ) + } + + private fun handleStartLocalRecording(call: MethodCall, result: Result) { + val audioDeviceModule = flutterWebRTCPlugin.audioDeviceModule + if (audioDeviceModule == null) { + result.error("rejectedPlatformUnavailable", "audio device module is unavailable", null) + return + } + + val executor = audioDeviceModuleExecutorOrError(result, "rejectedPlatformUnavailable") ?: return + val options = audioProcessingOptions(call) + try { + executor.execute { + try { + // prewarmRecording applies Android platform AP and prepares recording + // without setting the client-start flag. WebRTC exposes this as void, + // so only thrown failures can be surfaced here. + audioDeviceModule.prewarmRecording(options) + mainHandler.post { + result.success(null) + } + } catch (error: Throwable) { + mainHandler.post { + result.error("applyFailed", error.message, null) + } + } + } + } catch (error: RejectedExecutionException) { + result.error("rejectedPlatformUnavailable", "audio device module executor is unavailable", null) + } + } + + private fun handleStopLocalRecording(result: Result) { + val audioDeviceModule = flutterWebRTCPlugin.audioDeviceModule + if (audioDeviceModule == null) { + result.error("stopLocalRecording", "audio device module is unavailable", null) + return + } + + val executor = audioDeviceModuleExecutorOrError(result, "stopLocalRecording") ?: return + try { + executor.execute { + try { + audioDeviceModule.requestStopRecording() + mainHandler.post { + result.success(null) + } + } catch (error: Throwable) { + mainHandler.post { + result.error("stopLocalRecording", error.message, null) + } + } + } + } catch (error: RejectedExecutionException) { + result.error("stopLocalRecording", "audio device module executor is unavailable", null) + } + } + + private fun audioDeviceModuleExecutorOrError(result: Result, code: String): ExecutorService? { + val executor = audioDeviceModuleExecutor + if (executor == null || executor.isShutdown) { + result.error(code, "audio device module executor is unavailable", null) + return null + } + return executor + } + + private fun audioProcessingOptions(call: MethodCall): AudioProcessingOptions = + AudioProcessingOptions( AudioProcessingComponentOptions( call.argument("echoCancellation") ?: true, audioProcessingMode(call.argument("echoCancellationMode")), @@ -255,16 +341,6 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { ), ) - val processingResult = mediaTrack.setAudioProcessingOptions(options) - result.success( - mapOf( - "result" to processingResult.isSuccess, - "code" to audioProcessingResultCodeString(processingResult.code), - "message" to processingResult.message, - ), - ) - } - private fun audioProcessingMode(value: String?): AudioProcessingMode = when (value) { "platform" -> AudioProcessingMode.PLATFORM "software" -> AudioProcessingMode.SOFTWARE @@ -274,7 +350,7 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private fun audioProcessingResultCodeString(code: AudioProcessingOptionsResult.Code): String = when (code) { AudioProcessingOptionsResult.Code.APPLIED -> "applied" AudioProcessingOptionsResult.Code.STORED -> "stored" - AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "rejectedRemoteTrack" + AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "unknown" AudioProcessingOptionsResult.Code.REJECTED_INVALID_COMBINATION -> "rejectedInvalidCombination" AudioProcessingOptionsResult.Code.REJECTED_PLATFORM_UNAVAILABLE -> "rejectedPlatformUnavailable" AudioProcessingOptionsResult.Code.APPLY_FAILED -> "applyFailed" @@ -352,6 +428,14 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { handleSetAudioProcessingOptions(call, result) } + "startLocalRecording" -> { + handleStartLocalRecording(call, result) + } + + "stopLocalRecording" -> { + handleStopLocalRecording(result) + } + "getAudioProcessingState" -> { handleGetAudioProcessingState(result) } @@ -388,6 +472,9 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { audioSwitchManager?.dispose() audioSwitchManager = null + audioDeviceModuleExecutor?.shutdown() + audioDeviceModuleExecutor = null + // Cleanup all processors audioProcessors.values.forEach { it.cleanup() } audioProcessors.clear() diff --git a/docs/audio.md b/docs/audio.md index 0f11b5c5e..387cb5c06 100644 --- a/docs/audio.md +++ b/docs/audio.md @@ -108,24 +108,34 @@ final now = AudioManager.instance.audioEngineState; Session management is one half of the audio stack. The other half is audio processing, the signal processing applied to captured audio such as echo cancellation, noise suppression, auto gain control, and the high pass filter. `AudioManager` is the home for both, and they compose cleanly. -The session intent decides how the platform treats audio. Processing options decide what happens to captured local microphone audio: +The session intent decides how the platform treats audio. Capture options decide what happens to local microphone audio: - A call usually uses the automatic `communication` session. The default `AudioCaptureOptions` enable echo cancellation, noise suppression, and auto gain control, while leaving the high pass filter off. - Use `AudioProcessingOptions.communication()` when you want all four voice filters on for an existing local audio track. - Use `AudioProcessingOptions.noProcessing()` for local capture where you want minimal processing, such as high quality recording or app-managed audio effects. -Processing is applied per local audio track. A malformed request (an incompatible mode, or a remote track) throws, while a request the platform could not honor comes back as an unsuccessful result: +Create-time processing is configured through `AudioCaptureOptions`. `LocalAudioTrack.create(...)` stores these options, and LiveKit prepares them when local recording starts, such as during publish or preconnect. If the exposed native platform API reports that capture-time setup failed, the start path throws `AudioProcessingException` before publish creates a server-side publication. + +```dart +final track = await LocalAudioTrack.create( + const AudioCaptureOptions( + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + highPassFilter: false, + ), +); +``` + +For an existing local audio track, call `setAudioProcessingOptions`. It returns when the native layer applies or stores the options. If the request is invalid, unsupported, or cannot be applied, it throws `AudioProcessingException` and the track keeps its previous processing options. ```dart try { - final result = await localAudioTrack.setAudioProcessingOptions( + await localAudioTrack.setAudioProcessingOptions( const AudioProcessingOptions.noProcessing(), ); - if (!result.isSuccess) { - print('audio processing not applied: ${result.code.value} ${result.message}'); - } } on AudioProcessingException catch (error) { - print('invalid audio processing request: ${error.code.value} ${error.message}'); + print('audio processing not applied: ${error.reason.name} ${error.message}'); } ``` diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart index 116b2d25f..dea966037 100644 --- a/lib/src/audio/audio_manager.dart +++ b/lib/src/audio/audio_manager.dart @@ -291,8 +291,8 @@ class AudioManager { /// /// The audio processing module is owned by the native peer connection factory /// and shared engine-wide, so this reflects what is actually applied across - /// the engine rather than any single track — use it to verify what a - /// `LocalAudioTrack.setAudioProcessingOptions` request resolved to. Returns + /// the engine rather than any single track. Use it to verify native state + /// after a `LocalAudioTrack.setAudioProcessingOptions` request. Returns /// `null` when the native side cannot provide it. Future getAudioProcessingState() async { final response = await Native.getAudioProcessingState(); diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index c5b5f86c1..d8e4d41fe 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -179,80 +179,94 @@ class LocalParticipant extends Participant { final audioEncoding = publishOptions.encoding ?? AudioEncoding.presetMusic; final List encodings = [audioEncoding.toRTCRtpEncoding()]; - final req = lk_rtc.AddTrackRequest( - cid: track.getCid(), - name: publishOptions.name ?? AudioPublishOptions.defaultMicrophoneName, - type: track.kind.toPBType(), - source: track.source.toPBType(), - muted: track.muted, - stream: buildStreamId(publishOptions, track.source), - disableDtx: !publishOptions.dtx, - disableRed: room.e2eeManager != null ? true : publishOptions.red ?? true, - encryption: room.roomOptions.lkEncryptionType, - ); + final shouldStopOnFailure = !track.isActive; + try { + // Start capture before signaling so create-time audio processing failures + // abort publish without creating a server-side publication. + await track.start(); - // Populate audio features (e.g., TF_NO_DTX, TF_PRECONNECT_BUFFER) - req.audioFeatures.addAll([ - if (!publishOptions.dtx) lk_models.AudioTrackFeature.TF_NO_DTX, - if (publishOptions.preConnect) lk_models.AudioTrackFeature.TF_PRECONNECT_BUFFER, - ]); + final req = lk_rtc.AddTrackRequest( + cid: track.getCid(), + name: publishOptions.name ?? AudioPublishOptions.defaultMicrophoneName, + type: track.kind.toPBType(), + source: track.source.toPBType(), + muted: track.muted, + stream: buildStreamId(publishOptions, track.source), + disableDtx: !publishOptions.dtx, + disableRed: room.e2eeManager != null ? true : publishOptions.red ?? true, + encryption: room.roomOptions.lkEncryptionType, + ); - Future negotiate() async { - track.transceiver = await room.engine.createTransceiverRTCRtpSender(track, publishOptions!, encodings); - await room.engine.negotiate(); - return lk_models.TrackInfo(); - } + // Populate audio features (e.g., TF_NO_DTX, TF_PRECONNECT_BUFFER) + req.audioFeatures.addAll([ + if (!publishOptions.dtx) lk_models.AudioTrackFeature.TF_NO_DTX, + if (publishOptions.preConnect) lk_models.AudioTrackFeature.TF_PRECONNECT_BUFFER, + ]); - late lk_models.TrackInfo trackInfo; - if (room.engine.enabledPublishCodecs?.isNotEmpty ?? false) { - final rets = await Future.wait([room.engine.addTrack(req), negotiate()]); - trackInfo = rets[0]; - } else { - trackInfo = await room.engine.addTrack(req); + Future negotiate() async { + track.transceiver = await room.engine.createTransceiverRTCRtpSender(track, publishOptions!, encodings); + await room.engine.negotiate(); + return lk_models.TrackInfo(); + } - final transceiverInit = rtc.RTCRtpTransceiverInit( - direction: rtc.TransceiverDirection.SendOnly, - sendEncodings: encodings, - ); - // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web) - track.transceiver = await room.engine.publisher?.pc.addTransceiver( - track: track.mediaStreamTrack, - kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio, - init: transceiverInit, - ); + late lk_models.TrackInfo trackInfo; + if (room.engine.enabledPublishCodecs?.isNotEmpty ?? false) { + final rets = await Future.wait([room.engine.addTrack(req), negotiate()]); + trackInfo = rets[0]; + } else { + trackInfo = await room.engine.addTrack(req); - await room.engine.negotiate(); - } + final transceiverInit = rtc.RTCRtpTransceiverInit( + direction: rtc.TransceiverDirection.SendOnly, + sendEncodings: encodings, + ); + // addTransceiver cannot pass in a kind parameter due to a bug in flutter-webrtc (web) + track.transceiver = await room.engine.publisher?.pc.addTransceiver( + track: track.mediaStreamTrack, + kind: rtc.RTCRtpMediaType.RTCRtpMediaTypeAudio, + init: transceiverInit, + ); - logger.fine('publishAudioTrack engine.addTrack response: ${trackInfo}'); + await room.engine.negotiate(); + } - track.lastPublishOptions = publishOptions; + logger.fine('publishAudioTrack engine.addTrack response: ${trackInfo}'); - final pub = LocalTrackPublication( - participant: this, - info: trackInfo, - track: track, - ); - addTrackPublication(pub); + track.lastPublishOptions = publishOptions; - // did publish - await track.onPublish(); - await track.processor?.onPublish(room); + final pub = LocalTrackPublication( + participant: this, + info: trackInfo, + track: track, + ); + addTrackPublication(pub); - final listener = track.createListener(); - listener.on((TrackEndedEvent event) async { - logger.fine('TrackEndedEvent: ${event.track}'); - await removePublishedTrack(pub.sid); - }); + // did publish + await track.onPublish(); + await track.processor?.onPublish(room); - [events, room.events].emit(LocalTrackPublishedEvent( - participant: this, - publication: pub, - )); + final listener = track.createListener(); + listener.on((TrackEndedEvent event) async { + logger.fine('TrackEndedEvent: ${event.track}'); + await removePublishedTrack(pub.sid); + }); - await track.start(); + [events, room.events].emit(LocalTrackPublishedEvent( + participant: this, + publication: pub, + )); - return pub; + return pub; + } catch (error) { + if (shouldStopOnFailure) { + try { + await track.stop(); + } catch (stopError) { + logger.warning('failed to stop audio track after publish failure: $stopError'); + } + } + rethrow; + } } /// Publish a [LocalVideoTrack] to the [Room]. diff --git a/lib/src/preconnect/pre_connect_audio_buffer.dart b/lib/src/preconnect/pre_connect_audio_buffer.dart index 39cd2fb5a..c04eca8fd 100644 --- a/lib/src/preconnect/pre_connect_audio_buffer.dart +++ b/lib/src/preconnect/pre_connect_audio_buffer.dart @@ -14,9 +14,6 @@ import 'dart:async'; -import 'package:flutter/foundation.dart' show kIsWeb; - -import 'package:flutter_webrtc/flutter_webrtc.dart' as webrtc; import 'package:uuid/uuid.dart'; import '../audio/audio_frame_capture.dart'; @@ -25,6 +22,8 @@ import '../events.dart'; import '../logger.dart'; import '../participant/local.dart'; import '../support/byte_ring_buffer.dart'; +import '../support/native.dart'; +import '../support/platform.dart'; import '../support/reusable_completer.dart'; import '../track/local/audio.dart'; import '../types/data_stream.dart'; @@ -145,9 +144,16 @@ class PreConnectAudioBuffer { throw error; } - if (!kIsWeb) { - await webrtc.NativeAudioManagement.startLocalRecording(); - _nativeRecordingStarted = true; + try { + await _localTrack!.start(); + _nativeRecordingStarted = lkPlatformSupportsExplicitAudioRecordingStart(); + } catch (error) { + logger.severe('[Preconnect audio] failed to start local recording: $error'); + _onError?.call(error); + await stopRecording(withError: error); + await _localTrack?.stop(); + _localTrack = null; + rethrow; } logger.info('startAudioRenderer result: $result'); @@ -212,7 +218,7 @@ class PreConnectAudioBuffer { // Only stop native recording on error, the room's mic track still uses it. if (withError != null && _nativeRecordingStarted) { - await webrtc.NativeAudioManagement.stopLocalRecording(); + await Native.stopLocalRecording(); } _nativeRecordingStarted = false; diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index b71dd18ab..6817a9705 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -14,7 +14,7 @@ import 'dart:async'; -import 'package:flutter/services.dart' show MethodChannel, MethodCall; +import 'package:flutter/services.dart' show MethodChannel, MethodCall, MissingPluginException, PlatformException; import 'package:meta/meta.dart'; @@ -72,24 +72,87 @@ class Native { /// /// Resolved natively against the underlying WebRTC audio track owned by /// flutter_webrtc; [options] is the serialized [AudioProcessingOptions] map. - /// Returns the native result map (`result`/`code`/`message`) so the caller - /// can surface typed rejections. Channel errors propagate to the caller. + /// Returns the native result map (`result`/`code`/`message`) so the Dart + /// track API can translate native outcomes into typed exceptions. + /// This plugin is registered on platforms that do not implement runtime audio + /// processing, so missing or explicitly unimplemented hooks are normalized to + /// `rejectedPlatformUnavailable`. Other channel errors propagate because they + /// indicate unexpected native failures rather than an unsupported platform + /// capability. @internal static Future> setAudioProcessingOptions( String trackId, Map options, ) async { - final response = await channel.invokeMethod( - 'setAudioProcessingOptions', - { - 'trackId': trackId, - ...options, - }, - ); - if (response is Map) { - return response.map((key, value) => MapEntry(key.toString(), value)); + try { + final response = await channel.invokeMethod( + 'setAudioProcessingOptions', + { + 'trackId': trackId, + ...options, + }, + ); + if (response is Map) { + return response.map((key, value) => MapEntry(key.toString(), value)); + } + return {}; + } on MissingPluginException { + return _audioProcessingPlatformUnavailable(); + } on PlatformException catch (error) { + // Web registers the plugin but returns `Unimplemented` for methods it + // does not support. Treat only that narrow case like a missing native + // implementation; do not hide unrelated native/channel failures. + if (error.code == 'Unimplemented') { + return _audioProcessingPlatformUnavailable(); + } + rethrow; + } + } + + static Map _audioProcessingPlatformUnavailable() => { + 'result': false, + 'code': 'rejectedPlatformUnavailable', + 'message': 'Audio processing options are unavailable on this platform.', + }; + + static PlatformException _audioProcessingPlatformUnavailableException() => PlatformException( + code: 'rejectedPlatformUnavailable', + message: 'Audio processing options are unavailable on this platform.', + ); + + /// Starts the native WebRTC audio device module recording path with the + /// capture-time audio processing options for the local microphone track. + @internal + static Future startLocalRecording(Map audioProcessingOptions) async { + try { + await channel.invokeMethod( + 'startLocalRecording', + audioProcessingOptions, + ); + } on PlatformException catch (error) { + if (error.code == 'Unimplemented') { + throw _audioProcessingPlatformUnavailableException(); + } + rethrow; + } on MissingPluginException { + throw _audioProcessingPlatformUnavailableException(); + } + } + + /// Stops recording that was explicitly started through [startLocalRecording]. + @internal + static Future stopLocalRecording() async { + try { + await channel.invokeMethod('stopLocalRecording', {}); + } on PlatformException catch (error) { + if (error.code == 'Unimplemented') { + logger.warning('stopLocalRecording is not implemented on this platform'); + return; + } + logger.warning('stopLocalRecording did throw ${error.code}: ${error.message}'); + } on MissingPluginException { + logger.warning('stopLocalRecording is not available on this platform'); } - return {}; } /// Reads the engine-wide audio processing state from the native peer diff --git a/lib/src/support/platform.dart b/lib/src/support/platform.dart index 3c3bb05f5..242c0ff01 100644 --- a/lib/src/support/platform.dart +++ b/lib/src/support/platform.dart @@ -29,6 +29,9 @@ bool lkPlatformIsDesktop() => [ PlatformType.linux, ].contains(lkPlatform()); +bool lkPlatformSupportsExplicitAudioRecordingStart() => + !lkPlatformIsTest() && [PlatformType.iOS, PlatformType.macOS, PlatformType.android].contains(lkPlatform()); + bool lkPlatformSupportsE2EE() => lkE2EESupportedImplementation(); bool lkPlatformIsTest() => lkPlatformIsTestImplementation(); diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index 48468b8cc..77966d2b5 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -14,6 +14,8 @@ import 'dart:async'; +import 'package:flutter/services.dart' show PlatformException; + import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; @@ -25,6 +27,7 @@ import '../../options.dart'; import '../../stats/audio_source_stats.dart'; import '../../stats/stats.dart'; import '../../support/native.dart'; +import '../../support/platform.dart'; import '../../types/other.dart'; import '../audio_management.dart'; import '../options.dart' as track_options; @@ -47,36 +50,26 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi } } - Future setAudioProcessingOptions( - track_options.AudioProcessingOptions options) async { + /// Applies runtime audio processing options to this local audio track. + /// + /// On success, updates [currentOptions] and emits + /// [LocalTrackOptionsUpdatedEvent]. When the native layer cannot apply the + /// options, throws [track_options.AudioProcessingException] and leaves + /// [currentOptions] unchanged. + Future setAudioProcessingOptions(track_options.AudioProcessingOptions options) async { final nextOptions = currentOptions.copyWith(processing: options); final response = await Native.setAudioProcessingOptions( mediaStreamTrack.id!, options.toMap(), ); - final code = track_options.AudioProcessingOptionsResultCode.fromValue(response['code'] as String?); - final message = (response['message'] as String?) ?? ''; - - // Malformed requests (incompatible modes, or a non-local track) are caller - // bugs — surface them loudly rather than as a silently-unsuccessful result. - if (code == track_options.AudioProcessingOptionsResultCode.rejectedInvalidCombination || - code == track_options.AudioProcessingOptionsResultCode.rejectedRemoteTrack) { - throw track_options.AudioProcessingException( - code, - message.isNotEmpty ? message : 'Unable to apply audio processing options', - ); - } + _throwIfAudioProcessingFailed(response); - final result = track_options.AudioProcessingApplyResult(code, message); - if (result.isSuccess) { - currentOptions = nextOptions; - events.emit(LocalTrackOptionsUpdatedEvent( - track: this, - options: currentOptions, - )); - } - return result; + currentOptions = nextOptions; + events.emit(LocalTrackOptionsUpdatedEvent( + track: this, + options: currentOptions, + )); } num? _currentBitrate; @@ -84,6 +77,23 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi AudioSenderStats? prevStats; + @override + Future startCapture() async { + await super.startCapture(); + if (lkPlatformSupportsExplicitAudioRecordingStart()) { + try { + // Match Swift: start the ADM before publishing so capture-time audio + // processing options are applied before WebRTC opens the microphone. + await Native.startLocalRecording(currentOptions.processing.toMap()); + } on PlatformException catch (error) { + throw track_options.AudioProcessingException( + _audioProcessingFailureReason(error.code), + error.message ?? '', + ); + } + } + } + @override Future monitorStats() async { if (events.isDisposed || !isActive) { @@ -172,20 +182,69 @@ class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMi options, ); - if (options.processor != null) { - await track.setProcessor(options.processor); - } - - // Per-component processing modes are not part of standard capture - // constraints; apply them through the native audio processing path. - final processing = options.processing; - if (processing.echoCancellationMode != track_options.AudioProcessingMode.automatic || - processing.noiseSuppressionMode != track_options.AudioProcessingMode.automatic || - processing.autoGainControlMode != track_options.AudioProcessingMode.automatic || - processing.highPassFilterMode != track_options.AudioProcessingMode.automatic) { - await track.setAudioProcessingOptions(processing); + try { + if (options.processor != null) { + await track.setProcessor(options.processor); + } + } catch (error, stackTrace) { + try { + await track.stop(); + } catch (stopError) { + logger.warning('failed to stop audio track after processor setup failure: $stopError'); + } + Error.throwWithStackTrace(error, stackTrace); } return track; } } + +void _throwIfAudioProcessingFailed(Map response) { + final code = response['code'] as String?; + final message = (response['message'] as String?) ?? ''; + + final reason = _audioProcessingFailureReason(code); + switch (code) { + case 'applied': + case 'stored': + return; + case 'rejectedInvalidCombination': + case 'rejectedPlatformUnavailable': + case 'applyFailed': + case 'unknown': + case 'rejectedRemoteTrack': + throw track_options.AudioProcessingException( + reason, + message, + ); + default: + throw track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.unknown, + _unknownAudioProcessingMessage(code, message), + ); + } +} + +track_options.AudioProcessingFailureReason _audioProcessingFailureReason(String? code) { + switch (code) { + case 'rejectedInvalidCombination': + return track_options.AudioProcessingFailureReason.invalidCombination; + case 'rejectedPlatformUnavailable': + return track_options.AudioProcessingFailureReason.platformUnavailable; + case 'applyFailed': + return track_options.AudioProcessingFailureReason.applyFailed; + default: + return track_options.AudioProcessingFailureReason.unknown; + } +} + +String _unknownAudioProcessingMessage(String? code, String message) { + final trimmed = message.trim(); + if (trimmed.isNotEmpty) { + return trimmed; + } + if (code != null && code.isNotEmpty) { + return 'Unknown audio processing result code: $code.'; + } + return ''; +} diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index 1b197c50f..3a3cea8c6 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -96,13 +96,13 @@ mixin AudioTrack on Track { } @override - Future onStarted() async { - logger.fine('AudioTrack.onStarted()'); + Future startCapture() async { + logger.fine('AudioTrack.startCapture()'); } @override - Future onStopped() async { - logger.fine('AudioTrack.onStopped()'); + Future stopCapture() async { + logger.fine('AudioTrack.stopCapture()'); for (final group in _captureGroups.values) { await group.stop(); } diff --git a/lib/src/track/options.dart b/lib/src/track/options.dart index cc717a957..58aaa6e1a 100644 --- a/lib/src/track/options.dart +++ b/lib/src/track/options.dart @@ -509,55 +509,49 @@ class AudioOutputOptions { } } -/// Result code from applying [AudioProcessingOptions], mirroring the native -/// `AudioProcessingOptionsResult`. `applied`/`stored` are success; the rest are -/// rejections. -enum AudioProcessingOptionsResultCode { - applied('applied'), - stored('stored'), - rejectedRemoteTrack('rejectedRemoteTrack'), - rejectedInvalidCombination('rejectedInvalidCombination'), - rejectedPlatformUnavailable('rejectedPlatformUnavailable'), - applyFailed('applyFailed'); - - const AudioProcessingOptionsResultCode(this.value); - - final String value; - - static AudioProcessingOptionsResultCode fromValue(String? value) => - AudioProcessingOptionsResultCode.values.firstWhere( - (e) => e.value == value, - orElse: () => AudioProcessingOptionsResultCode.applyFailed, - ); +/// Reason that applying [AudioProcessingOptions] failed. +enum AudioProcessingFailureReason { + /// The requested mode combination is invalid for the native audio module. + invalidCombination, - bool get isSuccess => - this == AudioProcessingOptionsResultCode.applied || this == AudioProcessingOptionsResultCode.stored; -} + /// The platform or device cannot provide the requested processing path. + platformUnavailable, -/// Thrown when the native layer rejects requested [AudioProcessingOptions] -/// (e.g. an invalid platform/software combination, or platform processing that -/// is unavailable on the device). -class AudioProcessingException implements Exception { - AudioProcessingException(this.code, this.message); + /// The native layer attempted to apply the options but failed. + applyFailed, - final AudioProcessingOptionsResultCode code; - final String message; + /// The native layer returned an unrecognized or malformed result. + unknown, +} - @override - String toString() => 'AudioProcessingException(${code.value}): $message'; +String _defaultAudioProcessingMessage(AudioProcessingFailureReason reason) { + switch (reason) { + case AudioProcessingFailureReason.invalidCombination: + return 'The requested audio processing mode combination is invalid.'; + case AudioProcessingFailureReason.platformUnavailable: + return 'Audio processing options are unavailable on this platform or device.'; + case AudioProcessingFailureReason.applyFailed: + return 'The native WebRTC audio processing module could not apply the requested options.'; + case AudioProcessingFailureReason.unknown: + return 'Audio processing options failed for an unknown reason.'; + } +} + +String _audioProcessingMessageOrDefault(AudioProcessingFailureReason reason, String message) { + final trimmed = message.trim(); + return trimmed.isEmpty ? _defaultAudioProcessingMessage(reason) : trimmed; } -/// Outcome of applying [AudioProcessingOptions]. +/// Thrown when [AudioProcessingOptions] cannot be applied. /// -/// Returned for operational outcomes — `applied`/`stored` (success), and the -/// device-capability rejections `rejectedPlatformUnavailable`/`applyFailed` -/// (inspect [isSuccess]). A malformed request (incompatible modes, or a -/// non-local track) throws [AudioProcessingException] instead. -class AudioProcessingApplyResult { - AudioProcessingApplyResult(this.code, this.message); - - final AudioProcessingOptionsResultCode code; +/// [reason] is a stable SDK category. [message] carries native details when +/// available, or an SDK-provided fallback when native does not include details. +class AudioProcessingException implements Exception { + AudioProcessingException(this.reason, String message) : message = _audioProcessingMessageOrDefault(reason, message); + + final AudioProcessingFailureReason reason; final String message; - bool get isSuccess => code.isSuccess; + @override + String toString() => 'AudioProcessingException(${reason.name}): $message'; } diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index 520a69169..3e029b82b 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -110,11 +110,10 @@ abstract class Track extends DisposableChangeNotifier with EventsEmittable monitorStats(); + /// Called by [start] before this track is marked active. @internal - Future onStarted() async {} + Future startCapture() async {} + /// Called by [stop] while this track is still active, before the media track + /// is stopped. @internal - Future onStopped() async {} + Future stopCapture() async {} @internal void startMonitor() { diff --git a/pubspec.lock b/pubspec.lock index c78edefdc..4e81c9ecc 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -284,10 +284,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: c4e8db6ed337b8c30d76cd7cd8c91693f5495e1aeb556cbcb73f1e5d5bdcf020 + sha256: "0f89dee7f4c35dab5611f351c3897a3bfe9f7057720cc7da209b52455af4316e" url: "https://pub.dev" source: hosted - version: "1.5.1" + version: "1.5.2" frontend_server_client: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 0665dc206..72277ce22 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -46,7 +46,7 @@ dependencies: json_annotation: ^4.9.0 # Fix version to avoid version conflicts between WebRTC-SDK pods, which both this package and flutter_webrtc depend on. - flutter_webrtc: 1.5.1 + flutter_webrtc: 1.5.2 dart_webrtc: ^1.8.0 dev_dependencies: diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index 8a93f8431..2c6ae0ecf 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -429,7 +429,62 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { return } - let options = RTCAudioProcessingOptions( + let options = LiveKitPlugin.audioProcessingOptions(from: args) + let processingResult = audioTrack.setAudioProcessingOptions(options) + result([ + "result": processingResult.isSuccess, + "code": LiveKitPlugin.audioProcessingResultCodeString(processingResult.code), + "message": processingResult.message, + ]) + } + + public func handleStartLocalRecording(args: [String: Any?], result: @escaping FlutterResult) { + guard let adm = FlutterWebRTCPlugin.sharedSingleton()?.peerConnectionFactory?.audioDeviceModule else { + result(FlutterError(code: "rejectedPlatformUnavailable", message: "audio device module is unavailable", details: nil)) + return + } + + let options = LiveKitPlugin.audioProcessingOptions(from: args) + DispatchQueue.global(qos: .userInitiated).async { + let admResult = adm.initAndStartRecording(audioProcessingOptions: options) + DispatchQueue.main.async { + if admResult == 0 { + result(nil) + } else { + result(FlutterError( + code: "applyFailed", + message: "Audio engine returned error code: \(admResult)", + details: nil + )) + } + } + } + } + + public func handleStopLocalRecording(result: @escaping FlutterResult) { + guard let adm = FlutterWebRTCPlugin.sharedSingleton()?.peerConnectionFactory?.audioDeviceModule else { + result(FlutterError(code: "stopLocalRecording", message: "audio device module is unavailable", details: nil)) + return + } + + DispatchQueue.global(qos: .userInitiated).async { + let admResult = adm.stopRecording() + DispatchQueue.main.async { + if admResult == 0 { + result(nil) + } else { + result(FlutterError( + code: "stopLocalRecording", + message: "Audio engine returned error code: \(admResult)", + details: nil + )) + } + } + } + } + + static func audioProcessingOptions(from args: [String: Any?]) -> RTCAudioProcessingOptions { + RTCAudioProcessingOptions( echoCancellationOptions: RTCAudioProcessingComponentOptions( enabled: (args["echoCancellation"] as? Bool) ?? true, mode: LiveKitPlugin.audioProcessingMode(from: args["echoCancellationMode"] as? String) @@ -447,13 +502,6 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { mode: LiveKitPlugin.audioProcessingMode(from: args["highPassFilterMode"] as? String) ) ) - - let processingResult = audioTrack.setAudioProcessingOptions(options) - result([ - "result": processingResult.isSuccess, - "code": LiveKitPlugin.audioProcessingResultCodeString(processingResult.code), - "message": processingResult.message, - ]) } static func audioProcessingMode(from string: String?) -> RTCAudioProcessingMode { @@ -468,11 +516,11 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { switch code { case .applied: return "applied" case .stored: return "stored" - case .rejectedRemoteTrack: return "rejectedRemoteTrack" + case .rejectedRemoteTrack: return "unknown" case .rejectedInvalidCombination: return "rejectedInvalidCombination" case .rejectedPlatformUnavailable: return "rejectedPlatformUnavailable" case .applyFailed: return "applyFailed" - @unknown default: return "applyFailed" + @unknown default: return "unknown" } } @@ -552,6 +600,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { handleStartAudioRenderer(args: args, result: result) case "stopAudioRenderer": handleStopAudioRenderer(args: args, result: result) + case "startLocalRecording": + handleStartLocalRecording(args: args, result: result) + case "stopLocalRecording": + handleStopLocalRecording(result: result) case "setAudioProcessingOptions": handleSetAudioProcessingOptions(args: args, result: result) case "getAudioProcessingState": diff --git a/test/audio/audio_session_test.dart b/test/audio/audio_session_test.dart index 0705d57c6..7487d6af2 100644 --- a/test/audio/audio_session_test.dart +++ b/test/audio/audio_session_test.dart @@ -22,6 +22,7 @@ import 'package:livekit_client/src/audio/audio_session.dart'; import 'package:livekit_client/src/audio/audio_session_policy.dart'; import 'package:livekit_client/src/support/native.dart'; import 'package:livekit_client/src/support/native_audio.dart' as native_audio; +import 'package:livekit_client/src/track/options.dart' as track_options; void main() { TestWidgetsFlutterBinding.ensureInitialized(); @@ -197,6 +198,53 @@ void main() { }); }); + group('AudioProcessingException', () { + test('uses fallback messages when native omits details', () { + final invalid = track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.invalidCombination, + '', + ); + final platformUnavailable = track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.platformUnavailable, + ' ', + ); + final applyFailed = track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.applyFailed, + '', + ); + final unknown = track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.unknown, + '', + ); + + expect(invalid.message, 'The requested audio processing mode combination is invalid.'); + expect(platformUnavailable.message, 'Audio processing options are unavailable on this platform or device.'); + expect( + applyFailed.message, + 'The native WebRTC audio processing module could not apply the requested options.', + ); + expect(unknown.message, 'Audio processing options failed for an unknown reason.'); + }); + + test('preserves native messages when provided', () { + final error = track_options.AudioProcessingException( + track_options.AudioProcessingFailureReason.applyFailed, + ' native detail ', + ); + + expect(error.reason, track_options.AudioProcessingFailureReason.applyFailed); + expect(error.message, 'native detail'); + expect(error.toString(), 'AudioProcessingException(applyFailed): native detail'); + }); + + test('exposes unknown failure reason', () { + expect( + track_options.AudioProcessingFailureReason.values, + contains(track_options.AudioProcessingFailureReason.unknown), + ); + }); + }); + group('AudioManager', () { test('management mode can be set independently from options', () async { final manager = AudioManager.instance; @@ -544,6 +592,132 @@ void main() { containsPair('forceSpeakerOutput', true), ); }); + + test('returns platform unavailable when audio processing channel is missing', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + null, + ); + + final result = await Native.setAudioProcessingOptions( + 'track-id', + {'echoCancellation': true}, + ); + + expect( + result, + { + 'result': false, + 'code': 'rejectedPlatformUnavailable', + 'message': 'Audio processing options are unavailable on this platform.', + }, + ); + }); + + test('returns platform unavailable when audio processing method is unimplemented', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + calls.add(call); + throw PlatformException( + code: 'Unimplemented', + details: 'livekit for web does not implement ${call.method}', + ); + }, + ); + + final result = await Native.setAudioProcessingOptions( + 'track-id', + {'echoCancellation': true}, + ); + + expect(calls.single.method, 'setAudioProcessingOptions'); + expect(calls.single.arguments, containsPair('trackId', 'track-id')); + expect( + result, + { + 'result': false, + 'code': 'rejectedPlatformUnavailable', + 'message': 'Audio processing options are unavailable on this platform.', + }, + ); + }); + + test('propagates other audio processing channel failures', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + throw PlatformException(code: 'nativeFailure', message: 'boom'); + }, + ); + + await expectLater( + Native.setAudioProcessingOptions( + 'track-id', + {'echoCancellation': true}, + ), + throwsA(isA().having((error) => error.code, 'code', 'nativeFailure')), + ); + }); + + test('throws platform unavailable when startLocalRecording channel is missing', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + null, + ); + + await expectLater( + Native.startLocalRecording({'echoCancellation': true}), + throwsA(isA().having( + (error) => error.code, + 'code', + 'rejectedPlatformUnavailable', + )), + ); + }); + + test('throws platform unavailable when startLocalRecording is unimplemented', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + calls.add(call); + throw PlatformException(code: 'Unimplemented'); + }, + ); + + await expectLater( + Native.startLocalRecording({'echoCancellation': true}), + throwsA(isA().having( + (error) => error.code, + 'code', + 'rejectedPlatformUnavailable', + )), + ); + expect(calls.single.method, 'startLocalRecording'); + }); + + test('ignores stopLocalRecording platform failures', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + calls.add(call); + throw PlatformException(code: 'nativeFailure', message: 'boom'); + }, + ); + + await Native.stopLocalRecording(); + + expect(calls.single.method, 'stopLocalRecording'); + }); + + test('ignores stopLocalRecording missing plugin', () async { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + null, + ); + + await Native.stopLocalRecording(); + }); }); group('androidAudioSessionConfigurationToMap', () { From faa5b0b0e94a0a383f2b545dff00d934bc0751a7 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sat, 20 Jun 2026 08:58:42 +0900 Subject: [PATCH 16/18] Add Android initial audio session options (#1117) ## Summary - Add `LiveKitClient.initialize(initialAudioSessionOptions: ...)` for Android WebRTC audio-device initialization. - Seed Android automatic audio-session policy from the same initial options so playback-first apps do not need to immediately call `AudioManager.setAudioSessionOptions(...)` with the same preset. - Rename the playback preset to `AudioSessionOptions.mediaPlayback()` for clearer intent. - Avoid sticky Android speaker routing when updating speaker preference. - Update audio session docs and change entry. ## Runtime behavior Android uses `AudioSessionOptions` in two places with different timing: | API | Timing | What changes | | --- | --- | --- | | `LiveKitClient.initialize(initialAudioSessionOptions: ...)` | Before WebRTC initializes. | Passes Android audio configuration into flutter_webrtc/WebRTC ADM creation. This is where WebRTC playout `AudioAttributes` such as `usageType` and `contentType` are set. It also seeds LiveKit's automatic Android session policy. | | `AudioManager.instance.setAudioSessionOptions(...)` | Runtime. | Replaces LiveKit's stored session policy, switches to manual management, and applies LiveKit's platform session behavior: Android audio mode, audio focus mode, stream type, focus ownership, routing handler policy, and iOS category/options/mode. | The important Android limitation is that WebRTC playout `AudioAttributes` are still initialization-time configuration. Calling `AudioManager.setAudioSessionOptions(...)` at runtime applies LiveKit's platform session policy, but it does not mutate the `AudioAttributes` on an already-created WebRTC audio device module. For playback-first Android apps that need media volume/routing from WebRTC playout, pass `AudioSessionOptions.mediaPlayback()` to `LiveKitClient.initialize(...)` before connecting. ## Validation - `flutter test --no-pub test/audio/audio_session_test.dart` - `flutter analyze --no-pub --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .changes/android-media-routing | 3 + .../io/livekit/plugin/LKAudioSwitchManager.kt | 15 +-- docs/audio.md | 41 +++++-- lib/src/audio/audio_manager.dart | 15 +++ lib/src/audio/audio_session.dart | 11 +- lib/src/audio/audio_session_policy.dart | 7 +- lib/src/livekit.dart | 40 +++++-- .../support/webrtc_initialize_options.dart | 30 +++++ test/audio/audio_session_test.dart | 106 +++++++++++++++--- 9 files changed, 216 insertions(+), 52 deletions(-) create mode 100644 .changes/android-media-routing create mode 100644 lib/src/support/webrtc_initialize_options.dart diff --git a/.changes/android-media-routing b/.changes/android-media-routing new file mode 100644 index 000000000..ce41071ba --- /dev/null +++ b/.changes/android-media-routing @@ -0,0 +1,3 @@ +patch type="fixed" "Apply Android media audio attributes during WebRTC initialization" +patch type="fixed" "Use initialization audio options as the default Android session policy" +patch type="fixed" "Avoid sticky Android speaker routing when updating route preference" diff --git a/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt index 463d5ceec..37f928df0 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt @@ -195,20 +195,14 @@ internal class LKAudioSwitchManager(private val context: Context) { } private fun applySpeakerRouting(switch: AbstractAudioSwitch, speakerRouting: SpeakerRouting) { + // AudioSwitch treats selectDevice(null) as "select no device"; it does not + // recompute the best route from the preferred-device list. Keep routing + // automatic here so normal preference and forced-speaker priority both + // follow device hot-plug changes without leaving a sticky selected device. switch.setPreferredDeviceList(speakerRouting.preferredDeviceList) - val forcedSpeaker = if (speakerRouting.speakerOutputForced) { - switch.availableAudioDevices.firstOrNull { it is AudioDevice.Speakerphone } - } else { - null - } - // AudioSwitch selections are sticky. Use them only for forced speaker output. - // Clearing the selection lets the preferred-device list handle normal routing - // and headset hot-plug priority. - switch.selectDevice(forcedSpeaker) } private fun speakerRoutingSnapshot() = SpeakerRouting( - speakerOutputForced = speakerOutputForced, preferredDeviceList = preferredDeviceList( speakerOutputPreferred = speakerOutputPreferred, speakerOutputForced = speakerOutputForced, @@ -253,7 +247,6 @@ internal class LKAudioSwitchManager(private val context: Context) { ) private data class SpeakerRouting( - val speakerOutputForced: Boolean, val preferredDeviceList: List>, ) } diff --git a/docs/audio.md b/docs/audio.md index 387cb5c06..47718bfb4 100644 --- a/docs/audio.md +++ b/docs/audio.md @@ -12,6 +12,8 @@ LiveKit disables flutter_webrtc's own native audio management automatically when Speaker output is preferred by default, but a wired or Bluetooth headset still wins over the speaker. Forced speaker output is off, so the speaker is never forced over a connected headset unless you ask for it. +The media playback preset (`AudioSessionOptions.mediaPlayback()`) is for playback-first experiences such as viewer-only live streams. On Android, pass it to `LiveKitClient.initialize` before WebRTC initializes when you need the WebRTC audio device module to use media mode and media volume. This also seeds LiveKit's automatic runtime session policy until you explicitly replace it with `AudioManager.instance.setAudioSessionOptions(...)`. Runtime session updates apply LiveKit's platform session policy, but WebRTC playout `AudioAttributes` are currently initialized when the audio device module is created. + The default audio capture options apply standard voice processing, so echo cancellation, noise suppression, and auto gain control are on and the high pass filter is off. You can change this per track with `AudioProcessingOptions`. On macOS the audio engine state is reported but no `AVAudioSession` is configured. On web, Windows, and Linux the session APIs do not configure native audio. Speaker switching is available only on iOS and Android, where `AudioManager.instance.canSwitchSpeakerphone` is true. @@ -27,24 +29,45 @@ import 'package:livekit_client/livekit_client.dart'; // Take manual control and apply a media playback session. await AudioManager.instance.setAudioSessionOptions( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), +); +``` + +On Android, media output type has one WebRTC initialization-time piece and one LiveKit runtime-session piece. For playback-first apps, initialize WebRTC with the media intent before connecting: + +```dart +await LiveKitClient.initialize( + initialAudioSessionOptions: const AudioSessionOptions.mediaPlayback(), ); ``` -See the next section for the full rule. Apply options before connecting when you can. +This seeds both WebRTC's initialization-time playout attributes and LiveKit's automatic runtime session policy. See the next section for the full rule. Apply options before connecting when you can. + +## Configuration timing + +`AudioSessionOptions` are used in two places on Android: + +| API | Timing | What it can update | +| --- | --- | --- | +| `LiveKitClient.initialize(initialAudioSessionOptions: ...)` | Before WebRTC initializes. | WebRTC audio device module playout `AudioAttributes`, and LiveKit's initial automatic runtime session policy. Today WebRTC uses the Android `usageType` and `contentType` fields, for example `USAGE_MEDIA` and `CONTENT_TYPE_UNKNOWN` from `AudioSessionOptions.mediaPlayback()`. | +| `AudioManager.instance.setAudioSessionOptions(...)` | Runtime. | Explicitly replaces the stored session options, switches to manual management, and applies LiveKit's platform session policy: Android audio mode, audio focus mode, stream type, focus ownership, routing handler policy, and iOS category/options/mode. | + +Most fields in `AudioSessionOptions` are runtime session policy and can be applied again with `AudioManager.instance.setAudioSessionOptions(...)`. The exception is Android WebRTC playout attributes: changing `AndroidAudioSessionConfiguration.usageType` or `contentType` at runtime updates LiveKit's session/focus handler, but it does not change the `AudioAttributes` of an already-created WebRTC audio device module. Pass those options to `LiveKitClient.initialize(...)` before WebRTC initializes when they must affect playout volume/routing. You do not need to call `setAudioSessionOptions(...)` with the same options just to make LiveKit use them during automatic session management. + +We plan to make Android WebRTC playout attributes runtime-updatable in a future SDK/WebRTC integration if the native layer can safely update the stored attributes and recreate playout with acceptable behavior. Until then, treat WebRTC playout `AudioAttributes` as initialization-time configuration. ## Automatic vs manual mode The two modes differ in who owns the session lifecycle. -In automatic mode (the default) LiveKit manages the session from room, connect, and engine lifecycle and chooses the configuration for you. It does not take session options in this mode. +In automatic mode (the default) LiveKit manages the session from room, connect, and engine lifecycle. On iOS it derives the active category/mode from the current audio engine state. On Android it uses the current session intent, defaulting to communication and optionally seeded by `LiveKitClient.initialize(initialAudioSessionOptions: ...)`. In manual mode LiveKit does not touch the session on its own, and your app owns it. Enter manual mode when you need to apply a fixed platform configuration or deactivate the session yourself. ```dart // Apply a fixed config. This enters manual mode. await AudioManager.instance.setAudioSessionOptions( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), ); // Later, hand control back to LiveKit. @@ -84,7 +107,9 @@ await AudioManager.instance.setSpeakerOutputPreferred(true, force: true); await AudioManager.instance.setSpeakerOutputPreferred(false); ``` -Speaker routing is independent of the management mode and does not switch it. On Android and in iOS automatic mode, LiveKit applies the preference through its managed route policy. In iOS manual mode, the fixed Apple config you apply owns non-forced receiver vs speaker behavior. `force: true` still uses Apple's speaker override when the active category is `playAndRecord`. Read the current preference through `AudioManager.instance.isSpeakerOutputPreferred` and `AudioManager.instance.isSpeakerOutputForced`. `AudioManager.instance.canSwitchSpeakerphone` is true on iOS and Android. +Speaker routing is independent of the management mode and does not switch it. In Android communication/call sessions and in iOS automatic mode, LiveKit applies the preference through its managed route policy. In iOS manual mode, the fixed Apple config you apply owns non-forced receiver vs speaker behavior. `force: true` still uses Apple's speaker override when the active category is `playAndRecord`. Read the current preference through `AudioManager.instance.isSpeakerOutputPreferred` and `AudioManager.instance.isSpeakerOutputForced`. `AudioManager.instance.canSwitchSpeakerphone` is true on iOS and Android. + +On Android, LiveKit speaker routing is a communication/call routing policy. Media sessions use Android's normal media routing. Pass `AudioSessionOptions.mediaPlayback()` to `LiveKitClient.initialize` before WebRTC initializes when the WebRTC audio device module should use media attributes. `Room.setSpeakerOn(...)` is deprecated and forwards to `AudioManager.instance.setSpeakerOutputPreferred`. You can also set an initial preference through `RoomOptions` (`defaultAudioOutputOptions.speakerOn`) before connecting, which LiveKit applies when the session starts. @@ -148,7 +173,7 @@ print('echo cancellation in effect: ${state?.echoCancellation.effective}'); ## Per platform overrides -When the preset constructors are not enough you can pin exact platform values. Supplying options through `setAudioSessionOptions` switches to manual mode, so these configs are a manual-mode tool. `AudioSessionOptions.communication()` and `AudioSessionOptions.media()` pre-fill Apple and Android configs. Passing `apple` or `android` replaces that platform config rather than merging with the preset. +When the preset constructors are not enough you can pin exact platform values. Supplying options through `setAudioSessionOptions` switches to manual mode, so these configs are a manual-mode tool. `AudioSessionOptions.communication()` and `AudioSessionOptions.mediaPlayback()` pre-fill Apple and Android configs. Passing `apple` or `android` replaces that platform config rather than merging with the preset. ```dart await AudioManager.instance.setAudioSessionOptions( @@ -188,7 +213,7 @@ final clearedMode = updated.copyWith( ); ``` -Create a new `AudioSessionOptions.communication()` or `AudioSessionOptions.media()` when you want to start from a different preset config. +Create a new `AudioSessionOptions.communication()` or `AudioSessionOptions.mediaPlayback()` when you want to start from a different preset config. ## Platform support @@ -196,7 +221,7 @@ Create a new `AudioSessionOptions.communication()` or `AudioSessionOptions.media | --- | --- | --- | --- | | iOS | Automatic mode follows live engine state. Manual mode applies your Apple config verbatim. | Yes. Normal preference respects wired and Bluetooth devices. Forced speaker uses Apple's speaker override while the active category is `playAndRecord`. | Yes, from native WebRTC engine events. | | macOS | Not configured. There is no `AVAudioSession`. | No. `canSwitchSpeakerphone` is false. | Yes, the same engine events are reported. | -| Android | Automatic mode uses the communication session (in-communication mode, voice call stream). A media session is available in manual mode. Managed through LiveKit's AudioSwitch handler. | Yes. Normal preference orders headsets before the speaker. Forced speaker selects the speaker device. | Not reported, the Dart state stays idle. | +| Android | Automatic mode uses the current session intent, seeded by `LiveKitClient.initialize(initialAudioSessionOptions: ...)` and defaulting to communication. Pass media options before WebRTC initializes when the WebRTC audio device module should use media mode/volume. | Yes for communication/call routing. Media playback follows Android media routing. | Not reported, the Dart state stays idle. | | Web, Windows, Linux | Not configured. | No. `canSwitchSpeakerphone` is false. | Not reported. | On iOS automatic mode, listen only playout uses `playback`. When recording starts, LiveKit reapplies the session as `playAndRecord`. In manual mode, non-forced receiver vs speaker behavior comes from the Apple config you applied. diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart index dea966037..efb4a8d32 100644 --- a/lib/src/audio/audio_manager.dart +++ b/lib/src/audio/audio_manager.dart @@ -136,6 +136,21 @@ class AudioManager { _audioEngineStateController.add(nextState); } + /// Seeds the initial session intent without taking over the session lifecycle. + /// + /// `LiveKitClient.initialize(initialAudioSessionOptions: ...)` uses this so the + /// WebRTC initialization-time Android audio attributes and LiveKit's automatic + /// runtime session policy start from the same intent. Unlike + /// [setAudioSessionOptions], this keeps automatic management enabled and does + /// not apply native session changes immediately. + @internal + void setInitialAudioSessionOptions(AudioSessionOptions options) { + if (_managementMode != AudioSessionManagementMode.automatic) { + return; + } + _options = options; + } + /// Applies an explicit audio session configuration and switches to manual mode. /// /// Calling this puts [AudioManager] in [AudioSessionManagementMode.manual]: diff --git a/lib/src/audio/audio_session.dart b/lib/src/audio/audio_session.dart index 506488931..942f130d0 100644 --- a/lib/src/audio/audio_session.dart +++ b/lib/src/audio/audio_session.dart @@ -55,11 +55,12 @@ class AudioSessionOptions { /// One-way media playback preset. /// - /// This pre-fills playback-oriented platform policies. Apple playback policy - /// leaves routing to the platform, while Android speaker routing remains a - /// runtime preference. Override [apple] or [android] for exact platform - /// behavior. - const AudioSessionOptions.media({ + /// This pre-fills playback-oriented platform policies. Apple and Android + /// media routing are platform-owned. On Android, pass this to + /// `LiveKitClient.initialize` before WebRTC initializes when WebRTC playout + /// should use media `AudioAttributes`; the same value seeds LiveKit's initial + /// automatic runtime media session policy. + const AudioSessionOptions.mediaPlayback({ AppleAudioSessionConfiguration apple = AppleAudioSessionConfiguration.media, AndroidAudioSessionConfiguration android = AndroidAudioSessionConfiguration.media, }) : this._(apple: apple, android: android); diff --git a/lib/src/audio/audio_session_policy.dart b/lib/src/audio/audio_session_policy.dart index bc2abe410..4a870277d 100644 --- a/lib/src/audio/audio_session_policy.dart +++ b/lib/src/audio/audio_session_policy.dart @@ -53,9 +53,10 @@ class ResolvedAudioSessionPolicy { } AndroidAudioSessionConfiguration get androidConfiguration { - if (automatic) { - return AndroidAudioSessionConfiguration.communication; - } + // In automatic mode LiveKit still owns activation timing, focus/routing + // lifecycle, and speaker preference. The Android session intent itself is + // the current AudioSessionOptions value, seeded by LiveKitClient.initialize + // or replaced explicitly by AudioManager.setAudioSessionOptions. return options.android; } } diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 498091bae..c50ede2f8 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -15,8 +15,10 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'audio/audio_manager.dart'; +import 'audio/audio_session.dart'; import 'support/native.dart'; -import 'support/platform.dart' show lkPlatformIsMobile; +import 'support/platform.dart' show PlatformType, lkPlatformIs, lkPlatformIsMobile; +import 'support/webrtc_initialize_options.dart'; /// Main entry point to connect to a room. /// {@category Room} @@ -26,25 +28,45 @@ class LiveKitClient { /// Initialize the WebRTC plugin. /// /// Optional: call once at startup to enable [bypassVoiceProcessing] before - /// connecting. Otherwise WebRTC initializes lazily with defaults. + /// connecting, or to apply Android [initialAudioSessionOptions] before WebRTC + /// creates its audio device module. Otherwise WebRTC initializes lazily with + /// defaults. /// /// LiveKit owns the platform audio session, and flutter_webrtc's own native /// audio management is disabled automatically when the LiveKit plugin loads /// (done natively at registration), so that does not depend on this call. /// - /// Configure audio-session behavior through [AudioManager] before connecting, - /// e.g. `await AudioManager.instance.setAudioSessionManagementMode(...)` and + /// Configure explicit runtime audio-session behavior through [AudioManager] + /// before connecting, e.g. + /// `await AudioManager.instance.setAudioSessionManagementMode(...)` and /// `await AudioManager.instance.setAudioSessionOptions(...)`. + /// + /// [initialAudioSessionOptions] currently affects Android's WebRTC + /// initialization-time playout attributes, such as media vs voice + /// communication usage. It also seeds [AudioManager]'s initial automatic + /// runtime session policy until the app explicitly replaces it with + /// [AudioManager.setAudioSessionOptions]. A future SDK/WebRTC integration may + /// make those Android playout attributes runtime-updatable; for now, pass them + /// here before WebRTC initializes. static Future initialize({ bool bypassVoiceProcessing = false, + AudioSessionOptions? initialAudioSessionOptions, }) async { if (lkPlatformIsMobile()) { - // bypassVoiceProcessing controls only WebRTC voice processing, not the - // session intent. The audio session is owned by AudioManager. + // bypassVoiceProcessing controls only WebRTC voice processing. Android + // playout attributes are passed here because WebRTC reads them when it + // creates the audio device module. Native.bypassVoiceProcessing = bypassVoiceProcessing; - await rtc.WebRTC.initialize(options: { - if (bypassVoiceProcessing) 'bypassVoiceProcessing': bypassVoiceProcessing, - }); + await rtc.WebRTC.initialize( + options: liveKitWebRTCInitializeOptions( + bypassVoiceProcessing: bypassVoiceProcessing, + initialAudioSessionOptions: initialAudioSessionOptions, + includeAndroidAudioConfiguration: lkPlatformIs(PlatformType.android), + ), + ); + if (lkPlatformIs(PlatformType.android) && initialAudioSessionOptions != null) { + AudioManager.instance.setInitialAudioSessionOptions(initialAudioSessionOptions); + } } } } diff --git a/lib/src/support/webrtc_initialize_options.dart b/lib/src/support/webrtc_initialize_options.dart new file mode 100644 index 000000000..f902641b6 --- /dev/null +++ b/lib/src/support/webrtc_initialize_options.dart @@ -0,0 +1,30 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +import '../audio/android_audio_session_adapter.dart'; +import '../audio/audio_session.dart'; + +@internal +Map liveKitWebRTCInitializeOptions({ + required bool bypassVoiceProcessing, + required AudioSessionOptions? initialAudioSessionOptions, + required bool includeAndroidAudioConfiguration, +}) => + { + if (bypassVoiceProcessing) 'bypassVoiceProcessing': bypassVoiceProcessing, + if (includeAndroidAudioConfiguration && initialAudioSessionOptions != null) + 'androidAudioConfiguration': androidAudioSessionConfigurationToMap(initialAudioSessionOptions.android), + }; diff --git a/test/audio/audio_session_test.dart b/test/audio/audio_session_test.dart index 7487d6af2..583f27995 100644 --- a/test/audio/audio_session_test.dart +++ b/test/audio/audio_session_test.dart @@ -22,6 +22,7 @@ import 'package:livekit_client/src/audio/audio_session.dart'; import 'package:livekit_client/src/audio/audio_session_policy.dart'; import 'package:livekit_client/src/support/native.dart'; import 'package:livekit_client/src/support/native_audio.dart' as native_audio; +import 'package:livekit_client/src/support/webrtc_initialize_options.dart'; import 'package:livekit_client/src/track/options.dart' as track_options; void main() { @@ -86,14 +87,15 @@ void main() { expect(options.android.streamType, AndroidAudioStreamType.voiceCall); }); - test('media constructor pre-fills platform configs', () { - const options = AudioSessionOptions.media(); + test('mediaPlayback constructor pre-fills platform configs', () { + const options = AudioSessionOptions.mediaPlayback(); expect(options.apple.category, AppleAudioCategory.playback); expect(options.apple.categoryOptions, {AppleAudioCategoryOption.mixWithOthers}); expect(options.apple.mode, AppleAudioMode.spokenAudio); expect(options.android.audioMode, AndroidAudioMode.normal); expect(options.android.streamType, AndroidAudioStreamType.music); + expect(options.android.forceAudioRouting, isNull); }); test('copyWith replaces platform configs', () { @@ -261,13 +263,36 @@ void main() { final manager = AudioManager.instance; expect(manager.managementMode, AudioSessionManagementMode.automatic); - await manager.setAudioSessionOptions(const AudioSessionOptions.media()); + await manager.setAudioSessionOptions(const AudioSessionOptions.mediaPlayback()); expect(manager.managementMode, AudioSessionManagementMode.manual); expect(manager.options.apple.category, AppleAudioCategory.playback); expect(manager.options.android.streamType, AndroidAudioStreamType.music); }); + test('setInitialAudioSessionOptions seeds options without switching to manual', () { + final manager = AudioManager.instance; + + manager.setInitialAudioSessionOptions(const AudioSessionOptions.mediaPlayback()); + + expect(manager.managementMode, AudioSessionManagementMode.automatic); + expect(manager.options.android.streamType, AndroidAudioStreamType.music); + + final android = resolveAndroidPolicy(manager.options); + expect(android.audioMode, AndroidAudioMode.normal); + expect(android.streamType, AndroidAudioStreamType.music); + }); + + test('setInitialAudioSessionOptions does not replace manual options', () async { + final manager = AudioManager.instance; + await manager.setAudioSessionOptions(const AudioSessionOptions.communication()); + + manager.setInitialAudioSessionOptions(const AudioSessionOptions.mediaPlayback()); + + expect(manager.managementMode, AudioSessionManagementMode.manual); + expect(manager.options.android.streamType, AndroidAudioStreamType.voiceCall); + }); + test('deactivateAudioSession switches management to manual', () async { final manager = AudioManager.instance; expect(manager.managementMode, AudioSessionManagementMode.automatic); @@ -286,7 +311,7 @@ void main() { expect(manager.isSpeakerOutputPreferred, isTrue); await manager.setAudioSessionOptions( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), ); expect(manager.isSpeakerOutputPreferred, isTrue); @@ -324,7 +349,7 @@ void main() { test('automatic Apple policy ignores manual media options', () { final config = resolveApplePolicy( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), ); expect(config.appleAudioCategory, AppleAudioCategory.playAndRecord); @@ -341,7 +366,7 @@ void main() { test('resolves manual media Apple session policy as fixed playback', () { final config = resolveApplePolicy( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), automatic: false, ); @@ -352,7 +377,7 @@ void main() { test('forced speaker does not mutate Apple category options', () { final playback = resolveApplePolicy( - const AudioSessionOptions.media( + const AudioSessionOptions.mediaPlayback( apple: AppleAudioSessionConfiguration( category: AppleAudioCategory.playback, categoryOptions: {AppleAudioCategoryOption.mixWithOthers}, @@ -383,16 +408,16 @@ void main() { ); }); - test('resolves Android session policy from automatic mode or manual options', () { + test('resolves Android session policy from current options', () { final automaticMedia = resolveAndroidPolicy( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), ); - expect(automaticMedia.audioMode, AndroidAudioMode.inCommunication); - expect(automaticMedia.streamType, AndroidAudioStreamType.voiceCall); + expect(automaticMedia.audioMode, AndroidAudioMode.normal); + expect(automaticMedia.streamType, AndroidAudioStreamType.music); final media = resolveAndroidPolicy( - const AudioSessionOptions.media(), + const AudioSessionOptions.mediaPlayback(), automatic: false, ); @@ -413,10 +438,10 @@ void main() { expect(explicit.forceAudioRouting, isTrue); }); - test('automatic mode ignores stored manual options after switching back', () async { + test('automatic Apple policy ignores stored options while Android uses them', () async { final manager = AudioManager.instance; - await manager.setAudioSessionOptions(const AudioSessionOptions.media()); + await manager.setAudioSessionOptions(const AudioSessionOptions.mediaPlayback()); await manager.setAudioSessionManagementMode(AudioSessionManagementMode.automatic); final isAutomatic = manager.managementMode == AudioSessionManagementMode.automatic; @@ -440,8 +465,8 @@ void main() { AppleAudioCategoryOption.allowAirPlay, }, ); - expect(android.audioMode, AndroidAudioMode.inCommunication); - expect(android.streamType, AndroidAudioStreamType.voiceCall); + expect(android.audioMode, AndroidAudioMode.normal); + expect(android.streamType, AndroidAudioStreamType.music); }); test('handleAudioEngineState updates snapshot and stream', () async { @@ -509,6 +534,7 @@ void main() { expect(config.streamType, AndroidAudioStreamType.music); expect(config.usageType, AndroidAudioAttributesUsageType.media); expect(config.contentType, AndroidAudioAttributesContentType.unknown); + expect(config.forceAudioRouting, isNull); }); }); @@ -749,5 +775,53 @@ void main() { }, ); }); + + test('serializes media preset without forced routing', () { + expect( + androidAudioSessionConfigurationToMap(AndroidAudioSessionConfiguration.media), + { + 'manageAudioFocus': true, + 'androidAudioMode': 'normal', + 'androidAudioFocusMode': 'gain', + 'androidAudioStreamType': 'music', + 'androidAudioAttributesUsageType': 'media', + 'androidAudioAttributesContentType': 'unknown', + }, + ); + }); + }); + + group('liveKitWebRTCInitializeOptions', () { + test('includes Android audio configuration for Android startup', () { + expect( + liveKitWebRTCInitializeOptions( + bypassVoiceProcessing: true, + initialAudioSessionOptions: const AudioSessionOptions.mediaPlayback(), + includeAndroidAudioConfiguration: true, + ), + { + 'bypassVoiceProcessing': true, + 'androidAudioConfiguration': { + 'manageAudioFocus': true, + 'androidAudioMode': 'normal', + 'androidAudioFocusMode': 'gain', + 'androidAudioStreamType': 'music', + 'androidAudioAttributesUsageType': 'media', + 'androidAudioAttributesContentType': 'unknown', + }, + }, + ); + }); + + test('omits Android audio configuration on non-Android startup', () { + expect( + liveKitWebRTCInitializeOptions( + bypassVoiceProcessing: false, + initialAudioSessionOptions: const AudioSessionOptions.mediaPlayback(), + includeAndroidAudioConfiguration: false, + ), + isEmpty, + ); + }); }); } From 6bdbc3e3136a55434fcdc6b8676c83de19028070 Mon Sep 17 00:00:00 2001 From: xianshijing-lk Date: Tue, 30 Jun 2026 11:24:07 +0800 Subject: [PATCH 17/18] fix: improve initial video quality by setting x-google-start-bitrate for all video codecs (#1120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Apply `x-google-start-bitrate` SDP hint to all video codecs (VP8, VP9, AV1, H264, H265), not just SVC codecs - Use 90% of target bitrate as start bitrate to prevent initial blurriness - Degradation preference already defaults to `maintainResolution` ## Problem Video starts blurry for 5-15 seconds before improving. This is caused by WebRTC's bandwidth estimator starting at ~300kbps and slowly ramping up to the target bitrate. ## Solution **x-google-start-bitrate**: Tell WebRTC to start at 90% of target bitrate instead of ramping up from ~300kbps. Applied consistently to all video codecs. Note, The PR only addresses the x-google-start-bitrate fix (changing from SVC-only to all video codecs, and using 0.9 multiplier) - The maintainResolution default was already in place in Flutter SDK, so no change was needed for that part ## Test plan - [ ] Verify video quality is sharp from the start when publishing - [ ] Test with VP8, VP9, H264, AV1 codecs - [ ] Verify bandwidth estimator adapts properly if network can't handle the start bitrate 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.5 --- lib/src/core/transport.dart | 20 ++++++++++++-------- lib/src/participant/local.dart | 8 +++++--- lib/src/utils.dart | 2 ++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/src/core/transport.dart b/lib/src/core/transport.dart index 743e9eeb7..bb7ecd3dd 100644 --- a/lib/src/core/transport.dart +++ b/lib/src/core/transport.dart @@ -29,13 +29,17 @@ import '../utils.dart'; const ddExtensionURI = 'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'; -/* The svc codec (av1/vp9) would use a very low bitrate at the begining and -increase slowly by the bandwidth estimator until it reach the target bitrate. The -process commonly cost more than 10 seconds cause subscriber will get blur video at -the first few seconds. So we use a 70% of target bitrate here as the start bitrate to -eliminate this issue. -*/ -const startBitrateForSVC = 0.7; +/* + * Video codecs use a very low bitrate at the beginning and increase slowly by + * the bandwidth estimator until they reach the target bitrate. The process commonly + * costs more than 10 seconds causing subscribers to get blurry video at the first + * few seconds. We use x-google-start-bitrate to hint the BWE to start higher. + * + * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. + * Why same for all codecs: Target bitrate already accounts for codec efficiency + * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). + */ +const startBitrateMultiplier = 0.9; class TrackBitrateInfo { String? cid; @@ -204,7 +208,7 @@ class Transport extends Disposable { for (var fmtp in media['fmtp']) { if (fmtp['payload'] == codecPayload) { if (!(fmtp['config'] as String).contains('x-google-start-bitrate')) { - fmtp['config'] += ';x-google-start-bitrate=${(trackbr.maxbr * startBitrateForSVC).toInt()}'; + fmtp['config'] += ';x-google-start-bitrate=${(trackbr.maxbr * startBitrateMultiplier).toInt()}'; } break; } diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index d8e4d41fe..189dd839f 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -56,7 +56,7 @@ import '../types/other.dart'; import '../types/participant_permissions.dart'; import '../types/rpc.dart'; import '../types/video_dimensions.dart'; -import '../utils.dart' show buildStreamId, mimeTypeToVideoCodecString, Utils, compareVersions, isSVCCodec; +import '../utils.dart' show buildStreamId, mimeTypeToVideoCodecString, Utils, compareVersions, isSVCCodec, isVideoCodec; import 'participant.dart'; /// Represents the current participant in the room. Instance of [LocalParticipant] is automatically @@ -404,7 +404,8 @@ class LocalParticipant extends Participant { if (kIsWeb && lkBrowser() == BrowserType.firefox && track.kind == TrackType.AUDIO) { //TOOD: - } else if (isSVCCodec(publishOptions.videoCodec) && encodings?.first.maxBitrate != null) { + } else if (isVideoCodec(publishOptions.videoCodec) && encodings?.first.maxBitrate != null) { + // Apply start bitrate for all video codecs to prevent initial blurriness room.engine.publisher?.setTrackBitrateInfo(TrackBitrateInfo( cid: track.getCid(), transceiver: track.transceiver, @@ -502,7 +503,8 @@ class LocalParticipant extends Participant { if (kIsWeb && lkBrowser() == BrowserType.firefox && track.kind == TrackType.AUDIO) { //TOOD: - } else if (isSVCCodec(publishOptions.videoCodec) && encodings?.first.maxBitrate != null) { + } else if (isVideoCodec(publishOptions.videoCodec) && encodings?.first.maxBitrate != null) { + // Apply start bitrate for all video codecs to prevent initial blurriness room.engine.publisher?.setTrackBitrateInfo(TrackBitrateInfo( cid: track.getCid(), transceiver: track.transceiver, diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 1da153515..8b2b58e6f 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -626,6 +626,8 @@ const refreshSubscribedCodecAfterNewCodec = 5000; bool isSVCCodec(String codec) => ['vp9', 'av1'].contains(codec.toLowerCase()); +bool isVideoCodec(String codec) => ['vp8', 'vp9', 'av1', 'h264', 'h265'].contains(codec.toLowerCase()); + bool isAV1Codec(String codec) => codec.toLowerCase() == 'av1'; class ScalabilityMode { From f862833bdc6a27259ea06e1b6737092de5c4d9f8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 30 Jun 2026 09:14:32 +0000 Subject: [PATCH 18/18] fix(room): drop redundant track retry loop in favor of upstream pending-track queue The fork's EngineTrackAddedEvent handler added a blocking retry loop that re-looked-up the remote participant up to 5x (1s apart) when a track arrived before the participant's metadata was known. Upstream already handles this race via _pendingTrackQueue.enqueue + _flushPendingTracks, which subscribes early-arriving tracks immediately once the participant appears. The retry loop awaited before enqueuing, defeating upstream's queue and adding up to 5s of latency. It also broke the upstream e2e test 'tracks arriving before participant metadata are handled once metadata arrives' (1s timeout). Remove the retry loop and rely on the queue. The [audioElementLogs] diagnostics are kept (incl. the 'will enqueue for later' warning). Co-authored-by: td --- lib/src/core/room.dart | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index cb4f60ff0..e9e315561 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -643,19 +643,12 @@ class Room extends DisposableChangeNotifier with EventsEmittable { trackSid = streamId; } - RemoteParticipant? participant = _remoteParticipants.bySid[participantSid]; + // When a track arrives before its participant's metadata is known, + // upstream's `_pendingTrackQueue` defers the track and `_flushPendingTracks` + // subscribes it as soon as the participant appears, so no manual retry loop + // is needed here. We keep the `[audioElementLogs]` diagnostics below. + final participant = _remoteParticipants.bySid[participantSid]; - if (participant == null) { - logger.warning( - '[audioElementLogs] _setupEngineFailed could not find remoteParticipant, will try again, $participantSid'); - - for (var i = 0; i < 5; i++) { - if (participant != null) break; - logger.info('[audioElementLogs] _getRemoteParticipantBySid try: $i, sid: $participantSid'); - await Future.delayed(Duration(seconds: 1)); - participant = _remoteParticipants.bySid[participantSid]; - } - } try { if (trackSid == null || trackSid.isEmpty) { throw TrackSubscriptionExceptionEvent(