From ece5ae49caf87818e3d761efcff63c9235c99287 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 8 Jul 2025 08:23:39 -0700 Subject: [PATCH 1/9] add developer api to live --- .../firebase_ai/example/lib/main.dart | 33 ++++++-------- .../firebase_ai/lib/src/base_model.dart | 7 +++ .../firebase_ai/lib/src/firebase_ai.dart | 5 +-- .../firebase_ai/lib/src/live_model.dart | 44 ++++++++++++++----- .../lib/src/firebase_vertexai.dart | 1 + 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart index a7fd0363aba7..46602e08d3f3 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/main.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/main.dart @@ -192,12 +192,8 @@ class _HomeScreenState extends State { case 8: return VideoPage(title: 'Video Prompt', model: currentModel); case 9: - if (useVertexBackend) { - return BidiPage(title: 'Live Stream', model: currentModel); - } else { - // Fallback to the first page in case of an unexpected index - return ChatPage(title: 'Chat', model: currentModel); - } + return BidiPage(title: 'Live Stream', model: currentModel); + default: // Fallback to the first page in case of an unexpected index return ChatPage(title: 'Chat', model: currentModel); @@ -270,48 +266,48 @@ class _HomeScreenState extends State { unselectedItemColor: widget.useVertexBackend ? Theme.of(context).colorScheme.onSurface.withValues(alpha: 0.7) : Colors.grey, - items: [ - const BottomNavigationBarItem( + items: const [ + BottomNavigationBarItem( icon: Icon(Icons.chat), label: 'Chat', tooltip: 'Chat', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.mic), label: 'Audio', tooltip: 'Audio Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.numbers), label: 'Tokens', tooltip: 'Token Count', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.functions), label: 'Functions', tooltip: 'Function Calling', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.image), label: 'Image', tooltip: 'Image Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.image_search), label: 'Imagen', tooltip: 'Imagen Model', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.schema), label: 'Schema', tooltip: 'Schema Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.edit_document), label: 'Document', tooltip: 'Document Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.video_collection), label: 'Video', tooltip: 'Video Prompt', @@ -319,12 +315,9 @@ class _HomeScreenState extends State { BottomNavigationBarItem( icon: Icon( Icons.stream, - color: widget.useVertexBackend ? null : Colors.grey, ), label: 'Live', - tooltip: widget.useVertexBackend - ? 'Live Stream' - : 'Live Stream (Currently Disabled)', + tooltip: 'Live Stream', ), ], currentIndex: widget.selectedIndex, diff --git a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart index f63c883e7015..25cae93f8c59 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart @@ -56,6 +56,7 @@ enum Task { abstract interface class _ModelUri { String get baseAuthority; + String get apiVersion; Uri taskUri(Task task); ({String prefix, String name}) get model; } @@ -96,6 +97,9 @@ final class _VertexUri implements _ModelUri { @override String get baseAuthority => _baseAuthority; + @override + String get apiVersion => _apiVersion; + @override Uri taskUri(Task task) { return _projectUri.replace( @@ -135,6 +139,9 @@ final class _GoogleAIUri implements _ModelUri { @override String get baseAuthority => _baseAuthority; + @override + String get apiVersion => _apiVersion; + @override Uri taskUri(Task task) => _baseUri.replace( pathSegments: _baseUri.pathSegments diff --git a/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart index 232d052658d0..3739cfec6a9c 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart @@ -175,14 +175,11 @@ class FirebaseAI extends FirebasePluginPlatform { List? tools, Content? systemInstruction, }) { - if (!_useVertexBackend) { - throw FirebaseAISdkException( - 'LiveGenerativeModel is currently only supported with the VertexAI backend.'); - } return createLiveGenerativeModel( app: app, location: location, model: model, + useVertexBackend: _useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction, diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index 4787d0ea97ed..45ab1c6657f9 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -15,7 +15,8 @@ part of 'base_model.dart'; const _apiUrl = 'ws/google.firebase.vertexai'; -const _apiUrlSuffix = 'LlmBidiService/BidiGenerateContent/locations'; +const _apiUrlSuffixVertexAI = 'LlmBidiService/BidiGenerateContent/locations'; +const _apiUrlSuffixGoogleAI = 'GenerativeService/BidiGenerateContent'; /// A live, generative AI model for real-time interaction. /// @@ -32,6 +33,7 @@ final class LiveGenerativeModel extends BaseModel { {required String model, required String location, required FirebaseApp app, + required bool useVertexBackend, FirebaseAppCheck? appCheck, FirebaseAuth? auth, LiveGenerationConfig? liveGenerationConfig, @@ -39,6 +41,7 @@ final class LiveGenerativeModel extends BaseModel { Content? systemInstruction}) : _app = app, _location = location, + _useVertexBackend = useVertexBackend, _appCheck = appCheck, _auth = auth, _liveGenerationConfig = liveGenerationConfig, @@ -46,22 +49,39 @@ final class LiveGenerativeModel extends BaseModel { _systemInstruction = systemInstruction, super._( serializationStrategy: VertexSerialization(), - modelUri: _VertexUri( - model: model, - app: app, - location: location, - ), + modelUri: useVertexBackend + ? _VertexUri( + model: model, + app: app, + location: location, + ) + : _GoogleAIUri( + model: model, + app: app, + ), ); - static const _apiVersion = 'v1beta'; final FirebaseApp _app; final String _location; + final bool _useVertexBackend; final FirebaseAppCheck? _appCheck; final FirebaseAuth? _auth; final LiveGenerationConfig? _liveGenerationConfig; final List? _tools; final Content? _systemInstruction; + String _vertexAIUri() => 'wss://${_modelUri.baseAuthority}/' + '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixVertexAI/' + '$_location?key=${_app.options.apiKey}'; + + String _vertexAIModelString() => 'projects/${_app.options.projectId}/' + 'locations/$_location/publishers/google/models/${model.name}'; + + String _googleAIUri() => 'wss://${_modelUri.baseAuthority}/' + '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixGoogleAI?key=${_app.options.apiKey}'; + + String _googleAIModelString() => 'models/${model.name}'; + /// Establishes a connection to a live generation service. /// /// This function handles the WebSocket connection setup and returns an [LiveSession] @@ -70,11 +90,9 @@ final class LiveGenerativeModel extends BaseModel { /// Returns a [Future] that resolves to an [LiveSession] object upon successful /// connection. Future connect() async { - final uri = 'wss://${_modelUri.baseAuthority}/' - '$_apiUrl.$_apiVersion.$_apiUrlSuffix/' - '$_location?key=${_app.options.apiKey}'; - final modelString = 'projects/${_app.options.projectId}/' - 'locations/$_location/publishers/google/models/${model.name}'; + final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri(); + final modelString = + _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); final setupJson = { 'setup': { @@ -105,6 +123,7 @@ LiveGenerativeModel createLiveGenerativeModel({ required FirebaseApp app, required String location, required String model, + required bool useVertexBackend, FirebaseAppCheck? appCheck, FirebaseAuth? auth, LiveGenerationConfig? liveGenerationConfig, @@ -117,6 +136,7 @@ LiveGenerativeModel createLiveGenerativeModel({ appCheck: appCheck, auth: auth, location: location, + useVertexBackend: useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart index f7b3952e649c..9edd832302f6 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart @@ -176,6 +176,7 @@ class FirebaseVertexAI extends FirebasePluginPlatform { app: app, location: location, model: model, + useVertexBackend: _useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction, From d48a4ab31d1625977eab268ecd7bafa68b5583d2 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 8 Jul 2025 10:25:56 -0700 Subject: [PATCH 2/9] make the whole process going through --- .../firebase_ai/example/lib/main.dart | 10 +++---- .../example/lib/pages/bidi_page.dart | 29 ++++++++++++++----- .../firebase_ai/lib/src/live_model.dart | 7 ++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart index 46602e08d3f3..b8dbdaaec19c 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/main.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/main.dart @@ -154,10 +154,6 @@ class HomeScreen extends StatefulWidget { class _HomeScreenState extends State { void _onItemTapped(int index) { - if (index == 9 && !widget.useVertexBackend) { - // Live Stream feature only works with Vertex AI now. - return; - } widget.onSelectedIndexChanged(index); } @@ -192,7 +188,11 @@ class _HomeScreenState extends State { case 8: return VideoPage(title: 'Video Prompt', model: currentModel); case 9: - return BidiPage(title: 'Live Stream', model: currentModel); + return BidiPage( + title: 'Live Stream', + model: currentModel, + useVertexBackend: useVertexBackend, + ); default: // Fallback to the first page in case of an unexpected index diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index cb221329d28f..e72c6505474e 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -22,10 +22,15 @@ import '../utils/audio_output.dart'; import '../widgets/message_widget.dart'; class BidiPage extends StatefulWidget { - const BidiPage({super.key, required this.title, required this.model}); + const BidiPage( + {super.key, + required this.title, + required this.model, + required this.useVertexBackend}); final String title; final GenerativeModel model; + final bool useVertexBackend; @override State createState() => _BidiPageState(); @@ -64,13 +69,21 @@ class _BidiPageState extends State { ); // ignore: deprecated_member_use - _liveModel = FirebaseAI.vertexAI().liveGenerativeModel( - model: 'gemini-2.0-flash-exp', - liveGenerationConfig: config, - tools: [ - Tool.functionDeclarations([lightControlTool]), - ], - ); + _liveModel = widget.useVertexBackend + ? FirebaseAI.vertexAI().liveGenerativeModel( + model: 'gemini-2.0-flash-exp', + liveGenerationConfig: config, + tools: [ + Tool.functionDeclarations([lightControlTool]), + ], + ) + : FirebaseAI.googleAI().liveGenerativeModel( + model: 'gemini-2.0-flash-live-001', + liveGenerationConfig: config, + tools: [ + Tool.functionDeclarations([lightControlTool]), + ], + ); _initAudio(); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index 45ab1c6657f9..86106ad9a4ec 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -80,7 +80,8 @@ final class LiveGenerativeModel extends BaseModel { String _googleAIUri() => 'wss://${_modelUri.baseAuthority}/' '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixGoogleAI?key=${_app.options.apiKey}'; - String _googleAIModelString() => 'models/${model.name}'; + String _googleAIModelString() => + 'projects/${_app.options.projectId}/models/${model.name}'; /// Establishes a connection to a live generation service. /// @@ -113,7 +114,11 @@ final class LiveGenerativeModel extends BaseModel { : IOWebSocketChannel.connect(Uri.parse(uri), headers: headers); await ws.ready; + print('websocket connect with uri $uri'); + ws.sink.add(request); + + print('setup request sent: $setupJson'); return LiveSession(ws); } } From 8e454a6fbd57b5cf0c3676bab4d847cda2324c3c Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 02:26:52 +0000 Subject: [PATCH 3/9] feat(firebase_ai): handle unknown parts when parsing content --- .../firebase_ai/lib/src/content.dart | 86 ++++++++++++------- .../firebase_ai/test/content_test.dart | 21 +++-- 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart index ac47ed996069..e82ecc218b24 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/content.dart @@ -13,7 +13,9 @@ // limitations under the License. import 'dart:convert'; +import 'dart:developer'; import 'dart:typed_data'; + import 'error.dart'; /// The base structured datatype containing multi-part content of a message. @@ -81,37 +83,51 @@ Content parseContent(Object jsonObject) { /// Parse the [Part] from json object. Part parsePart(Object? jsonObject) { - if (jsonObject is Map && jsonObject.containsKey('functionCall')) { - final functionCall = jsonObject['functionCall']; - if (functionCall is Map && - functionCall.containsKey('name') && - functionCall.containsKey('args')) { - return FunctionCall( - functionCall['name'] as String, - functionCall['args'] as Map, - id: functionCall['id'] as String?, - ); - } else { - throw unhandledFormat('functionCall', functionCall); - } + if (jsonObject is! Map) { + log('Unhandled part format: $jsonObject'); + return UnknownPart({ + 'unhandled': jsonObject, + }); } - return switch (jsonObject) { - {'text': final String text} => TextPart(text), - { - 'file_data': { - 'file_uri': final String fileUri, - 'mime_type': final String mimeType + try { + if (jsonObject.containsKey('functionCall')) { + final functionCall = jsonObject['functionCall']; + if (functionCall is Map && + functionCall.containsKey('name') && + functionCall.containsKey('args')) { + return FunctionCall( + functionCall['name'] as String, + functionCall['args'] as Map, + id: functionCall['id'] as String?, + ); + } else { + throw unhandledFormat('functionCall', functionCall); } - } => - FileData(mimeType, fileUri), - { - 'functionResponse': {'name': String _, 'response': Map _} - } => - throw UnimplementedError('FunctionResponse part not yet supported'), - {'inlineData': {'mimeType': String mimeType, 'data': String bytes}} => - InlineDataPart(mimeType, base64Decode(bytes)), - _ => throw unhandledFormat('Part', jsonObject), - }; + } + return switch (jsonObject) { + {'text': final String text} => TextPart(text), + { + 'file_data': { + 'file_uri': final String fileUri, + 'mime_type': final String mimeType + } + } => + FileData(mimeType, fileUri), + { + 'functionResponse': { + 'name': String _, + 'response': Map _ + } + } => + throw UnimplementedError('FunctionResponse part not yet supported'), + {'inlineData': {'mimeType': String mimeType, 'data': String bytes}} => + InlineDataPart(mimeType, base64Decode(bytes)), + _ => throw unhandledFormat('Part', jsonObject), + }; + } on Object catch (e) { + log('unhandled part format: $jsonObject, $e'); + return UnknownPart(jsonObject); + } } /// A datatype containing media that is part of a multi-part [Content] message. @@ -120,6 +136,18 @@ sealed class Part { Object toJson(); } +/// A [Part] that contains unparsable data. +final class UnknownPart implements Part { + // ignore: public_member_api_docs + UnknownPart(this.data); + + /// The unparsed data. + final Map data; + + @override + Object toJson() => data; +} + /// A [Part] with the text content. final class TextPart implements Part { // ignore: public_member_api_docs diff --git a/packages/firebase_ai/firebase_ai/test/content_test.dart b/packages/firebase_ai/firebase_ai/test/content_test.dart index fc2957d89d73..f9c6f8c8424e 100644 --- a/packages/firebase_ai/firebase_ai/test/content_test.dart +++ b/packages/firebase_ai/firebase_ai/test/content_test.dart @@ -199,17 +199,26 @@ void main() { expect(() => parsePart(json), throwsA(isA())); }); - test('throws unhandledFormat for invalid JSON', () { + test('returns UnknownPart for invalid JSON', () { final json = {'invalid': 'data'}; - expect(() => parsePart(json), throwsA(isA())); + final result = parsePart(json); + expect(result, isA()); + final unknownPart = result as UnknownPart; + expect(unknownPart.data, json); }); - test('throws unhandledFormat for null input', () { - expect(() => parsePart(null), throwsA(isA())); + test('returns UnknownPart for null input', () { + final result = parsePart(null); + expect(result, isA()); + final unknownPart = result as UnknownPart; + expect(unknownPart.data, {'unhandled': null}); }); - test('throws unhandledFormat for empty map', () { - expect(() => parsePart({}), throwsA(isA())); + test('returns UnknownPart for empty map', () { + final result = parsePart({}); + expect(result, isA()); + final unknownPart = result as UnknownPart; + expect(unknownPart.data, {}); }); }); } From 206747e9883b03514201b57645fabbe15ebff0e2 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 15 Jul 2025 20:07:00 -0700 Subject: [PATCH 4/9] tweak the content test --- packages/firebase_ai/firebase_ai/lib/src/content.dart | 7 ------- packages/firebase_ai/firebase_ai/test/content_test.dart | 9 ++++++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart index e82ecc218b24..9dc5511f12be 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/content.dart @@ -113,13 +113,6 @@ Part parsePart(Object? jsonObject) { } } => FileData(mimeType, fileUri), - { - 'functionResponse': { - 'name': String _, - 'response': Map _ - } - } => - throw UnimplementedError('FunctionResponse part not yet supported'), {'inlineData': {'mimeType': String mimeType, 'data': String bytes}} => InlineDataPart(mimeType, base64Decode(bytes)), _ => throw unhandledFormat('Part', jsonObject), diff --git a/packages/firebase_ai/firebase_ai/test/content_test.dart b/packages/firebase_ai/firebase_ai/test/content_test.dart index f9c6f8c8424e..ddbbacfd50c2 100644 --- a/packages/firebase_ai/firebase_ai/test/content_test.dart +++ b/packages/firebase_ai/firebase_ai/test/content_test.dart @@ -192,11 +192,14 @@ void main() { expect(inlineData.bytes, [1, 2, 3]); }); - test('throws UnimplementedError for functionResponse', () { + test('returns UnknownPart for functionResponse', () { final json = { 'functionResponse': {'name': 'test', 'response': {}} }; - expect(() => parsePart(json), throwsA(isA())); + final result = parsePart(json); + expect(result, isA()); + final unknownPart = result as UnknownPart; + expect(unknownPart.data, json); }); test('returns UnknownPart for invalid JSON', () { @@ -218,7 +221,7 @@ void main() { final result = parsePart({}); expect(result, isA()); final unknownPart = result as UnknownPart; - expect(unknownPart.data, {}); + expect(unknownPart.data, {'unhandled': {}}); }); }); } From 6523b1db436229578c0cdf685443d457eec8cb03 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 15 Jul 2025 20:58:02 -0700 Subject: [PATCH 5/9] remove the extra exception --- .../firebase_ai/lib/src/content.dart | 59 +++++++++---------- .../firebase_ai/test/content_test.dart | 1 - 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart index 9dc5511f12be..b2661d05c880 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/content.dart @@ -89,38 +89,37 @@ Part parsePart(Object? jsonObject) { 'unhandled': jsonObject, }); } - try { - if (jsonObject.containsKey('functionCall')) { - final functionCall = jsonObject['functionCall']; - if (functionCall is Map && - functionCall.containsKey('name') && - functionCall.containsKey('args')) { - return FunctionCall( - functionCall['name'] as String, - functionCall['args'] as Map, - id: functionCall['id'] as String?, - ); - } else { - throw unhandledFormat('functionCall', functionCall); - } + + if (jsonObject.containsKey('functionCall')) { + final functionCall = jsonObject['functionCall']; + if (functionCall is Map && + functionCall.containsKey('name') && + functionCall.containsKey('args')) { + return FunctionCall( + functionCall['name'] as String, + functionCall['args'] as Map, + id: functionCall['id'] as String?, + ); + } else { + throw unhandledFormat('functionCall', functionCall); } - return switch (jsonObject) { - {'text': final String text} => TextPart(text), - { - 'file_data': { - 'file_uri': final String fileUri, - 'mime_type': final String mimeType - } - } => - FileData(mimeType, fileUri), - {'inlineData': {'mimeType': String mimeType, 'data': String bytes}} => - InlineDataPart(mimeType, base64Decode(bytes)), - _ => throw unhandledFormat('Part', jsonObject), - }; - } on Object catch (e) { - log('unhandled part format: $jsonObject, $e'); - return UnknownPart(jsonObject); } + return switch (jsonObject) { + {'text': final String text} => TextPart(text), + { + 'file_data': { + 'file_uri': final String fileUri, + 'mime_type': final String mimeType + } + } => + FileData(mimeType, fileUri), + {'inlineData': {'mimeType': String mimeType, 'data': String bytes}} => + InlineDataPart(mimeType, base64Decode(bytes)), + _ => () { + log('unhandled part format: $jsonObject'); + return UnknownPart(jsonObject); + }(), + }; } /// A datatype containing media that is part of a multi-part [Content] message. diff --git a/packages/firebase_ai/firebase_ai/test/content_test.dart b/packages/firebase_ai/firebase_ai/test/content_test.dart index ddbbacfd50c2..59a68bd6a198 100644 --- a/packages/firebase_ai/firebase_ai/test/content_test.dart +++ b/packages/firebase_ai/firebase_ai/test/content_test.dart @@ -16,7 +16,6 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:firebase_ai/src/content.dart'; -import 'package:firebase_ai/src/error.dart'; import 'package:flutter_test/flutter_test.dart'; // Mock google_ai classes (if needed) From d9f32e762b849f65f79dde283562bd342c15ca59 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 31 Jul 2025 16:56:17 -0700 Subject: [PATCH 6/9] Make Live API sending tool response for function calling --- .../firebase_ai/example/lib/pages/bidi_page.dart | 10 +++++++--- packages/firebase_ai/firebase_ai/lib/src/content.dart | 5 +++-- .../firebase_ai/firebase_ai/lib/src/live_api.dart | 10 +++++++++- .../firebase_ai/firebase_ai/lib/src/live_session.dart | 11 +++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index e72c6505474e..2fb5502e3e95 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -402,9 +402,13 @@ class _BidiPageState extends State { brightness: brightness, colorTemperature: color, ); - await _session.send( - input: Content.functionResponse(functionCall.name, functionResult), - ); + await _session.sendToolResponse([ + FunctionResponse( + functionCall.name, + functionResult, + id: functionCall.id, + ), + ]); } else { throw UnimplementedError( 'Function not declared to the model: ${functionCall.name}', diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart index b2661d05c880..fb627a9871c1 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/content.dart @@ -48,8 +48,9 @@ final class Content { static Content model(Iterable parts) => Content('model', [...parts]); /// Return a [Content] with [FunctionResponse]. - static Content functionResponse(String name, Map response) => - Content('function', [FunctionResponse(name, response)]); + static Content functionResponse(String name, Map response, + {String? id}) => + Content('function', [FunctionResponse(name, response, id: id)]); /// Return a [Content] with multiple [FunctionResponse]. static Content functionResponses(Iterable responses) => diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 8ae67a65051f..c5358101c52d 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -234,7 +234,15 @@ class LiveClientToolResponse { final List? functionResponses; // ignore: public_member_api_docs Map toJson() => { - 'functionResponses': functionResponses?.map((e) => e.toJson()).toList(), + 'toolResponse': { + 'functionResponses': functionResponses + ?.map((e) => { + 'name': e.name, + 'response': e.response, + if (e.id != null) 'id': e.id, + }) + .toList(), + }, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 34835ff11247..20e700bc82bf 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -61,7 +61,18 @@ class LiveSession { ? LiveClientContent(turns: [input], turnComplete: turnComplete) : LiveClientContent(turnComplete: turnComplete); var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + /// Sends tool responses for function calling to the server. + /// + /// [functionResponses] (optional): The list of function responses. + Future sendToolResponse( + List? functionResponses) async { + final toolResponse = + LiveClientToolResponse(functionResponses: functionResponses); + _checkWsStatus(); + var clientJson = jsonEncode(toolResponse.toJson()); _ws.sink.add(clientJson); } From 31d8aa4a547635311836e0202f9e2c61dc17d302 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 1 Aug 2025 16:05:27 -0700 Subject: [PATCH 7/9] update bidi model for googleAI in example --- .../firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 2fb5502e3e95..6796fb996cab 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -78,7 +78,7 @@ class _BidiPageState extends State { ], ) : FirebaseAI.googleAI().liveGenerativeModel( - model: 'gemini-2.0-flash-live-001', + model: 'gemini-live-2.5-flash-preview', liveGenerationConfig: config, tools: [ Tool.functionDeclarations([lightControlTool]), From 952878603c1a34426f6274b1ba0cc65ca269ccee Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 7 Aug 2025 17:33:51 -0700 Subject: [PATCH 8/9] fix analyzer and test --- .../firebase_ai/example/lib/pages/bidi_page.dart | 11 ++++++----- .../firebase_ai/lib/src/live_model.dart | 3 --- .../firebase_vertexai/test/live_test.dart | 14 ++++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index 6796fb996cab..d50241d29e36 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -22,11 +22,12 @@ import '../utils/audio_output.dart'; import '../widgets/message_widget.dart'; class BidiPage extends StatefulWidget { - const BidiPage( - {super.key, - required this.title, - required this.model, - required this.useVertexBackend}); + const BidiPage({ + super.key, + required this.title, + required this.model, + required this.useVertexBackend, + }); final String title; final GenerativeModel model; diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index 86106ad9a4ec..19174fc92d08 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -114,11 +114,8 @@ final class LiveGenerativeModel extends BaseModel { : IOWebSocketChannel.connect(Uri.parse(uri), headers: headers); await ws.ready; - print('websocket connect with uri $uri'); - ws.sink.add(request); - print('setup request sent: $setupJson'); return LiveSession(ws); } } diff --git a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart index 14909bcbc0eb..bdb4fd37a021 100644 --- a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart +++ b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart @@ -155,15 +155,17 @@ void main() { final response = FunctionResponse('test', {}); final message = LiveClientToolResponse(functionResponses: [response]); expect(message.toJson(), { - 'functionResponses': [ - { - 'functionResponse': {'name': 'test', 'response': {}} - } - ] + 'toolResponse': { + 'functionResponses': [ + {'name': 'test', 'response': {}} + ] + } }); final message2 = LiveClientToolResponse(); - expect(message2.toJson(), {'functionResponses': null}); + expect(message2.toJson(), { + 'toolResponse': {'functionResponses': null} + }); }); test('parseServerMessage parses serverContent message correctly', () { From 09c84001a21ef4caab3e0063b1477d4c06442cb4 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 7 Aug 2025 18:08:46 -0700 Subject: [PATCH 9/9] more test fixing --- .../firebase_ai/firebase_ai/test/live_test.dart | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/test/live_test.dart b/packages/firebase_ai/firebase_ai/test/live_test.dart index 090a2cbda469..0f9c9b00e6d5 100644 --- a/packages/firebase_ai/firebase_ai/test/live_test.dart +++ b/packages/firebase_ai/firebase_ai/test/live_test.dart @@ -154,15 +154,17 @@ void main() { final response = FunctionResponse('test', {}); final message = LiveClientToolResponse(functionResponses: [response]); expect(message.toJson(), { - 'functionResponses': [ - { - 'functionResponse': {'name': 'test', 'response': {}} - } - ] + 'toolResponse': { + 'functionResponses': [ + {'name': 'test', 'response': {}} + ] + } }); final message2 = LiveClientToolResponse(); - expect(message2.toJson(), {'functionResponses': null}); + expect(message2.toJson(), { + 'toolResponse': {'functionResponses': null} + }); }); test('parseServerMessage parses serverContent message correctly', () {