From 11360adebb5465f1ce05f993738cc68599cca49d Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 22:26:52 +0000 Subject: [PATCH 1/6] feat: Add support for Imagen editing --- .../example/lib/pages/imagen_page.dart | 51 ++ .../firebase_ai/lib/src/imagen_edit.dart | 435 ++++++++++++++++++ .../firebase_ai/lib/src/imagen_model.dart | 86 ++++ .../firebase_ai/test/imagen_edit_test.dart | 107 +++++ 4 files changed, 679 insertions(+) create mode 100644 packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart create mode 100644 packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart index c957f207278e..d9500a28e115 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart @@ -105,6 +105,19 @@ class _ImagenPageState extends State { ) else const CircularProgressIndicator(), + if (!_loading) + IconButton( + onPressed: () async { + await _testImagenEdit(); + }, + icon: Icon( + Icons.edit, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Imagen edit', + ) + else + const CircularProgressIndicator(), // NOTE: Keep this API private until future release. // if (!_loading) // IconButton( @@ -211,4 +224,42 @@ class _ImagenPageState extends State { }, ); } + + Future _testImagenEdit() async { + setState(() { + _loading = true; + }); + + try { + // TODO: Add a way to select an image from the gallery + final image = ImagenInlineImage(data: []); + var response = await widget.model.inpaintImage( + image, + 'a dog', + ImagenBackgroundMask(), + ); + + if (response.images.isNotEmpty) { + var imagenImage = response.images[0]; + + _generatedContent.add( + MessageData( + image: Image.memory(imagenImage.bytesBase64Encoded), + text: 'a dog', + fromUser: false, + ), + ); + } else { + // Handle the case where no images were generated + _showError('Error: No images were generated.'); + } + } catch (e) { + _showError(e.toString()); + } + + setState(() { + _loading = false; + _scrollDown(); + }); + } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart new file mode 100644 index 000000000000..68ff2e990b42 --- /dev/null +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart @@ -0,0 +1,435 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +part of 'base_model.dart'; + +/// The desired outcome of the image editing. +@experimental +final class ImagenEditMode { + const ImagenEditMode._(this._jsonString); + + final String _jsonString; + + /// The result of the editing will be an insertion of the prompt in the masked + /// region. + static const ImagenEditMode inpaintInsertion = + ImagenEditMode._('inpaint-insertion'); + + /// The result of the editing will be a removal of the masked region. + static const ImagenEditMode inpaintRemoval = + ImagenEditMode._('inpaint-removal'); + + /// The result of the editing will be an outpainting of the source image. + static const ImagenEditMode outpaint = ImagenEditMode._('outpaint'); + + @override + String toString() => _jsonString; +} + +/// The type of the subject in the image. +@experimental +final class ImagenSubjectReferenceType { + const ImagenSubjectReferenceType._(this._jsonString); + + final String _jsonString; + + /// The subject is a person. + static const ImagenSubjectReferenceType person = + ImagenSubjectReferenceType._('person'); + + /// The subject is an animal. + static const ImagenSubjectReferenceType animal = + ImagenSubjectReferenceType._('animal'); + + /// The subject is a product. + static const ImagenSubjectReferenceType product = + ImagenSubjectReferenceType._('product'); + + @override + String toString() => _jsonString; +} + +/// The type of control image. +@experimental +final class ImagenControlType { + const ImagenControlType._(this._jsonString); + + final String _jsonString; + + /// Canny edge detection. + static const ImagenControlType canny = ImagenControlType._('canny'); + + /// Scribble. + static const ImagenControlType scribble = ImagenControlType._('scribble'); + + /// Face mesh. + static const ImagenControlType faceMesh = ImagenControlType._('face-mesh'); + + /// Color superpixel. + static const ImagenControlType colorSuperpixel = + ImagenControlType._('color-superpixel'); + + @override + String toString() => _jsonString; +} + +/// The mode of the mask. +@experimental +final class ImagenMaskMode { + const ImagenMaskMode._(this._jsonString); + + final String _jsonString; + + /// The mask is user provided. + static const ImagenMaskMode userProvided = ImagenMaskMode._('user-provided'); + + /// The mask is the background. + static const ImagenMaskMode background = ImagenMaskMode._('background'); + + /// The mask is the foreground. + static const ImagenMaskMode foreground = ImagenMaskMode._('foreground'); + + /// The mask is semantic. + static const ImagenMaskMode semantic = ImagenMaskMode._('semantic'); + + @override + String toString() => _jsonString; +} + +/// The configuration for the mask. +@experimental +final class ImagenMaskConfig { + ImagenMaskConfig({ + required this.maskType, + this.maskDilation, + }); + + final ImagenMaskMode maskType; + final double? maskDilation; +} + +/// The configuration for the subject. +@experimental +final class ImagenSubjectConfig { + ImagenSubjectConfig({ + this.description, + this.type, + }); + + final String? description; + final ImagenSubjectReferenceType? type; +} + +/// The configuration for the style. +@experimental +final class ImagenStyleConfig { + ImagenStyleConfig({ + this.description, + }); + + final String? description; +} + +/// The configuration for the control. +@experimental +final class ImagenControlConfig { + ImagenControlConfig({ + required this.controlType, + this.enableComputation, + this.superpixelRegionSize, + this.superpixelRuler, + }); + + final ImagenControlType controlType; + final bool? enableComputation; + final int? superpixelRegionSize; + final int? superpixelRuler; +} + +/// A reference image for image editing. +@experimental +sealed class ImagenReferenceImage { + ImagenReferenceImage({ + this.maskConfig, + this.subjectConfig, + this.styleConfig, + this.controlConfig, + this.image, + this.referenceId, + }); + + final ImagenMaskConfig? maskConfig; + final ImagenSubjectConfig? subjectConfig; + final ImagenStyleConfig? styleConfig; + final ImagenControlConfig? controlConfig; + final ImagenInlineImage? image; + final int? referenceId; + + Map toJson() { + final json = {}; + if (image != null) { + json['image'] = image!.toJson(); + } + if (referenceId != null) { + json['referenceId'] = referenceId; + } + if (maskConfig != null) { + json['mask'] = { + 'type': maskConfig!.maskType.toString(), + if (maskConfig!.maskDilation != null) + 'dilation': maskConfig!.maskDilation, + }; + } + if (subjectConfig != null) { + json['subject'] = { + if (subjectConfig!.description != null) + 'description': subjectConfig!.description, + if (subjectConfig!.type != null) 'type': subjectConfig!.type.toString(), + }; + } + if (styleConfig != null) { + json['style'] = { + if (styleConfig!.description != null) + 'description': styleConfig!.description, + }; + } + if (controlConfig != null) { + json['control'] = { + 'type': controlConfig!.controlType.toString(), + if (controlConfig!.enableComputation != null) + 'enableComputation': controlConfig!.enableComputation, + if (controlConfig!.superpixelRegionSize != null) + 'superpixelRegionSize': controlConfig!.superpixelRegionSize, + if (controlConfig!.superpixelRuler != null) + 'superpixelRuler': controlConfig!.superpixelRuler, + }; + } + return json; + } +} + +/// A reference image that is a mask. +@experimental +sealed class ImagenMaskReference extends ImagenReferenceImage { + ImagenMaskReference({ + super.maskConfig, + super.image, + }); + + /// Generates a mask and pads the image for outpainting. + static List generateMaskAndPadForOutpainting({ + required ImagenInlineImage image, + required Dimensions newDimensions, + ImagenImagePlacement newPosition = ImagenImagePlacement.center, + }) { + // TODO: implement + return []; + } +} + +/// A raw image. +@experimental +final class ImagenRawImage extends ImagenReferenceImage { + ImagenRawImage({ + required ImagenInlineImage image, + }) : super(image: image); +} + +/// A raw mask. +@experimental +final class ImagenRawMask extends ImagenMaskReference { + ImagenRawMask({ + required ImagenInlineImage mask, + double? dilation, + }) : super( + image: mask, + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.userProvided, + maskDilation: dilation, + ), + ); +} + +/// A semantic mask. +@experimental +final class ImagenSemanticMask extends ImagenMaskReference { + ImagenSemanticMask({ + required List classes, + double? dilation, + }) : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.semantic, + maskDilation: dilation, + ), + ); +} + +/// A background mask. +@experimental +final class ImagenBackgroundMask extends ImagenMaskReference { + ImagenBackgroundMask({ + double? dilation, + }) : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.background, + maskDilation: dilation, + ), + ); +} + +/// A foreground mask. +@experimental +final class ImagenForegroundMask extends ImagenMaskReference { + ImagenForegroundMask({ + double? dilation, + }) : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.foreground, + maskDilation: dilation, + ), + ); +} + +/// A subject reference. +@experimental +final class ImagenSubjectReference extends ImagenReferenceImage { + ImagenSubjectReference({ + required ImagenInlineImage image, + int? referenceId, + String? description, + ImagenSubjectReferenceType? subjectType, + }) : super( + image: image, + referenceId: referenceId, + subjectConfig: ImagenSubjectConfig( + description: description, + type: subjectType, + ), + ); +} + +/// A style reference. +@experimental +final class ImagenStyleReference extends ImagenReferenceImage { + ImagenStyleReference({ + required ImagenInlineImage image, + int? referenceId, + String? description, + }) : super( + image: image, + referenceId: referenceId, + styleConfig: ImagenStyleConfig( + description: description, + ), + ); +} + +/// A control reference. +@experimental +final class ImagenControlReference extends ImagenReferenceImage { + ImagenControlReference({ + required ImagenControlType controlType, + ImagenInlineImage? image, + int? referenceId, + bool? enableComputation, + int? superpixelRegionSize, + int? superpixelRuler, + }) : super( + image: image, + referenceId: referenceId, + controlConfig: ImagenControlConfig( + controlType: controlType, + enableComputation: enableComputation, + superpixelRegionSize: superpixelRegionSize, + superpixelRuler: superpixelRuler, + ), + ); +} + +/// The configuration for image editing. +@experimental +final class ImagenEditingConfig { + ImagenEditingConfig({ + this.editMode, + this.editSteps, + }); + + final ImagenEditMode? editMode; + final int? editSteps; +} + +/// The dimensions of an image. +@experimental +final class Dimensions { + Dimensions({ + required this.width, + required this.height, + }); + + final int width; + final int height; +} + +/// The placement of an image. +@experimental +final class ImagenImagePlacement { + const ImagenImagePlacement._(this.x, this.y); + + final int? x; + final int? y; + + /// Creates a placement from a coordinate. + static ImagenImagePlacement fromCoordinate(int x, int y) => + ImagenImagePlacement._(x, y); + + /// The center of the image. + static const ImagenImagePlacement center = ImagenImagePlacement._(null, null); + + /// The top center of the image. + static const ImagenImagePlacement topCenter = + ImagenImagePlacement._(null, null); + + /// The bottom center of the image. + static const ImagenImagePlacement bottomCenter = + ImagenImagePlacement._(null, null); + + /// The left center of the image. + static const ImagenImagePlacement leftCenter = + ImagenImagePlacement._(null, null); + + /// The right center of the image. + static const ImagenImagePlacement rightCenter = + ImagenImagePlacement._(null, null); + + /// The top left of the image. + static const ImagenImagePlacement topLeft = ImagenImagePlacement._(0, 0); + + /// The top right of the image. + static const ImagenImagePlacement topRight = + ImagenImagePlacement._(null, null); + + /// The bottom left of the image. + static const ImagenImagePlacement bottomLeft = + ImagenImagePlacement._(null, null); + + /// The bottom right of the image. + static const ImagenImagePlacement bottomRight = + ImagenImagePlacement._(null, null); +} + +extension on Bitmap { + /// Converts a [Bitmap] to an [ImagenInlineImage]. + ImagenInlineImage get toImagenInlineImage => + ImagenInlineImage(data: asUint8List()); +} diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart index bf4731a3b264..36f9d327b475 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart @@ -110,6 +110,92 @@ final class ImagenModel extends BaseApiClientModel { (jsonObject) => parseImagenGenerationResponse(jsonObject), ); + + /// Edits an image based on a prompt and a list of reference images. + @experimental + Future> editImage( + List referenceImages, + String prompt, { + ImagenEditingConfig? config, + }) => + makeRequest( + Task.predict, + _generateImagenEditRequest( + referenceImages, + prompt, + config: config, + ), + (jsonObject) => + parseImagenGenerationResponse(jsonObject), + ); + + /// Inpaints an image based on a prompt and a mask. + @experimental + Future> inpaintImage( + ImagenInlineImage image, + String prompt, + ImagenMaskReference mask, { + ImagenEditingConfig? config, + }) => + editImage( + [ + ImagenRawImage(image: image), + mask, + ], + prompt, + config: config, + ); + + /// Outpaints an image based on a prompt and new dimensions. + @experimental + Future> outpaintImage( + ImagenInlineImage image, + Dimensions newDimensions, { + ImagenImagePlacement newPosition = ImagenImagePlacement.center, + String prompt = '', + ImagenEditingConfig? config, + }) => + editImage( + ImagenMaskReference.generateMaskAndPadForOutpainting( + image: image, + newDimensions: newDimensions, + newPosition: newPosition, + ), + prompt, + config: config, + ); + + Map _generateImagenEditRequest( + List images, + String prompt, { + ImagenEditingConfig? config, + }) { + final parameters = { + 'sampleCount': _generationConfig?.numberOfImages ?? 1, + if (config?.editMode case final editMode?) 'editMode': editMode.toString(), + if (config?.editSteps case final editSteps?) 'editSteps': editSteps, + if (_generationConfig?.negativePrompt case final negativePrompt?) + 'negativePrompt': negativePrompt, + if (_generationConfig?.addWatermark case final addWatermark?) + 'addWatermark': addWatermark, + if (_generationConfig?.imageFormat case final imageFormat?) + 'outputOption': imageFormat.toJson(), + if (_safetySettings?.personFilterLevel case final personFilterLevel?) + 'personGeneration': personFilterLevel.toJson(), + if (_safetySettings?.safetyFilterLevel case final safetyFilterLevel?) + 'safetySetting': safetyFilterLevel.toJson(), + }; + + return { + 'instances': [ + { + 'prompt': prompt, + 'images': images.map((i) => i.toJson()).toList(), + } + ], + 'parameters': parameters, + }; + } } /// Returns a [ImagenModel] using it's private constructor. diff --git a/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart new file mode 100644 index 000000000000..c727517f658e --- /dev/null +++ b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart @@ -0,0 +1,107 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may +// obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:convert'; + +import 'package:firebase_ai/firebase_ai.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('ImagenReferenceImage', () { + test('ImagenRawImage toJson', () { + final image = ImagenRawImage(image: ImagenInlineImage(data: [])); + final json = image.toJson(); + expect(json, { + 'image': {'data': 'AA=='} + }); + }); + + test('ImagenRawMask toJson', () { + final image = ImagenRawMask(mask: ImagenInlineImage(data: [])); + final json = image.toJson(); + expect(json, { + 'image': {'data': 'AA=='}, + 'mask': {'type': 'user-provided'} + }); + }); + + test('ImagenSemanticMask toJson', () { + final image = ImagenSemanticMask(classes: [1, 2]); + final json = image.toJson(); + expect(json, { + 'mask': {'type': 'semantic'} + }); + }); + + test('ImagenBackgroundMask toJson', () { + final image = ImagenBackgroundMask(); + final json = image.toJson(); + expect(json, { + 'mask': {'type': 'background'} + }); + }); + + test('ImagenForegroundMask toJson', () { + final image = ImagenForegroundMask(); + final json = image.toJson(); + expect(json, { + 'mask': {'type': 'foreground'} + }); + }); + + test('ImagenSubjectReference toJson', () { + final image = ImagenSubjectReference( + image: ImagenInlineImage(data: []), + referenceId: 1, + description: 'a cat', + subjectType: ImagenSubjectReferenceType.animal, + ); + final json = image.toJson(); + expect(json, { + 'image': {'data': 'AA=='}, + 'referenceId': 1, + 'subject': {'description': 'a cat', 'type': 'animal'} + }); + }); + + test('ImagenStyleReference toJson', () { + final image = ImagenStyleReference( + image: ImagenInlineImage(data: []), + referenceId: 1, + description: 'van gogh style', + ); + final json = image.toJson(); + expect(json, { + 'image': {'data': 'AA=='}, + 'referenceId': 1, + 'style': {'description': 'van gogh style'} + }); + }); + + test('ImagenControlReference toJson', () { + final image = ImagenControlReference( + controlType: ImagenControlType.canny, + image: ImagenInlineImage(data: []), + referenceId: 1, + ); + final json = image.toJson(); + expect(json, { + 'image': {'data': 'AA=='}, + 'referenceId': 1, + 'control': {'type': 'canny'} + }); + }); + }); +} From 16b053540c298b2121aa24cf4b945d55471db202 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 23 Jul 2025 21:31:10 -0700 Subject: [PATCH 2/6] some level of working imagen editing --- .../firebase_ai/example/lib/main.dart | 2 +- .../example/lib/pages/imagen_page.dart | 374 +++++++++++---- .../macos/Runner/DebugProfile.entitlements | 2 + .../example/macos/Runner/Info.plist | 2 + .../firebase_ai/example/pubspec.yaml | 1 + .../firebase_ai/lib/firebase_ai.dart | 31 +- .../firebase_ai/lib/src/base_model.dart | 8 +- .../firebase_ai/lib/src/client.dart | 6 +- .../lib/src/{ => imagen}/imagen_api.dart | 0 .../lib/src/{ => imagen}/imagen_content.dart | 9 +- .../lib/src/imagen/imagen_edit.dart | 273 +++++++++++ .../lib/src/{ => imagen}/imagen_model.dart | 34 +- .../lib/src/imagen/imagen_reference.dart | 319 +++++++++++++ .../firebase_ai/lib/src/imagen_edit.dart | 435 ------------------ .../firebase_ai/test/imagen_edit_test.dart | 31 +- .../firebase_ai/test/imagen_test.dart | 4 +- 16 files changed, 971 insertions(+), 560 deletions(-) rename packages/firebase_ai/firebase_ai/lib/src/{ => imagen}/imagen_api.dart (100%) rename packages/firebase_ai/firebase_ai/lib/src/{ => imagen}/imagen_content.dart (94%) create mode 100644 packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart rename packages/firebase_ai/firebase_ai/lib/src/{ => imagen}/imagen_model.dart (92%) create mode 100644 packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart delete mode 100644 packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart index a7fd0363aba7..584edfa8079f 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/main.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/main.dart @@ -80,7 +80,7 @@ class _GenerativeAISampleState extends State { imageFormat: ImagenFormat.jpeg(compressionQuality: 75), ); return instance.imagenModel( - model: 'imagen-3.0-generate-002', + model: 'imagen-3.0-capability-001', generationConfig: generationConfig, safetySettings: ImagenSafetySettings( ImagenSafetyFilterLevel.blockLowAndAbove, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart index d9500a28e115..57b89b64ca67 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart @@ -12,8 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'package:flutter/material.dart'; +import 'dart:typed_data'; + +import 'package:image_picker/image_picker.dart'; import 'package:firebase_ai/firebase_ai.dart'; + +import 'package:flutter/material.dart'; //import 'package:firebase_storage/firebase_storage.dart'; import '../widgets/message_widget.dart'; @@ -38,6 +42,10 @@ class _ImagenPageState extends State { final List _generatedContent = []; bool _loading = false; + // For image picking + ImagenInlineImage? _sourceImage; + ImagenInlineImage? _maskImageForEditing; + void _scrollDown() { WidgetsBinding.instance.addPostFrameCallback( (_) => _scrollController.animateTo( @@ -80,58 +88,89 @@ class _ImagenPageState extends State { vertical: 25, horizontal: 15, ), - child: Row( + child: Column( children: [ - Expanded( - child: TextField( - autofocus: true, - focusNode: _textFieldFocus, - controller: _textController, - ), - ), - const SizedBox.square( - dimension: 15, - ), - if (!_loading) - IconButton( - onPressed: () async { - await _testImagen(_textController.text); - }, - icon: Icon( - Icons.image_search, - color: Theme.of(context).colorScheme.primary, + // Generate Image Row + Row( + children: [ + Expanded( + child: TextField( + autofocus: true, + focusNode: _textFieldFocus, + decoration: const InputDecoration( + hintText: 'Enter a prompt...', + ), + controller: _textController, + ), + ), + const SizedBox.square(dimension: 15), + IconButton( + onPressed: () async { + await _pickSourceImage(); + }, + icon: Icon( + Icons.add_a_photo, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Pick Source Image', + ), + IconButton( + onPressed: () async { + await _pickMaskImage(); + }, + icon: Icon( + Icons.add_to_photos, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Pick mask', ), - tooltip: 'Imagen raw data', - ) - else - const CircularProgressIndicator(), - if (!_loading) - IconButton( - onPressed: () async { - await _testImagenEdit(); - }, - icon: Icon( - Icons.edit, - color: Theme.of(context).colorScheme.primary, + IconButton( + onPressed: () async { + await _editWithStyle(); + }, + icon: Icon( + Icons.edit, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Edit with Style', ), - tooltip: 'Imagen edit', - ) - else - const CircularProgressIndicator(), - // NOTE: Keep this API private until future release. - // if (!_loading) - // IconButton( - // onPressed: () async { - // await _testImagenGCS(_textController.text); - // }, - // icon: Icon( - // Icons.imagesearch_roller, - // color: Theme.of(context).colorScheme.primary, - // ), - // tooltip: 'Imagen GCS', - // ) - // else - // const CircularProgressIndicator(), + IconButton( + onPressed: () async { + await _outpaintImageHappyPath(); + }, + icon: Icon( + Icons.masks, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Outpaint', + ), + IconButton( + onPressed: () async { + await _inpaintImageHappyPath(); + }, + icon: Icon( + Icons.plus_one, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Inpaint', + ), + if (!_loading) + IconButton( + onPressed: () async { + await _generateImageFromPrompt( + _textController.text, + ); + }, + icon: Icon( + Icons.image_search, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Generate Image', + ) + else + const CircularProgressIndicator(), + ], + ), ], ), ), @@ -141,7 +180,202 @@ class _ImagenPageState extends State { ); } - Future _testImagen(String prompt) async { + Future _pickImage() async { + final ImagePicker picker = ImagePicker(); + try { + final XFile? imageFile = + await picker.pickImage(source: ImageSource.gallery); + if (imageFile != null) { + // Attempt to get mimeType, default if null. + // Note: imageFile.mimeType might be null on some platforms or for some files. + final String mimeType = imageFile.mimeType ?? 'image/jpeg'; + final Uint8List imageBytes = await imageFile.readAsBytes(); + return ImagenInlineImage( + bytesBase64Encoded: imageBytes, mimeType: mimeType); + } + } catch (e) { + _showError('Error picking image: $e'); + } + return null; + } + + Future _pickSourceImage() async { + final pickedImage = await _pickImage(); + if (pickedImage != null) { + setState(() { + _sourceImage = pickedImage; + }); + } + } + + Future _pickMaskImage() async { + final pickedImage = await _pickImage(); + if (pickedImage != null) { + setState(() { + _maskImageForEditing = pickedImage; + }); + } + } + + Future _inpaintImageHappyPath() async { + if (_sourceImage == null) { + _showError('Please pick a source image for inpaint insertion.'); + return; + } + setState(() { + _loading = true; + }); + + final String prompt = _textController.text; + + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: 'Try to inpaint image with prompt: $prompt', + fromUser: true, + ), + ); + _scrollDown(); + }); + + try { + final response = await widget.model.inpaintImage( + _sourceImage!, + prompt, + ImagenBackgroundMask(referenceId: 1), + config: ImagenEditingConfig(editMode: ImagenEditMode.inpaintInsertion), + ); + if (response.images.isNotEmpty) { + final inpaintImage = response.images[0]; + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(inpaintImage.bytesBase64Encoded), + text: 'Inpaint image result with prompt: $prompt', + fromUser: false, + ), + ); + _scrollDown(); + }); + } else { + _showError('No image was returned from inpaint.'); + } + } catch (e) { + _showError('Error inpaint image: $e'); + } + + setState(() { + _loading = false; + }); + } + + Future _outpaintImageHappyPath() async { + if (_sourceImage == null) { + _showError('Please pick a source image for outpainting.'); + return; + } + setState(() { + _loading = true; + }); + + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: 'Outpaint the picture to 1400*1400', + fromUser: true, + ), + ); + _scrollDown(); + }); + + try { + final response = await widget.model.outpaintImage( + _sourceImage!, + ImagenDimensions(width: 1400, height: 1400), + ); + if (response.images.isNotEmpty) { + final editedImage = response.images[0]; + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(editedImage.bytesBase64Encoded), + text: 'Edited image Outpaint 1400*1400', + fromUser: false, + ), + ); + _scrollDown(); + }); + } else { + _showError('No image was returned from editing.'); + } + } catch (e) { + _showError('Error editing image: $e'); + } + setState(() { + _loading = false; + }); + } + + Future _editWithStyle() async { + if (_sourceImage == null) { + _showError('Please pick a source image for style editing.'); + return; + } + setState(() { + _loading = true; + }); + + final String prompt = _textController.text; + + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: prompt, + fromUser: true, + ), + ); + _scrollDown(); + }); + + try { + final response = await widget.model.editImage( + [ + ImagenStyleReference( + image: _sourceImage!, + referenceId: 1, + description: 'van goh style', + ), + ], + prompt, + config: ImagenEditingConfig(editSteps: 50), + ); + if (response.images.isNotEmpty) { + final editedImage = response.images[0]; + setState(() { + _generatedContent.add( + MessageData( + image: Image.memory(editedImage.bytesBase64Encoded), + text: 'Edited image with style: $prompt', + fromUser: false, + ), + ); + _scrollDown(); + }); + } else { + _showError('No image was returned from style editing.'); + } + } catch (e) { + _showError('Error performing style edit: $e'); + } + setState(() { + _loading = false; + }); + } + + Future _generateImageFromPrompt(String prompt) async { setState(() { _loading = true; }); @@ -224,42 +458,4 @@ class _ImagenPageState extends State { }, ); } - - Future _testImagenEdit() async { - setState(() { - _loading = true; - }); - - try { - // TODO: Add a way to select an image from the gallery - final image = ImagenInlineImage(data: []); - var response = await widget.model.inpaintImage( - image, - 'a dog', - ImagenBackgroundMask(), - ); - - if (response.images.isNotEmpty) { - var imagenImage = response.images[0]; - - _generatedContent.add( - MessageData( - image: Image.memory(imagenImage.bytesBase64Encoded), - text: 'a dog', - fromUser: false, - ), - ); - } else { - // Handle the case where no images were generated - _showError('Error: No images were generated.'); - } - } catch (e) { - _showError(e.toString()); - } - - setState(() { - _loading = false; - _scrollDown(); - }); - } } diff --git a/packages/firebase_ai/firebase_ai/example/macos/Runner/DebugProfile.entitlements b/packages/firebase_ai/firebase_ai/example/macos/Runner/DebugProfile.entitlements index b4bd9ee174a1..8560da29b687 100644 --- a/packages/firebase_ai/firebase_ai/example/macos/Runner/DebugProfile.entitlements +++ b/packages/firebase_ai/firebase_ai/example/macos/Runner/DebugProfile.entitlements @@ -14,5 +14,7 @@ com.apple.security.device.audio-input + com.apple.security.files.user-selected.read-only + diff --git a/packages/firebase_ai/firebase_ai/example/macos/Runner/Info.plist b/packages/firebase_ai/firebase_ai/example/macos/Runner/Info.plist index a81b3fd0d617..d4369e6253fa 100644 --- a/packages/firebase_ai/firebase_ai/example/macos/Runner/Info.plist +++ b/packages/firebase_ai/firebase_ai/example/macos/Runner/Info.plist @@ -30,5 +30,7 @@ NSApplication NSMicrophoneUsageDescription Permission to Record audio + NSPhotoLibraryUsageDescription + This app needs access to your photo library to let you select a profile picture. diff --git a/packages/firebase_ai/firebase_ai/example/pubspec.yaml b/packages/firebase_ai/firebase_ai/example/pubspec.yaml index 4ebad97543bb..01f710410276 100644 --- a/packages/firebase_ai/firebase_ai/example/pubspec.yaml +++ b/packages/firebase_ai/firebase_ai/example/pubspec.yaml @@ -27,6 +27,7 @@ dependencies: sdk: flutter flutter_markdown: ^0.6.20 flutter_soloud: ^3.1.6 + image_picker: ^1.1.2 path_provider: ^2.1.5 record: ^5.2.1 diff --git a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart index 0587c156f9a5..7c51a41a389c 100644 --- a/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/firebase_ai.dart @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'src/imagen/imagen_reference.dart'; + export 'src/api.dart' show BlockReason, @@ -51,7 +53,7 @@ export 'src/error.dart' ServerException, UnsupportedUserLocation; export 'src/firebase_ai.dart' show FirebaseAI; -export 'src/imagen_api.dart' +export 'src/imagen/imagen_api.dart' show ImagenSafetySettings, ImagenFormat, @@ -59,7 +61,32 @@ export 'src/imagen_api.dart' ImagenPersonFilterLevel, ImagenGenerationConfig, ImagenAspectRatio; -export 'src/imagen_content.dart' show ImagenInlineImage; +export 'src/imagen/imagen_content.dart' show ImagenInlineImage; +export 'src/imagen/imagen_edit.dart' + show + ImagenEditMode, + ImagenSubjectReferenceType, + ImagenControlType, + ImagenMaskMode, + ImagenMaskConfig, + ImagenSubjectConfig, + ImagenStyleConfig, + ImagenControlConfig, + ImagenEditingConfig, + ImagenDimensions, + ImagenImagePlacement; +export 'src/imagen/imagen_reference.dart' + show + ImagenReferenceImage, + ImagenMaskReference, + ImagenRawImage, + ImagenRawMask, + ImagenSemanticMask, + ImagenBackgroundMask, + ImagenForegroundMask, + ImagenSubjectReference, + ImagenStyleReference, + ImagenControlReference; export 'src/live_api.dart' show LiveGenerationConfig, diff --git a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart index 413af8ba49eb..de62a972dbfe 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart @@ -28,15 +28,17 @@ import 'api.dart'; import 'client.dart'; import 'content.dart'; import 'developer/api.dart'; -import 'imagen_api.dart'; -import 'imagen_content.dart'; +import 'imagen/imagen_api.dart'; +import 'imagen/imagen_content.dart'; +import 'imagen/imagen_edit.dart'; +import 'imagen/imagen_reference.dart'; import 'live_api.dart'; import 'live_session.dart'; import 'tool.dart'; import 'vertex_version.dart'; part 'generative_model.dart'; -part 'imagen_model.dart'; +part 'imagen/imagen_model.dart'; part 'live_model.dart'; /// [Task] enum class for [GenerativeModel] to make request. diff --git a/packages/firebase_ai/firebase_ai/lib/src/client.dart b/packages/firebase_ai/firebase_ai/lib/src/client.dart index ba3eed67b6fe..464698ac9eeb 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/client.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/client.dart @@ -63,9 +63,13 @@ final class HttpApiClient implements ApiClient { @override Future> makeRequest( Uri uri, Map body) async { + print(uri); + final headers = await _headers(); + print(headers); + print(body); final response = await (_httpClient?.post ?? http.post)( uri, - headers: await _headers(), + headers: headers, body: _utf8Json.encode(body), ); if (response.statusCode >= 500) { diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_api.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_api.dart similarity index 100% rename from packages/firebase_ai/firebase_ai/lib/src/imagen_api.dart rename to packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_api.dart diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_content.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_content.dart similarity index 94% rename from packages/firebase_ai/firebase_ai/lib/src/imagen_content.dart rename to packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_content.dart index 525cbeef44ed..2aafcf5e6c0b 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen_content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_content.dart @@ -13,8 +13,9 @@ // limitations under the License. import 'dart:convert'; import 'dart:typed_data'; +import 'dart:ui' as ui; import 'package:meta/meta.dart'; -import 'error.dart'; +import '../error.dart'; /// Base type of Imagen Image. sealed class ImagenImage { @@ -59,6 +60,12 @@ final class ImagenInlineImage implements ImagenImage { 'mimeType': mimeType, 'bytesBase64Encoded': base64Encode(bytesBase64Encoded), }; + // Helper to decode bytes into a dart:ui.Image. + Future asUiImage() async { + final codec = await ui.instantiateImageCodec(bytesBase64Encoded); + final frame = await codec.getNextFrame(); + return frame.image; + } } /// Represents an image stored in Google Cloud Storage. diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart new file mode 100644 index 000000000000..27449e9dfc10 --- /dev/null +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart @@ -0,0 +1,273 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +/// The desired outcome of the image editing. +@experimental +enum ImagenEditMode { + /// The result of the editing will be an insertion of the prompt in the masked + /// region. + inpaintInsertion('EDIT_MODE_INPAINT_INSERTION'), + + /// The result of the editing will be a removal of the masked region. + inpaintRemoval('EDIT_MODE_INPAINT_REMOVAL'), + + /// The result of the editing will be an outpainting of the source image. + outpaint('EDIT_MODE_OUTPAINT'); + + const ImagenEditMode(this._jsonString); + final String _jsonString; + // ignore: public_member_api_docs + String toJson() => _jsonString; +} + +/// The type of the subject in the image. +@experimental +enum ImagenSubjectReferenceType { + /// The subject is a person. + person('SUBJECT_TYPE_PERSON'), + + /// The subject is an animal. + animal('SUBJECT_TYPE_ANIMAL'), + + /// The subject is a product. + product('SUBJECT_TYPE_PRODUCT'); + + const ImagenSubjectReferenceType(this._jsonString); + final String _jsonString; + + // ignore: public_member_api_docs + String toJson() => _jsonString; +} + +/// The type of control image. +@experimental +enum ImagenControlType { + /// Use edge detection to ensure the new image follow the same outlines. + canny('CONTROL_TYPE_CANNY'), + + /// Use enhanced edge detection to ensure the new image follow similar + /// outlines. + scribble('CONTROL_TYPE_SCRIBBLE'), + + /// Use face mesh control to ensure that the new image has the same facial + /// expressions. + faceMesh('CONTROL_TYPE_FACE_MESH'), + + /// Use color superpixels to ensure that the new image is similar in shape + /// and color to the original. + colorSuperpixel('CONTROL_TYPE_COLOR_SUPERPIXEL'); + + const ImagenControlType(this._jsonString); + final String _jsonString; + + // ignore: public_member_api_docs + String toJson() => _jsonString; +} + +/// The mode of the mask. +@experimental +enum ImagenMaskMode { + /// The mask is user provided. + userProvided('MASK_MODE_USER_PROVIDED'), + + /// The mask is the background. + background('MASK_MODE_BACKGROUND'), + + /// The mask is the foreground. + foreground('MASK_MODE_FOREGROUND'), + + /// The mask is semantic. + semantic('MASK_MODE_SEMANTIC'); + + const ImagenMaskMode(this._jsonString); + final String _jsonString; + // ignore: public_member_api_docs + String toJson() => _jsonString; +} + +sealed class ImagenReferenceConfig { + /// Convert the [ImagenReferenceConfig] content to json format. + Map toJson(); +} + +/// The configuration for the mask. +@experimental +final class ImagenMaskConfig extends ImagenReferenceConfig { + ImagenMaskConfig({ + required this.maskType, + this.maskDilation, + this.maskClasses, + }); + + final ImagenMaskMode maskType; + final double? maskDilation; + final List? maskClasses; + + @override + Map toJson() => { + 'maskImageConfig': { + 'maskMode': maskType.toJson(), + if (maskDilation != null) 'dilation': maskDilation, + if (maskClasses != null) 'maskClasses': maskClasses.toString(), + }, + }; +} + +/// The configuration for the subject. +@experimental +final class ImagenSubjectConfig extends ImagenReferenceConfig { + ImagenSubjectConfig({ + this.description, + this.type, + }); + + final String? description; + final ImagenSubjectReferenceType? type; + + @override + Map toJson() => { + 'subjectImageConfig': { + if (description != null) 'subjectDescription': description, + if (type != null) 'subjectType': type!.toJson(), + }, + }; +} + +/// The configuration for the style. +@experimental +final class ImagenStyleConfig extends ImagenReferenceConfig { + ImagenStyleConfig({ + this.description, + }); + + final String? description; + @override + Map toJson() => { + 'styleImageConfig': { + if (description != null) 'subjectDescription': description, + }, + }; +} + +/// The configuration for the control. +@experimental +final class ImagenControlConfig extends ImagenReferenceConfig { + ImagenControlConfig({ + required this.controlType, + this.enableComputation, + this.superpixelRegionSize, + this.superpixelRuler, + }); + + final ImagenControlType controlType; + final bool? enableComputation; + final int? superpixelRegionSize; + final int? superpixelRuler; + @override + Map toJson() => { + 'controlImageConfig': { + 'controlType': controlType.toJson(), + if (enableComputation != null) + 'enableControlImageComputation': enableComputation, + if (superpixelRegionSize != null) + 'superpixelRegionSize': superpixelRegionSize, + if (superpixelRuler != null) 'superpixelRuler': superpixelRuler, + }, + }; +} + +/// The configuration for image editing. +@experimental +final class ImagenEditingConfig { + ImagenEditingConfig({ + this.editMode, + this.editSteps, + }); + + final ImagenEditMode? editMode; + final int? editSteps; +} + +/// The dimensions of an image. +@experimental +final class ImagenDimensions { + ImagenDimensions({ + required this.width, + required this.height, + }); + + final int width; + final int height; +} + +/// The placement of an image. +@experimental +final class ImagenImagePlacement { + const ImagenImagePlacement._(this.x, this.y); + + final int? x; + final int? y; + + /// Creates a placement from a coordinate. + static ImagenImagePlacement fromCoordinate(int x, int y) => + ImagenImagePlacement._(x, y); + + /// The center of the image. + static const ImagenImagePlacement center = ImagenImagePlacement._(null, null); + + /// The top center of the image. + static const ImagenImagePlacement topCenter = + ImagenImagePlacement._(null, null); + + /// The bottom center of the image. + static const ImagenImagePlacement bottomCenter = + ImagenImagePlacement._(null, null); + + /// The left center of the image. + static const ImagenImagePlacement leftCenter = + ImagenImagePlacement._(null, null); + + /// The right center of the image. + static const ImagenImagePlacement rightCenter = + ImagenImagePlacement._(null, null); + + /// The top left of the image. + static const ImagenImagePlacement topLeft = ImagenImagePlacement._(0, 0); + + /// The top right of the image. + static const ImagenImagePlacement topRight = + ImagenImagePlacement._(null, null); + + /// The bottom left of the image. + static const ImagenImagePlacement bottomLeft = + ImagenImagePlacement._(null, null); + + /// The bottom right of the image. + static const ImagenImagePlacement bottomRight = + ImagenImagePlacement._(null, null); + + /// A mock normalization function. + ImagenImagePlacement normalizeToDimensions( + ImagenDimensions original, + ImagenDimensions newDim, + ) { + // In a real implementation, this would calculate the top-left (x, y) + // based on the placement strategy (e.g., center, top-left). + final x = (newDim.width - original.width) / 2; + final y = (newDim.height - original.height) / 2; + return ImagenImagePlacement.fromCoordinate(x.toInt(), y.toInt()); + } +} diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart similarity index 92% rename from packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart rename to packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart index 36f9d327b475..77e71e569887 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -part of 'base_model.dart'; +part of '../base_model.dart'; /// Represents a remote Imagen model with the ability to generate images using /// text prompts. @@ -139,7 +139,7 @@ final class ImagenModel extends BaseApiClientModel { }) => editImage( [ - ImagenRawImage(image: image), + ImagenRawImage(image: image, referenceId: 0), mask, ], prompt, @@ -150,20 +150,23 @@ final class ImagenModel extends BaseApiClientModel { @experimental Future> outpaintImage( ImagenInlineImage image, - Dimensions newDimensions, { + ImagenDimensions newDimensions, { ImagenImagePlacement newPosition = ImagenImagePlacement.center, String prompt = '', ImagenEditingConfig? config, - }) => - editImage( - ImagenMaskReference.generateMaskAndPadForOutpainting( - image: image, - newDimensions: newDimensions, - newPosition: newPosition, - ), - prompt, - config: config, - ); + }) async { + final referenceImages = + await ImagenMaskReference.generateMaskAndPadForOutpainting( + image: image, + newDimensions: newDimensions, + newPosition: newPosition, + ); + return editImage( + referenceImages, + prompt, + config: config, + ); + } Map _generateImagenEditRequest( List images, @@ -172,7 +175,8 @@ final class ImagenModel extends BaseApiClientModel { }) { final parameters = { 'sampleCount': _generationConfig?.numberOfImages ?? 1, - if (config?.editMode case final editMode?) 'editMode': editMode.toString(), + if (config?.editMode case final editMode?) + 'editMode': editMode.toString(), if (config?.editSteps case final editSteps?) 'editSteps': editSteps, if (_generationConfig?.negativePrompt case final negativePrompt?) 'negativePrompt': negativePrompt, @@ -190,7 +194,7 @@ final class ImagenModel extends BaseApiClientModel { 'instances': [ { 'prompt': prompt, - 'images': images.map((i) => i.toJson()).toList(), + 'referenceImages': images.map((i) => i.toJson()).toList(), } ], 'parameters': parameters, diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart new file mode 100644 index 000000000000..1b09256c10ef --- /dev/null +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart @@ -0,0 +1,319 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import 'dart:ui' as ui; + +import 'package:flutter/material.dart'; +import 'package:meta/meta.dart'; + +import 'imagen_content.dart'; +import 'imagen_edit.dart'; + +enum ReferenceType { + UNSPECIFIED('REFERENCE_TYPE_UNSPECIFIED'), + RAW('REFERENCE_TYPE_RAW'), + MASK('REFERENCE_TYPE_MASK'), + CONTROL('REFERENCE_TYPE_CONTROL'), + STYLE('REFERENCE_TYPE_STYLE'), + SUBJECT('REFERENCE_TYPE_SUBJECT'), + MASKED_SUBJECT('REFERENCE_TYPE_MASKED_SUBJECT'), + PRODUCT('REFERENCE_TYPE_PRODUCT'); + + const ReferenceType(this._jsonString); + final String _jsonString; + + @override + String toString() => _jsonString; +} + +/// A reference image for image editing. +@experimental +sealed class ImagenReferenceImage { + ImagenReferenceImage({ + this.referenceConfig, + this.image, + this.referenceId, + required this.referenceType, + }); + + final ImagenReferenceConfig? referenceConfig; + final ImagenInlineImage? image; + final int? referenceId; + final ReferenceType referenceType; + + Map toJson() { + final json = {}; + json['referenceType'] = referenceType.toString(); + if (image != null) { + json['referenceImage'] = image!.toJson(); + } + if (referenceId != null) { + json['referenceId'] = referenceId; + } + if (referenceConfig != null) { + json.addAll(referenceConfig!.toJson()); + } + + return json; + } +} + +/// A reference image that is a mask. +@experimental +sealed class ImagenMaskReference extends ImagenReferenceImage { + ImagenMaskReference({ + ImagenMaskConfig? maskConfig, + super.image, + super.referenceId, + }) : super(referenceType: ReferenceType.MASK, referenceConfig: maskConfig); + + /// Generates a mask and pads the image for outpainting. + static Future> generateMaskAndPadForOutpainting({ + required ImagenInlineImage image, + required ImagenDimensions newDimensions, + ImagenImagePlacement newPosition = ImagenImagePlacement.center, + }) async { + final originalImage = await image.asUiImage(); + + // Validate that the new dimensions are strictly larger. + if (originalImage.width >= newDimensions.width || + originalImage.height >= newDimensions.height) { + throw ArgumentError( + 'New Dimensions must be strictly larger than original image dimensions. ' + 'Original image is: ${originalImage.width}x${originalImage.height}, ' + 'new dimensions are ${newDimensions.width}x${newDimensions.height}', + ); + } + + // Calculate the position of the original image on the new canvas. + final normalizedPosition = newPosition.normalizeToDimensions( + ImagenDimensions( + width: originalImage.width, height: originalImage.height), + newDimensions, + ); + + final x = normalizedPosition.x; + final y = normalizedPosition.y; + + if (x == null || y == null) { + throw StateError('Error normalizing position for mask and padding.'); + } + + // Define the rectangle where the original image will be drawn. + final imageRect = ui.Rect.fromLTWH( + x.toDouble(), + y.toDouble(), + originalImage.width.toDouble(), + originalImage.height.toDouble(), + ); + + // Create both the mask and the new image concurrently. + final results = await Future.wait([ + // Future to create the mask + _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + // Fill the mask with white, then draw a black rectangle where the image is. + canvas.drawPaint(Paint()..color = Colors.white); + canvas.drawRect(imageRect, Paint()..color = Colors.black); + }, + ), + // Future to create the new padded image + _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + // Fill the new image with black padding. + canvas.drawPaint(Paint()..color = Colors.black); + // Draw the original image into the corresponding spot. + canvas.drawImageRect( + originalImage, + ui.Rect.fromLTWH(0, 0, originalImage.width.toDouble(), + originalImage.height.toDouble()), + imageRect, + Paint(), + ); + }, + ), + ]); + + final newPaddedUiImage = results[1]; + final maskUiImage = results[0]; + + // Convert the generated ui.Image objects back to byte data. + final newImageBytes = + await newPaddedUiImage.toByteData(format: ui.ImageByteFormat.png); + final maskBytes = + await maskUiImage.toByteData(format: ui.ImageByteFormat.png); + + if (newImageBytes == null || maskBytes == null) { + throw StateError('Failed to encode generated images.'); + } + + return [ + ImagenRawImage( + referenceId: 1, + image: ImagenInlineImage( + bytesBase64Encoded: newImageBytes.buffer.asUint8List(), + mimeType: image.mimeType, + ), + ), + ImagenRawMask( + referenceId: 2, + mask: ImagenInlineImage( + bytesBase64Encoded: maskBytes.buffer.asUint8List(), + mimeType: image.mimeType, + ), + ), + ]; + } + + /// Helper function to create a ui.Image by drawing on a Canvas. + static Future _createImageFromPainter({ + required int width, + required int height, + required void Function(ui.Canvas canvas, ui.Size size) painter, + }) { + final recorder = ui.PictureRecorder(); + final canvas = ui.Canvas(recorder); + final size = ui.Size(width.toDouble(), height.toDouble()); + + painter(canvas, size); + + final picture = recorder.endRecording(); + return picture.toImage(width, height); + } +} + +/// A raw image. +@experimental +final class ImagenRawImage extends ImagenReferenceImage { + ImagenRawImage({ + required ImagenInlineImage image, + super.referenceId, + }) : super(image: image, referenceType: ReferenceType.RAW); +} + +/// A raw mask. +@experimental +final class ImagenRawMask extends ImagenMaskReference { + ImagenRawMask({ + required ImagenInlineImage mask, + double? dilation, + super.referenceId, + }) : super( + image: mask, + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.userProvided, + maskDilation: dilation, + ), + ); +} + +/// A semantic mask. +@experimental +final class ImagenSemanticMask extends ImagenMaskReference { + ImagenSemanticMask({ + required List classes, + double? dilation, + super.referenceId, + }) : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.semantic, + maskDilation: dilation, + ), + ); +} + +/// A background mask. +@experimental +final class ImagenBackgroundMask extends ImagenMaskReference { + ImagenBackgroundMask({double? dilation, super.referenceId}) + : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.background, + maskDilation: dilation, + ), + ); +} + +/// A foreground mask. +@experimental +final class ImagenForegroundMask extends ImagenMaskReference { + ImagenForegroundMask({ + double? dilation, + super.referenceId, + }) : super( + maskConfig: ImagenMaskConfig( + maskType: ImagenMaskMode.foreground, + maskDilation: dilation, + ), + ); +} + +/// A subject reference. +@experimental +final class ImagenSubjectReference extends ImagenReferenceImage { + ImagenSubjectReference({ + required ImagenInlineImage image, + super.referenceId, + String? description, + ImagenSubjectReferenceType? subjectType, + }) : super( + image: image, + referenceConfig: ImagenSubjectConfig( + description: description, + type: subjectType, + ), + referenceType: ReferenceType.SUBJECT, + ); +} + +/// A style reference. +@experimental +final class ImagenStyleReference extends ImagenReferenceImage { + ImagenStyleReference({ + required ImagenInlineImage image, + super.referenceId, + String? description, + }) : super( + image: image, + referenceConfig: ImagenStyleConfig( + description: description, + ), + referenceType: ReferenceType.STYLE, + ); +} + +/// A control reference. +@experimental +final class ImagenControlReference extends ImagenReferenceImage { + ImagenControlReference({ + required ImagenControlType controlType, + ImagenInlineImage? image, + super.referenceId, + bool? enableComputation, + int? superpixelRegionSize, + int? superpixelRuler, + }) : super( + image: image, + referenceConfig: ImagenControlConfig( + controlType: controlType, + enableComputation: enableComputation, + superpixelRegionSize: superpixelRegionSize, + superpixelRuler: superpixelRuler, + ), + referenceType: ReferenceType.CONTROL, + ); +} diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart deleted file mode 100644 index 68ff2e990b42..000000000000 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen_edit.dart +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -part of 'base_model.dart'; - -/// The desired outcome of the image editing. -@experimental -final class ImagenEditMode { - const ImagenEditMode._(this._jsonString); - - final String _jsonString; - - /// The result of the editing will be an insertion of the prompt in the masked - /// region. - static const ImagenEditMode inpaintInsertion = - ImagenEditMode._('inpaint-insertion'); - - /// The result of the editing will be a removal of the masked region. - static const ImagenEditMode inpaintRemoval = - ImagenEditMode._('inpaint-removal'); - - /// The result of the editing will be an outpainting of the source image. - static const ImagenEditMode outpaint = ImagenEditMode._('outpaint'); - - @override - String toString() => _jsonString; -} - -/// The type of the subject in the image. -@experimental -final class ImagenSubjectReferenceType { - const ImagenSubjectReferenceType._(this._jsonString); - - final String _jsonString; - - /// The subject is a person. - static const ImagenSubjectReferenceType person = - ImagenSubjectReferenceType._('person'); - - /// The subject is an animal. - static const ImagenSubjectReferenceType animal = - ImagenSubjectReferenceType._('animal'); - - /// The subject is a product. - static const ImagenSubjectReferenceType product = - ImagenSubjectReferenceType._('product'); - - @override - String toString() => _jsonString; -} - -/// The type of control image. -@experimental -final class ImagenControlType { - const ImagenControlType._(this._jsonString); - - final String _jsonString; - - /// Canny edge detection. - static const ImagenControlType canny = ImagenControlType._('canny'); - - /// Scribble. - static const ImagenControlType scribble = ImagenControlType._('scribble'); - - /// Face mesh. - static const ImagenControlType faceMesh = ImagenControlType._('face-mesh'); - - /// Color superpixel. - static const ImagenControlType colorSuperpixel = - ImagenControlType._('color-superpixel'); - - @override - String toString() => _jsonString; -} - -/// The mode of the mask. -@experimental -final class ImagenMaskMode { - const ImagenMaskMode._(this._jsonString); - - final String _jsonString; - - /// The mask is user provided. - static const ImagenMaskMode userProvided = ImagenMaskMode._('user-provided'); - - /// The mask is the background. - static const ImagenMaskMode background = ImagenMaskMode._('background'); - - /// The mask is the foreground. - static const ImagenMaskMode foreground = ImagenMaskMode._('foreground'); - - /// The mask is semantic. - static const ImagenMaskMode semantic = ImagenMaskMode._('semantic'); - - @override - String toString() => _jsonString; -} - -/// The configuration for the mask. -@experimental -final class ImagenMaskConfig { - ImagenMaskConfig({ - required this.maskType, - this.maskDilation, - }); - - final ImagenMaskMode maskType; - final double? maskDilation; -} - -/// The configuration for the subject. -@experimental -final class ImagenSubjectConfig { - ImagenSubjectConfig({ - this.description, - this.type, - }); - - final String? description; - final ImagenSubjectReferenceType? type; -} - -/// The configuration for the style. -@experimental -final class ImagenStyleConfig { - ImagenStyleConfig({ - this.description, - }); - - final String? description; -} - -/// The configuration for the control. -@experimental -final class ImagenControlConfig { - ImagenControlConfig({ - required this.controlType, - this.enableComputation, - this.superpixelRegionSize, - this.superpixelRuler, - }); - - final ImagenControlType controlType; - final bool? enableComputation; - final int? superpixelRegionSize; - final int? superpixelRuler; -} - -/// A reference image for image editing. -@experimental -sealed class ImagenReferenceImage { - ImagenReferenceImage({ - this.maskConfig, - this.subjectConfig, - this.styleConfig, - this.controlConfig, - this.image, - this.referenceId, - }); - - final ImagenMaskConfig? maskConfig; - final ImagenSubjectConfig? subjectConfig; - final ImagenStyleConfig? styleConfig; - final ImagenControlConfig? controlConfig; - final ImagenInlineImage? image; - final int? referenceId; - - Map toJson() { - final json = {}; - if (image != null) { - json['image'] = image!.toJson(); - } - if (referenceId != null) { - json['referenceId'] = referenceId; - } - if (maskConfig != null) { - json['mask'] = { - 'type': maskConfig!.maskType.toString(), - if (maskConfig!.maskDilation != null) - 'dilation': maskConfig!.maskDilation, - }; - } - if (subjectConfig != null) { - json['subject'] = { - if (subjectConfig!.description != null) - 'description': subjectConfig!.description, - if (subjectConfig!.type != null) 'type': subjectConfig!.type.toString(), - }; - } - if (styleConfig != null) { - json['style'] = { - if (styleConfig!.description != null) - 'description': styleConfig!.description, - }; - } - if (controlConfig != null) { - json['control'] = { - 'type': controlConfig!.controlType.toString(), - if (controlConfig!.enableComputation != null) - 'enableComputation': controlConfig!.enableComputation, - if (controlConfig!.superpixelRegionSize != null) - 'superpixelRegionSize': controlConfig!.superpixelRegionSize, - if (controlConfig!.superpixelRuler != null) - 'superpixelRuler': controlConfig!.superpixelRuler, - }; - } - return json; - } -} - -/// A reference image that is a mask. -@experimental -sealed class ImagenMaskReference extends ImagenReferenceImage { - ImagenMaskReference({ - super.maskConfig, - super.image, - }); - - /// Generates a mask and pads the image for outpainting. - static List generateMaskAndPadForOutpainting({ - required ImagenInlineImage image, - required Dimensions newDimensions, - ImagenImagePlacement newPosition = ImagenImagePlacement.center, - }) { - // TODO: implement - return []; - } -} - -/// A raw image. -@experimental -final class ImagenRawImage extends ImagenReferenceImage { - ImagenRawImage({ - required ImagenInlineImage image, - }) : super(image: image); -} - -/// A raw mask. -@experimental -final class ImagenRawMask extends ImagenMaskReference { - ImagenRawMask({ - required ImagenInlineImage mask, - double? dilation, - }) : super( - image: mask, - maskConfig: ImagenMaskConfig( - maskType: ImagenMaskMode.userProvided, - maskDilation: dilation, - ), - ); -} - -/// A semantic mask. -@experimental -final class ImagenSemanticMask extends ImagenMaskReference { - ImagenSemanticMask({ - required List classes, - double? dilation, - }) : super( - maskConfig: ImagenMaskConfig( - maskType: ImagenMaskMode.semantic, - maskDilation: dilation, - ), - ); -} - -/// A background mask. -@experimental -final class ImagenBackgroundMask extends ImagenMaskReference { - ImagenBackgroundMask({ - double? dilation, - }) : super( - maskConfig: ImagenMaskConfig( - maskType: ImagenMaskMode.background, - maskDilation: dilation, - ), - ); -} - -/// A foreground mask. -@experimental -final class ImagenForegroundMask extends ImagenMaskReference { - ImagenForegroundMask({ - double? dilation, - }) : super( - maskConfig: ImagenMaskConfig( - maskType: ImagenMaskMode.foreground, - maskDilation: dilation, - ), - ); -} - -/// A subject reference. -@experimental -final class ImagenSubjectReference extends ImagenReferenceImage { - ImagenSubjectReference({ - required ImagenInlineImage image, - int? referenceId, - String? description, - ImagenSubjectReferenceType? subjectType, - }) : super( - image: image, - referenceId: referenceId, - subjectConfig: ImagenSubjectConfig( - description: description, - type: subjectType, - ), - ); -} - -/// A style reference. -@experimental -final class ImagenStyleReference extends ImagenReferenceImage { - ImagenStyleReference({ - required ImagenInlineImage image, - int? referenceId, - String? description, - }) : super( - image: image, - referenceId: referenceId, - styleConfig: ImagenStyleConfig( - description: description, - ), - ); -} - -/// A control reference. -@experimental -final class ImagenControlReference extends ImagenReferenceImage { - ImagenControlReference({ - required ImagenControlType controlType, - ImagenInlineImage? image, - int? referenceId, - bool? enableComputation, - int? superpixelRegionSize, - int? superpixelRuler, - }) : super( - image: image, - referenceId: referenceId, - controlConfig: ImagenControlConfig( - controlType: controlType, - enableComputation: enableComputation, - superpixelRegionSize: superpixelRegionSize, - superpixelRuler: superpixelRuler, - ), - ); -} - -/// The configuration for image editing. -@experimental -final class ImagenEditingConfig { - ImagenEditingConfig({ - this.editMode, - this.editSteps, - }); - - final ImagenEditMode? editMode; - final int? editSteps; -} - -/// The dimensions of an image. -@experimental -final class Dimensions { - Dimensions({ - required this.width, - required this.height, - }); - - final int width; - final int height; -} - -/// The placement of an image. -@experimental -final class ImagenImagePlacement { - const ImagenImagePlacement._(this.x, this.y); - - final int? x; - final int? y; - - /// Creates a placement from a coordinate. - static ImagenImagePlacement fromCoordinate(int x, int y) => - ImagenImagePlacement._(x, y); - - /// The center of the image. - static const ImagenImagePlacement center = ImagenImagePlacement._(null, null); - - /// The top center of the image. - static const ImagenImagePlacement topCenter = - ImagenImagePlacement._(null, null); - - /// The bottom center of the image. - static const ImagenImagePlacement bottomCenter = - ImagenImagePlacement._(null, null); - - /// The left center of the image. - static const ImagenImagePlacement leftCenter = - ImagenImagePlacement._(null, null); - - /// The right center of the image. - static const ImagenImagePlacement rightCenter = - ImagenImagePlacement._(null, null); - - /// The top left of the image. - static const ImagenImagePlacement topLeft = ImagenImagePlacement._(0, 0); - - /// The top right of the image. - static const ImagenImagePlacement topRight = - ImagenImagePlacement._(null, null); - - /// The bottom left of the image. - static const ImagenImagePlacement bottomLeft = - ImagenImagePlacement._(null, null); - - /// The bottom right of the image. - static const ImagenImagePlacement bottomRight = - ImagenImagePlacement._(null, null); -} - -extension on Bitmap { - /// Converts a [Bitmap] to an [ImagenInlineImage]. - ImagenInlineImage get toImagenInlineImage => - ImagenInlineImage(data: asUint8List()); -} diff --git a/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart index c727517f658e..61691060d1c1 100644 --- a/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart +++ b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'dart:convert'; +import 'dart:typed_data'; import 'package:firebase_ai/firebase_ai.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -21,18 +21,24 @@ import 'package:flutter_test/flutter_test.dart'; void main() { group('ImagenReferenceImage', () { test('ImagenRawImage toJson', () { - final image = ImagenRawImage(image: ImagenInlineImage(data: [])); + final image = ImagenRawImage( + image: ImagenInlineImage( + bytesBase64Encoded: Uint8List.fromList([]), + mimeType: 'image/jpeg')); final json = image.toJson(); expect(json, { - 'image': {'data': 'AA=='} + 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'} }); }); test('ImagenRawMask toJson', () { - final image = ImagenRawMask(mask: ImagenInlineImage(data: [])); + final image = ImagenRawMask( + mask: ImagenInlineImage( + bytesBase64Encoded: Uint8List.fromList([]), + mimeType: 'image/jpeg')); final json = image.toJson(); expect(json, { - 'image': {'data': 'AA=='}, + 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, 'mask': {'type': 'user-provided'} }); }); @@ -63,14 +69,15 @@ void main() { test('ImagenSubjectReference toJson', () { final image = ImagenSubjectReference( - image: ImagenInlineImage(data: []), + image: ImagenInlineImage( + bytesBase64Encoded: Uint8List.fromList([]), mimeType: 'image/jpeg'), referenceId: 1, description: 'a cat', subjectType: ImagenSubjectReferenceType.animal, ); final json = image.toJson(); expect(json, { - 'image': {'data': 'AA=='}, + 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, 'referenceId': 1, 'subject': {'description': 'a cat', 'type': 'animal'} }); @@ -78,13 +85,14 @@ void main() { test('ImagenStyleReference toJson', () { final image = ImagenStyleReference( - image: ImagenInlineImage(data: []), + image: ImagenInlineImage( + bytesBase64Encoded: Uint8List.fromList([]), mimeType: 'image/jpeg'), referenceId: 1, description: 'van gogh style', ); final json = image.toJson(); expect(json, { - 'image': {'data': 'AA=='}, + 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, 'referenceId': 1, 'style': {'description': 'van gogh style'} }); @@ -93,12 +101,13 @@ void main() { test('ImagenControlReference toJson', () { final image = ImagenControlReference( controlType: ImagenControlType.canny, - image: ImagenInlineImage(data: []), + image: ImagenInlineImage( + bytesBase64Encoded: Uint8List.fromList([]), mimeType: 'image/jpeg'), referenceId: 1, ); final json = image.toJson(); expect(json, { - 'image': {'data': 'AA=='}, + 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, 'referenceId': 1, 'control': {'type': 'canny'} }); diff --git a/packages/firebase_ai/firebase_ai/test/imagen_test.dart b/packages/firebase_ai/firebase_ai/test/imagen_test.dart index bdb6200593c1..6f26baffd333 100644 --- a/packages/firebase_ai/firebase_ai/test/imagen_test.dart +++ b/packages/firebase_ai/firebase_ai/test/imagen_test.dart @@ -16,8 +16,8 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:firebase_ai/src/error.dart'; -import 'package:firebase_ai/src/imagen_api.dart'; -import 'package:firebase_ai/src/imagen_content.dart'; +import 'package:firebase_ai/src/imagen/imagen_api.dart'; +import 'package:firebase_ai/src/imagen/imagen_content.dart'; import 'package:flutter_test/flutter_test.dart'; void main() { From 4d21169715f29a5e575c4b020a32875dcc84e6b0 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Mon, 28 Jul 2025 22:46:23 -0700 Subject: [PATCH 3/6] inpaint background change working --- .../example/lib/pages/imagen_page.dart | 3 +- .../lib/src/imagen/imagen_edit.dart | 6 +- .../lib/src/imagen/imagen_model.dart | 17 +++-- .../lib/src/imagen/imagen_reference.dart | 62 ++++++++++++------- .../firebase_ai/test/imagen_edit_test.dart | 59 ++++++++++++------ 5 files changed, 94 insertions(+), 53 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart index 57b89b64ca67..ccd780c3f7df 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart @@ -243,7 +243,7 @@ class _ImagenPageState extends State { final response = await widget.model.inpaintImage( _sourceImage!, prompt, - ImagenBackgroundMask(referenceId: 1), + ImagenBackgroundMask(), config: ImagenEditingConfig(editMode: ImagenEditMode.inpaintInsertion), ); if (response.images.isNotEmpty) { @@ -345,7 +345,6 @@ class _ImagenPageState extends State { [ ImagenStyleReference( image: _sourceImage!, - referenceId: 1, description: 'van goh style', ), ], diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart index 27449e9dfc10..58a8769a1482 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_edit.dart @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'dart:convert'; + import 'package:meta/meta.dart'; /// The desired outcome of the image editing. @@ -121,7 +123,7 @@ final class ImagenMaskConfig extends ImagenReferenceConfig { 'maskImageConfig': { 'maskMode': maskType.toJson(), if (maskDilation != null) 'dilation': maskDilation, - if (maskClasses != null) 'maskClasses': maskClasses.toString(), + if (maskClasses != null) 'maskClasses': jsonEncode(maskClasses), }, }; } @@ -157,7 +159,7 @@ final class ImagenStyleConfig extends ImagenReferenceConfig { @override Map toJson() => { 'styleImageConfig': { - if (description != null) 'subjectDescription': description, + if (description != null) 'styleDescription': description, }, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart index 77e71e569887..a19967c862b1 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart @@ -139,8 +139,8 @@ final class ImagenModel extends BaseApiClientModel { }) => editImage( [ - ImagenRawImage(image: image, referenceId: 0), mask, + ImagenRawImage(image: image), ], prompt, config: config, @@ -175,9 +175,9 @@ final class ImagenModel extends BaseApiClientModel { }) { final parameters = { 'sampleCount': _generationConfig?.numberOfImages ?? 1, - if (config?.editMode case final editMode?) - 'editMode': editMode.toString(), - if (config?.editSteps case final editSteps?) 'editSteps': editSteps, + if (config?.editMode case final editMode?) 'editMode': editMode.toJson(), + if (config?.editSteps case final editSteps?) + 'editConfig': {'baseSteps': editSteps}, if (_generationConfig?.negativePrompt case final negativePrompt?) 'negativePrompt': negativePrompt, if (_generationConfig?.addWatermark case final addWatermark?) @@ -191,13 +191,18 @@ final class ImagenModel extends BaseApiClientModel { }; return { + 'parameters': parameters, 'instances': [ { 'prompt': prompt, - 'referenceImages': images.map((i) => i.toJson()).toList(), + 'referenceImages': images.asMap().entries.map((entry) { + int index = entry.key; + var image = entry.value; + return image.toJson( + referenceIdOverrideIfNull: index + images.length); + }).toList(), } ], - 'parameters': parameters, }; } } diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart index 1b09256c10ef..a17bbebcc06a 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart @@ -19,7 +19,7 @@ import 'package:meta/meta.dart'; import 'imagen_content.dart'; import 'imagen_edit.dart'; -enum ReferenceType { +enum _ReferenceType { UNSPECIFIED('REFERENCE_TYPE_UNSPECIFIED'), RAW('REFERENCE_TYPE_RAW'), MASK('REFERENCE_TYPE_MASK'), @@ -29,7 +29,7 @@ enum ReferenceType { MASKED_SUBJECT('REFERENCE_TYPE_MASKED_SUBJECT'), PRODUCT('REFERENCE_TYPE_PRODUCT'); - const ReferenceType(this._jsonString); + const _ReferenceType(this._jsonString); final String _jsonString; @override @@ -39,26 +39,36 @@ enum ReferenceType { /// A reference image for image editing. @experimental sealed class ImagenReferenceImage { - ImagenReferenceImage({ + ImagenReferenceImage._({ this.referenceConfig, this.image, - this.referenceId, required this.referenceType, + this.referenceId, }); + /// A config describing the reference image. final ImagenReferenceConfig? referenceConfig; + + /// The actual image data of the reference image. final ImagenInlineImage? image; + + /// The type of the reference image. + final _ReferenceType referenceType; + + /// The reference ID of the image. final int? referenceId; - final ReferenceType referenceType; - Map toJson() { + // ignore: public_member_api_docs + Map toJson({int referenceIdOverrideIfNull = 0}) { final json = {}; json['referenceType'] = referenceType.toString(); - if (image != null) { - json['referenceImage'] = image!.toJson(); - } if (referenceId != null) { json['referenceId'] = referenceId; + } else { + json['referenceId'] = referenceIdOverrideIfNull; + } + if (image != null) { + json['referenceImage'] = image!.toJson(); } if (referenceConfig != null) { json.addAll(referenceConfig!.toJson()); @@ -75,7 +85,10 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { ImagenMaskConfig? maskConfig, super.image, super.referenceId, - }) : super(referenceType: ReferenceType.MASK, referenceConfig: maskConfig); + }) : super._( + referenceType: _ReferenceType.MASK, + referenceConfig: maskConfig, + ); /// Generates a mask and pads the image for outpainting. static Future> generateMaskAndPadForOutpainting({ @@ -163,14 +176,12 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { return [ ImagenRawImage( - referenceId: 1, image: ImagenInlineImage( bytesBase64Encoded: newImageBytes.buffer.asUint8List(), mimeType: image.mimeType, ), ), ImagenRawMask( - referenceId: 2, mask: ImagenInlineImage( bytesBase64Encoded: maskBytes.buffer.asUint8List(), mimeType: image.mimeType, @@ -202,7 +213,7 @@ final class ImagenRawImage extends ImagenReferenceImage { ImagenRawImage({ required ImagenInlineImage image, super.referenceId, - }) : super(image: image, referenceType: ReferenceType.RAW); + }) : super._(image: image, referenceType: _ReferenceType.RAW); } /// A raw mask. @@ -232,6 +243,7 @@ final class ImagenSemanticMask extends ImagenMaskReference { maskConfig: ImagenMaskConfig( maskType: ImagenMaskMode.semantic, maskDilation: dilation, + maskClasses: classes, ), ); } @@ -239,8 +251,10 @@ final class ImagenSemanticMask extends ImagenMaskReference { /// A background mask. @experimental final class ImagenBackgroundMask extends ImagenMaskReference { - ImagenBackgroundMask({double? dilation, super.referenceId}) - : super( + ImagenBackgroundMask({ + double? dilation, + super.referenceId, + }) : super( maskConfig: ImagenMaskConfig( maskType: ImagenMaskMode.background, maskDilation: dilation, @@ -267,16 +281,16 @@ final class ImagenForegroundMask extends ImagenMaskReference { final class ImagenSubjectReference extends ImagenReferenceImage { ImagenSubjectReference({ required ImagenInlineImage image, - super.referenceId, String? description, ImagenSubjectReferenceType? subjectType, - }) : super( + super.referenceId, + }) : super._( image: image, referenceConfig: ImagenSubjectConfig( description: description, type: subjectType, ), - referenceType: ReferenceType.SUBJECT, + referenceType: _ReferenceType.SUBJECT, ); } @@ -285,14 +299,14 @@ final class ImagenSubjectReference extends ImagenReferenceImage { final class ImagenStyleReference extends ImagenReferenceImage { ImagenStyleReference({ required ImagenInlineImage image, - super.referenceId, String? description, - }) : super( + super.referenceId, + }) : super._( image: image, referenceConfig: ImagenStyleConfig( description: description, ), - referenceType: ReferenceType.STYLE, + referenceType: _ReferenceType.STYLE, ); } @@ -302,11 +316,11 @@ final class ImagenControlReference extends ImagenReferenceImage { ImagenControlReference({ required ImagenControlType controlType, ImagenInlineImage? image, - super.referenceId, bool? enableComputation, int? superpixelRegionSize, int? superpixelRuler, - }) : super( + super.referenceId, + }) : super._( image: image, referenceConfig: ImagenControlConfig( controlType: controlType, @@ -314,6 +328,6 @@ final class ImagenControlReference extends ImagenReferenceImage { superpixelRegionSize: superpixelRegionSize, superpixelRuler: superpixelRuler, ), - referenceType: ReferenceType.CONTROL, + referenceType: _ReferenceType.CONTROL, ); } diff --git a/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart index 61691060d1c1..6d425a09ee3d 100644 --- a/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart +++ b/packages/firebase_ai/firebase_ai/test/imagen_edit_test.dart @@ -24,10 +24,13 @@ void main() { final image = ImagenRawImage( image: ImagenInlineImage( bytesBase64Encoded: Uint8List.fromList([]), - mimeType: 'image/jpeg')); + mimeType: 'image/jpeg'), + referenceId: 1); final json = image.toJson(); expect(json, { - 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'} + 'referenceType': 'REFERENCE_TYPE_RAW', + 'referenceId': 1, + 'referenceImage': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'} }); }); @@ -35,35 +38,47 @@ void main() { final image = ImagenRawMask( mask: ImagenInlineImage( bytesBase64Encoded: Uint8List.fromList([]), - mimeType: 'image/jpeg')); + mimeType: 'image/jpeg'), + referenceId: 1); final json = image.toJson(); expect(json, { - 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, - 'mask': {'type': 'user-provided'} + 'referenceType': 'REFERENCE_TYPE_MASK', + 'referenceId': 1, + 'referenceImage': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'maskImageConfig': {'maskMode': 'MASK_MODE_USER_PROVIDED'} }); }); test('ImagenSemanticMask toJson', () { - final image = ImagenSemanticMask(classes: [1, 2]); + final image = ImagenSemanticMask(classes: [1, 2], referenceId: 1); final json = image.toJson(); expect(json, { - 'mask': {'type': 'semantic'} + 'referenceType': 'REFERENCE_TYPE_MASK', + 'referenceId': 1, + 'maskImageConfig': { + 'maskMode': 'MASK_MODE_SEMANTIC', + 'maskClasses': '[1,2]' + } }); }); test('ImagenBackgroundMask toJson', () { - final image = ImagenBackgroundMask(); + final image = ImagenBackgroundMask(referenceId: 1); final json = image.toJson(); expect(json, { - 'mask': {'type': 'background'} + 'referenceType': 'REFERENCE_TYPE_MASK', + 'referenceId': 1, + 'maskImageConfig': {'maskMode': 'MASK_MODE_BACKGROUND'} }); }); test('ImagenForegroundMask toJson', () { - final image = ImagenForegroundMask(); + final image = ImagenForegroundMask(referenceId: 1); final json = image.toJson(); expect(json, { - 'mask': {'type': 'foreground'} + 'referenceType': 'REFERENCE_TYPE_MASK', + 'referenceId': 1, + 'maskImageConfig': {'maskMode': 'MASK_MODE_FOREGROUND'} }); }); @@ -71,15 +86,19 @@ void main() { final image = ImagenSubjectReference( image: ImagenInlineImage( bytesBase64Encoded: Uint8List.fromList([]), mimeType: 'image/jpeg'), - referenceId: 1, description: 'a cat', subjectType: ImagenSubjectReferenceType.animal, + referenceId: 1, ); final json = image.toJson(); expect(json, { - 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'referenceType': 'REFERENCE_TYPE_SUBJECT', 'referenceId': 1, - 'subject': {'description': 'a cat', 'type': 'animal'} + 'referenceImage': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'subjectImageConfig': { + 'subjectDescription': 'a cat', + 'subjectType': 'SUBJECT_TYPE_ANIMAL' + } }); }); @@ -87,14 +106,15 @@ void main() { final image = ImagenStyleReference( image: ImagenInlineImage( bytesBase64Encoded: Uint8List.fromList([]), mimeType: 'image/jpeg'), - referenceId: 1, description: 'van gogh style', + referenceId: 1, ); final json = image.toJson(); expect(json, { - 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'referenceType': 'REFERENCE_TYPE_STYLE', 'referenceId': 1, - 'style': {'description': 'van gogh style'} + 'referenceImage': {'mimeType': 'image/jpeg', 'bytesBase64Encoded': ''}, + 'styleImageConfig': {'styleDescription': 'van gogh style'} }); }); @@ -107,9 +127,10 @@ void main() { ); final json = image.toJson(); expect(json, { - 'image': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'referenceType': 'REFERENCE_TYPE_CONTROL', 'referenceId': 1, - 'control': {'type': 'canny'} + 'referenceImage': {'bytesBase64Encoded': '', 'mimeType': 'image/jpeg'}, + 'controlImageConfig': {'controlType': 'CONTROL_TYPE_CANNY'} }); }); }); From 2f7f3d90a33434c1242ef2740fd24acb4da13ffd Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Wed, 30 Jul 2025 22:35:45 -0700 Subject: [PATCH 4/6] some profiling fix --- .../example/lib/pages/imagen_page.dart | 147 ++++++++------- .../lib/src/imagen/imagen_model.dart | 3 +- .../lib/src/imagen/imagen_reference.dart | 174 +++++++++--------- 3 files changed, 165 insertions(+), 159 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart index ccd780c3f7df..7a14fca77043 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart @@ -21,6 +21,9 @@ import 'package:flutter/material.dart'; //import 'package:firebase_storage/firebase_storage.dart'; import '../widgets/message_widget.dart'; +// Define a constant for the history limit +const int _MAX_HISTORY = 4; + class ImagenPage extends StatefulWidget { const ImagenPage({ super.key, @@ -227,17 +230,13 @@ class _ImagenPageState extends State { }); final String prompt = _textController.text; + final promptMessage = MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: 'Try to inpaint image with prompt: $prompt', + fromUser: true, + ); - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), - text: 'Try to inpaint image with prompt: $prompt', - fromUser: true, - ), - ); - _scrollDown(); - }); + MessageData? resultMessage; try { final response = await widget.model.inpaintImage( @@ -248,16 +247,11 @@ class _ImagenPageState extends State { ); if (response.images.isNotEmpty) { final inpaintImage = response.images[0]; - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(inpaintImage.bytesBase64Encoded), - text: 'Inpaint image result with prompt: $prompt', - fromUser: false, - ), - ); - _scrollDown(); - }); + resultMessage = MessageData( + image: Image.memory(inpaintImage.bytesBase64Encoded), + text: 'Inpaint image result with prompt: $prompt', + fromUser: false, + ); } else { _showError('No image was returned from inpaint.'); } @@ -266,7 +260,16 @@ class _ImagenPageState extends State { } setState(() { + _generatedContent.add(promptMessage); + if (resultMessage != null) { + _generatedContent.add(resultMessage); + } + // Apply history limit here + while (_generatedContent.length > _MAX_HISTORY) { + _generatedContent.removeAt(0); + } _loading = false; + _scrollDown(); }); } @@ -279,17 +282,13 @@ class _ImagenPageState extends State { _loading = true; }); - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), - text: 'Outpaint the picture to 1400*1400', - fromUser: true, - ), - ); - _scrollDown(); - }); + final promptMessage = MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: 'Outpaint the picture to 1400*1400', + fromUser: true, + ); + MessageData? resultMessage; try { final response = await widget.model.outpaintImage( _sourceImage!, @@ -297,24 +296,29 @@ class _ImagenPageState extends State { ); if (response.images.isNotEmpty) { final editedImage = response.images[0]; - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(editedImage.bytesBase64Encoded), - text: 'Edited image Outpaint 1400*1400', - fromUser: false, - ), - ); - _scrollDown(); - }); + resultMessage = MessageData( + image: Image.memory(editedImage.bytesBase64Encoded), + text: 'Edited image Outpaint 1400*1400', + fromUser: false, + ); } else { _showError('No image was returned from editing.'); } } catch (e) { _showError('Error editing image: $e'); } + setState(() { + _generatedContent.add(promptMessage); + if (resultMessage != null) { + _generatedContent.add(resultMessage); + } + // Apply history limit here + while (_generatedContent.length > _MAX_HISTORY) { + _generatedContent.removeAt(0); + } _loading = false; + _scrollDown(); }); } @@ -328,18 +332,12 @@ class _ImagenPageState extends State { }); final String prompt = _textController.text; - - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), - text: prompt, - fromUser: true, - ), - ); - _scrollDown(); - }); - + final promptMessage = MessageData( + image: Image.memory(_sourceImage!.bytesBase64Encoded), + text: prompt, + fromUser: true, + ); + MessageData? resultMessage; try { final response = await widget.model.editImage( [ @@ -353,24 +351,30 @@ class _ImagenPageState extends State { ); if (response.images.isNotEmpty) { final editedImage = response.images[0]; - setState(() { - _generatedContent.add( - MessageData( - image: Image.memory(editedImage.bytesBase64Encoded), - text: 'Edited image with style: $prompt', - fromUser: false, - ), - ); - _scrollDown(); - }); + + resultMessage = MessageData( + image: Image.memory(editedImage.bytesBase64Encoded), + text: 'Edited image with style: $prompt', + fromUser: false, + ); } else { _showError('No image was returned from style editing.'); } } catch (e) { _showError('Error performing style edit: $e'); } + setState(() { + _generatedContent.add(promptMessage); + if (resultMessage != null) { + _generatedContent.add(resultMessage); + } + // Apply history limit here + while (_generatedContent.length > _MAX_HISTORY) { + _generatedContent.removeAt(0); + } _loading = false; + _scrollDown(); }); } @@ -378,19 +382,17 @@ class _ImagenPageState extends State { setState(() { _loading = true; }); - + MessageData? resultMessage; try { var response = await widget.model.generateImages(prompt); if (response.images.isNotEmpty) { var imagenImage = response.images[0]; - _generatedContent.add( - MessageData( - image: Image.memory(imagenImage.bytesBase64Encoded), - text: prompt, - fromUser: false, - ), + resultMessage = MessageData( + image: Image.memory(imagenImage.bytesBase64Encoded), + text: prompt, + fromUser: false, ); } else { // Handle the case where no images were generated @@ -401,6 +403,13 @@ class _ImagenPageState extends State { } setState(() { + if (resultMessage != null) { + _generatedContent.add(resultMessage); + } + // Apply history limit here + while (_generatedContent.length > _MAX_HISTORY) { + _generatedContent.removeAt(0); + } _loading = false; _scrollDown(); }); diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart index a19967c862b1..b4c64016824e 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_model.dart @@ -164,7 +164,8 @@ final class ImagenModel extends BaseApiClientModel { return editImage( referenceImages, prompt, - config: config, + config: ImagenEditingConfig( + editMode: ImagenEditMode.outpaint, editSteps: config?.editSteps), ); } diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart index a17bbebcc06a..53298b9e2b35 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart @@ -97,97 +97,93 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { ImagenImagePlacement newPosition = ImagenImagePlacement.center, }) async { final originalImage = await image.asUiImage(); - - // Validate that the new dimensions are strictly larger. - if (originalImage.width >= newDimensions.width || - originalImage.height >= newDimensions.height) { - throw ArgumentError( - 'New Dimensions must be strictly larger than original image dimensions. ' - 'Original image is: ${originalImage.width}x${originalImage.height}, ' - 'new dimensions are ${newDimensions.width}x${newDimensions.height}', - ); - } - - // Calculate the position of the original image on the new canvas. - final normalizedPosition = newPosition.normalizeToDimensions( - ImagenDimensions( - width: originalImage.width, height: originalImage.height), - newDimensions, - ); - - final x = normalizedPosition.x; - final y = normalizedPosition.y; - - if (x == null || y == null) { - throw StateError('Error normalizing position for mask and padding.'); - } - - // Define the rectangle where the original image will be drawn. - final imageRect = ui.Rect.fromLTWH( - x.toDouble(), - y.toDouble(), - originalImage.width.toDouble(), - originalImage.height.toDouble(), - ); - - // Create both the mask and the new image concurrently. - final results = await Future.wait([ - // Future to create the mask - _createImageFromPainter( - width: newDimensions.width, - height: newDimensions.height, - painter: (canvas, size) { - // Fill the mask with white, then draw a black rectangle where the image is. - canvas.drawPaint(Paint()..color = Colors.white); - canvas.drawRect(imageRect, Paint()..color = Colors.black); - }, - ), - // Future to create the new padded image - _createImageFromPainter( - width: newDimensions.width, - height: newDimensions.height, - painter: (canvas, size) { - // Fill the new image with black padding. - canvas.drawPaint(Paint()..color = Colors.black); - // Draw the original image into the corresponding spot. - canvas.drawImageRect( - originalImage, - ui.Rect.fromLTWH(0, 0, originalImage.width.toDouble(), - originalImage.height.toDouble()), - imageRect, - Paint(), - ); - }, - ), - ]); - - final newPaddedUiImage = results[1]; - final maskUiImage = results[0]; - - // Convert the generated ui.Image objects back to byte data. - final newImageBytes = - await newPaddedUiImage.toByteData(format: ui.ImageByteFormat.png); - final maskBytes = - await maskUiImage.toByteData(format: ui.ImageByteFormat.png); - - if (newImageBytes == null || maskBytes == null) { - throw StateError('Failed to encode generated images.'); - } - - return [ - ImagenRawImage( - image: ImagenInlineImage( - bytesBase64Encoded: newImageBytes.buffer.asUint8List(), - mimeType: image.mimeType, + ui.Image? maskImage; + ui.Image? paddedImage; + + try { + // Validate that the new dimensions are strictly larger. + if (originalImage.width >= newDimensions.width || + originalImage.height >= newDimensions.height) { + throw ArgumentError( + 'New Dimensions must be strictly larger than original image dimensions. ' + 'Original image is: ${originalImage.width}x${originalImage.height}, ' + 'new dimensions are ${newDimensions.width}x${newDimensions.height}', + ); + } + + // Calculate the position of the original image on the new canvas. + final originalDimensions = ImagenDimensions( + width: originalImage.width, height: originalImage.height); + final normalizedPosition = + newPosition.normalizeToDimensions(originalDimensions, newDimensions); + + final x = normalizedPosition.x?.toDouble(); + final y = normalizedPosition.y?.toDouble(); + + if (x == null || y == null) { + throw StateError('Error normalizing position for mask and padding.'); + } + + final sourceRect = ui.Rect.fromLTWH(0, 0, originalImage.width.toDouble(), + originalImage.height.toDouble()); + final destRect = ui.Rect.fromLTWH(x, y, originalImage.width.toDouble(), + originalImage.height.toDouble()); + + final whitePaint = Paint()..color = Colors.white; + final blackPaint = Paint()..color = Colors.black; + + // 3. Use Dart 3's record pattern for concurrent image creation. + // This is much more readable and safer than accessing results by index. + final [createMask, createdPaddedImage] = await Future.wait([ + _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + canvas.drawPaint(whitePaint); + canvas.drawRect(destRect, blackPaint); + }, ), - ), - ImagenRawMask( - mask: ImagenInlineImage( - bytesBase64Encoded: maskBytes.buffer.asUint8List(), - mimeType: image.mimeType, + _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + canvas.drawPaint(blackPaint); + canvas.drawImageRect(originalImage, sourceRect, destRect, Paint()); + }, + ), + ]); + maskImage = createMask; + paddedImage = createdPaddedImage; + + final maskBytes = + await maskImage.toByteData(format: ui.ImageByteFormat.png); + final paddedBytes = + await paddedImage.toByteData(format: ui.ImageByteFormat.png); + + if (paddedBytes == null || maskBytes == null) { + throw StateError('Failed to encode generated images.'); + } + + // 5. Return a cleaner, more readable list + return [ + ImagenRawImage( + image: ImagenInlineImage( + bytesBase64Encoded: paddedBytes.buffer.asUint8List(), + mimeType: image.mimeType, + ), ), - ), - ]; + ImagenRawMask( + mask: ImagenInlineImage( + bytesBase64Encoded: maskBytes.buffer.asUint8List(), + mimeType: image.mimeType, + ), + ), + ]; + } finally { + originalImage.dispose(); + maskImage?.dispose(); + paddedImage?.dispose(); + } } /// Helper function to create a ui.Image by drawing on a Canvas. From 01c9a04fefd2f95d73f20edd3aef703c5fd6db21 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 31 Jul 2025 16:08:38 -0700 Subject: [PATCH 5/6] make MessageData hold image bytes instead of Image object --- .../example/lib/pages/audio_page.dart | 6 ++- .../example/lib/pages/bidi_page.dart | 6 ++- .../example/lib/pages/chat_page.dart | 6 ++- .../example/lib/pages/image_prompt_page.dart | 10 ++-- .../example/lib/pages/imagen_page.dart | 20 ++++---- .../example/lib/widgets/message_widget.dart | 6 +-- .../lib/src/imagen/imagen_reference.dart | 47 ++++++++----------- 7 files changed, 57 insertions(+), 44 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/audio_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/audio_page.dart index 8708bcb01c15..be04d6a2db30 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/audio_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/audio_page.dart @@ -137,7 +137,11 @@ class _AudioPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _messages[idx].text, - image: _messages[idx].image, + image: Image.memory( + _messages[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ), isFromUser: _messages[idx].fromUser ?? false, ); }, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index cb221329d28f..3b7cfd334bc5 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -119,7 +119,11 @@ class _BidiPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _messages[idx].text, - image: _messages[idx].image, + image: Image.memory( + _messages[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ), isFromUser: _messages[idx].fromUser ?? false, ); }, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart index df0afea88482..eb8e6128f2fc 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/chat_page.dart @@ -70,7 +70,11 @@ class _ChatPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _messages[idx].text, - image: _messages[idx].image, + image: Image.memory( + _messages[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ), isFromUser: _messages[idx].fromUser ?? false, ); }, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/image_prompt_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/image_prompt_page.dart index 5dff25a2efe1..5409c264450b 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/image_prompt_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/image_prompt_page.dart @@ -65,7 +65,11 @@ class _ImagePromptPageState extends State { var content = _generatedContent[idx]; return MessageWidget( text: content.text, - image: content.image, + image: Image.memory( + content.imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ), isFromUser: content.fromUser ?? false, ); }, @@ -137,14 +141,14 @@ class _ImagePromptPageState extends State { ]; _generatedContent.add( MessageData( - image: Image.asset('assets/images/cat.jpg'), + imageBytes: catBytes.buffer.asUint8List(), text: message, fromUser: true, ), ); _generatedContent.add( MessageData( - image: Image.asset('assets/images/scones.jpg'), + imageBytes: sconeBytes.buffer.asUint8List(), fromUser: true, ), ); diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart index 7a14fca77043..ed016fb03d86 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/imagen_page.dart @@ -79,7 +79,11 @@ class _ImagenPageState extends State { itemBuilder: (context, idx) { return MessageWidget( text: _generatedContent[idx].text, - image: _generatedContent[idx].image, + image: Image.memory( + _generatedContent[idx].imageBytes!, + cacheWidth: 400, + cacheHeight: 400, + ), isFromUser: _generatedContent[idx].fromUser ?? false, ); }, @@ -231,7 +235,7 @@ class _ImagenPageState extends State { final String prompt = _textController.text; final promptMessage = MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), + imageBytes: _sourceImage!.bytesBase64Encoded, text: 'Try to inpaint image with prompt: $prompt', fromUser: true, ); @@ -248,7 +252,7 @@ class _ImagenPageState extends State { if (response.images.isNotEmpty) { final inpaintImage = response.images[0]; resultMessage = MessageData( - image: Image.memory(inpaintImage.bytesBase64Encoded), + imageBytes: inpaintImage.bytesBase64Encoded, text: 'Inpaint image result with prompt: $prompt', fromUser: false, ); @@ -283,7 +287,7 @@ class _ImagenPageState extends State { }); final promptMessage = MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), + imageBytes: _sourceImage!.bytesBase64Encoded, text: 'Outpaint the picture to 1400*1400', fromUser: true, ); @@ -297,7 +301,7 @@ class _ImagenPageState extends State { if (response.images.isNotEmpty) { final editedImage = response.images[0]; resultMessage = MessageData( - image: Image.memory(editedImage.bytesBase64Encoded), + imageBytes: editedImage.bytesBase64Encoded, text: 'Edited image Outpaint 1400*1400', fromUser: false, ); @@ -333,7 +337,7 @@ class _ImagenPageState extends State { final String prompt = _textController.text; final promptMessage = MessageData( - image: Image.memory(_sourceImage!.bytesBase64Encoded), + imageBytes: _sourceImage!.bytesBase64Encoded, text: prompt, fromUser: true, ); @@ -353,7 +357,7 @@ class _ImagenPageState extends State { final editedImage = response.images[0]; resultMessage = MessageData( - image: Image.memory(editedImage.bytesBase64Encoded), + imageBytes: editedImage.bytesBase64Encoded, text: 'Edited image with style: $prompt', fromUser: false, ); @@ -390,7 +394,7 @@ class _ImagenPageState extends State { var imagenImage = response.images[0]; resultMessage = MessageData( - image: Image.memory(imagenImage.bytesBase64Encoded), + imageBytes: imagenImage.bytesBase64Encoded, text: prompt, fromUser: false, ); diff --git a/packages/firebase_ai/firebase_ai/example/lib/widgets/message_widget.dart b/packages/firebase_ai/firebase_ai/example/lib/widgets/message_widget.dart index b8a0f23ce03b..368dfc1fea88 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/widgets/message_widget.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/widgets/message_widget.dart @@ -11,13 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:flutter_markdown/flutter_markdown.dart'; class MessageData { - MessageData({this.image, this.text, this.fromUser}); - final Image? image; + MessageData({this.imageBytes, this.text, this.fromUser}); + final Uint8List? imageBytes; final String? text; final bool? fromUser; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart index 53298b9e2b35..7a03b0323796 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart @@ -97,8 +97,6 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { ImagenImagePlacement newPosition = ImagenImagePlacement.center, }) async { final originalImage = await image.asUiImage(); - ui.Image? maskImage; - ui.Image? paddedImage; try { // Validate that the new dimensions are strictly larger. @@ -132,33 +130,30 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { final whitePaint = Paint()..color = Colors.white; final blackPaint = Paint()..color = Colors.black; - // 3. Use Dart 3's record pattern for concurrent image creation. - // This is much more readable and safer than accessing results by index. - final [createMask, createdPaddedImage] = await Future.wait([ - _createImageFromPainter( - width: newDimensions.width, - height: newDimensions.height, - painter: (canvas, size) { - canvas.drawPaint(whitePaint); - canvas.drawRect(destRect, blackPaint); - }, - ), - _createImageFromPainter( - width: newDimensions.width, - height: newDimensions.height, - painter: (canvas, size) { - canvas.drawPaint(blackPaint); - canvas.drawImageRect(originalImage, sourceRect, destRect, Paint()); - }, - ), - ]); - maskImage = createMask; - paddedImage = createdPaddedImage; - + final maskImage = await _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + canvas.drawPaint(whitePaint); + canvas.drawRect(destRect, blackPaint); + }, + ); final maskBytes = await maskImage.toByteData(format: ui.ImageByteFormat.png); + maskImage.dispose(); // Dispose right away + + // 2. Create, encode, and immediately dispose of the padded image + final paddedImage = await _createImageFromPainter( + width: newDimensions.width, + height: newDimensions.height, + painter: (canvas, size) { + canvas.drawPaint(blackPaint); + canvas.drawImageRect(originalImage, sourceRect, destRect, Paint()); + }, + ); final paddedBytes = await paddedImage.toByteData(format: ui.ImageByteFormat.png); + paddedImage.dispose(); // Dispose right away if (paddedBytes == null || maskBytes == null) { throw StateError('Failed to encode generated images.'); @@ -181,8 +176,6 @@ sealed class ImagenMaskReference extends ImagenReferenceImage { ]; } finally { originalImage.dispose(); - maskImage?.dispose(); - paddedImage?.dispose(); } } From 59c75e96ac2fd4518f05ea958364692fa0e71cf5 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 1 Aug 2025 17:29:44 -0700 Subject: [PATCH 6/6] rename toString to toJson --- .../firebase_ai/lib/src/imagen/imagen_reference.dart | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart index 7a03b0323796..f3eef99878a6 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/imagen/imagen_reference.dart @@ -31,9 +31,7 @@ enum _ReferenceType { const _ReferenceType(this._jsonString); final String _jsonString; - - @override - String toString() => _jsonString; + String toJson() => _jsonString; } /// A reference image for image editing. @@ -61,7 +59,7 @@ sealed class ImagenReferenceImage { // ignore: public_member_api_docs Map toJson({int referenceIdOverrideIfNull = 0}) { final json = {}; - json['referenceType'] = referenceType.toString(); + json['referenceType'] = referenceType.toJson(); if (referenceId != null) { json['referenceId'] = referenceId; } else {