From 998992fe1fc7f24de14e1e16807dcfaa3675880e Mon Sep 17 00:00:00 2001 From: chenxue Date: Tue, 27 May 2025 21:02:02 +0800 Subject: [PATCH] feat: aihubmix painting support imagen (#6525) * add imagen * feat: support imagen model * update proxy notice --------- Co-authored-by: zhaochenxue --- src/renderer/src/i18n/locales/en-us.json | 12 ++++- src/renderer/src/i18n/locales/ja-jp.json | 12 ++++- src/renderer/src/i18n/locales/ru-ru.json | 12 ++++- src/renderer/src/i18n/locales/zh-cn.json | 12 ++++- src/renderer/src/i18n/locales/zh-tw.json | 12 ++++- .../src/pages/paintings/AihubmixPage.tsx | 33 +++++++++++-- .../pages/paintings/config/aihubmixConfig.tsx | 47 ++++++++++++++++++- .../src/pages/paintings/config/constants.ts | 6 +++ .../providers/AiProvider/AihubmixProvider.ts | 8 ++-- .../providers/AiProvider/GeminiProvider.ts | 27 +++++++++-- .../src/providers/AiProvider/index.ts | 5 +- src/renderer/src/types/index.ts | 4 +- 12 files changed, 167 insertions(+), 23 deletions(-) diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index ec1b3c7cdb..250e2be90c 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -827,7 +827,8 @@ "learn_more": "Learn More", "paint_course": "tutorial", "prompt_placeholder_edit": "Enter your image description, text drawing uses \"double quotes\" to wrap", - "proxy_required": "Currently, you need to open a proxy to view the generated images, it will be supported in the future", + "prompt_placeholder_en": "Enter your image description, currently Imagen only supports English prompts", + "proxy_required": "Open the proxy and enable “TUN mode” to view generated images or copy them to the browser for opening. In the future, domestic direct connection will be supported", "image_file_required": "Please upload an image first", "image_file_retry": "Please re-upload an image first", "image_placeholder": "No image available", @@ -866,6 +867,11 @@ "portrait": "Portrait", "landscape": "Landscape" }, + "person_generation_options": { + "allow_all": "Allow all", + "allow_adult": "Allow adult", + "allow_none": "Not allowed" + }, "quality": "Quality", "moderation": "Moderation", "background": "Background", @@ -882,7 +888,9 @@ "negative_prompt_tip": "Describe unwanted elements, only for V_1, V_1_TURBO, V_2, and V_2_TURBO", "magic_prompt_option_tip": "Intelligently enhances prompts for better results", "style_type_tip": "Image generation style for V_2 and above", - "rendering_speed_tip": "Controls rendering speed vs. quality trade-off, only available for V_3" + "rendering_speed_tip": "Controls rendering speed vs. quality trade-off, only available for V_3", + "person_generation": "Generate person", + "person_generation_tip": "Allow model to generate person images" }, "edit": { "image_file": "Edited Image", diff --git a/src/renderer/src/i18n/locales/ja-jp.json b/src/renderer/src/i18n/locales/ja-jp.json index 7535d37709..f7d35fd904 100644 --- a/src/renderer/src/i18n/locales/ja-jp.json +++ b/src/renderer/src/i18n/locales/ja-jp.json @@ -825,8 +825,9 @@ "style_type": "スタイル", "learn_more": "詳しくはこちら", "prompt_placeholder_edit": "画像の説明を入力します。テキスト描画には '二重引用符' を使用します", + "prompt_placeholder_en": "「英語」の説明を入力します。Imagenは現在、英語のプロンプト語のみをサポートしています", "paint_course": "チュートリアル", - "proxy_required": "現在、プロキシを開く必要があります。これは、将来サポートされる予定です", + "proxy_required": "打開代理並開啟”TUN模式“查看生成圖片或複製到瀏覽器開啟，後續會支持國內直連", "image_file_required": "画像を先にアップロードしてください", "image_file_retry": "画像を先にアップロードしてください", "image_placeholder": "画像がありません", @@ -864,6 +865,11 @@ "portrait": "縦図", "landscape": "横図" }, + "person_generation_options": { + "allow_all": "許可する", + "allow_adult": "許可する", + "allow_none": "許可しない" + }, "quality": "品質", "moderation": "敏感度", "background": "背景", @@ -880,7 +886,9 @@ "negative_prompt_tip": "画像に含めたくない内容を説明します", "magic_prompt_option_tip": "生成効果を向上させるための提示詞を最適化します", "style_type_tip": "画像生成スタイル、V_2 以上のバージョンでのみ適用", - "rendering_speed_tip": "レンダリング速度と品質のバランスを調整します。V_3バージョンでのみ利用可能です" + "rendering_speed_tip": "レンダリング速度と品質のバランスを調整します。V_3バージョンでのみ利用可能です", + "person_generation": "人物生成", + "person_generation_tip": "人物画像を生成する" }, "edit": { "image_file": "編集画像", diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index 5550262b27..960b242728 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -826,8 +826,9 @@ "rendering_speed": "Скорость рендеринга", "learn_more": "Узнать больше", "prompt_placeholder_edit": "Введите ваше описание изображения, текстовая отрисовка использует двойные кавычки для обертки", + "prompt_placeholder_en": "Введите” английский “описание изображения, текстовая отрисовка использует двойные кавычки для обертки", "paint_course": "Руководство / Учебник", - "proxy_required": "Сейчас необходимо открыть прокси для просмотра сгенерированных изображений, в будущем будет поддерживаться прямое соединение", + "proxy_required": "Открыть прокси и включить “TUN режим” для просмотра сгенерированных изображений или скопировать их в браузер для открытия. В будущем будет поддерживаться прямое соединение", "image_file_required": "Пожалуйста, сначала загрузите изображение", "image_file_retry": "Пожалуйста, сначала загрузите изображение", "image_placeholder": "Изображение недоступно", @@ -866,6 +867,11 @@ "portrait": "Портрет", "landscape": "Пейзаж" }, + "person_generation_options": { + "allow_all": "Разрешено все", + "allow_adult": "Разрешено взрослые", + "allow_none": "Не разрешено" + }, "quality": "Качество", "moderation": "Сенсорность", "background": "Фон", @@ -882,7 +888,9 @@ "negative_prompt_tip": "Описывает, что вы не хотите видеть в изображении", "magic_prompt_option_tip": "Интеллектуально оптимизирует подсказки для улучшения эффекта генерации", "style_type_tip": "Стиль генерации изображений, доступен только для версий V_2 и выше", - "rendering_speed_tip": "Управляет балансом между скоростью рендеринга и качеством, доступно только для V_3" + "rendering_speed_tip": "Управляет балансом между скоростью рендеринга и качеством, доступно только для V_3", + "person_generation": "Генерация персонажа", + "person_generation_tip": "Разрешить модель генерировать изображения людей" }, "edit": { "image_file": "Изображение для редактирования", diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index f224e15466..b88c9f7996 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -827,7 +827,8 @@ "learn_more": "了解更多", "paint_course": "教程", "prompt_placeholder_edit": "输入你的图片描述，文本绘制用 \"双引号\" 包裹", - "proxy_required": "目前需要打开代理才能查看生成图片，后续会支持国内直连", + "prompt_placeholder_en": "输入”英文“图片描述，目前 Imagen 仅支持英文提示词", + "proxy_required": "打开代理并开启”TUN模式“查看生成图片或复制到浏览器打开，后续会支持国内直连", "image_file_required": "请先上传图片", "image_file_retry": "请重新上传图片", "image_placeholder": "暂无图片", @@ -861,6 +862,11 @@ "transparent": "透明", "opaque": "不透明" }, + "person_generation_options": { + "allow_all": "允许所有", + "allow_adult": "允许成人", + "allow_none": "不允许" + }, "aspect_ratios": { "square": "方形", "portrait": "竖图", @@ -882,7 +888,9 @@ "negative_prompt_tip": "描述不想在图像中出现的元素，仅支持 V_1、V_1_TURBO、V_2 和 V_2_TURBO 版本", "magic_prompt_option_tip": "智能优化提示词以提升生成效果", "style_type_tip": "图像生成风格，仅适用于 V_2 及以上版本", - "rendering_speed_tip": "控制渲染速度与质量的平衡，仅适用于 V_3 版本" + "rendering_speed_tip": "控制渲染速度与质量的平衡，仅适用于 V_3 版本", + "person_generation": "生成人物", + "person_generation_tip": "允许模型生成人物图像" }, "edit": { "image_file": "编辑的图像", diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index 4b702dc955..d8eabe45f1 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -826,8 +826,9 @@ "style_type": "風格", "learn_more": "了解更多", "prompt_placeholder_edit": "輸入你的圖片描述，文本繪製用 '雙引號' 包裹", + "prompt_placeholder_en": "輸入”英文“圖片描述，目前 Imagen 僅支持英文提示詞", "paint_course": "教程", - "proxy_required": "目前需要打開代理才能查看生成圖片，後續會支持國內直連", + "proxy_required": "打開代理並開啟”TUN模式“查看生成圖片或複製到瀏覽器開啟，後續會支持國內直連", "image_file_required": "請先上傳圖片", "image_file_retry": "請重新上傳圖片", "image_placeholder": "無圖片", @@ -866,6 +867,11 @@ "portrait": "豎圖", "landscape": "橫圖" }, + "person_generation_options": { + "allow_all": "允許所有", + "allow_adult": "允許成人", + "allow_none": "不允許" + }, "quality": "品質", "moderation": "敏感度", "background": "背景", @@ -882,7 +888,9 @@ "negative_prompt_tip": "描述不想在圖像中出現的內容", "magic_prompt_option_tip": "智能優化生成效果的提示詞", "style_type_tip": "圖像生成風格，僅適用於 V_2 及以上版本", - "rendering_speed_tip": "控制渲染速度與品質之間的平衡，僅適用於V_3版本" + "rendering_speed_tip": "控制渲染速度與品質之間的平衡，僅適用於V_3版本", + "person_generation": "人物生成", + "person_generation_tip": "允許模型生成人物圖像" }, "edit": { "image_file": "編輯圖像", diff --git a/src/renderer/src/pages/paintings/AihubmixPage.tsx b/src/renderer/src/pages/paintings/AihubmixPage.tsx index 666ed1d72f..8e411fcae1 100644 --- a/src/renderer/src/pages/paintings/AihubmixPage.tsx +++ b/src/renderer/src/pages/paintings/AihubmixPage.tsx @@ -11,6 +11,7 @@ import { usePaintings } from '@renderer/hooks/usePaintings' import { useAllProviders } from '@renderer/hooks/useProvider' import { useRuntime } from '@renderer/hooks/useRuntime' import { useSettings } from '@renderer/hooks/useSettings' +import AiProvider from '@renderer/providers/AiProvider' import FileManager from '@renderer/services/FileManager' import { translateText } from '@renderer/services/TranslateService' import { useAppDispatch } from '@renderer/store' @@ -68,7 +69,6 @@ const AihubmixPage: FC<{ Options: string[] }> = ({ Options }) => { const modeOptions = [ { label: t('paintings.mode.generate'), value: 'generate' }, - // { label: t('paintings.mode.edit'), value: 'edit' }, { label: t('paintings.mode.remix'), value: 'remix' }, { label: t('paintings.mode.upscale'), value: 'upscale' } ] @@ -177,7 +177,28 @@ const AihubmixPage: FC<{ Options: string[] }> = ({ Options }) => { try { if (mode === 'generate') { - if (painting.model === 'V_3') { + if (painting.model.startsWith('imagen-')) { + const AI = new AiProvider(aihubmixProvider) + const base64s = await AI.generateImage({ + prompt, + model: painting.model, + config: { + aspectRatio: painting.aspectRatio?.replace('ASPECT_', '').replace('_', ':'), + numberOfImages: painting.model.startsWith('imagen-4.0-ultra-generate-exp') ? 1 : painting.numberOfImages, + personGeneration: painting.personGeneration + } + }) + if (base64s?.length > 0) { + const validFiles = await Promise.all( + base64s.map(async (base64) => { + return await window.api.file.saveBase64Image(base64) + }) + ) + await FileManager.addFiles(validFiles) + updatePaintingState({ files: validFiles, urls: validFiles.map((file) => file.name) }) + } + return + } else if (painting.model === 'V_3') { // V3 API uses different endpoint and parameters format const formData = new FormData() formData.append('prompt', prompt) @@ -825,7 +846,13 @@ const AihubmixPage: FC<{ Options: string[] }> = ({ Options }) => { value={painting.prompt} spellCheck={false} onChange={(e) => updatePaintingState({ prompt: e.target.value })} - placeholder={isTranslating ? t('paintings.translating') : t('paintings.prompt_placeholder_edit')} + placeholder={ + isTranslating + ? t('paintings.translating') + : painting.model?.startsWith('imagen-') + ? t('paintings.prompt_placeholder_en') + : t('paintings.prompt_placeholder_edit') + } onKeyDown={handleKeyDown} /> diff --git a/src/renderer/src/pages/paintings/config/aihubmixConfig.tsx b/src/renderer/src/pages/paintings/config/aihubmixConfig.tsx index 45868ad054..6aaded1c2b 100644 --- a/src/renderer/src/pages/paintings/config/aihubmixConfig.tsx +++ b/src/renderer/src/pages/paintings/config/aihubmixConfig.tsx @@ -4,6 +4,7 @@ import { ASPECT_RATIOS, BACKGROUND_OPTIONS, MODERATION_OPTIONS, + PERSON_GENERATION_OPTIONS, QUALITY_OPTIONS, RENDERING_SPEED_OPTIONS, STYLE_TYPES, @@ -67,6 +68,15 @@ export const createModeConfigs = (): Record => { title: 'OpenAI', options: [{ label: 'gpt-image-1', value: 'gpt-image-1' }] }, + { + label: 'Gemini', + title: 'Gemini', + options: [ + { label: 'imagen-4.0-preview', value: 'imagen-4.0-generate-preview-05-20' }, + { label: 'imagen-4.0-ultra-exp', value: 'imagen-4.0-ultra-generate-exp-05-20' }, + { label: 'imagen-3.0', value: 'imagen-3.0-generate-001' } + ] + }, { label: 'ideogram', title: 'ideogram', @@ -186,6 +196,40 @@ export const createModeConfigs = (): Record => { options: BACKGROUND_OPTIONS, initialValue: 'auto', condition: (painting) => painting.model === 'gpt-image-1' + }, + { + type: 'slider', + key: 'numberOfImages', + title: 'paintings.number_images', + tooltip: 'paintings.generate.number_images_tip', + min: 1, + max: 4, + initialValue: 4, + condition: (painting) => + Boolean(painting.model?.startsWith('imagen-') && painting.model !== 'imagen-4.0-ultra-generate-exp-05-20') + }, + { + type: 'select', + key: 'aspectRatio', + title: 'paintings.aspect_ratio', + options: [ + { label: '1:1', value: 'ASPECT_1_1' }, + { label: '3:4', value: 'ASPECT_3_4' }, + { label: '4:3', value: 'ASPECT_4_3' }, + { label: '9:16', value: 'ASPECT_9_16' }, + { label: '16:9', value: 'ASPECT_16_9' } + ], + initialValue: 'ASPECT_1_1', + condition: (painting) => Boolean(painting.model?.startsWith('imagen-')) + }, + { + type: 'select', + key: 'personGeneration', + title: 'paintings.generate.person_generation', + tooltip: 'paintings.generate.person_generation_tip', + options: PERSON_GENERATION_OPTIONS, + initialValue: 'ALLOW_ALL', + condition: (painting) => Boolean(painting.model?.startsWith('imagen-')) } ], remix: [ @@ -340,5 +384,6 @@ export const DEFAULT_PAINTING: PaintingAction = { background: 'auto', quality: 'auto', moderation: 'auto', - n: 1 + n: 1, + numberOfImages: 4 } diff --git a/src/renderer/src/pages/paintings/config/constants.ts b/src/renderer/src/pages/paintings/config/constants.ts index 16dd9d2fb8..377f06533f 100644 --- a/src/renderer/src/pages/paintings/config/constants.ts +++ b/src/renderer/src/pages/paintings/config/constants.ts @@ -141,3 +141,9 @@ export const BACKGROUND_OPTIONS = [ { label: 'paintings.background_options.transparent', value: 'transparent' }, { label: 'paintings.background_options.opaque', value: 'opaque' } ] + +export const PERSON_GENERATION_OPTIONS = [ + { label: 'paintings.person_generation_options.allow_all', value: 'ALLOW_ALL' }, + { label: 'paintings.person_generation_options.allow_adult', value: 'ALLOW_ADULT' }, + { label: 'paintings.person_generation_options.allow_none', value: 'DONT_ALLOW' } +] diff --git a/src/renderer/src/providers/AiProvider/AihubmixProvider.ts b/src/renderer/src/providers/AiProvider/AihubmixProvider.ts index 900c39d90d..e42a4e2039 100644 --- a/src/renderer/src/providers/AiProvider/AihubmixProvider.ts +++ b/src/renderer/src/providers/AiProvider/AihubmixProvider.ts @@ -43,8 +43,8 @@ export default class AihubmixProvider extends BaseProvider { if (id.startsWith('claude')) { return this.providers.get('claude')! } - // gemini开头 且不以-nothink、-search结尾 - if (id.startsWith('gemini') && !id.endsWith('-nothink') && !id.endsWith('-search')) { + // gemini开头 或 imagen开头 且不以-nothink、-search结尾 + if ((id.startsWith('gemini') || id.startsWith('imagen')) && !id.endsWith('-nothink') && !id.endsWith('-search')) { return this.providers.get('gemini')! } if (isOpenAILLMModel(model)) { @@ -64,7 +64,9 @@ export default class AihubmixProvider extends BaseProvider { } public async generateImage(params: any): Promise { - return this.defaultProvider.generateImage(params) + return this.getProvider({ + id: params.model + } as unknown as Model).generateImage(params) } public async generateImageByChat(params: any): Promise { diff --git a/src/renderer/src/providers/AiProvider/GeminiProvider.ts b/src/renderer/src/providers/AiProvider/GeminiProvider.ts index 9a6515b00a..e19c47ea3c 100644 --- a/src/renderer/src/providers/AiProvider/GeminiProvider.ts +++ b/src/renderer/src/providers/AiProvider/GeminiProvider.ts @@ -6,6 +6,7 @@ import { FunctionCall, GenerateContentConfig, GenerateContentResponse, + GenerateImagesParameters, GoogleGenAI, HarmBlockThreshold, HarmCategory, @@ -887,10 +888,30 @@ export default class GeminiProvider extends BaseProvider { /** * Generate an image - * @returns The generated image + * @param params - The parameters for image generation + * @returns The generated image URLs */ - public async generateImage(): Promise { - return [] + public async generateImage(params: GenerateImagesParameters): Promise { + try { + console.log('[GeminiProvider] generateImage params:', params) + const response = await this.sdk.models.generateImages(params) + + if (!response.generatedImages || response.generatedImages.length === 0) { + return [] + } + + const images = response.generatedImages + .filter((image) => image.image?.imageBytes) + .map((image) => { + const dataPrefix = `data:${image.image?.mimeType || 'image/png'};base64,` + return dataPrefix + image.image?.imageBytes + }) + // console.log(response?.generatedImages?.[0]?.image?.imageBytes); + return images + } catch (error) { + console.error('[generateImage] error:', error) + throw error + } } /** diff --git a/src/renderer/src/providers/AiProvider/index.ts b/src/renderer/src/providers/AiProvider/index.ts index 1c56572cd6..ef77dff14b 100644 --- a/src/renderer/src/providers/AiProvider/index.ts +++ b/src/renderer/src/providers/AiProvider/index.ts @@ -1,3 +1,4 @@ +import { GenerateImagesParameters } from '@google/genai' import BaseProvider from '@renderer/providers/AiProvider/BaseProvider' import ProviderFactory from '@renderer/providers/AiProvider/ProviderFactory' import type { Assistant, GenerateImageParams, MCPTool, Model, Provider, Suggestion } from '@renderer/types' @@ -70,8 +71,8 @@ export default class AiProvider { return this.sdk.getApiKey() } - public async generateImage(params: GenerateImageParams): Promise { - return this.sdk.generateImage(params) + public async generateImage(params: GenerateImageParams | GenerateImagesParameters): Promise { + return this.sdk.generateImage(params as GenerateImageParams) } public async generateImageByChat({ diff --git a/src/renderer/src/types/index.ts b/src/renderer/src/types/index.ts index f54f2a4be9..1e463e67c5 100644 --- a/src/renderer/src/types/index.ts +++ b/src/renderer/src/types/index.ts @@ -1,5 +1,5 @@ import type { WebSearchResultBlock } from '@anthropic-ai/sdk/resources' -import type { GroundingMetadata } from '@google/genai' +import type { GenerateImagesConfig, GroundingMetadata } from '@google/genai' import type OpenAI from 'openai' import type { CSSProperties } from 'react' @@ -214,6 +214,8 @@ export interface GeneratePainting extends PaintingParams { n?: number size?: string background?: string + personGeneration?: GenerateImagesConfig['personGeneration'] + numberOfImages?: number } export interface EditPainting extends PaintingParams {