diff --git a/src/renderer/src/config/models.ts b/src/renderer/src/config/models.ts index 65ed25a04f..6555211f14 100644 --- a/src/renderer/src/config/models.ts +++ b/src/renderer/src/config/models.ts @@ -189,7 +189,7 @@ export const TEXT_TO_IMAGE_REGEX = /flux|diffusion|stabilityai|sd-|dall|cogview| // Reasoning models export const REASONING_REGEX = - /^(o\d+(?:-[\w-]+)?|.*\b(?:reasoner|thinking)\b.*|.*-[rR]\d+.*|.*\bqwq(?:-[\w-]+)?\b.*|.*\bhunyuan-t1(?:-[\w-]+)?\b.*|.*\bglm-zero-preview\b.*|.*\bgrok-3-mini(?:-[\w-]+)?\b.*)$/i + /^(o\d+(?:-[\w-]+)?|.*\b(?:reasoning|reasoner|thinking)\b.*|.*-[rR]\d+.*|.*\bqwq(?:-[\w-]+)?\b.*|.*\bhunyuan-t1(?:-[\w-]+)?\b.*|.*\bglm-zero-preview\b.*|.*\bgrok-3-mini(?:-[\w-]+)?\b.*)$/i // Embedding models export const EMBEDDING_REGEX = @@ -206,7 +206,7 @@ export const FUNCTION_CALLING_MODELS = [ 'gpt-4o-mini', 'gpt-4', 'gpt-4.5', - 'o1(?:-[\\w-]+)?', + 'o(1|3|4)(?:-[\\w-]+)?', 'claude', 'qwen', 'qwen3', @@ -2153,11 +2153,11 @@ export const TEXT_TO_IMAGES_MODELS_SUPPORT_IMAGE_ENHANCEMENT = [ export const GENERATE_IMAGE_MODELS = [ 'gemini-2.0-flash-exp-image-generation', + 'gemini-2.0-flash-preview-image-generation', 'gemini-2.0-flash-exp', 'grok-2-image-1212', 'grok-2-image', 'grok-2-image-latest', - 'gpt-4o-image', 'gpt-image-1' ] @@ -2172,6 +2172,7 @@ export const GEMINI_SEARCH_MODELS = [ 'gemini-2.5-pro-exp-03-25', 'gemini-2.5-pro-preview', 'gemini-2.5-pro-preview-03-25', + 'gemini-2.5-pro-preview-05-06', 'gemini-2.5-flash-preview', 'gemini-2.5-flash-preview-04-17' ] diff --git a/src/renderer/src/pages/home/Inputbar/ThinkingButton.tsx b/src/renderer/src/pages/home/Inputbar/ThinkingButton.tsx index 702c1d2823..2caef6c158 100644 --- a/src/renderer/src/pages/home/Inputbar/ThinkingButton.tsx +++ b/src/renderer/src/pages/home/Inputbar/ThinkingButton.tsx @@ -35,7 +35,7 @@ const MODEL_SUPPORTED_OPTIONS: Record = { default: ['off', 'low', 'medium', 'high'], grok: ['off', 'low', 'high'], gemini: ['off', 'low', 'medium', 'high', 'auto'], - qwen: ['off', 'low', 'medium', 'high', 'auto'] + qwen: ['off', 'low', 'medium', 'high'] } // 选项转换映射表：当选项不支持时使用的替代选项 diff --git a/src/renderer/src/pages/home/Messages/Blocks/ErrorBlock.tsx b/src/renderer/src/pages/home/Messages/Blocks/ErrorBlock.tsx index 2c8161397d..f55a3ab6b1 100644 --- a/src/renderer/src/pages/home/Messages/Blocks/ErrorBlock.tsx +++ b/src/renderer/src/pages/home/Messages/Blocks/ErrorBlock.tsx @@ -16,7 +16,7 @@ const MessageErrorInfo: React.FC<{ block: ErrorMessageBlock }> = ({ block }) => const HTTP_ERROR_CODES = [400, 401, 403, 404, 429, 500, 502, 503, 504] if (block.error && HTTP_ERROR_CODES.includes(block.error?.status)) { - return + return } if (block?.error?.message) { const errorKey = `error.${block.error.message}` diff --git a/src/renderer/src/pages/home/Messages/Blocks/ThinkingBlock.tsx b/src/renderer/src/pages/home/Messages/Blocks/ThinkingBlock.tsx index e95f47a493..caf7d3f764 100644 --- a/src/renderer/src/pages/home/Messages/Blocks/ThinkingBlock.tsx +++ b/src/renderer/src/pages/home/Messages/Blocks/ThinkingBlock.tsx @@ -4,7 +4,7 @@ import { MessageBlockStatus, type ThinkingMessageBlock } from '@renderer/types/n import { Collapse, message as antdMessage, Tooltip } from 'antd' import { Lightbulb } from 'lucide-react' import { motion } from 'motion/react' -import { memo, useCallback, useEffect, useMemo, useState } from 'react' +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import styled from 'styled-components' @@ -40,6 +40,8 @@ const ThinkingBlock: React.FC = ({ block }) => { const { t } = useTranslation() const { messageFont, fontSize, thoughtAutoCollapse } = useSettings() const [activeKey, setActiveKey] = useState<'thought' | ''>(thoughtAutoCollapse ? '' : 'thought') + const [thinkingTime, setThinkingTime] = useState(block.thinking_millsec || 0) + const intervalId = useRef(null) const isThinking = useMemo(() => block.status === MessageBlockStatus.STREAMING, [block.status]) @@ -73,13 +75,31 @@ const ThinkingBlock: React.FC = ({ block }) => { } }, [block.content, t]) + // FIXME: 这里统计的和请求处统计的有一定误差 + useEffect(() => { + if (isThinking) { + intervalId.current = setInterval(() => { + setThinkingTime((prev) => prev + 100) + }, 100) + } else if (intervalId.current) { + // 立即清除计时器 + clearInterval(intervalId.current) + intervalId.current = null + } + + return () => { + if (intervalId.current) { + window.clearInterval(intervalId.current) + } + } + }, [isThinking]) + + const thinkingTimeSeconds = useMemo(() => (thinkingTime / 1000).toFixed(1), [thinkingTime]) + if (!block.content) { return null } - const thinkingTime = block.thinking_millsec || 0 - const thinkingTimeSeconds = (thinkingTime / 1000).toFixed(1) - return ( = ({ message }) => { - // const { t } = useTranslation() - // if (message.status === 'pending') { - // return ( - - // ) - // } - - // if (message.status === 'searching') { - // return ( - // - // - // {t('message.searching')} - // - // - // ) - // } - - // if (message.status === 'error') { - // return - // } - - // if (message.type === '@' && model) { - // const content = `[@${model.name}](#) ${getBriefInfo(message.content)}` - // return - // } - // const toolUseRegex = /([\s\S]*?)<\/tool_use>/g - - // console.log('message', message) - return ( <> diff --git a/src/renderer/src/pages/home/Messages/MessageError.tsx b/src/renderer/src/pages/home/Messages/MessageError.tsx deleted file mode 100644 index e0c0ea00bd..0000000000 --- a/src/renderer/src/pages/home/Messages/MessageError.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import type { ErrorMessageBlock } from '@renderer/types/newMessage' -import { Alert as AntdAlert } from 'antd' -import { FC } from 'react' -import { useTranslation } from 'react-i18next' -import styled from 'styled-components' - -const MessageError: FC<{ block: ErrorMessageBlock }> = ({ block }) => { - return ( - <> - {/* - {block.error && ( - - )} */} - - - ) -} - -const MessageErrorInfo: FC<{ block: ErrorMessageBlock }> = ({ block }) => { - const { t } = useTranslation() - - const HTTP_ERROR_CODES = [400, 401, 403, 404, 429, 500, 502, 503, 504] - console.log('block', block) - if (block.error && HTTP_ERROR_CODES.includes(block.error?.status)) { - return - } - if (block?.error?.message) { - return - } - - return -} - -const Alert = styled(AntdAlert)` - margin: 15px 0 8px; - padding: 10px; - font-size: 12px; -` - -export default MessageError diff --git a/src/renderer/src/providers/AiProvider/AnthropicProvider.ts b/src/renderer/src/providers/AiProvider/AnthropicProvider.ts index 1bb58c22ab..3f2929bdd0 100644 --- a/src/renderer/src/providers/AiProvider/AnthropicProvider.ts +++ b/src/renderer/src/providers/AiProvider/AnthropicProvider.ts @@ -30,10 +30,12 @@ import { MCPCallToolResponse, MCPTool, MCPToolResponse, + Metrics, Model, Provider, Suggestion, ToolCallResponse, + Usage, WebSearchSource } from '@renderer/types' import { ChunkType } from '@renderer/types/chunk' @@ -47,7 +49,7 @@ import { } from '@renderer/utils/mcp-tools' import { findFileBlocks, findImageBlocks, getMainTextContent } from '@renderer/utils/messageUtils/find' import { buildSystemPrompt } from '@renderer/utils/prompt' -import { first, flatten, sum, takeRight } from 'lodash' +import { first, flatten, takeRight } from 'lodash' import OpenAI from 'openai' import { CompletionsParams } from '.' @@ -270,77 +272,82 @@ export default class AnthropicProvider extends BaseProvider { ...this.getCustomParameters(assistant) } - let time_first_token_millsec = 0 - let time_first_content_millsec = 0 - let checkThinkingContent = false - let thinking_content = '' - const start_time_millsec = new Date().getTime() - - if (!streamOutput) { - const message = await this.sdk.messages.create({ ...body, stream: false }) - const time_completion_millsec = new Date().getTime() - start_time_millsec - - let text = '' - let reasoning_content = '' - - if (message.content && message.content.length > 0) { - const thinkingBlock = message.content.find((block) => block.type === 'thinking') - const textBlock = message.content.find((block) => block.type === 'text') - - if (thinkingBlock && 'thinking' in thinkingBlock) { - reasoning_content = thinkingBlock.thinking - } - - if (textBlock && 'text' in textBlock) { - text = textBlock.text - } - } - - return onChunk({ - type: ChunkType.BLOCK_COMPLETE, - response: { - text, - reasoning_content, - usage: message.usage as any, - metrics: { - completion_tokens: message.usage.output_tokens, - time_completion_millsec, - time_first_token_millsec: 0 - } - } - }) - } - const { abortController, cleanup } = this.createAbortController(lastUserMessage?.id) const { signal } = abortController + + const finalUsage: Usage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + } + + const finalMetrics: Metrics = { + completion_tokens: 0, + time_completion_millsec: 0, + time_first_token_millsec: 0 + } const toolResponses: MCPToolResponse[] = [] - const processStream = (body: MessageCreateParamsNonStreaming, idx: number) => { + const processStream = async (body: MessageCreateParamsNonStreaming, idx: number) => { + let time_first_token_millsec = 0 + const start_time_millsec = new Date().getTime() + + if (!streamOutput) { + const message = await this.sdk.messages.create({ ...body, stream: false }) + const time_completion_millsec = new Date().getTime() - start_time_millsec + + let text = '' + let reasoning_content = '' + + if (message.content && message.content.length > 0) { + const thinkingBlock = message.content.find((block) => block.type === 'thinking') + const textBlock = message.content.find((block) => block.type === 'text') + + if (thinkingBlock && 'thinking' in thinkingBlock) { + reasoning_content = thinkingBlock.thinking + } + + if (textBlock && 'text' in textBlock) { + text = textBlock.text + } + } + + return onChunk({ + type: ChunkType.BLOCK_COMPLETE, + response: { + text, + reasoning_content, + usage: message.usage as any, + metrics: { + completion_tokens: message.usage.output_tokens, + time_completion_millsec, + time_first_token_millsec: 0 + } + } + }) + } + + let thinking_content = '' + let isFirstChunk = true + return new Promise((resolve, reject) => { // 等待接口返回流 const toolCalls: ToolUseBlock[] = [] - let hasThinkingContent = false + this.sdk.messages .stream({ ...body, stream: true }, { signal, timeout: 5 * 60 * 1000 }) .on('text', (text) => { - if (hasThinkingContent && !checkThinkingContent) { - checkThinkingContent = true - onChunk({ - type: ChunkType.THINKING_COMPLETE, - text: thinking_content, - thinking_millsec: new Date().getTime() - time_first_content_millsec - }) - } - if (time_first_token_millsec == 0) { - time_first_token_millsec = new Date().getTime() - } - - thinking_content = '' - checkThinkingContent = false - hasThinkingContent = false - - if (!hasThinkingContent && time_first_content_millsec === 0) { - time_first_content_millsec = new Date().getTime() + if (isFirstChunk) { + isFirstChunk = false + if (time_first_token_millsec == 0) { + time_first_token_millsec = new Date().getTime() + } else { + onChunk({ + type: ChunkType.THINKING_COMPLETE, + text: thinking_content, + thinking_millsec: new Date().getTime() - time_first_token_millsec + }) + } } onChunk({ type: ChunkType.TEXT_DELTA, text }) @@ -372,34 +379,22 @@ export default class AnthropicProvider extends BaseProvider { }) } } + if (block.type === 'tool_use') { + toolCalls.push(block) + } }) .on('thinking', (thinking) => { - hasThinkingContent = true - const currentTime = new Date().getTime() // Get current time for each chunk - if (time_first_token_millsec == 0) { - time_first_token_millsec = currentTime + time_first_token_millsec = new Date().getTime() } - // Set time_first_content_millsec ONLY when the first content (thinking or text) arrives - if (time_first_content_millsec === 0) { - time_first_content_millsec = currentTime - } - - // Calculate thinking time as time elapsed since start until this chunk - const thinking_time = currentTime - time_first_content_millsec onChunk({ type: ChunkType.THINKING_DELTA, text: thinking, - thinking_millsec: thinking_time + thinking_millsec: new Date().getTime() - time_first_token_millsec }) thinking_content += thinking }) - .on('contentBlock', (content) => { - if (content.type === 'tool_use') { - toolCalls.push(content) - } - }) .on('finalMessage', async (message) => { const toolResults: Awaited> = [] // tool call @@ -458,29 +453,28 @@ export default class AnthropicProvider extends BaseProvider { newBody.messages = userMessages onChunk({ type: ChunkType.LLM_RESPONSE_CREATED }) - await processStream(newBody, idx + 1) + try { + await processStream(newBody, idx + 1) + } catch (error) { + console.error('Error processing stream:', error) + reject(error) + } } - const time_completion_millsec = new Date().getTime() - start_time_millsec + finalUsage.prompt_tokens += message.usage.input_tokens + finalUsage.completion_tokens += message.usage.output_tokens + finalUsage.total_tokens += finalUsage.prompt_tokens + finalUsage.completion_tokens + finalMetrics.completion_tokens = finalUsage.completion_tokens + finalMetrics.time_completion_millsec += new Date().getTime() - start_time_millsec + finalMetrics.time_first_token_millsec = time_first_token_millsec - start_time_millsec onChunk({ type: ChunkType.BLOCK_COMPLETE, response: { - usage: { - prompt_tokens: message.usage.input_tokens, - completion_tokens: message.usage.output_tokens, - total_tokens: sum(Object.values(message.usage)) - }, - metrics: { - completion_tokens: message.usage.output_tokens, - time_completion_millsec, - time_first_token_millsec: time_first_token_millsec - start_time_millsec - } + usage: finalUsage, + metrics: finalMetrics } }) - // FIXME: 临时方案，重置时间戳和思考内容 - time_first_token_millsec = 0 - time_first_content_millsec = 0 resolve() }) .on('error', (error) => reject(error)) diff --git a/src/renderer/src/providers/AiProvider/GeminiProvider.ts b/src/renderer/src/providers/AiProvider/GeminiProvider.ts index db30fa6c4b..0a49ebe573 100644 --- a/src/renderer/src/providers/AiProvider/GeminiProvider.ts +++ b/src/renderer/src/providers/AiProvider/GeminiProvider.ts @@ -40,6 +40,7 @@ import { MCPCallToolResponse, MCPTool, MCPToolResponse, + Metrics, Model, Provider, Suggestion, @@ -126,7 +127,6 @@ export default class GeminiProvider extends BaseProvider { * @returns The message contents */ private async getMessageContents(message: Message): Promise { - console.log('getMessageContents', message) const role = message.role === 'user' ? 'user' : 'model' const parts: Part[] = [{ text: await this.getMessageContent(message) }] // Add any generated images from previous responses @@ -153,6 +153,16 @@ export default class GeminiProvider extends BaseProvider { } } } + const file = imageBlock.file + if (file) { + const base64Data = await window.api.file.base64Image(file.id + file.ext) + parts.push({ + inlineData: { + data: base64Data.base64, + mimeType: base64Data.mime + } as Part['inlineData'] + }) + } } const fileBlocks = findFileBlocks(message) @@ -186,6 +196,50 @@ export default class GeminiProvider extends BaseProvider { } } + private async getImageFileContents(message: Message): Promise { + const role = message.role === 'user' ? 'user' : 'model' + const content = getMainTextContent(message) + const parts: Part[] = [{ text: content }] + const imageBlocks = findImageBlocks(message) + for (const imageBlock of imageBlocks) { + if ( + imageBlock.metadata?.generateImageResponse?.images && + imageBlock.metadata.generateImageResponse.images.length > 0 + ) { + for (const imageUrl of imageBlock.metadata.generateImageResponse.images) { + if (imageUrl && imageUrl.startsWith('data:')) { + // Extract base64 data and mime type from the data URL + const matches = imageUrl.match(/^data:(.+);base64,(.*)$/) + if (matches && matches.length === 3) { + const mimeType = matches[1] + const base64Data = matches[2] + parts.push({ + inlineData: { + data: base64Data, + mimeType: mimeType + } as Part['inlineData'] + }) + } + } + } + } + const file = imageBlock.file + if (file) { + const base64Data = await window.api.file.base64Image(file.id + file.ext) + parts.push({ + inlineData: { + data: base64Data.base64, + mimeType: base64Data.mime + } as Part['inlineData'] + }) + } + } + return { + role, + parts: parts + } + } + /** * Get the safety settings * @returns The safety settings @@ -273,6 +327,18 @@ export default class GeminiProvider extends BaseProvider { }: CompletionsParams): Promise { const defaultModel = getDefaultModel() const model = assistant.model || defaultModel + let canGenerateImage = false + if (isGenerateImageModel(model)) { + if (model.id === 'gemini-2.0-flash-exp') { + canGenerateImage = assistant.enableGenerateImage! + } else { + canGenerateImage = true + } + } + if (canGenerateImage) { + await this.generateImageByChat({ messages, assistant, onChunk }) + return + } const { contextCount, maxTokens, streamOutput, enableToolUse } = getAssistantSettings(assistant) const userMessages = filterUserRoleStartMessages( @@ -309,21 +375,10 @@ export default class GeminiProvider extends BaseProvider { }) } - let canGenerateImage = false - if (isGenerateImageModel(model)) { - if (model.id === 'gemini-2.0-flash-exp') { - canGenerateImage = assistant.enableGenerateImage! - } else { - canGenerateImage = true - } - } - const generateContentConfig: GenerateContentConfig = { - responseModalities: canGenerateImage ? [Modality.TEXT, Modality.IMAGE] : undefined, - responseMimeType: canGenerateImage ? 'text/plain' : undefined, safetySettings: this.getSafetySettings(), // generate image don't need system instruction - systemInstruction: isGemmaModel(model) || canGenerateImage ? undefined : systemInstruction, + systemInstruction: isGemmaModel(model) ? undefined : systemInstruction, temperature: assistant?.settings?.temperature, topP: assistant?.settings?.topP, maxOutputTokens: maxTokens, @@ -360,8 +415,17 @@ export default class GeminiProvider extends BaseProvider { } } - const start_time_millsec = new Date().getTime() - let time_first_token_millsec = 0 + const finalUsage: Usage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + } + + const finalMetrics: Metrics = { + completion_tokens: 0, + time_completion_millsec: 0, + time_first_token_millsec: 0 + } const { cleanup, abortController } = this.createAbortController(userLastMessage?.id, true) @@ -435,6 +499,8 @@ export default class GeminiProvider extends BaseProvider { history.push(messageContents) let functionCalls: FunctionCall[] = [] + let time_first_token_millsec = 0 + const start_time_millsec = new Date().getTime() if (stream instanceof GenerateContentResponse) { let content = '' @@ -494,45 +560,28 @@ export default class GeminiProvider extends BaseProvider { } as BlockCompleteChunk) } else { let content = '' - let final_time_completion_millsec = 0 - let lastUsage: Usage | undefined = undefined for await (const chunk of stream) { if (window.keyv.get(EVENT_NAMES.CHAT_COMPLETION_PAUSED)) break - // --- Calculate Metrics --- - if (time_first_token_millsec == 0 && chunk.text !== undefined) { - // Update based on text arrival - time_first_token_millsec = new Date().getTime() - start_time_millsec + if (time_first_token_millsec == 0) { + time_first_token_millsec = new Date().getTime() } - // 1. Text Content if (chunk.text !== undefined) { content += chunk.text onChunk({ type: ChunkType.TEXT_DELTA, text: chunk.text }) } - // 2. Usage Data - if (chunk.usageMetadata) { - lastUsage = { - prompt_tokens: chunk.usageMetadata.promptTokenCount || 0, - completion_tokens: chunk.usageMetadata.candidatesTokenCount || 0, - total_tokens: chunk.usageMetadata.totalTokenCount || 0 - } - final_time_completion_millsec = new Date().getTime() - start_time_millsec - } - - // 4. Image Generation - const generateImage = this.processGeminiImageResponse(chunk, onChunk) - if (generateImage?.images?.length) { - onChunk({ type: ChunkType.IMAGE_COMPLETE, image: generateImage }) - } - if (chunk.candidates?.[0]?.finishReason) { if (chunk.text) { onChunk({ type: ChunkType.TEXT_COMPLETE, text: content }) } + if (chunk.usageMetadata) { + finalUsage.prompt_tokens += chunk.usageMetadata.promptTokenCount || 0 + finalUsage.completion_tokens += chunk.usageMetadata.candidatesTokenCount || 0 + finalUsage.total_tokens += chunk.usageMetadata.totalTokenCount || 0 + } if (chunk.candidates?.[0]?.groundingMetadata) { - // 3. Grounding/Search Metadata const groundingMetadata = chunk.candidates?.[0]?.groundingMetadata onChunk({ type: ChunkType.LLM_WEB_SEARCH_COMPLETE, @@ -551,35 +600,37 @@ export default class GeminiProvider extends BaseProvider { functionCalls = functionCalls.concat(chunk.functionCalls) } - onChunk({ - type: ChunkType.BLOCK_COMPLETE, - response: { - metrics: { - completion_tokens: lastUsage?.completion_tokens, - time_completion_millsec: final_time_completion_millsec, - time_first_token_millsec - }, - usage: lastUsage - } - }) - } - - // --- End Incremental onChunk calls --- - - // Call processToolUses AFTER potentially processing text content in this chunk - // This assumes tools might be specified within the text stream - // Note: parseAndCallTools inside should handle its own onChunk for tool responses - let toolResults: Awaited> = [] - if (functionCalls.length) { - toolResults = await processToolCalls(functionCalls) - } - if (content.length) { - toolResults = toolResults.concat(await processToolUses(content)) - } - if (toolResults.length) { - await processToolResults(toolResults, idx) + finalMetrics.completion_tokens = finalUsage.completion_tokens + finalMetrics.time_completion_millsec += new Date().getTime() - start_time_millsec + finalMetrics.time_first_token_millsec = + (finalMetrics.time_first_token_millsec || 0) + (time_first_token_millsec - start_time_millsec) } } + + // --- End Incremental onChunk calls --- + + // Call processToolUses AFTER potentially processing text content in this chunk + // This assumes tools might be specified within the text stream + // Note: parseAndCallTools inside should handle its own onChunk for tool responses + let toolResults: Awaited> = [] + if (functionCalls.length) { + toolResults = await processToolCalls(functionCalls) + } + if (content.length) { + toolResults = toolResults.concat(await processToolUses(content)) + } + if (toolResults.length) { + await processToolResults(toolResults, idx) + } + + // FIXME: 由于递归，会发送n次 + onChunk({ + type: ChunkType.BLOCK_COMPLETE, + response: { + usage: finalUsage, + metrics: finalMetrics + } + }) } } @@ -605,17 +656,6 @@ export default class GeminiProvider extends BaseProvider { }) await processStream(userMessagesStream, 0).finally(cleanup) - - const final_time_completion_millsec = new Date().getTime() - start_time_millsec - onChunk({ - type: ChunkType.BLOCK_COMPLETE, - response: { - metrics: { - time_completion_millsec: final_time_completion_millsec, - time_first_token_millsec - } - } - }) } /** @@ -949,8 +989,97 @@ export default class GeminiProvider extends BaseProvider { return data.embeddings?.[0]?.values?.length || 0 } - public generateImageByChat(): Promise { - throw new Error('Method not implemented.') + public async generateImageByChat({ messages, assistant, onChunk }): Promise { + const defaultModel = getDefaultModel() + const model = assistant.model || defaultModel + const { contextCount, maxTokens } = getAssistantSettings(assistant) + const userMessages = filterUserRoleStartMessages( + filterEmptyMessages(filterContextMessages(takeRight(messages, contextCount + 2))) + ) + + const userLastMessage = userMessages.pop() + const { abortController } = this.createAbortController(userLastMessage?.id, true) + const { signal } = abortController + const generateContentConfig: GenerateContentConfig = { + responseModalities: [Modality.TEXT, Modality.IMAGE], + responseMimeType: 'text/plain', + safetySettings: this.getSafetySettings(), + temperature: assistant?.settings?.temperature, + topP: assistant?.settings?.top_p, + maxOutputTokens: maxTokens, + abortSignal: signal, + ...this.getCustomParameters(assistant) + } + const history: Content[] = [] + try { + for (const message of userMessages) { + history.push(await this.getImageFileContents(message)) + } + + let time_first_token_millsec = 0 + const start_time_millsec = new Date().getTime() + onChunk({ type: ChunkType.LLM_RESPONSE_CREATED }) + const chat = this.sdk.chats.create({ + model: model.id, + config: generateContentConfig, + history: history + }) + let content = '' + const finalUsage: Usage = { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0 + } + const userMessage: Content = await this.getImageFileContents(userLastMessage!) + const response = await chat.sendMessageStream({ + message: userMessage.parts!, + config: { + ...generateContentConfig, + abortSignal: signal + } + }) + for await (const chunk of response as AsyncGenerator) { + if (time_first_token_millsec == 0) { + time_first_token_millsec = new Date().getTime() + } + + if (chunk.text !== undefined) { + content += chunk.text + onChunk({ type: ChunkType.TEXT_DELTA, text: chunk.text }) + } + const generateImage = this.processGeminiImageResponse(chunk, onChunk) + if (generateImage?.images?.length) { + onChunk({ type: ChunkType.IMAGE_COMPLETE, image: generateImage }) + } + if (chunk.candidates?.[0]?.finishReason) { + if (chunk.text) { + onChunk({ type: ChunkType.TEXT_COMPLETE, text: content }) + } + if (chunk.usageMetadata) { + finalUsage.prompt_tokens = chunk.usageMetadata.promptTokenCount || 0 + finalUsage.completion_tokens = chunk.usageMetadata.candidatesTokenCount || 0 + finalUsage.total_tokens = chunk.usageMetadata.totalTokenCount || 0 + } + } + } + onChunk({ + type: ChunkType.BLOCK_COMPLETE, + response: { + usage: finalUsage, + metrics: { + completion_tokens: finalUsage.completion_tokens, + time_completion_millsec: new Date().getTime() - start_time_millsec, + time_first_token_millsec: time_first_token_millsec - start_time_millsec + } + } + }) + } catch (error) { + console.error('[generateImageByChat] error', error) + onChunk({ + type: ChunkType.ERROR, + error + }) + } } public convertMcpTools(mcpTools: MCPTool[]): T[] { diff --git a/src/renderer/src/providers/AiProvider/OpenAICompatibleProvider.ts b/src/renderer/src/providers/AiProvider/OpenAICompatibleProvider.ts index cfb617d46a..f9a9042074 100644 --- a/src/renderer/src/providers/AiProvider/OpenAICompatibleProvider.ts +++ b/src/renderer/src/providers/AiProvider/OpenAICompatibleProvider.ts @@ -34,6 +34,7 @@ import { MCPCallToolResponse, MCPTool, MCPToolResponse, + Metrics, Model, Provider, Suggestion, @@ -395,7 +396,6 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { return streamOutput } - const start_time_millsec = new Date().getTime() const lastUserMessage = _messages.findLast((m) => m.role === 'user') const { abortController, cleanup, signalPromise } = this.createAbortController(lastUserMessage?.id, true) const { signal } = abortController @@ -423,6 +423,18 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { reqMessages = [systemMessage, ...userMessages].filter(Boolean) as ChatCompletionMessageParam[] } + let finalUsage: Usage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + } + + const finalMetrics: Metrics = { + completion_tokens: 0, + time_completion_millsec: 0, + time_first_token_millsec: 0 + } + const toolResponses: MCPToolResponse[] = [] const processToolResults = async (toolResults: Awaited>, idx: number) => { @@ -505,18 +517,17 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { const processStream = async (stream: any, idx: number) => { const toolCalls: ChatCompletionMessageToolCall[] = [] + let time_first_token_millsec = 0 + const start_time_millsec = new Date().getTime() + // Handle non-streaming case (already returns early, no change needed here) if (!isSupportStreamOutput()) { - const time_completion_millsec = new Date().getTime() - start_time_millsec // Calculate final metrics once - const finalMetrics = { - completion_tokens: stream.usage?.completion_tokens, - time_completion_millsec, - time_first_token_millsec: 0 // Non-streaming, first token time is not relevant - } + finalMetrics.completion_tokens = stream.usage?.completion_tokens + finalMetrics.time_completion_millsec = new Date().getTime() - start_time_millsec // Create a synthetic usage object if stream.usage is undefined - const finalUsage = stream.usage + finalUsage = { ...stream.usage } // Separate onChunk calls for text and usage/metrics let content = '' stream.choices.forEach((choice) => { @@ -526,7 +537,7 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { onChunk({ type: ChunkType.THINKING_COMPLETE, text: choice.message.reasoning, - thinking_millsec: time_completion_millsec + thinking_millsec: new Date().getTime() - start_time_millsec }) } // text @@ -576,20 +587,9 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { return } - let content = '' // Accumulate content for tool processing if needed + let content = '' let thinkingContent = '' - // 记录最终的完成时间差 - let final_time_completion_millsec_delta = 0 - let final_time_thinking_millsec_delta = 0 - // Variable to store the last received usage object - let lastUsage: Usage | undefined = undefined - // let isThinkingInContent: ThoughtProcessor | undefined = undefined - // const processThinkingChunk = this.handleThinkingTags() let isFirstChunk = true - let time_first_token_millsec = 0 - let time_first_token_millsec_delta = 0 - let time_first_content_millsec = 0 - let time_thinking_start = 0 // 1. 初始化中间件 const reasoningTags = [ @@ -640,25 +640,24 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { // 3. 消费 processedStream，分发 onChunk for await (const chunk of readableStreamAsyncIterable(processedStream)) { - const currentTime = new Date().getTime() const delta = chunk.type === 'finish' ? chunk.delta : chunk const rawChunk = chunk.type === 'finish' ? chunk.chunk : chunk switch (chunk.type) { case 'reasoning': { - if (time_thinking_start === 0) { - time_thinking_start = currentTime - time_first_token_millsec = currentTime - time_first_token_millsec_delta = currentTime - start_time_millsec + if (time_first_token_millsec === 0) { + time_first_token_millsec = new Date().getTime() } thinkingContent += chunk.textDelta - const thinking_time = currentTime - time_thinking_start - onChunk({ type: ChunkType.THINKING_DELTA, text: chunk.textDelta, thinking_millsec: thinking_time }) + onChunk({ + type: ChunkType.THINKING_DELTA, + text: chunk.textDelta, + thinking_millsec: new Date().getTime() - time_first_token_millsec + }) break } case 'text-delta': { let textDelta = chunk.textDelta - if (assistant.enableWebSearch && delta) { const originalDelta = rawChunk?.choices?.[0]?.delta @@ -676,25 +675,32 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { if (isFirstChunk) { isFirstChunk = false if (time_first_token_millsec === 0) { - time_first_token_millsec = currentTime - time_first_token_millsec_delta = currentTime - start_time_millsec + time_first_token_millsec = new Date().getTime() + } else { + onChunk({ + type: ChunkType.THINKING_COMPLETE, + text: thinkingContent, + thinking_millsec: new Date().getTime() - time_first_token_millsec + }) } } content += textDelta - if (time_thinking_start > 0 && time_first_content_millsec === 0) { - time_first_content_millsec = currentTime - final_time_thinking_millsec_delta = time_first_content_millsec - time_thinking_start - - onChunk({ - type: ChunkType.THINKING_COMPLETE, - text: thinkingContent, - thinking_millsec: final_time_thinking_millsec_delta - }) - } onChunk({ type: ChunkType.TEXT_DELTA, text: textDelta }) break } case 'tool-calls': { + if (isFirstChunk) { + isFirstChunk = false + if (time_first_token_millsec === 0) { + time_first_token_millsec = new Date().getTime() + } else { + onChunk({ + type: ChunkType.THINKING_COMPLETE, + text: thinkingContent, + thinking_millsec: new Date().getTime() - time_first_token_millsec + }) + } + } chunk.delta.tool_calls.forEach((toolCall) => { const { id, index, type, function: fun } = toolCall if (id && type === 'function' && fun) { @@ -721,10 +727,14 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { if (!isEmpty(finishReason)) { onChunk({ type: ChunkType.TEXT_COMPLETE, text: content }) - final_time_completion_millsec_delta = currentTime - start_time_millsec if (usage) { - lastUsage = usage + finalUsage.completion_tokens += usage.completion_tokens || 0 + finalUsage.prompt_tokens += usage.prompt_tokens || 0 + finalUsage.total_tokens += usage.total_tokens || 0 + finalMetrics.completion_tokens += usage.completion_tokens || 0 } + finalMetrics.time_completion_millsec += new Date().getTime() - start_time_millsec + finalMetrics.time_first_token_millsec = time_first_token_millsec - start_time_millsec if (originalFinishDelta?.annotations) { onChunk({ type: ChunkType.LLM_WEB_SEARCH_COMPLETE, @@ -774,49 +784,46 @@ export default class OpenAICompatibleProvider extends BaseOpenAiProvider { } as LLMWebSearchCompleteChunk) } } - reqMessages.push({ - role: 'assistant', - content: content, - tool_calls: toolCalls.length - ? toolCalls.map((toolCall) => ({ - id: toolCall.id, - function: { - ...toolCall.function, - arguments: - typeof toolCall.function.arguments === 'string' - ? toolCall.function.arguments - : JSON.stringify(toolCall.function.arguments) - }, - type: 'function' - })) - : undefined - }) - let toolResults: Awaited> = [] - if (toolCalls.length) { - toolResults = await processToolCalls(mcpTools, toolCalls) - } - if (content.length) { - toolResults = toolResults.concat(await processToolUses(content)) - } - if (toolResults.length) { - await processToolResults(toolResults, idx) - } - onChunk({ - type: ChunkType.BLOCK_COMPLETE, - response: { - usage: lastUsage, - metrics: { - completion_tokens: lastUsage?.completion_tokens, - time_completion_millsec: final_time_completion_millsec_delta, - time_first_token_millsec: time_first_token_millsec_delta, - time_thinking_millsec: final_time_thinking_millsec_delta - } - } - }) break } } } + + reqMessages.push({ + role: 'assistant', + content: content, + tool_calls: toolCalls.length + ? toolCalls.map((toolCall) => ({ + id: toolCall.id, + function: { + ...toolCall.function, + arguments: + typeof toolCall.function.arguments === 'string' + ? toolCall.function.arguments + : JSON.stringify(toolCall.function.arguments) + }, + type: 'function' + })) + : undefined + }) + let toolResults: Awaited> = [] + if (toolCalls.length) { + toolResults = await processToolCalls(mcpTools, toolCalls) + } + if (content.length) { + toolResults = toolResults.concat(await processToolUses(content)) + } + if (toolResults.length) { + await processToolResults(toolResults, idx) + } + + onChunk({ + type: ChunkType.BLOCK_COMPLETE, + response: { + usage: finalUsage, + metrics: finalMetrics + } + }) } reqMessages = processReqMessages(model, reqMessages) diff --git a/src/renderer/src/providers/AiProvider/OpenAIProvider.ts b/src/renderer/src/providers/AiProvider/OpenAIProvider.ts index 51c690ee41..154b1a7357 100644 --- a/src/renderer/src/providers/AiProvider/OpenAIProvider.ts +++ b/src/renderer/src/providers/AiProvider/OpenAIProvider.ts @@ -24,6 +24,7 @@ import { MCPCallToolResponse, MCPTool, MCPToolResponse, + Metrics, Model, Provider, Suggestion, @@ -332,7 +333,6 @@ export abstract class BaseOpenAiProvider extends BaseProvider { const lastUserMessage = _messages.findLast((m) => m.role === 'user') const { abortController, cleanup, signalPromise } = this.createAbortController(lastUserMessage?.id, true) const { signal } = abortController - let time_first_token_millsec_delta = 0 const start_time_millsec = new Date().getTime() const response = await this.sdk.chat.completions // @ts-ignore key is not typed @@ -354,8 +354,17 @@ export abstract class BaseOpenAiProvider extends BaseProvider { const processStream = async (stream: any) => { let content = '' let isFirstChunk = true - let final_time_completion_millsec_delta = 0 - let lastUsage: Usage | undefined = undefined + const finalUsage: Usage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + } + + const finalMetrics: Metrics = { + completion_tokens: 0, + time_completion_millsec: 0, + time_first_token_millsec: 0 + } for await (const chunk of stream as any) { if (window.keyv.get(EVENT_NAMES.CHAT_COMPLETION_PAUSED)) { break @@ -368,17 +377,21 @@ export abstract class BaseOpenAiProvider extends BaseProvider { } if (isFirstChunk) { isFirstChunk = false - time_first_token_millsec_delta = new Date().getTime() - start_time_millsec + finalMetrics.time_first_token_millsec = new Date().getTime() - start_time_millsec } content += delta.content onChunk({ type: ChunkType.TEXT_DELTA, text: delta.content }) } if (!isEmpty(finishReason) || chunk?.annotations) { onChunk({ type: ChunkType.TEXT_COMPLETE, text: content }) - final_time_completion_millsec_delta = new Date().getTime() - start_time_millsec + finalMetrics.time_completion_millsec = new Date().getTime() - start_time_millsec if (chunk.usage) { - lastUsage = chunk.usage + const usage = chunk.usage as OpenAI.Completions.CompletionUsage + finalUsage.completion_tokens = usage.completion_tokens + finalUsage.prompt_tokens = usage.prompt_tokens + finalUsage.total_tokens = usage.total_tokens } + finalMetrics.completion_tokens = finalUsage.completion_tokens } if (delta?.annotations) { onChunk({ @@ -393,12 +406,8 @@ export abstract class BaseOpenAiProvider extends BaseProvider { onChunk({ type: ChunkType.BLOCK_COMPLETE, response: { - usage: lastUsage, - metrics: { - completion_tokens: lastUsage?.completion_tokens, - time_completion_millsec: final_time_completion_millsec_delta, - time_first_token_millsec: time_first_token_millsec_delta - } + usage: finalUsage, + metrics: finalMetrics } }) } @@ -428,7 +437,6 @@ export abstract class BaseOpenAiProvider extends BaseProvider { type: 'input_text' } if (isSupportedReasoningEffortOpenAIModel(model)) { - systemMessageInput.text = `Formatting re-enabled${systemMessageInput.text ? '\n' + systemMessageInput.text : ''}` systemMessage.role = 'developer' } @@ -455,9 +463,6 @@ export abstract class BaseOpenAiProvider extends BaseProvider { userMessage.push(await this.getReponseMessageParam(message, model)) } - let time_first_token_millsec = 0 - const start_time_millsec = new Date().getTime() - const lastUserMessage = _messages.findLast((m) => m.role === 'user') const { abortController, cleanup, signalPromise } = this.createAbortController(lastUserMessage?.id, true) const { signal } = abortController @@ -470,6 +475,18 @@ export abstract class BaseOpenAiProvider extends BaseProvider { reqMessages = [systemMessage, ...userMessage].filter(Boolean) as OpenAI.Responses.EasyInputMessage[] } + const finalUsage: Usage = { + completion_tokens: 0, + prompt_tokens: 0, + total_tokens: 0 + } + + const finalMetrics: Metrics = { + completion_tokens: 0, + time_completion_millsec: 0, + time_first_token_millsec: 0 + } + const toolResponses: MCPToolResponse[] = [] const processToolResults = async (toolResults: Awaited>, idx: number) => { @@ -549,6 +566,8 @@ export abstract class BaseOpenAiProvider extends BaseProvider { idx: number ) => { const toolCalls: OpenAI.Responses.ResponseFunctionToolCall[] = [] + let time_first_token_millsec = 0 + const start_time_millsec = new Date().getTime() if (!streamOutput) { const nonStream = stream as OpenAI.Responses.Response @@ -633,17 +652,15 @@ export abstract class BaseOpenAiProvider extends BaseProvider { const outputItems: OpenAI.Responses.ResponseOutputItem[] = [] - let lastUsage: Usage | undefined = undefined - let final_time_completion_millsec_delta = 0 for await (const chunk of stream as Stream) { if (window.keyv.get(EVENT_NAMES.CHAT_COMPLETION_PAUSED)) { break } switch (chunk.type) { - case 'response.created': - time_first_token_millsec = new Date().getTime() - break case 'response.output_item.added': + if (time_first_token_millsec === 0) { + time_first_token_millsec = new Date().getTime() + } if (chunk.item.type === 'function_call') { outputItems.push(chunk.item) } @@ -708,18 +725,18 @@ export abstract class BaseOpenAiProvider extends BaseProvider { } break case 'response.completed': { - final_time_completion_millsec_delta = new Date().getTime() - start_time_millsec const completion_tokens = (chunk.response.usage?.output_tokens || 0) + (chunk.response.usage?.output_tokens_details.reasoning_tokens ?? 0) const total_tokens = (chunk.response.usage?.total_tokens || 0) + (chunk.response.usage?.output_tokens_details.reasoning_tokens ?? 0) - lastUsage = { - completion_tokens, - prompt_tokens: chunk.response.usage?.input_tokens || 0, - total_tokens - } + finalUsage.completion_tokens += completion_tokens + finalUsage.prompt_tokens += chunk.response.usage?.input_tokens || 0 + finalUsage.total_tokens += total_tokens + finalMetrics.completion_tokens += completion_tokens + finalMetrics.time_completion_millsec += new Date().getTime() - start_time_millsec + finalMetrics.time_first_token_millsec = time_first_token_millsec - start_time_millsec break } case 'error': @@ -761,12 +778,8 @@ export abstract class BaseOpenAiProvider extends BaseProvider { onChunk({ type: ChunkType.BLOCK_COMPLETE, response: { - usage: lastUsage, - metrics: { - completion_tokens: lastUsage?.completion_tokens, - time_completion_millsec: final_time_completion_millsec_delta, - time_first_token_millsec: time_first_token_millsec - start_time_millsec - } + usage: finalUsage, + metrics: finalMetrics } }) } diff --git a/src/renderer/src/services/StreamProcessingService.ts b/src/renderer/src/services/StreamProcessingService.ts index 1d8bef15b3..818345523e 100644 --- a/src/renderer/src/services/StreamProcessingService.ts +++ b/src/renderer/src/services/StreamProcessingService.ts @@ -59,7 +59,7 @@ export function createStreamProcessor(callbacks: StreamProcessorCallbacks = {}) callbacks.onTextComplete(data.text) } if (data.type === ChunkType.THINKING_DELTA && callbacks.onThinkingChunk) { - callbacks.onThinkingChunk(data.text, data.thinking_millsec) + callbacks.onThinkingChunk(data.text) } if (data.type === ChunkType.THINKING_COMPLETE && callbacks.onThinkingComplete) { callbacks.onThinkingComplete(data.text, data.thinking_millsec) diff --git a/src/renderer/src/store/index.ts b/src/renderer/src/store/index.ts index df7af60f69..abfc334986 100644 --- a/src/renderer/src/store/index.ts +++ b/src/renderer/src/store/index.ts @@ -46,7 +46,7 @@ const persistedReducer = persistReducer( { key: 'cherry-studio', storage, - version: 99, + version: 100, blacklist: ['runtime', 'messages', 'messageBlocks'], migrate }, diff --git a/src/renderer/src/store/migrate.ts b/src/renderer/src/store/migrate.ts index ab852b18f6..b2c94ce561 100644 --- a/src/renderer/src/store/migrate.ts +++ b/src/renderer/src/store/migrate.ts @@ -1296,6 +1296,16 @@ const migrateConfig = { } catch (error) { return state } + }, + '100': (state: RootState) => { + try { + if (!state.settings.zoomFactor) { + state.settings.zoomFactor = 1 + } + return state + } catch (error) { + return state + } } } diff --git a/src/renderer/src/store/thunk/messageThunk.ts b/src/renderer/src/store/thunk/messageThunk.ts index 5dd8d5984d..5d40439626 100644 --- a/src/renderer/src/store/thunk/messageThunk.ts +++ b/src/renderer/src/store/thunk/messageThunk.ts @@ -398,7 +398,7 @@ const fetchAndProcessAssistantResponseImpl = async ( } else { const newBlock = createThinkingBlock(assistantMsgId, accumulatedThinking, { status: MessageBlockStatus.STREAMING, - thinking_millsec: thinking_millsec + thinking_millsec: 0 }) handleBlockTransition(newBlock, MessageBlockType.THINKING) } @@ -565,7 +565,7 @@ const fetchAndProcessAssistantResponseImpl = async ( message: pauseErrorLanguagePlaceholder || error.message || 'Stream processing error', originalMessage: error.message, stack: error.stack, - status: error.status, + status: error.status || error.code, requestId: error.request_id } if (lastBlockId) { @@ -609,13 +609,13 @@ const fetchAndProcessAssistantResponseImpl = async ( // 更新topic的name autoRenameTopic(assistant, topicId) - if (response && !response.usage) { + if (response && response.usage?.total_tokens === 0) { const usage = await estimateMessagesUsage({ assistant, messages: finalContextWithAssistant }) response.usage = usage } } if (response && response.metrics) { - if (!response.metrics.completion_tokens && response.usage) { + if (response.metrics.completion_tokens === 0 && response.usage?.completion_tokens) { response = { ...response, metrics: { diff --git a/src/renderer/src/types/index.ts b/src/renderer/src/types/index.ts index e09e826b1b..0873fc0a99 100644 --- a/src/renderer/src/types/index.ts +++ b/src/renderer/src/types/index.ts @@ -121,8 +121,8 @@ export type Usage = OpenAI.Completions.CompletionUsage & { } export type Metrics = { - completion_tokens?: number - time_completion_millsec?: number + completion_tokens: number + time_completion_millsec: number time_first_token_millsec?: number time_thinking_millsec?: number }