diff --git a/src/renderer/src/aiCore/index_new.ts b/src/renderer/src/aiCore/index_new.ts index b1f0819891..2131eebaae 100644 --- a/src/renderer/src/aiCore/index_new.ts +++ b/src/renderer/src/aiCore/index_new.ts @@ -14,7 +14,13 @@ import { addSpan, endSpan } from '@renderer/services/SpanManagerService' import { StartSpanParams } from '@renderer/trace/types/ModelSpanEntity' import type { Assistant, GenerateImageParams, Model, Provider } from '@renderer/types' import type { AiSdkModel, StreamTextParams } from '@renderer/types/aiCoreTypes' -import { CreateVideoParams, CreateVideoResult, RetrieveVideoParams, RetrieveVideoResult } from '@renderer/types/video' +import { + CreateVideoParams, + CreateVideoResult, + RetrieveVideoContentResult, + RetrieveVideoParams, + RetrieveVideoResult +} from '@renderer/types/video' import { buildClaudeCodeSystemModelMessage } from '@shared/anthropic' import { type ImageModel, type LanguageModel, type Provider as AiSdkProvider, wrapLanguageModel } from 'ai' @@ -515,6 +521,13 @@ export default class ModernAiProvider { return this.legacyProvider.retrieveVideo(params) } + /** + * We manually implement this method before aisdk supports it well + */ + public async retrieveVideoContent(params: RetrieveVideoParams): Promise { + return this.legacyProvider.retrieveVideoContent(params) + } + public getBaseURL(): string { return this.legacyProvider.getBaseURL() } diff --git a/src/renderer/src/aiCore/legacy/clients/openai/OpenAIResponseAPIClient.ts b/src/renderer/src/aiCore/legacy/clients/openai/OpenAIResponseAPIClient.ts index b462e184ab..766e1455e3 100644 --- a/src/renderer/src/aiCore/legacy/clients/openai/OpenAIResponseAPIClient.ts +++ b/src/renderer/src/aiCore/legacy/clients/openai/OpenAIResponseAPIClient.ts @@ -36,7 +36,7 @@ import { OpenAIResponseSdkTool, OpenAIResponseSdkToolCall } from '@renderer/types/sdk' -import { CreateVideoParams, RetrieveVideoParams } from '@renderer/types/video' +import { CreateVideoParams, RetrieveVideoContentParams, RetrieveVideoParams } from '@renderer/types/video' import { addImageFileToContents } from '@renderer/utils/formats' import { isSupportedToolUse, @@ -163,6 +163,11 @@ export class OpenAIResponseAPIClient extends OpenAIBaseClient< return sdk.videos.retrieve(params.videoId, params.options) } + public async retrieveVideoContent(params: RetrieveVideoContentParams): Promise { + const sdk = await this.getSdkInstance() + return sdk.videos.downloadContent(params.videoId, params.query, params.options) + } + private async handlePdfFile(file: FileMetadata): Promise { if (file.size > 32 * MB) return undefined try { diff --git a/src/renderer/src/aiCore/legacy/index.ts b/src/renderer/src/aiCore/legacy/index.ts index aa08d5194b..206027f7cd 100644 --- a/src/renderer/src/aiCore/legacy/index.ts +++ b/src/renderer/src/aiCore/legacy/index.ts @@ -7,7 +7,13 @@ import { withSpanResult } from '@renderer/services/SpanManagerService' import { StartSpanParams } from '@renderer/trace/types/ModelSpanEntity' import type { GenerateImageParams, Model, Provider } from '@renderer/types' import type { RequestOptions, SdkModel } from '@renderer/types/sdk' -import { CreateVideoParams, CreateVideoResult, RetrieveVideoParams, RetrieveVideoResult } from '@renderer/types/video' +import { + CreateVideoParams, + CreateVideoResult, + RetrieveVideoContentResult, + RetrieveVideoParams, + RetrieveVideoResult +} from '@renderer/types/video' import { isSupportedToolUse } from '@renderer/utils/mcp-tools' import { AihubmixAPIClient } from './clients/aihubmix/AihubmixAPIClient' @@ -181,7 +187,7 @@ export default class AiProvider { } public async createVideo(params: CreateVideoParams): Promise { - if (this.apiClient instanceof OpenAIResponseAPIClient) { + if (this.apiClient instanceof OpenAIResponseAPIClient && params.type === 'openai') { const video = await this.apiClient.createVideo(params) return { type: 'openai', @@ -204,6 +210,18 @@ export default class AiProvider { } } + public async retrieveVideoContent(params: RetrieveVideoParams): Promise { + if (this.apiClient instanceof OpenAIResponseAPIClient && params.type === 'openai') { + const response = await this.apiClient.retrieveVideoContent(params) + return { + type: 'openai', + response + } + } else { + throw new Error('Video generation is not supported by this provider') + } + } + public getBaseURL(): string { return this.apiClient.getBaseURL() } diff --git a/src/renderer/src/services/ApiService.ts b/src/renderer/src/services/ApiService.ts index 5c2b844f96..51caec8fd3 100644 --- a/src/renderer/src/services/ApiService.ts +++ b/src/renderer/src/services/ApiService.ts @@ -16,7 +16,13 @@ import type { StreamTextParams } from '@renderer/types/aiCoreTypes' import { type Chunk, ChunkType } from '@renderer/types/chunk' import { Message } from '@renderer/types/newMessage' import { SdkModel } from '@renderer/types/sdk' -import { CreateVideoParams, CreateVideoResult, RetrieveVideoParams, RetrieveVideoResult } from '@renderer/types/video' +import { + CreateVideoParams, + CreateVideoResult, + RetrieveVideoContentResult, + RetrieveVideoParams, + RetrieveVideoResult +} from '@renderer/types/video' import { removeSpecialCharactersForTopicName, uuid } from '@renderer/utils' import { abortCompletion, readyToAbort } from '@renderer/utils/abortController' import { isAbortError } from '@renderer/utils/error' @@ -408,6 +414,11 @@ export async function retrieveVideo(params: RetrieveVideoParams): Promise { + const ai = new AiProviderNew(params.provider) + return ai.retrieveVideoContent(params) +} + export function hasApiKey(provider: Provider) { if (!provider) return false if (['ollama', 'lmstudio', 'vertexai', 'cherryai'].includes(provider.id)) return true diff --git a/src/renderer/src/types/video.ts b/src/renderer/src/types/video.ts index d2b6d4ec19..92dd34aa09 100644 --- a/src/renderer/src/types/video.ts +++ b/src/renderer/src/types/video.ts @@ -54,3 +54,29 @@ export interface OpenAIRetrieveVideoResult extends RetrieveVideoBaseResult { } export type RetrieveVideoResult = OpenAIRetrieveVideoResult + +// Retrieve Video Content +interface RetrieveVideoContentBaseParams { + type: VideoEndpointType + provider: Provider +} + +export interface OpenAIRetrieveVideoContentParams extends RetrieveVideoContentBaseParams { + type: 'openai' + videoId: string + query?: OpenAI.Videos.VideoDownloadContentParams + options?: OpenAI.RequestOptions +} + +export type RetrieveVideoContentParams = OpenAIRetrieveVideoContentParams + +interface RetrieveVideoContentBaseResult { + type: VideoEndpointType +} + +export interface OpenAIRetrieveVideoContentResult extends RetrieveVideoContentBaseResult { + type: 'openai' + response: Response +} + +export type RetrieveVideoContentResult = OpenAIRetrieveVideoContentResult