diff --git a/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch b/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch index cf4219719c..d79275839c 100644 --- a/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch +++ b/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch @@ -37,3 +37,49 @@ index 7d19f5578040afa004bc887aab1725e8703d2bac..59ec725b6142299a62798ac4bdedb63b if (status === 400) { return new BadRequestError(status, error, message, headers); } +diff --git a/resources/embeddings.js b/resources/embeddings.js +index aae578404cb2d09a39ac33fc416f1c215c45eecd..25c54b05bdae64d5c3b36fbb30dc7c8221b14034 100644 +--- a/resources/embeddings.js ++++ b/resources/embeddings.js +@@ -36,6 +36,9 @@ class Embeddings extends resource_1.APIResource { + // No encoding_format specified, defaulting to base64 for performance reasons + // See https://github.com/openai/openai-node/pull/1312 + let encoding_format = hasUserProvidedEncodingFormat ? body.encoding_format : 'base64'; ++ if (body.model.includes('jina')) { ++ encoding_format = undefined; ++ } + if (hasUserProvidedEncodingFormat) { + Core.debug('Request', 'User defined encoding_format:', body.encoding_format); + } +@@ -47,7 +50,7 @@ class Embeddings extends resource_1.APIResource { + ...options, + }); + // if the user specified an encoding_format, return the response as-is +- if (hasUserProvidedEncodingFormat) { ++ if (hasUserProvidedEncodingFormat || body.model.includes('jina')) { + return response; + } + // in this stage, we are sure the user did not specify an encoding_format +diff --git a/resources/embeddings.mjs b/resources/embeddings.mjs +index 0df3c6cc79a520e54acb4c2b5f77c43b774035ff..aa488b8a11b2c413c0a663d9a6059d286d7b5faf 100644 +--- a/resources/embeddings.mjs ++++ b/resources/embeddings.mjs +@@ -10,6 +10,9 @@ export class Embeddings extends APIResource { + // No encoding_format specified, defaulting to base64 for performance reasons + // See https://github.com/openai/openai-node/pull/1312 + let encoding_format = hasUserProvidedEncodingFormat ? body.encoding_format : 'base64'; ++ if (body.model.includes('jina')) { ++ encoding_format = undefined; ++ } + if (hasUserProvidedEncodingFormat) { + Core.debug('Request', 'User defined encoding_format:', body.encoding_format); + } +@@ -21,7 +24,7 @@ export class Embeddings extends APIResource { + ...options, + }); + // if the user specified an encoding_format, return the response as-is +- if (hasUserProvidedEncodingFormat) { ++ if (hasUserProvidedEncodingFormat || body.model.includes('jina')) { + return response; + } + // in this stage, we are sure the user did not specify an encoding_format diff --git a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx index f8535bdd19..368de200a3 100644 --- a/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx +++ b/src/renderer/src/pages/knowledge/components/AddKnowledgePopup.tsx @@ -14,7 +14,7 @@ import { getErrorMessage } from '@renderer/utils/error' import { Form, Input, Modal, Select, Slider } from 'antd' import { find, sortBy } from 'lodash' import { nanoid } from 'nanoid' -import { useRef, useState } from 'react' +import { useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' interface ShowParams { @@ -40,59 +40,69 @@ const PopupContainer: React.FC = ({ title, resolve }) => { const { providers } = useProviders() const { addKnowledgeBase } = useKnowledgeBases() - const allModels = providers - .map((p) => p.models) - .flat() - .filter((model) => isEmbeddingModel(model) && !isRerankModel(model)) + const embeddingModels = useMemo(() => { + return providers + .map((p) => p.models) + .flat() + .filter((model) => isEmbeddingModel(model)) + }, [providers]) - const rerankModels = providers - .map((p) => p.models) - .flat() - .filter((model) => isRerankModel(model)) + const rerankModels = useMemo(() => { + return providers + .map((p) => p.models) + .flat() + .filter((model) => isRerankModel(model)) + }, [providers]) const nameInputRef = useRef(null) - const selectOptions = providers - .filter((p) => p.models.length > 0) - .map((p) => ({ - label: p.isSystem ? t(`provider.${p.id}`) : p.name, - title: p.name, - options: sortBy(p.models, 'name') - .filter((model) => isEmbeddingModel(model) && !isRerankModel(model)) - .map((m) => ({ - label: m.name, - value: getModelUniqId(m) - })) - })) - .filter((group) => group.options.length > 0) + const embeddingSelectOptions = useMemo(() => { + console.log(providers) + return providers + .filter((p) => p.models.length > 0) + .map((p) => ({ + label: p.isSystem ? t(`provider.${p.id}`) : p.name, + title: p.name, + options: sortBy(p.models, 'name') + .filter((model) => isEmbeddingModel(model)) + .map((m) => ({ + label: m.name, + value: getModelUniqId(m), + key: `${p.id}-${m.id}` + })) + })) + .filter((group) => group.options.length > 0) + }, [providers, t]) - const rerankSelectOptions = providers - .filter((p) => p.models.length > 0) - .filter((p) => !NOT_SUPPORTED_REANK_PROVIDERS.includes(p.id)) - .map((p) => ({ - label: p.isSystem ? t(`provider.${p.id}`) : p.name, - title: p.name, - options: sortBy(p.models, 'name') - .filter((model) => isRerankModel(model)) - .map((m) => ({ - label: m.name, - value: getModelUniqId(m) - })) - })) - .filter((group) => group.options.length > 0) + const rerankSelectOptions = useMemo(() => { + return providers + .filter((p) => p.models.length > 0) + .filter((p) => !NOT_SUPPORTED_REANK_PROVIDERS.includes(p.id)) + .map((p) => ({ + label: p.isSystem ? t(`provider.${p.id}`) : p.name, + title: p.name, + options: sortBy(p.models, 'name') + .filter((model) => isRerankModel(model)) + .map((m) => ({ + label: m.name, + value: getModelUniqId(m) + })) + })) + .filter((group) => group.options.length > 0) + }, [providers, t]) const onOk = async () => { try { const values = await form.validateFields() - const selectedModel = find(allModels, JSON.parse(values.model)) as Model + const selectedEmbeddingModel = find(embeddingModels, JSON.parse(values.model)) as Model const selectedRerankModel = values.rerankModel ? (find(rerankModels, JSON.parse(values.rerankModel)) as Model) : undefined - if (selectedModel) { + if (selectedEmbeddingModel) { setLoading(true) - const provider = providers.find((p) => p.id === selectedModel.provider) + const provider = providers.find((p) => p.id === selectedEmbeddingModel.provider) if (!provider) { return @@ -102,7 +112,7 @@ const PopupContainer: React.FC = ({ title, resolve }) => { let dimensions = 0 try { - dimensions = await aiProvider.getEmbeddingDimensions(selectedModel) + dimensions = await aiProvider.getEmbeddingDimensions(selectedEmbeddingModel) } catch (error) { console.error('Error getting embedding dimensions:', error) window.message.error(t('message.error.get_embedding_dimensions') + '\n' + getErrorMessage(error)) @@ -113,7 +123,7 @@ const PopupContainer: React.FC = ({ title, resolve }) => { const newBase = { id: nanoid(), name: values.name, - model: selectedModel, + model: selectedEmbeddingModel, rerankModel: selectedRerankModel, dimensions, documentCount: values.documentCount || DEFAULT_KNOWLEDGE_DOCUMENT_COUNT, @@ -166,7 +176,7 @@ const PopupContainer: React.FC = ({ title, resolve }) => { label={t('models.embedding_model')} tooltip={{ title: t('models.embedding_model_tooltip'), placement: 'right' }} rules={[{ required: true, message: t('message.error.enter.model') }]}> -