diff --git a/package.json b/package.json index 97adcc809c..b866ba7138 100644 --- a/package.json +++ b/package.json @@ -203,6 +203,7 @@ "husky": "^9.1.7", "i18next": "^23.11.5", "iconv-lite": "^0.6.3", + "isbinaryfile": "5.0.4", "jaison": "^2.0.2", "jest-styled-components": "^7.2.0", "linguist-languages": "^8.0.0", diff --git a/packages/shared/IpcChannel.ts b/packages/shared/IpcChannel.ts index 20bc852e7e..56ebfb3d58 100644 --- a/packages/shared/IpcChannel.ts +++ b/packages/shared/IpcChannel.ts @@ -157,6 +157,7 @@ export enum IpcChannel { File_GetPdfInfo = 'file:getPdfInfo', Fs_Read = 'fs:read', File_OpenWithRelativePath = 'file:openWithRelativePath', + File_IsTextFile = 'file:isTextFile', // file service FileService_Upload = 'file-service:upload', diff --git a/src/main/ipc.ts b/src/main/ipc.ts index f094e8d580..8689ab2c3b 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -444,6 +444,7 @@ export function registerIpc(mainWindow: BrowserWindow, app: Electron.App) { ipcMain.handle(IpcChannel.File_Copy, fileManager.copyFile.bind(fileManager)) ipcMain.handle(IpcChannel.File_BinaryImage, fileManager.binaryImage.bind(fileManager)) ipcMain.handle(IpcChannel.File_OpenWithRelativePath, fileManager.openFileWithRelativePath.bind(fileManager)) + ipcMain.handle(IpcChannel.File_IsTextFile, fileManager.isTextFile.bind(fileManager)) // file service ipcMain.handle(IpcChannel.FileService_Upload, async (_, provider: Provider, file: FileMetadata) => { diff --git a/src/main/services/FileStorage.ts b/src/main/services/FileStorage.ts index f5df9ed3f7..39a16713d7 100644 --- a/src/main/services/FileStorage.ts +++ b/src/main/services/FileStorage.ts @@ -1,7 +1,8 @@ import { loggerService } from '@logger' import { getFilesDir, getFileType, getTempDir, readTextFileWithAutoEncoding } from '@main/utils/file' -import { documentExts, imageExts, MB } from '@shared/config/constant' +import { documentExts, imageExts, KB, MB } from '@shared/config/constant' import { FileMetadata } from '@types' +import chardet from 'chardet' import * as crypto from 'crypto' import { dialog, @@ -15,6 +16,7 @@ import { import * as fs from 'fs' import { writeFileSync } from 'fs' import { readFile } from 'fs/promises' +import { isBinaryFile } from 'isbinaryfile' import officeParser from 'officeparser' import * as path from 'path' import { PDFDocument } from 'pdf-lib' @@ -630,6 +632,34 @@ class FileStorage { public getFilePathById(file: FileMetadata): string { return path.join(this.storageDir, file.id + file.ext) } + + public isTextFile = async (_: Electron.IpcMainInvokeEvent, filePath: string): Promise => { + try { + const isBinary = await isBinaryFile(filePath) + if (isBinary) { + return false + } + + const length = 8 * KB + const fileHandle = await fs.promises.open(filePath, 'r') + const buffer = Buffer.alloc(length) + const { bytesRead } = await fileHandle.read(buffer, 0, length, 0) + await fileHandle.close() + + const sampleBuffer = buffer.subarray(0, bytesRead) + const matches = chardet.analyse(sampleBuffer) + + // 如果检测到的编码置信度较高,认为是文本文件 + if (matches.length > 0 && matches[0].confidence > 0.8) { + return true + } + + return false + } catch (error) { + logger.error('Failed to check if file is text:', error as Error) + return false + } + } } export const fileStorage = new FileStorage() diff --git a/src/preload/index.ts b/src/preload/index.ts index 1a630a19ee..1059826224 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -170,7 +170,8 @@ const api = { base64File: (fileId: string) => ipcRenderer.invoke(IpcChannel.File_Base64File, fileId), pdfInfo: (fileId: string) => ipcRenderer.invoke(IpcChannel.File_GetPdfInfo, fileId), getPathForFile: (file: File) => webUtils.getPathForFile(file), - openFileWithRelativePath: (file: FileMetadata) => ipcRenderer.invoke(IpcChannel.File_OpenWithRelativePath, file) + openFileWithRelativePath: (file: FileMetadata) => ipcRenderer.invoke(IpcChannel.File_OpenWithRelativePath, file), + isTextFile: (filePath: string): Promise => ipcRenderer.invoke(IpcChannel.File_IsTextFile, filePath) }, fs: { read: (pathOrUrl: string, encoding?: BufferEncoding) => ipcRenderer.invoke(IpcChannel.Fs_Read, pathOrUrl, encoding) diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index eef4f124aa..1841716293 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -322,6 +322,7 @@ "expand": "Expand", "file_error": "Error processing file", "file_not_supported": "Model does not support this file type", + "file_not_supported_count": "{{count}} files are not supported", "generate_image": "Generate image", "generate_image_not_supported": "The model does not support generating images.", "knowledge_base": "Knowledge Base", diff --git a/src/renderer/src/i18n/locales/ja-jp.json b/src/renderer/src/i18n/locales/ja-jp.json index 289914cca6..5842f6b24b 100644 --- a/src/renderer/src/i18n/locales/ja-jp.json +++ b/src/renderer/src/i18n/locales/ja-jp.json @@ -322,6 +322,7 @@ "expand": "展開", "file_error": "ファイル処理エラー", "file_not_supported": "モデルはこのファイルタイプをサポートしません", + "file_not_supported_count": "{{count}} 個のファイルはサポートされていません", "generate_image": "画像を生成する", "generate_image_not_supported": "モデルは画像の生成をサポートしていません。", "knowledge_base": "ナレッジベース", diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index ca219486dd..d1a8fff8e5 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -322,6 +322,7 @@ "expand": "Развернуть", "file_error": "Ошибка обработки файла", "file_not_supported": "Модель не поддерживает этот тип файла", + "file_not_supported_count": "{{count}} файлов не поддерживаются", "generate_image": "Сгенерировать изображение", "generate_image_not_supported": "Модель не поддерживает генерацию изображений.", "knowledge_base": "База знаний", diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index ff692f3bbd..812d182c94 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -322,6 +322,7 @@ "expand": "展开", "file_error": "文件处理出错", "file_not_supported": "模型不支持此文件类型", + "file_not_supported_count": "{{count}} 个文件不被支持", "generate_image": "生成图片", "generate_image_not_supported": "模型不支持生成图片", "knowledge_base": "知识库", diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index aef5a324dd..5b0bc186ca 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -322,6 +322,7 @@ "expand": "展開", "file_error": "檔案處理錯誤", "file_not_supported": "模型不支援此檔案類型", + "file_not_supported_count": "{{count}} 個檔案不被支援", "generate_image": "生成圖片", "generate_image_not_supported": "模型不支援生成圖片", "knowledge_base": "知識庫", diff --git a/src/renderer/src/i18n/translate/el-gr.json b/src/renderer/src/i18n/translate/el-gr.json index 7e60d79b49..05c1efa2cf 100644 --- a/src/renderer/src/i18n/translate/el-gr.json +++ b/src/renderer/src/i18n/translate/el-gr.json @@ -322,6 +322,7 @@ "expand": "Επεκτάση", "file_error": "Σφάλμα κατά την επεξεργασία του αρχείου", "file_not_supported": "Το μοντέλο δεν υποστηρίζει αυτό το είδος αρχείων", + "file_not_supported_count": "{{count}} αρχεία δεν υποστηρίζονται", "generate_image": "Δημιουργία εικόνας", "generate_image_not_supported": "Το μοντέλο δεν υποστηρίζει τη δημιουργία εικόνων", "knowledge_base": "Βάση γνώσεων", diff --git a/src/renderer/src/i18n/translate/es-es.json b/src/renderer/src/i18n/translate/es-es.json index ef4234f334..d63cda4c58 100644 --- a/src/renderer/src/i18n/translate/es-es.json +++ b/src/renderer/src/i18n/translate/es-es.json @@ -322,6 +322,7 @@ "expand": "Expandir", "file_error": "Error al procesar el archivo", "file_not_supported": "El modelo no admite este tipo de archivo", + "file_not_supported_count": "{{count}} archivos no soportados", "generate_image": "Generar imagen", "generate_image_not_supported": "El modelo no soporta la generación de imágenes", "knowledge_base": "Base de conocimientos", diff --git a/src/renderer/src/i18n/translate/fr-fr.json b/src/renderer/src/i18n/translate/fr-fr.json index 9e2f724425..a7575df024 100644 --- a/src/renderer/src/i18n/translate/fr-fr.json +++ b/src/renderer/src/i18n/translate/fr-fr.json @@ -322,6 +322,7 @@ "expand": "Développer", "file_error": "Erreur lors du traitement du fichier", "file_not_supported": "Le modèle ne prend pas en charge ce type de fichier", + "file_not_supported_count": "{{count}} fichiers non pris en charge", "generate_image": "Générer une image", "generate_image_not_supported": "Le modèle ne supporte pas la génération d'images", "knowledge_base": "Base de connaissances", diff --git a/src/renderer/src/i18n/translate/pt-pt.json b/src/renderer/src/i18n/translate/pt-pt.json index 9bed40be77..3234ad8215 100644 --- a/src/renderer/src/i18n/translate/pt-pt.json +++ b/src/renderer/src/i18n/translate/pt-pt.json @@ -322,6 +322,7 @@ "expand": "Expandir", "file_error": "Erro ao processar o arquivo", "file_not_supported": "O modelo não suporta este tipo de arquivo", + "file_not_supported_count": "{{count}} arquivos não suportados", "generate_image": "Gerar imagem", "generate_image_not_supported": "Modelo não suporta geração de imagem", "knowledge_base": "Base de conhecimento", diff --git a/src/renderer/src/pages/home/Inputbar/AttachmentButton.tsx b/src/renderer/src/pages/home/Inputbar/AttachmentButton.tsx index dc439e79c7..7e6d583fc3 100644 --- a/src/renderer/src/pages/home/Inputbar/AttachmentButton.tsx +++ b/src/renderer/src/pages/home/Inputbar/AttachmentButton.tsx @@ -1,4 +1,5 @@ -import { FileType } from '@renderer/types' +import { FileMetadata, FileType } from '@renderer/types' +import { filterSupportedFiles } from '@renderer/utils/file' import { Tooltip } from 'antd' import { Paperclip } from 'lucide-react' import { FC, useCallback, useImperativeHandle } from 'react' @@ -30,20 +31,39 @@ const AttachmentButton: FC = ({ const { t } = useTranslation() const onSelectFile = useCallback(async () => { - const _files = await window.api.file.select({ + // when the number of extensions is greater than 20, use *.* to avoid selecting window lag + const useAllFiles = extensions.length > 20 + + const _files: FileMetadata[] = await window.api.file.select({ properties: ['openFile', 'multiSelections'], filters: [ { name: 'Files', - extensions: extensions.map((i) => i.replace('.', '')) + extensions: useAllFiles ? ['*'] : extensions.map((i) => i.replace('.', '')) } ] }) if (_files) { - setFiles([...files, ..._files]) + if (!useAllFiles) { + setFiles([...files, ..._files]) + return + } + const supportedFiles = await filterSupportedFiles(_files, extensions) + if (supportedFiles.length > 0) { + setFiles([...files, ...supportedFiles]) + } + + if (supportedFiles.length !== _files.length) { + window.message.info({ + key: 'file_not_supported', + content: t('chat.input.file_not_supported_count', { + count: _files.length - supportedFiles.length + }) + }) + } } - }, [extensions, files, setFiles]) + }, [extensions, files, setFiles, t]) const openQuickPanel = useCallback(() => { onSelectFile() diff --git a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx index 82a157adba..f36af1635e 100644 --- a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx +++ b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx @@ -37,7 +37,7 @@ import { setSearching } from '@renderer/store/runtime' import { sendMessage as _sendMessage } from '@renderer/store/thunk/messageThunk' import { Assistant, FileType, FileTypes, KnowledgeBase, KnowledgeItem, Model, Topic } from '@renderer/types' import type { MessageInputBaseParams } from '@renderer/types/newMessage' -import { classNames, delay, formatFileSize } from '@renderer/utils' +import { classNames, delay, filterSupportedFiles, formatFileSize } from '@renderer/utils' import { formatQuotedText } from '@renderer/utils/formats' import { getFilesFromDropEvent, @@ -585,26 +585,20 @@ const Inputbar: FC = ({ assistant: _assistant, setActiveTopic, topic }) = setText(text + data) - const files = await getFilesFromDropEvent(e).catch((err) => { + const droppedFiles = await getFilesFromDropEvent(e).catch((err) => { logger.error('handleDrop:', err) return null }) - if (files) { - let supportedFiles = 0 - - files.forEach((file) => { - if (supportedExts.includes(file.ext)) { - setFiles((prevFiles) => [...prevFiles, file]) - supportedFiles++ - } - }) - - // 如果有文件,但都不支持 - if (files.length > 0 && supportedFiles === 0) { + if (droppedFiles) { + const supportedFiles = await filterSupportedFiles(droppedFiles, supportedExts) + supportedFiles.length > 0 && setFiles((prevFiles) => [...prevFiles, ...supportedFiles]) + if (droppedFiles.length > 0 && supportedFiles.length !== droppedFiles.length) { window.message.info({ key: 'file_not_supported', - content: t('chat.input.file_not_supported') + content: t('chat.input.file_not_supported_count', { + count: droppedFiles.length - supportedFiles.length + }) }) } } diff --git a/src/renderer/src/services/FileManager.ts b/src/renderer/src/services/FileManager.ts index f6b6945df4..4b780cfa94 100644 --- a/src/renderer/src/services/FileManager.ts +++ b/src/renderer/src/services/FileManager.ts @@ -132,7 +132,9 @@ class FileManager { } static getSafePath(file: FileMetadata) { - return this.isDangerFile(file) ? getFileDirectory(this.getFilePath(file)) : this.getFilePath(file) + // use the path from the file metadata instead + // this function is used to get path for files which are not in the filestorage + return this.isDangerFile(file) ? getFileDirectory(file.path) : file.path } static getFileUrl(file: FileMetadata) { diff --git a/src/renderer/src/services/PasteService.ts b/src/renderer/src/services/PasteService.ts index 845ec536dd..277bc9ef66 100644 --- a/src/renderer/src/services/PasteService.ts +++ b/src/renderer/src/services/PasteService.ts @@ -1,6 +1,6 @@ import { loggerService } from '@logger' import { FileMetadata } from '@renderer/types' -import { getFileExtension } from '@renderer/utils' +import { getFileExtension, isSupportedFile } from '@renderer/utils' const logger = loggerService.withContext('PasteService') @@ -60,6 +60,7 @@ export const handlePaste = async ( // 2. 文件/图片粘贴(仅在无文本时处理) if (event.clipboardData?.files && event.clipboardData.files.length > 0) { event.preventDefault() + const extensionSet = new Set(supportExts) try { for (const file of event.clipboardData.files) { // 使用新的API获取文件路径 @@ -90,7 +91,7 @@ export const handlePaste = async ( } // 有路径的情况 - if (supportExts.includes(getFileExtension(filePath))) { + if (await isSupportedFile(filePath, extensionSet)) { const selectedFile = await window.api.file.get(filePath) if (selectedFile) { setFiles((prevFiles) => [...prevFiles, selectedFile]) diff --git a/src/renderer/src/utils/file.ts b/src/renderer/src/utils/file.ts index 0c0f5039a6..6218e3f5c1 100644 --- a/src/renderer/src/utils/file.ts +++ b/src/renderer/src/utils/file.ts @@ -1,3 +1,4 @@ +import { FileMetadata } from '@renderer/types' import { KB, MB } from '@shared/config/constant' /** @@ -55,3 +56,30 @@ export function removeSpecialCharactersForFileName(str: string): string { .replace(/[\r\n]+/g, ' ') .trim() } + +export async function isSupportedFile(filePath: string, supportExts: Set): Promise { + try { + if (supportExts.has(getFileExtension(filePath))) { + return true + } + + if (await window.api.file.isTextFile(filePath)) { + return true + } + + return false + } catch (error) { + return false + } +} + +export async function filterSupportedFiles(files: FileMetadata[], supportExts: string[]): Promise { + const extensionSet = new Set(supportExts) + const validationResults = await Promise.all( + files.map(async (file) => ({ + file, + isValid: await isSupportedFile(file.path, extensionSet) + })) + ) + return validationResults.filter((result) => result.isValid).map((result) => result.file) +} diff --git a/yarn.lock b/yarn.lock index c0950f0a4a..b9833e0cc3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8510,6 +8510,7 @@ __metadata: husky: "npm:^9.1.7" i18next: "npm:^23.11.5" iconv-lite: "npm:^0.6.3" + isbinaryfile: "npm:5.0.4" jaison: "npm:^2.0.2" jest-styled-components: "npm:^7.2.0" jsdom: "npm:26.1.0" @@ -14389,6 +14390,13 @@ __metadata: languageName: node linkType: hard +"isbinaryfile@npm:5.0.4, isbinaryfile@npm:^5.0.0": + version: 5.0.4 + resolution: "isbinaryfile@npm:5.0.4" + checksum: 10c0/fea255bfae67ff4827e8dd2238d6700d4803d02b4d892b72eeac4541487284e901251a3427966af5018d4eb29fa155b036dcb75dd217634146a072991afbc2c2 + languageName: node + linkType: hard + "isbinaryfile@npm:^4.0.8": version: 4.0.10 resolution: "isbinaryfile@npm:4.0.10" @@ -14396,13 +14404,6 @@ __metadata: languageName: node linkType: hard -"isbinaryfile@npm:^5.0.0": - version: 5.0.4 - resolution: "isbinaryfile@npm:5.0.4" - checksum: 10c0/fea255bfae67ff4827e8dd2238d6700d4803d02b4d892b72eeac4541487284e901251a3427966af5018d4eb29fa155b036dcb75dd217634146a072991afbc2c2 - languageName: node - linkType: hard - "isexe@npm:^2.0.0": version: 2.0.0 resolution: "isexe@npm:2.0.0"