From 5771d0c9e8c39b61c7a00480562686d2b423515d Mon Sep 17 00:00:00 2001 From: beyondkmp Date: Mon, 11 Aug 2025 16:35:46 +0800 Subject: [PATCH] refactor: file path improve (#8990) * refactor(FileManager): streamline file path handling in FilesPage and ImageBlock components * refactor(file): implement getSafeFilePath utility for consistent file path handling across loaders and preprocessors * refactor(FileStorage): replace getSafeFilePath with fileStorage.getFilePathById for consistent file path retrieval across services * refactor(file): unify file path retrieval across loaders and preprocessors for improved consistency * refactor(Inputbar, MessageEditor): replace getFileExtension with file.ext for improved file type handling * refactor(FileStorage): simplify getFilePathById method by removing redundant checks for file path retrieval * fix(FileStorage): update getFilePathById to ensure file.path is consistent with generated filePath * refactor(FileStorage): simplify getFilePathById method by removing unnecessary file path consistency checks * fix(FileStorage): update duplicate file check to use file.path for accurate detection * fix(FileStorage): correct file path usage in uploadFile method for accurate duplicate detection * fix(loader): update file path retrieval to use file.path for consistency across loaders --- src/main/ipc.ts | 5 ++--- src/main/knowledge/loader/index.ts | 18 +++++++++------- .../preprocess/Doc2xPreprocessProvider.ts | 21 ++++++++++--------- .../preprocess/MineruPreprocessProvider.ts | 18 ++++++++-------- .../preprocess/MistralPreprocessProvider.ts | 10 +++++---- src/main/services/ExportService.ts | 8 +++---- src/main/services/FileStorage.ts | 15 ++++++++----- src/main/services/KnowledgeService.ts | 6 ++++-- src/main/services/remotefile/GeminiService.ts | 3 ++- .../services/remotefile/MistralService.ts | 3 ++- src/renderer/src/pages/files/FilesPage.tsx | 8 +++++-- .../src/pages/home/Inputbar/Inputbar.tsx | 4 ++-- .../pages/home/Messages/Blocks/ImageBlock.tsx | 5 +++-- .../src/pages/home/Messages/MessageEditor.tsx | 4 ++-- src/renderer/src/services/FileManager.ts | 2 +- 15 files changed, 73 insertions(+), 57 deletions(-) diff --git a/src/main/ipc.ts b/src/main/ipc.ts index d8897311f4..20f3f5ca96 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -20,7 +20,7 @@ import { configManager } from './services/ConfigManager' import CopilotService from './services/CopilotService' import DxtService from './services/DxtService' import { ExportService } from './services/ExportService' -import FileStorage from './services/FileStorage' +import { fileStorage as fileManager } from './services/FileStorage' import FileService from './services/FileSystemService' import KnowledgeService from './services/KnowledgeService' import mcpService from './services/MCPService' @@ -61,9 +61,8 @@ import { compress, decompress } from './utils/zip' const logger = loggerService.withContext('IPC') -const fileManager = new FileStorage() const backupManager = new BackupManager() -const exportService = new ExportService(fileManager) +const exportService = new ExportService() const obsidianVaultService = new ObsidianVaultService() const vertexAIService = VertexAIService.getInstance() const memoryService = MemoryService.getInstance() diff --git a/src/main/knowledge/loader/index.ts b/src/main/knowledge/loader/index.ts index 34ed9b5fda..6d9b4c7ace 100644 --- a/src/main/knowledge/loader/index.ts +++ b/src/main/knowledge/loader/index.ts @@ -73,17 +73,19 @@ export async function addFileLoader( // 获取文件类型，如果没有匹配则默认为文本类型 const loaderType = FILE_LOADER_MAP[file.ext.toLowerCase()] || 'text' let loaderReturn: AddLoaderReturn + // 使用文件的实际路径 + const filePath = file.path // JSON类型处理 let jsonObject = {} let jsonParsed = true - logger.info(`[KnowledgeBase] processing file ${file.path} as ${loaderType} type`) + logger.info(`[KnowledgeBase] processing file ${filePath} as ${loaderType} type`) switch (loaderType) { case 'common': // 内置类型处理 loaderReturn = await ragApplication.addLoader( new LocalPathLoader({ - path: file.path, + path: filePath, chunkSize: base.chunkSize, chunkOverlap: base.chunkOverlap }) as any, @@ -99,7 +101,7 @@ export async function addFileLoader( // epub类型处理 loaderReturn = await ragApplication.addLoader( new EpubLoader({ - filePath: file.path, + filePath: filePath, chunkSize: base.chunkSize ?? 1000, chunkOverlap: base.chunkOverlap ?? 200 }) as any, @@ -109,14 +111,14 @@ export async function addFileLoader( case 'drafts': // Drafts类型处理 - loaderReturn = await ragApplication.addLoader(new DraftsExportLoader(file.path) as any, forceReload) + loaderReturn = await ragApplication.addLoader(new DraftsExportLoader(filePath), forceReload) break case 'html': // HTML类型处理 loaderReturn = await ragApplication.addLoader( new WebLoader({ - urlOrContent: await readTextFileWithAutoEncoding(file.path), + urlOrContent: await readTextFileWithAutoEncoding(filePath), chunkSize: base.chunkSize, chunkOverlap: base.chunkOverlap }) as any, @@ -126,11 +128,11 @@ export async function addFileLoader( case 'json': try { - jsonObject = JSON.parse(await readTextFileWithAutoEncoding(file.path)) + jsonObject = JSON.parse(await readTextFileWithAutoEncoding(filePath)) } catch (error) { jsonParsed = false logger.warn( - `[KnowledgeBase] failed parsing json file, falling back to text processing: ${file.path}`, + `[KnowledgeBase] failed parsing json file, falling back to text processing: ${filePath}`, error as Error ) } @@ -145,7 +147,7 @@ export async function addFileLoader( // 如果是其他文本类型且尚未读取文件，则读取文件 loaderReturn = await ragApplication.addLoader( new TextLoader({ - text: await readTextFileWithAutoEncoding(file.path), + text: await readTextFileWithAutoEncoding(filePath), chunkSize: base.chunkSize, chunkOverlap: base.chunkOverlap }) as any, diff --git a/src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts b/src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts index 56349071c6..afc8d1ba9b 100644 --- a/src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts +++ b/src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts @@ -2,6 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { loggerService } from '@logger' +import { fileStorage } from '@main/services/FileStorage' import { FileMetadata, PreprocessProvider } from '@types' import AdmZip from 'adm-zip' import axios, { AxiosRequestConfig } from 'axios' @@ -54,20 +55,21 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider { public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> { try { - logger.info(`Preprocess processing started: ${file.path}`) + const filePath = fileStorage.getFilePathById(file) + logger.info(`Preprocess processing started: ${filePath}`) // 步骤1: 准备上传 const { uid, url } = await this.preupload() logger.info(`Preprocess preupload completed: uid=${uid}`) - await this.validateFile(file.path) + await this.validateFile(filePath) // 步骤2: 上传文件 - await this.putFile(file.path, url) + await this.putFile(filePath, url) // 步骤3: 等待处理完成 await this.waitForProcessing(sourceId, uid) - logger.info(`Preprocess parsing completed successfully for: ${file.path}`) + logger.info(`Preprocess parsing completed successfully for: ${filePath}`) // 步骤4: 导出文件 const { path: outputPath } = await this.exportFile(file, uid) @@ -77,9 +79,7 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider { processedFile: this.createProcessedFileInfo(file, outputPath) } } catch (error) { - logger.error( - `Preprocess processing failed for ${file.path}: ${error instanceof Error ? error.message : String(error)}` - ) + logger.error(`Preprocess processing failed for:`, error as Error) throw error } } @@ -102,11 +102,12 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider { * @returns 导出文件的路径 */ public async exportFile(file: FileMetadata, uid: string): Promise<{ path: string }> { - logger.info(`Exporting file: ${file.path}`) + const filePath = fileStorage.getFilePathById(file) + logger.info(`Exporting file: ${filePath}`) // 步骤1: 转换文件 - await this.convertFile(uid, file.path) - logger.info(`File conversion completed for: ${file.path}`) + await this.convertFile(uid, filePath) + logger.info(`File conversion completed for: ${filePath}`) // 步骤2: 等待导出并获取URL const exportUrl = await this.waitForExport(uid) diff --git a/src/main/knowledge/preprocess/MineruPreprocessProvider.ts b/src/main/knowledge/preprocess/MineruPreprocessProvider.ts index afc19ae34a..0e29a6443f 100644 --- a/src/main/knowledge/preprocess/MineruPreprocessProvider.ts +++ b/src/main/knowledge/preprocess/MineruPreprocessProvider.ts @@ -2,6 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { loggerService } from '@logger' +import { fileStorage } from '@main/services/FileStorage' import { FileMetadata, PreprocessProvider } from '@types' import AdmZip from 'adm-zip' import axios from 'axios' @@ -63,8 +64,9 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider { file: FileMetadata ): Promise<{ processedFile: FileMetadata; quota: number }> { try { - logger.info(`MinerU preprocess processing started: ${file.path}`) - await this.validateFile(file.path) + const filePath = fileStorage.getFilePathById(file) + logger.info(`MinerU preprocess processing started: ${filePath}`) + await this.validateFile(filePath) // 1. 获取上传URL并上传文件 const batchId = await this.uploadFile(file) @@ -86,7 +88,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider { quota } } catch (error: any) { - logger.error(`MinerU preprocess processing failed for ${file.path}: ${error.message}`) + logger.error(`MinerU preprocess processing failed for:`, error as Error) throw new Error(error.message) } } @@ -205,16 +207,14 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider { try { // 步骤1: 获取上传URL const { batchId, fileUrls } = await this.getBatchUploadUrls(file) - logger.debug(`Got upload URLs for batch: ${batchId}`) - - logger.debug(`batchId: ${batchId}, fileurls: ${fileUrls}`) // 步骤2: 上传文件到获取的URL - await this.putFileToUrl(file.path, fileUrls[0]) - logger.info(`File uploaded successfully: ${file.path}`) + const filePath = fileStorage.getFilePathById(file) + await this.putFileToUrl(filePath, fileUrls[0]) + logger.info(`File uploaded successfully: ${filePath}`, { batchId, fileUrls }) return batchId } catch (error: any) { - logger.error(`Failed to upload file ${file.path}: ${error.message}`) + logger.error(`Failed to upload file:`, error as Error) throw new Error(error.message) } } diff --git a/src/main/knowledge/preprocess/MistralPreprocessProvider.ts b/src/main/knowledge/preprocess/MistralPreprocessProvider.ts index 444e375dcd..d5ad3d4e14 100644 --- a/src/main/knowledge/preprocess/MistralPreprocessProvider.ts +++ b/src/main/knowledge/preprocess/MistralPreprocessProvider.ts @@ -1,6 +1,7 @@ import fs from 'node:fs' import { loggerService } from '@logger' +import { fileStorage } from '@main/services/FileStorage' import { MistralClientManager } from '@main/services/MistralClientManager' import { MistralService } from '@main/services/remotefile/MistralService' import { Mistral } from '@mistralai/mistralai' @@ -38,7 +39,8 @@ export default class MistralPreprocessProvider extends BasePreprocessProvider { private async preupload(file: FileMetadata): Promise { let document: PreuploadResponse - logger.info(`preprocess preupload started for local file: ${file.path}`) + const filePath = fileStorage.getFilePathById(file) + logger.info(`preprocess preupload started for local file: ${filePath}`) if (file.ext.toLowerCase() === '.pdf') { const uploadResponse = await this.fileService.uploadFile(file) @@ -58,7 +60,7 @@ export default class MistralPreprocessProvider extends BasePreprocessProvider { documentUrl: fileUrl.url } } else { - const base64Image = Buffer.from(fs.readFileSync(file.path)).toString('base64') + const base64Image = Buffer.from(fs.readFileSync(filePath)).toString('base64') document = { type: 'image_url', imageUrl: `data:image/png;base64,${base64Image}` @@ -97,8 +99,8 @@ export default class MistralPreprocessProvider extends BasePreprocessProvider { // 使用统一的存储路径：Data/Files/{file.id}/ const conversionId = file.id const outputPath = path.join(this.storageDir, file.id) - // const outputPath = this.storageDir - const outputFileName = path.basename(file.path, path.extname(file.path)) + const filePath = fileStorage.getFilePathById(file) + const outputFileName = path.basename(filePath, path.extname(filePath)) fs.mkdirSync(outputPath, { recursive: true }) const markdownParts: string[] = [] diff --git a/src/main/services/ExportService.ts b/src/main/services/ExportService.ts index c31982b60c..8aa10cb466 100644 --- a/src/main/services/ExportService.ts +++ b/src/main/services/ExportService.ts @@ -21,15 +21,13 @@ import { import { dialog } from 'electron' import MarkdownIt from 'markdown-it' -import FileStorage from './FileStorage' +import { fileStorage } from './FileStorage' const logger = loggerService.withContext('ExportService') export class ExportService { - private fileManager: FileStorage private md: MarkdownIt - constructor(fileManager: FileStorage) { - this.fileManager = fileManager + constructor() { this.md = new MarkdownIt() } @@ -399,7 +397,7 @@ export class ExportService { }) if (filePath) { - await this.fileManager.writeFile(_, filePath, buffer) + await fileStorage.writeFile(_, filePath, buffer) logger.debug('Document exported successfully') } } catch (error) { diff --git a/src/main/services/FileStorage.ts b/src/main/services/FileStorage.ts index 08cf6b9c44..e34c51f299 100644 --- a/src/main/services/FileStorage.ts +++ b/src/main/services/FileStorage.ts @@ -156,7 +156,8 @@ class FileStorage { } public uploadFile = async (_: Electron.IpcMainInvokeEvent, file: FileMetadata): Promise => { - const duplicateFile = await this.findDuplicateFile(file.path) + const filePath = file.path + const duplicateFile = await this.findDuplicateFile(filePath) if (duplicateFile) { return duplicateFile @@ -167,13 +168,13 @@ class FileStorage { const ext = path.extname(origin_name).toLowerCase() const destPath = path.join(this.storageDir, uuid + ext) - logger.info(`[FileStorage] Uploading file: ${file.path}`) + logger.info(`[FileStorage] Uploading file: ${filePath}`) // 根据文件类型选择处理方式 if (imageExts.includes(ext)) { - await this.compressImage(file.path, destPath) + await this.compressImage(filePath, destPath) } else { - await fs.promises.copyFile(file.path, destPath) + await fs.promises.copyFile(filePath, destPath) } const stats = await fs.promises.stat(destPath) @@ -624,6 +625,10 @@ class FileStorage { throw error } } + + public getFilePathById(file: FileMetadata): string { + return path.join(this.storageDir, file.id + file.ext) + } } -export default FileStorage +export const fileStorage = new FileStorage() diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index 78a7e2d4a3..99879390e4 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -27,6 +27,7 @@ import { addFileLoader } from '@main/knowledge/loader' import { NoteLoader } from '@main/knowledge/loader/noteLoader' import PreprocessProvider from '@main/knowledge/preprocess/PreprocessProvider' import Reranker from '@main/knowledge/reranker/Reranker' +import { fileStorage } from '@main/services/FileStorage' import { windowService } from '@main/services/WindowService' import { getDataPath } from '@main/utils' import { getAllFiles } from '@main/utils/file' @@ -689,15 +690,16 @@ class KnowledgeService { if (base.preprocessProvider && file.ext.toLowerCase() === '.pdf') { try { const provider = new PreprocessProvider(base.preprocessProvider.provider, userId) + const filePath = fileStorage.getFilePathById(file) // Check if file has already been preprocessed const alreadyProcessed = await provider.checkIfAlreadyProcessed(file) if (alreadyProcessed) { - logger.debug(`File already preprocess processed, using cached result: ${file.path}`) + logger.debug(`File already preprocess processed, using cached result: ${filePath}`) return alreadyProcessed } // Execute preprocessing - logger.debug(`Starting preprocess processing for scanned PDF: ${file.path}`) + logger.debug(`Starting preprocess processing for scanned PDF: ${filePath}`) const { processedFile, quota } = await provider.parseFile(item.id, file) fileToProcess = processedFile const mainWindow = windowService.getMainWindow() diff --git a/src/main/services/remotefile/GeminiService.ts b/src/main/services/remotefile/GeminiService.ts index b059094420..ba5a8fae80 100644 --- a/src/main/services/remotefile/GeminiService.ts +++ b/src/main/services/remotefile/GeminiService.ts @@ -1,5 +1,6 @@ import { File, Files, FileState, GoogleGenAI } from '@google/genai' import { loggerService } from '@logger' +import { fileStorage } from '@main/services/FileStorage' import { FileListResponse, FileMetadata, FileUploadResponse, Provider } from '@types' import { v4 as uuidv4 } from 'uuid' @@ -29,7 +30,7 @@ export class GeminiService extends BaseFileService { async uploadFile(file: FileMetadata): Promise { try { const uploadResult = await this.fileManager.upload({ - file: file.path, + file: fileStorage.getFilePathById(file), config: { mimeType: 'application/pdf', name: file.id, diff --git a/src/main/services/remotefile/MistralService.ts b/src/main/services/remotefile/MistralService.ts index 05bbf75814..d3867a619d 100644 --- a/src/main/services/remotefile/MistralService.ts +++ b/src/main/services/remotefile/MistralService.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises' import { loggerService } from '@logger' +import { fileStorage } from '@main/services/FileStorage' import { Mistral } from '@mistralai/mistralai' import { FileListResponse, FileMetadata, FileUploadResponse, Provider } from '@types' @@ -21,7 +22,7 @@ export class MistralService extends BaseFileService { async uploadFile(file: FileMetadata): Promise { try { - const fileBuffer = await fs.readFile(file.path) + const fileBuffer = await fs.readFile(fileStorage.getFilePathById(file)) const response = await this.client.files.upload({ file: { fileName: file.origin_name, diff --git a/src/renderer/src/pages/files/FilesPage.tsx b/src/renderer/src/pages/files/FilesPage.tsx index e3690826b5..e988efa6c2 100644 --- a/src/renderer/src/pages/files/FilesPage.tsx +++ b/src/renderer/src/pages/files/FilesPage.tsx @@ -46,11 +46,15 @@ const FilesPage: FC = () => { const dataSource = sortedFiles?.map((file) => { return { key: file.id, - file: window.api.file.openPath(file.path)}>{FileManager.formatFileName(file)}, + file: ( + window.api.file.openPath(FileManager.getFilePath(file))}> + {FileManager.formatFileName(file)} + + ), size: formatFileSize(file.size), size_bytes: file.size, count: file.count, - path: file.path, + path: FileManager.getFilePath(file), ext: file.ext, created_at: dayjs(file.created_at).format('MM-DD HH:mm'), created_at_unix: dayjs(file.created_at).unix(), diff --git a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx index c941b32dc8..b84f4d6274 100644 --- a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx +++ b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx @@ -35,7 +35,7 @@ import { setSearching } from '@renderer/store/runtime' import { sendMessage as _sendMessage } from '@renderer/store/thunk/messageThunk' import { Assistant, FileType, FileTypes, KnowledgeBase, KnowledgeItem, Model, Topic } from '@renderer/types' import type { MessageInputBaseParams } from '@renderer/types/newMessage' -import { classNames, delay, formatFileSize, getFileExtension } from '@renderer/utils' +import { classNames, delay, formatFileSize } from '@renderer/utils' import { formatQuotedText } from '@renderer/utils/formats' import { getFilesFromDropEvent, @@ -590,7 +590,7 @@ const Inputbar: FC = ({ assistant: _assistant, setActiveTopic, topic }) = let supportedFiles = 0 files.forEach((file) => { - if (supportedExts.includes(getFileExtension(file.path))) { + if (supportedExts.includes(file.ext)) { setFiles((prevFiles) => [...prevFiles, file]) supportedFiles++ } diff --git a/src/renderer/src/pages/home/Messages/Blocks/ImageBlock.tsx b/src/renderer/src/pages/home/Messages/Blocks/ImageBlock.tsx index ceb4bcceb8..4ac6b0b0b0 100644 --- a/src/renderer/src/pages/home/Messages/Blocks/ImageBlock.tsx +++ b/src/renderer/src/pages/home/Messages/Blocks/ImageBlock.tsx @@ -1,4 +1,5 @@ import ImageViewer from '@renderer/components/ImageViewer' +import FileManager from '@renderer/services/FileManager' import { type ImageMessageBlock, MessageBlockStatus } from '@renderer/types/newMessage' import { Skeleton } from 'antd' import React from 'react' @@ -13,8 +14,8 @@ const ImageBlock: React.FC = ({ block }) => { if (block.status === MessageBlockStatus.STREAMING || block.status === MessageBlockStatus.SUCCESS) { const images = block.metadata?.generateImageResponse?.images?.length ? block.metadata?.generateImageResponse?.images - : block?.file?.path - ? [`file://${block?.file?.path}`] + : block?.file + ? [`file://${FileManager.getFilePath(block?.file)}`] : [] return ( diff --git a/src/renderer/src/pages/home/Messages/MessageEditor.tsx b/src/renderer/src/pages/home/Messages/MessageEditor.tsx index 09f52a1a3d..170f754006 100644 --- a/src/renderer/src/pages/home/Messages/MessageEditor.tsx +++ b/src/renderer/src/pages/home/Messages/MessageEditor.tsx @@ -10,7 +10,7 @@ import { useAppSelector } from '@renderer/store' import { selectMessagesForTopic } from '@renderer/store/newMessage' import { FileMetadata, FileTypes } from '@renderer/types' import { Message, MessageBlock, MessageBlockStatus, MessageBlockType } from '@renderer/types/newMessage' -import { classNames, getFileExtension } from '@renderer/utils' +import { classNames } from '@renderer/utils' import { getFilesFromDropEvent, isSendMessageKeyPressed } from '@renderer/utils/input' import { createFileBlock, createImageBlock } from '@renderer/utils/messageUtils/create' import { findAllBlocks } from '@renderer/utils/messageUtils/find' @@ -173,7 +173,7 @@ const MessageBlockEditor: FC = ({ message, topicId, onSave, onResend, onC if (files) { let supportedFiles = 0 files.forEach((file) => { - if (extensions.includes(getFileExtension(file.path))) { + if (extensions.includes(file.ext)) { setFiles((prevFiles) => [...prevFiles, file]) supportedFiles++ } diff --git a/src/renderer/src/services/FileManager.ts b/src/renderer/src/services/FileManager.ts index d7a7acc811..f6b6945df4 100644 --- a/src/renderer/src/services/FileManager.ts +++ b/src/renderer/src/services/FileManager.ts @@ -132,7 +132,7 @@ class FileManager { } static getSafePath(file: FileMetadata) { - return this.isDangerFile(file) ? getFileDirectory(file.path) : file.path + return this.isDangerFile(file) ? getFileDirectory(this.getFilePath(file)) : this.getFilePath(file) } static getFileUrl(file: FileMetadata) {