diff --git a/electron-builder.yml b/electron-builder.yml index e3ab493666..f732860e94 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -66,6 +66,7 @@ asarUnpack: - resources/** - "**/*.{metal,exp,lib}" - "node_modules/@img/sharp-libvips-*/**" + - "node_modules/node-screenshots-*/**" # copy from node_modules/claude-code-plugins/plugins to resources/data/claude-code-pluginso extraResources: diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 172d48ca9a..ab45617ee6 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -116,7 +116,8 @@ export default defineConfig({ miniWindow: resolve(__dirname, 'src/renderer/miniWindow.html'), selectionToolbar: resolve(__dirname, 'src/renderer/selectionToolbar.html'), selectionAction: resolve(__dirname, 'src/renderer/selectionAction.html'), - traceWindow: resolve(__dirname, 'src/renderer/traceWindow.html') + traceWindow: resolve(__dirname, 'src/renderer/traceWindow.html'), + screenshotSelection: resolve(__dirname, 'src/renderer/screenshotSelection.html') }, onwarn(warning, warn) { if (warning.code === 'COMMONJS_VARIABLE_IN_ESM') return diff --git a/package.json b/package.json index a70663ffc8..932f01d555 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "gray-matter": "^4.0.3", "js-yaml": "^4.1.0", "jsdom": "26.1.0", + "node-screenshots": "0.2.1", "node-stream-zip": "^1.15.0", "officeparser": "^4.2.0", "os-proxy-config": "^1.1.2", diff --git a/packages/shared/IpcChannel.ts b/packages/shared/IpcChannel.ts index aec1d57b43..7e41e3e9a3 100644 --- a/packages/shared/IpcChannel.ts +++ b/packages/shared/IpcChannel.ts @@ -386,5 +386,13 @@ export enum IpcChannel { WebSocket_Stop = 'webSocket:stop', WebSocket_Status = 'webSocket:status', WebSocket_SendFile = 'webSocket:send-file', - WebSocket_GetAllCandidates = 'webSocket:get-all-candidates' + WebSocket_GetAllCandidates = 'webSocket:get-all-candidates', + + // Screenshot + Screenshot_CheckPermission = 'screenshot:check-permission', + Screenshot_Capture = 'screenshot:capture', + Screenshot_CaptureWithSelection = 'screenshot:capture-with-selection', + Screenshot_SelectionWindowReady = 'screenshot:selection-window-ready', + Screenshot_SelectionConfirm = 'screenshot:selection-confirm', + Screenshot_SelectionCancel = 'screenshot:selection-cancel' } diff --git a/src/main/ipc.ts b/src/main/ipc.ts index 4cb3402414..f7e503195e 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -61,6 +61,7 @@ import powerMonitorService from './services/PowerMonitorService' import { proxyManager } from './services/ProxyManager' import { pythonService } from './services/PythonService' import { FileServiceManager } from './services/remotefile/FileServiceManager' +import { screenshotService } from './services/ScreenshotService' import { searchService } from './services/SearchService' import { SelectionService } from './services/SelectionService' import { registerShortcuts, unregisterAllShortcuts } from './services/ShortcutService' @@ -615,6 +616,39 @@ export function registerIpc(mainWindow: BrowserWindow, app: Electron.App) { ipcMain.handle(IpcChannel.File_GetPdfInfo, fileManager.pdfPageCount.bind(fileManager)) ipcMain.handle(IpcChannel.File_Download, fileManager.downloadFile.bind(fileManager)) ipcMain.handle(IpcChannel.File_Copy, fileManager.copyFile.bind(fileManager)) + + // Screenshot + ipcMain.handle(IpcChannel.Screenshot_CheckPermission, async () => { + if (!isMac) { + return { status: 'granted' as const } + } + + const status = systemPreferences.getMediaAccessStatus('screen') + return { + status: status === 'granted' ? ('granted' as const) : ('denied' as const) + } + }) + + // Screenshot + ipcMain.handle(IpcChannel.Screenshot_Capture, async (_event, fileName: string) => { + return await screenshotService.capture(fileName) + }) + + ipcMain.handle(IpcChannel.Screenshot_CaptureWithSelection, async (_event, fileName: string) => { + return await screenshotService.captureWithSelection(fileName) + }) + + ipcMain.handle( + IpcChannel.Screenshot_SelectionConfirm, + async (_event, selection: { x: number; y: number; width: number; height: number }) => { + screenshotService.confirmSelection(selection) + } + ) + + ipcMain.handle(IpcChannel.Screenshot_SelectionCancel, async () => { + screenshotService.cancelSelection() + }) + ipcMain.handle(IpcChannel.File_BinaryImage, fileManager.binaryImage.bind(fileManager)) ipcMain.handle(IpcChannel.File_OpenWithRelativePath, fileManager.openFileWithRelativePath.bind(fileManager)) ipcMain.handle(IpcChannel.File_IsTextFile, fileManager.isTextFile.bind(fileManager)) diff --git a/src/main/services/ScreenshotService.ts b/src/main/services/ScreenshotService.ts new file mode 100644 index 0000000000..d685c5338d --- /dev/null +++ b/src/main/services/ScreenshotService.ts @@ -0,0 +1,353 @@ +import { loggerService } from '@logger' +import { isDev, isMac, isWin } from '@main/constant' +import { fileStorage } from '@main/services/FileStorage' +import { IpcChannel } from '@shared/IpcChannel' +import type { FileMetadata } from '@types' +import { FileTypes } from '@types' +import { BrowserWindow, screen, systemPreferences } from 'electron' +import fs from 'fs' +import { join } from 'path' +import path from 'path' +import { v4 as uuidv4 } from 'uuid' + +const logger = loggerService.withContext('ScreenshotService') + +type PermissionStatus = 'granted' | 'denied' + +type CaptureResult = + | { success: true; file: FileMetadata } + | { success: false; status: PermissionStatus; needsRestart: boolean; message: string } + +type SelectionCaptureResult = + | { success: true; file: FileMetadata } + | { success: false; status: 'cancelled' | 'denied' | 'error'; needsRestart?: boolean; message: string } + +interface Rectangle { + x: number + y: number + width: number + height: number +} + +class ScreenshotService { + private selectionWindow: BrowserWindow | null = null + private screenshotBuffer: Buffer | null = null + private screenshotData: string | null = null + private screenshotTempPath: string | null = null + private currentFileName: string | null = null + private selectionPromise: { + resolve: (value: SelectionCaptureResult) => void + reject: (reason?: any) => void + } | null = null + public async capture(fileName: string): Promise { + try { + const Screenshots = await import('node-screenshots') + + // Try to capture - this will trigger permission dialog if needed + const monitor = Screenshots.Monitor.fromPoint(0, 0) ?? Screenshots.Monitor.all()[0] + + if (!monitor) { + logger.error('No monitor found for screenshot') + return { + success: false, + status: 'denied', + needsRestart: false, + message: 'No monitor found' + } + } + + const image = await monitor.captureImage() + const buffer = await image.toPng() + + const ext = '.png' + const tempFilePath = await fileStorage.createTempFile({} as any, fileName || `screenshot${ext}`) + await fs.promises.writeFile(tempFilePath, buffer) + + const stats = await fs.promises.stat(tempFilePath) + const id = uuidv4() + const file: FileMetadata = { + id, + name: `${id}${ext}`, + origin_name: path.basename(fileName || 'screenshot.png'), + path: tempFilePath, + size: stats.size, + ext, + type: FileTypes.IMAGE, + created_at: new Date().toISOString(), + count: 1 + } + + return { success: true, file } + } catch (error) { + logger.error('Screenshot capture failed', error as Error) + + // Check if it's a permission issue on macOS + if (process.platform === 'darwin') { + const status = systemPreferences.getMediaAccessStatus('screen') + logger.info('Permission status after error:', { status }) + + if (status !== 'granted') { + return { + success: false, + status: 'denied', + needsRestart: status === 'not-determined' ? false : true, + message: 'Screen recording permission required' + } + } + } + + return { + success: false, + status: 'denied', + needsRestart: false, + message: error instanceof Error ? error.message : 'Screenshot capture failed' + } + } + } + + private createSelectionWindow(): BrowserWindow { + const display = screen.getPrimaryDisplay() + const { width, height } = display.bounds + + const window = new BrowserWindow({ + width, + height, + x: display.bounds.x, + y: display.bounds.y, + show: false, + frame: false, + transparent: true, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + minimizable: false, + maximizable: false, + fullscreenable: false, + movable: false, + hasShadow: false, + enableLargerThanScreen: true, + + // Platform specific settings + ...(isWin ? { type: 'toolbar', focusable: false } : { type: 'panel' }), + ...(isMac && { hiddenInMissionControl: true, acceptFirstMouse: true, visibleOnAllWorkspaces: true }), + + webPreferences: { + preload: join(__dirname, '../preload/index.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: false, + devTools: isDev ? true : false + } + }) + + // Clean up when closed + window.on('closed', async () => { + await this.cleanupSelection() + }) + + // Load the selection HTML + if (isDev) { + window.loadURL(`http://localhost:5173/screenshotSelection.html`) + } else { + window.loadFile(join(__dirname, '../renderer/screenshotSelection.html')) + } + + return window + } + + private async cleanupSelection() { + if (this.selectionWindow && !this.selectionWindow.isDestroyed()) { + this.selectionWindow.destroy() + } + this.selectionWindow = null + this.screenshotBuffer = null + this.screenshotData = null + this.currentFileName = null + this.selectionPromise = null + + // Clean up temporary file + if (this.screenshotTempPath) { + try { + await fs.promises.unlink(this.screenshotTempPath) + } catch (error) { + logger.warn('Failed to delete temporary screenshot file', error as Error) + } + this.screenshotTempPath = null + } + } + + public async confirmSelection(selection: Rectangle): Promise { + if (!this.selectionPromise) { + logger.warn('No active selection to confirm') + return + } + + // Validate selection + if (selection.width < 10 || selection.height < 10) { + this.selectionPromise.resolve({ + success: false, + status: 'error', + message: 'Selection too small (minimum 10×10 pixels)' + }) + await this.cleanupSelection() + return + } + + // Process the selection + this.processSelection(selection) + } + + public async cancelSelection(): Promise { + if (!this.selectionPromise) { + logger.warn('No active selection to cancel') + return + } + + this.selectionPromise.resolve({ + success: false, + status: 'cancelled', + message: 'User cancelled selection' + }) + await this.cleanupSelection() + } + + private async processSelection(selection: Rectangle): Promise { + if (!this.screenshotBuffer || !this.selectionPromise) { + logger.error('Missing screenshot buffer or selection promise') + this.cleanupSelection() + return + } + + try { + // Get display scale factor for HiDPI support + const cursorPoint = screen.getCursorScreenPoint() + const display = screen.getDisplayNearestPoint(cursorPoint) + const scaleFactor = display.scaleFactor + + // Crop the screenshot + const croppedBuffer = await this.cropScreenshot(this.screenshotBuffer, selection, scaleFactor) + + // Save the cropped image + const ext = '.png' + const fileName = this.currentFileName || `screenshot_${Date.now()}${ext}` + const tempFilePath = await fileStorage.createTempFile({} as any, fileName) + await fs.promises.writeFile(tempFilePath, croppedBuffer) + + const stats = await fs.promises.stat(tempFilePath) + const id = uuidv4() + const file: FileMetadata = { + id, + name: `${id}${ext}`, + origin_name: fileName, + path: tempFilePath, + size: stats.size, + ext, + type: FileTypes.IMAGE, + created_at: new Date().toISOString(), + count: 1 + } + + this.selectionPromise.resolve({ success: true, file }) + } catch (error) { + logger.error('Failed to process selection', error as Error) + this.selectionPromise.resolve({ + success: false, + status: 'error', + message: error instanceof Error ? error.message : 'Failed to process selection' + }) + } finally { + await this.cleanupSelection() + } + } + + private async cropScreenshot(buffer: Buffer, selection: Rectangle, scaleFactor: number): Promise { + const sharp = (await import('sharp')).default + + return sharp(buffer) + .extract({ + left: Math.round(selection.x * scaleFactor), + top: Math.round(selection.y * scaleFactor), + width: Math.round(selection.width * scaleFactor), + height: Math.round(selection.height * scaleFactor) + }) + .png() + .toBuffer() + } + + public async captureWithSelection(fileName: string): Promise { + try { + // Try to capture - this will trigger permission dialog if needed + const Screenshots = await import('node-screenshots') + const monitor = Screenshots.Monitor.fromPoint(0, 0) ?? Screenshots.Monitor.all()[0] + + if (!monitor) { + logger.error('No monitor found for screenshot') + return { + success: false, + status: 'error', + message: 'No monitor found' + } + } + + const image = await monitor.captureImage() + const buffer = await image.toPng() + + // Store the buffer and fileName for later cropping + this.screenshotBuffer = buffer + this.currentFileName = fileName + + // Write buffer to a temporary file and store the file URL for the renderer + const tempFilePath = await fileStorage.createTempFile({} as any, `screenshot-${uuidv4()}.png`) + await fs.promises.writeFile(tempFilePath, buffer) + this.screenshotTempPath = tempFilePath + this.screenshotData = `file://${tempFilePath}` + + // Create or show selection window + if (!this.selectionWindow || this.selectionWindow.isDestroyed()) { + this.selectionWindow = this.createSelectionWindow() + } + + // Return a promise that resolves when user confirms or cancels + return new Promise((resolve, reject) => { + this.selectionPromise = { resolve, reject } + + // Show the window and send screenshot data + this.selectionWindow!.once('ready-to-show', () => { + this.selectionWindow!.show() + this.selectionWindow!.focus() + + // Send screenshot data to renderer + this.selectionWindow!.webContents.send(IpcChannel.Screenshot_SelectionWindowReady, { + screenshotData: this.screenshotData + }) + }) + }) + } catch (error) { + logger.error('Screenshot capture with selection failed', error as Error) + await this.cleanupSelection() + + // Check if it's a permission issue on macOS + if (process.platform === 'darwin') { + const status = systemPreferences.getMediaAccessStatus('screen') + logger.info('Permission status after error:', { status }) + + if (status !== 'granted') { + return { + success: false, + status: 'denied', + needsRestart: status === 'not-determined' ? false : true, + message: 'Screen recording permission required' + } + } + } + + return { + success: false, + status: 'error', + message: error instanceof Error ? error.message : 'Screenshot capture failed' + } + } + } +} + +export const screenshotService = new ScreenshotService() diff --git a/src/preload/index.ts b/src/preload/index.ts index dc08e9a2df..5c4f68e728 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -137,6 +137,25 @@ const api = { compress: (text: string) => ipcRenderer.invoke(IpcChannel.Zip_Compress, text), decompress: (text: Buffer) => ipcRenderer.invoke(IpcChannel.Zip_Decompress, text) }, + screenshot: { + checkPermission: (): Promise<{ status: 'granted' | 'denied' }> => + ipcRenderer.invoke(IpcChannel.Screenshot_CheckPermission), + capture: ( + fileName: string + ): Promise< + | { success: true; file: FileMetadata } + | { success: false; status: 'granted' | 'denied'; needsRestart: boolean; message: string } + > => ipcRenderer.invoke(IpcChannel.Screenshot_Capture, fileName), + captureWithSelection: ( + fileName: string + ): Promise< + | { success: true; file: FileMetadata } + | { success: false; status: 'cancelled' | 'denied' | 'error'; needsRestart?: boolean; message: string } + > => ipcRenderer.invoke(IpcChannel.Screenshot_CaptureWithSelection, fileName), + confirmSelection: (selection: { x: number; y: number; width: number; height: number }): Promise => + ipcRenderer.invoke(IpcChannel.Screenshot_SelectionConfirm, selection), + cancelSelection: (): Promise => ipcRenderer.invoke(IpcChannel.Screenshot_SelectionCancel) + }, backup: { backup: (filename: string, content: string, path: string, skipBackupFile: boolean) => ipcRenderer.invoke(IpcChannel.Backup_Backup, filename, content, path, skipBackupFile), diff --git a/src/renderer/screenshotSelection.html b/src/renderer/screenshotSelection.html new file mode 100644 index 0000000000..1050d1e8c7 --- /dev/null +++ b/src/renderer/screenshotSelection.html @@ -0,0 +1,47 @@ + + + + + + + Cherry Studio Screenshot Selection + + + +
+ + + + diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index f4012363e3..b9da3a5a92 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -726,6 +726,18 @@ "pause": "Pause", "placeholder": "Type your message here, press {{key}} to send - @ to Select Model, / to Include Tools", "placeholder_without_triggers": "Type your message here, press {{key}} to send", + "screenshot": { + "cancel": "Cancel", + "capture_failed": "Failed to capture screenshot", + "full_screen": "Full Screen", + "open_settings": "Open Settings", + "permission_denied": "Screen recording permission required", + "permission_dialog_content": "Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "Screen Recording Permission Required", + "permission_granted_restart": "Permission has been granted. Please restart the application for the changes to take effect.", + "select_region": "Select Region", + "tooltip": "Take screenshot" + }, "send": "Send", "settings": "Settings", "slash_commands": { diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index 0e5b2f60e7..b70c2eaa4f 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -726,6 +726,18 @@ "pause": "暂停", "placeholder": "在这里输入消息，按 {{key}} 发送 - @ 选择模型， / 选择工具", "placeholder_without_triggers": "在这里输入消息，按 {{key}} 发送", + "screenshot": { + "cancel": "取消", + "capture_failed": "截屏失败", + "full_screen": "全屏截图", + "open_settings": "打开设置", + "permission_denied": "需要屏幕录制权限", + "permission_dialog_content": "使用截屏功能需要授予屏幕录制权限。是否要打开系统设置进行授权？", + "permission_dialog_title": "需要屏幕录制权限", + "permission_granted_restart": "权限已授予，请重启应用以使权限生效。", + "select_region": "选择区域", + "tooltip": "截屏" + }, "send": "发送", "settings": "设置", "slash_commands": { diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index 9625c68386..bfd627ad03 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -726,6 +726,18 @@ "pause": "暫停", "placeholder": "在此輸入您的訊息，按 {{key}} 傳送 - @ 選擇模型，/ 包含工具", "placeholder_without_triggers": "在此輸入您的訊息，按 {{key}} 傳送", + "screenshot": { + "cancel": "取消", + "capture_failed": "截圖失敗", + "full_screen": "全螢幕截圖", + "open_settings": "打開設定", + "permission_denied": "需要螢幕錄製權限", + "permission_dialog_content": "使用截圖功能需要授��螢幕錄製權限。是否要打開系統設定進行授權？", + "permission_dialog_title": "需要螢幕錄製權限", + "permission_granted_restart": "權限已授予，請重啟應用程式以使權限生效。", + "select_region": "選擇區域", + "tooltip": "截圖" + }, "send": "傳送", "settings": "設定", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/de-de.json b/src/renderer/src/i18n/translate/de-de.json index b3acb49950..a2e2a7b22e 100644 --- a/src/renderer/src/i18n/translate/de-de.json +++ b/src/renderer/src/i18n/translate/de-de.json @@ -726,6 +726,16 @@ "pause": "Pause", "placeholder": "Geben Sie hier eine Nachricht ein, drücken Sie {{key}} zum Senden - @ für Modellauswahl, / für Tools", "placeholder_without_triggers": "Geben Sie hier eine Nachricht ein, drücken Sie {{key}} zum Senden", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Senden", "settings": "Einstellungen", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/el-gr.json b/src/renderer/src/i18n/translate/el-gr.json index ae7b855646..82b02eabac 100644 --- a/src/renderer/src/i18n/translate/el-gr.json +++ b/src/renderer/src/i18n/translate/el-gr.json @@ -726,6 +726,16 @@ "pause": "Παύση", "placeholder": "Εισάγετε μήνυμα εδώ...", "placeholder_without_triggers": "Γράψτε το μήνυμά σας εδώ, πατήστε {{key}} για αποστολή", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Αποστολή", "settings": "Ρυθμίσεις", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/es-es.json b/src/renderer/src/i18n/translate/es-es.json index 26b499cba2..3d3a8a1db3 100644 --- a/src/renderer/src/i18n/translate/es-es.json +++ b/src/renderer/src/i18n/translate/es-es.json @@ -726,6 +726,16 @@ "pause": "Pausar", "placeholder": "Escribe aquí tu mensaje...", "placeholder_without_triggers": "Escribe tu mensaje aquí, presiona {{key}} para enviar", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Enviar", "settings": "Configuración", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/fr-fr.json b/src/renderer/src/i18n/translate/fr-fr.json index 4dff56d7e9..c1ce13dcdf 100644 --- a/src/renderer/src/i18n/translate/fr-fr.json +++ b/src/renderer/src/i18n/translate/fr-fr.json @@ -726,6 +726,16 @@ "pause": "Pause", "placeholder": "Entrez votre message ici...", "placeholder_without_triggers": "Tapez votre message ici, appuyez sur {{key}} pour envoyer", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Envoyer", "settings": "Paramètres", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/ja-jp.json b/src/renderer/src/i18n/translate/ja-jp.json index 090a1927cd..b2e5e2b30c 100644 --- a/src/renderer/src/i18n/translate/ja-jp.json +++ b/src/renderer/src/i18n/translate/ja-jp.json @@ -726,6 +726,16 @@ "pause": "一時停止", "placeholder": "ここにメッセージを入力し、{{key}} を押して送信...", "placeholder_without_triggers": "ここにメッセージを入力し、{{key}} を押して送信...", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "送信", "settings": "設定", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/pt-pt.json b/src/renderer/src/i18n/translate/pt-pt.json index 50cc4fae03..e91286f7c5 100644 --- a/src/renderer/src/i18n/translate/pt-pt.json +++ b/src/renderer/src/i18n/translate/pt-pt.json @@ -726,6 +726,16 @@ "pause": "Pausar", "placeholder": "Digite sua mensagem aqui...", "placeholder_without_triggers": "Escreve a tua mensagem aqui, pressiona {{key}} para enviar", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Enviar", "settings": "Configurações", "slash_commands": { diff --git a/src/renderer/src/i18n/translate/ru-ru.json b/src/renderer/src/i18n/translate/ru-ru.json index 8a6a781451..a93cc9a8d9 100644 --- a/src/renderer/src/i18n/translate/ru-ru.json +++ b/src/renderer/src/i18n/translate/ru-ru.json @@ -726,6 +726,16 @@ "pause": "Остановить", "placeholder": "Введите ваше сообщение здесь, нажмите {{key}} для отправки...", "placeholder_without_triggers": "Напишите сообщение здесь, нажмите {{key}} для отправки", + "screenshot": { + "cancel": "[to be translated]:Cancel", + "capture_failed": "[to be translated]:Failed to capture screenshot", + "open_settings": "[to be translated]:Open Settings", + "permission_denied": "[to be translated]:Screen recording permission required", + "permission_dialog_content": "[to be translated]:Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?", + "permission_dialog_title": "[to be translated]:Screen Recording Permission Required", + "permission_granted_restart": "[to be translated]:Permission has been granted. Please restart the application for the changes to take effect.", + "tooltip": "[to be translated]:Take screenshot" + }, "send": "Отправить", "settings": "Настройки", "slash_commands": { diff --git a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx index fc95082e50..b76f0f7cc6 100644 --- a/src/renderer/src/pages/home/Inputbar/Inputbar.tsx +++ b/src/renderer/src/pages/home/Inputbar/Inputbar.tsx @@ -188,25 +188,21 @@ const InputbarInner: FC = ({ assistant: initialAssistant, se }, [isGenerateImageSupported, isVisionSupported]) const supportedExts = useMemo(() => { - if (canAddImageFile && canAddTextFile) { - return [...imageExts, ...documentExts, ...textExts] - } - - if (canAddImageFile) { - return [...imageExts] - } - - if (canAddTextFile) { - return [...documentExts, ...textExts] - } - - return [] - }, [canAddImageFile, canAddTextFile]) + // Always allow images so screenshot files can be attached regardless of model caps + const image = imageExts + const text = canAddTextFile ? [...documentExts, ...textExts] : [] + return [...image, ...text] + }, [canAddTextFile]) useEffect(() => { setCouldAddImageFile(canAddImageFile) }, [canAddImageFile, setCouldAddImageFile]) + // Ensure screenshot files (images) are allowed + useEffect(() => { + setCouldAddImageFile(true) + }, [setCouldAddImageFile]) + const onUnmount = useEffectEvent((id: string) => { CacheService.set(getMentionedModelsCacheKey(id), mentionedModels, DRAFT_CACHE_TTL) }) diff --git a/src/renderer/src/pages/home/Inputbar/tools/index.ts b/src/renderer/src/pages/home/Inputbar/tools/index.ts index fade68e2ac..0ee20547b5 100644 --- a/src/renderer/src/pages/home/Inputbar/tools/index.ts +++ b/src/renderer/src/pages/home/Inputbar/tools/index.ts @@ -14,6 +14,7 @@ import './generateImageTool' import './clearTopicTool' import './toggleExpandTool' import './newContextTool' +import './screenshotTool' // Agent Session tools import './createSessionTool' import './slashCommandsTool' diff --git a/src/renderer/src/pages/home/Inputbar/tools/screenshotTool.tsx b/src/renderer/src/pages/home/Inputbar/tools/screenshotTool.tsx new file mode 100644 index 0000000000..caab2159e1 --- /dev/null +++ b/src/renderer/src/pages/home/Inputbar/tools/screenshotTool.tsx @@ -0,0 +1,160 @@ +import { loggerService } from '@logger' +import { ActionIconButton } from '@renderer/components/Buttons' +import { defineTool, registerTool, TopicType } from '@renderer/pages/home/Inputbar/types' +import type { FileType } from '@renderer/types' +import { FileTypes } from '@renderer/types' +import type { MenuProps } from 'antd' +import { Dropdown } from 'antd' +import { Camera, ChevronDown } from 'lucide-react' +import { useCallback, useState } from 'react' + +const logger = loggerService.withContext('ScreenshotTool') + +const ScreenshotTool = ({ context }) => { + const { actions, t } = context + const [isCapturing, setIsCapturing] = useState(false) + + const showPermissionDialog = useCallback( + (needsRestart: boolean = false) => { + const content = needsRestart + ? (t('chat.input.screenshot.permission_granted_restart') ?? + 'Permission has been granted. Please restart the application for the changes to take effect.') + : (t('chat.input.screenshot.permission_dialog_content') ?? + 'Screenshot feature requires screen recording permission. Would you like to open system settings to grant permission?') + + window.modal.confirm({ + title: t('chat.input.screenshot.permission_dialog_title') ?? 'Screen Recording Permission Required', + content, + centered: true, + okText: needsRestart ? (t('common.ok') ?? 'OK') : (t('chat.input.screenshot.open_settings') ?? 'Open Settings'), + cancelText: needsRestart ? undefined : (t('chat.input.screenshot.cancel') ?? 'Cancel'), + onOk: () => { + if (!needsRestart && window.electron.process.platform === 'darwin') { + void window.api.shell.openExternal( + 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture' + ) + } + } + }) + }, + [t] + ) + + const handleCapture = useCallback(async () => { + if (isCapturing) return + setIsCapturing(true) + try { + // Directly try to capture - this will trigger system permission dialog on first use + const fileName = `screenshot_${Date.now()}.png` + const result = await window.api.screenshot.capture(fileName) + + if (!result.success) { + // Handle permission errors + if (result.status === 'denied') { + if (result.needsRestart) { + showPermissionDialog(true) + } else { + showPermissionDialog(false) + } + } else { + logger.error('Screenshot capture failed', new Error(result.message)) + window.toast?.error(t('chat.input.screenshot.capture_failed') ?? 'Failed to capture screenshot') + } + return + } + + // Normalize to FileType + const nextFile: FileType = { + ...result.file, + type: FileTypes.IMAGE + } + + actions.setFiles((prev) => [...prev, nextFile]) + } catch (error: any) { + logger.error('Screenshot capture failed', error as Error) + window.toast?.error(t('chat.input.screenshot.capture_failed') ?? 'Failed to capture screenshot') + } finally { + setIsCapturing(false) + } + }, [actions, isCapturing, showPermissionDialog, t]) + + const handleCaptureWithSelection = useCallback(async () => { + if (isCapturing) return + setIsCapturing(true) + try { + const fileName = `screenshot_${Date.now()}.png` + const result = await window.api.screenshot.captureWithSelection(fileName) + + if (!result.success) { + // Handle different status types + if (result.status === 'denied') { + if (result.needsRestart) { + showPermissionDialog(true) + } else { + showPermissionDialog(false) + } + } else if (result.status === 'cancelled') { + logger.info('User cancelled screenshot selection') + // No toast for cancelled - user intentionally cancelled + } else { + logger.error('Screenshot selection failed', new Error(result.message)) + window.toast?.error(t('chat.input.screenshot.capture_failed') ?? 'Failed to capture screenshot') + } + return + } + + // Normalize to FileType + const nextFile: FileType = { + ...result.file, + type: FileTypes.IMAGE + } + + actions.setFiles((prev) => [...prev, nextFile]) + } catch (error: any) { + logger.error('Screenshot selection failed', error as Error) + window.toast?.error(t('chat.input.screenshot.capture_failed') ?? 'Failed to capture screenshot') + } finally { + setIsCapturing(false) + } + }, [actions, isCapturing, showPermissionDialog, t]) + + const menuItems: MenuProps['items'] = [ + { + key: 'full', + label: t('chat.input.screenshot.full_screen') ?? 'Full Screen', + onClick: handleCapture + }, + { + key: 'region', + label: t('chat.input.screenshot.select_region') ?? 'Select Region', + onClick: handleCaptureWithSelection + } + ] + + return ( + + + + + + + ) +} + +const screenshotTool = defineTool({ + key: 'screenshot', + label: (t) => t('chat.input.tools.screenshot') ?? 'Screenshot', + + visibleInScopes: [TopicType.Chat, TopicType.Session, 'mini-window'], + + dependencies: { + state: ['files'] as const, + actions: ['setFiles'] as const + }, + + render: (context) => +}) + +registerTool(screenshotTool) + +export default screenshotTool diff --git a/src/renderer/src/store/inputTools.ts b/src/renderer/src/store/inputTools.ts index aad87dba9f..42026e5c20 100644 --- a/src/renderer/src/store/inputTools.ts +++ b/src/renderer/src/store/inputTools.ts @@ -13,6 +13,7 @@ export const DEFAULT_TOOL_ORDER: ToolOrder = { visible: [ 'new_topic', 'attachment', + 'screenshot', 'thinking', 'web_search', 'url_context', @@ -30,11 +31,11 @@ export const DEFAULT_TOOL_ORDER: ToolOrder = { export const DEFAULT_TOOL_ORDER_BY_SCOPE: Record = { [TopicType.Chat]: DEFAULT_TOOL_ORDER, [TopicType.Session]: { - visible: ['create_session', 'slash_commands', 'attachment'], + visible: ['create_session', 'slash_commands', 'attachment', 'screenshot'], hidden: [] }, 'mini-window': { - visible: ['attachment', 'mention_models', 'quick_phrases'], + visible: ['attachment', 'screenshot', 'mention_models', 'quick_phrases'], hidden: [] } } diff --git a/src/renderer/src/types/chat.ts b/src/renderer/src/types/chat.ts index 75b15fae0c..2bb941c6f8 100644 --- a/src/renderer/src/types/chat.ts +++ b/src/renderer/src/types/chat.ts @@ -14,6 +14,7 @@ export type InputBarToolType = | 'clear_topic' | 'toggle_expand' | 'new_context' + | 'screenshot' // Agent Session tools | 'create_session' | 'slash_commands' diff --git a/src/renderer/src/windows/screenshot/ScreenshotSelection.tsx b/src/renderer/src/windows/screenshot/ScreenshotSelection.tsx new file mode 100644 index 0000000000..e8c4549773 --- /dev/null +++ b/src/renderer/src/windows/screenshot/ScreenshotSelection.tsx @@ -0,0 +1,205 @@ +import { loggerService } from '@logger' +import { IpcChannel } from '@shared/IpcChannel' +import { useCallback, useEffect, useRef, useState } from 'react' + +const logger = loggerService.withContext('ScreenshotSelection') + +interface SelectionState { + startX: number + startY: number + endX: number + endY: number + isDragging: boolean +} + +interface Rectangle { + x: number + y: number + width: number + height: number +} + +const ScreenshotSelection = () => { + const [screenshotData, setScreenshotData] = useState(null) + const [selection, setSelection] = useState({ + startX: 0, + startY: 0, + endX: 0, + endY: 0, + isDragging: false + }) + const canvasRef = useRef(null) + const containerRef = useRef(null) + + // Listen for screenshot data from main process + useEffect(() => { + const handler = (_event: any, data: { screenshotData: string }) => { + logger.info('Received screenshot data') + setScreenshotData(data.screenshotData) + } + + window.electron.ipcRenderer.on(IpcChannel.Screenshot_SelectionWindowReady, handler) + + return () => { + window.electron.ipcRenderer.removeListener(IpcChannel.Screenshot_SelectionWindowReady, handler) + } + }, []) + + // Draw screenshot on canvas when data is received + useEffect(() => { + if (!screenshotData || !canvasRef.current) return + + const canvas = canvasRef.current + const ctx = canvas.getContext('2d') + if (!ctx) return + + const img = new Image() + img.onload = () => { + // Set canvas size to match window + canvas.width = window.innerWidth + canvas.height = window.innerHeight + + // Draw the screenshot + ctx.drawImage(img, 0, 0, canvas.width, canvas.height) + logger.info('Screenshot drawn on canvas') + } + img.onerror = (e) => { + logger.error('Failed to load screenshot image', { error: e }) + // Close the selection window gracefully on image load error + window.api.screenshot.cancelSelection() + } + img.src = screenshotData + }, [screenshotData]) + + const getSelectionRectangle = useCallback((): Rectangle | null => { + if (!selection.isDragging && selection.endX === 0 && selection.endY === 0) { + return null + } + + const x = Math.min(selection.startX, selection.endX) + const y = Math.min(selection.startY, selection.endY) + const width = Math.abs(selection.endX - selection.startX) + const height = Math.abs(selection.endY - selection.startY) + + return { x, y, width, height } + }, [selection]) + + // Handle keyboard events + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + logger.info('User cancelled selection with ESC') + window.api.screenshot.cancelSelection() + } else if (e.key === 'Enter') { + const rect = getSelectionRectangle() + if (rect && rect.width >= 10 && rect.height >= 10) { + logger.info('User confirmed selection with ENTER', rect) + window.api.screenshot.confirmSelection(rect) + } + } + } + + window.addEventListener('keydown', handleKeyDown) + return () => window.removeEventListener('keydown', handleKeyDown) + }, [getSelectionRectangle]) + + const handleMouseDown = (e: React.MouseEvent) => { + const rect = containerRef.current?.getBoundingClientRect() + if (!rect) return + + const x = e.clientX - rect.left + const y = e.clientY - rect.top + + setSelection({ + startX: x, + startY: y, + endX: x, + endY: y, + isDragging: true + }) + } + + const handleMouseMove = (e: React.MouseEvent) => { + if (!selection.isDragging) return + + const rect = containerRef.current?.getBoundingClientRect() + if (!rect) return + + const x = e.clientX - rect.left + const y = e.clientY - rect.top + + setSelection((prev) => ({ + ...prev, + endX: x, + endY: y + })) + } + + const handleMouseUp = useCallback(() => { + if (!selection.isDragging) return + + const rect = getSelectionRectangle() + if (rect && rect.width >= 10 && rect.height >= 10) { + logger.info('Selection completed', rect) + window.api.screenshot.confirmSelection(rect) + } else { + // Selection too small, reset + setSelection((prev) => ({ ...prev, isDragging: false })) + } + }, [selection, getSelectionRectangle]) + + const rect = getSelectionRectangle() + const hasValidSelection = rect && rect.width > 0 && rect.height > 0 + + return ( +
+ {/* Canvas with screenshot */} + + + {/* Dark overlay */} +
+ {/* Clear area for selection */} + {hasValidSelection && rect && ( +
+ )} +
+ + {/* Dimension display */} + {hasValidSelection && rect && ( +
+ {Math.round(rect.width)} × {Math.round(rect.height)} +
+ )} + + {/* Control hints */} +
+ ESC to cancel | Drag to select | ENTER to confirm +
+
+ ) +} + +export default ScreenshotSelection diff --git a/src/renderer/src/windows/screenshot/entryPoint.tsx b/src/renderer/src/windows/screenshot/entryPoint.tsx new file mode 100644 index 0000000000..682ca3e4a7 --- /dev/null +++ b/src/renderer/src/windows/screenshot/entryPoint.tsx @@ -0,0 +1,18 @@ +import '@renderer/assets/styles/index.css' +import '@renderer/assets/styles/tailwind.css' +import '@ant-design/v5-patch-for-react-19' + +import { loggerService } from '@logger' +import { ThemeProvider } from '@renderer/context/ThemeProvider' +import { createRoot } from 'react-dom/client' + +import ScreenshotSelection from './ScreenshotSelection' + +loggerService.initWindowSource('ScreenshotSelection') + +const root = createRoot(document.getElementById('root') as HTMLElement) +root.render( + + + +) diff --git a/yarn.lock b/yarn.lock index d9d5ec1d6c..0fe3258d6d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10278,6 +10278,7 @@ __metadata: mime: "npm:^4.0.4" mime-types: "npm:^3.0.1" motion: "npm:^12.10.5" + node-screenshots: "npm:0.2.1" node-stream-zip: "npm:^1.15.0" notion-helper: "npm:^1.3.22" npx-scope-finder: "npm:^1.2.0" @@ -19871,6 +19872,95 @@ __metadata: languageName: node linkType: hard +"node-screenshots-darwin-arm64@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-darwin-arm64@npm:0.2.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"node-screenshots-darwin-universal@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-darwin-universal@npm:0.2.1" + conditions: os=darwin + languageName: node + linkType: hard + +"node-screenshots-darwin-x64@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-darwin-x64@npm:0.2.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"node-screenshots-linux-x64-gnu@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-linux-x64-gnu@npm:0.2.1" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"node-screenshots-linux-x64-musl@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-linux-x64-musl@npm:0.2.1" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"node-screenshots-win32-arm64-msvc@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-win32-arm64-msvc@npm:0.2.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"node-screenshots-win32-ia32-msvc@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-win32-ia32-msvc@npm:0.2.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"node-screenshots-win32-x64-msvc@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots-win32-x64-msvc@npm:0.2.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"node-screenshots@npm:0.2.1": + version: 0.2.1 + resolution: "node-screenshots@npm:0.2.1" + dependencies: + node-screenshots-darwin-arm64: "npm:0.2.1" + node-screenshots-darwin-universal: "npm:0.2.1" + node-screenshots-darwin-x64: "npm:0.2.1" + node-screenshots-linux-x64-gnu: "npm:0.2.1" + node-screenshots-linux-x64-musl: "npm:0.2.1" + node-screenshots-win32-arm64-msvc: "npm:0.2.1" + node-screenshots-win32-ia32-msvc: "npm:0.2.1" + node-screenshots-win32-x64-msvc: "npm:0.2.1" + dependenciesMeta: + node-screenshots-darwin-arm64: + optional: true + node-screenshots-darwin-universal: + optional: true + node-screenshots-darwin-x64: + optional: true + node-screenshots-linux-x64-gnu: + optional: true + node-screenshots-linux-x64-musl: + optional: true + node-screenshots-win32-arm64-msvc: + optional: true + node-screenshots-win32-ia32-msvc: + optional: true + node-screenshots-win32-x64-msvc: + optional: true + checksum: 10c0/f1c011dbd104e62776292d2efd1fcdc98bc75282d389e0e6fd057e80dc32fcfbaa1e217a8e55c7c329cba1a246f82501733db2cdd79e6d5cb0e05d4219e02edf + languageName: node + linkType: hard + "node-stream-zip@npm:^1.15.0": version: 1.15.0 resolution: "node-stream-zip@npm:1.15.0"