Northern_Lights/cherry-studio
1
0
Fork 0
mirror of https://github.com/CherryHQ/cherry-studio.git synced 2025-12-26 03:31:24 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

添加了 TTS 相关服务并更新了设置

Browse Source
This commit is contained in:
1600822305 2025-04-12 12:04:27 +08:00
parent 42c38b73ce
commit 808e5ef076
14 changed files with 159 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/main/services/MsTTSIpcHandler.ts
View File

@ -16,32 +16,35 @@ export function registerMsTTSIpcHandlers(): void {
)
// 流式合成语音
ipcMain.handle(IpcChannel.MsTTS_SynthesizeStream, async (event, requestId: string, text: string, voice: string, outputFormat: string) => {
const window = BrowserWindow.fromWebContents(event.sender)
if (!window) return
ipcMain.handle(
IpcChannel.MsTTS_SynthesizeStream,
async (event, requestId: string, text: string, voice: string, outputFormat: string) => {
const window = BrowserWindow.fromWebContents(event.sender)
if (!window) return
try {
await MsTTSService.synthesizeStream(
text,
voice,
outputFormat,
(chunk: Uint8Array) => {
// 发送音频数据块
if (!window.isDestroyed()) {
window.webContents.send(IpcChannel.MsTTS_StreamData, requestId, chunk)
try {
await MsTTSService.synthesizeStream(
text,
voice,
outputFormat,
(chunk: Uint8Array) => {
// 发送音频数据块
if (!window.isDestroyed()) {
window.webContents.send(IpcChannel.MsTTS_StreamData, requestId, chunk)
}
},
() => {
// 发送流结束信号
if (!window.isDestroyed()) {
window.webContents.send(IpcChannel.MsTTS_StreamEnd, requestId)
}
}
},
() => {
// 发送流结束信号
if (!window.isDestroyed()) {
window.webContents.send(IpcChannel.MsTTS_StreamEnd, requestId)
}
}
)
return { success: true }
} catch (error) {
console.error('流式TTS合成失败:', error)
return { success: false, error: error instanceof Error ? error.message : String(error) }
)
return { success: true }
} catch (error) {
console.error('流式TTS合成失败:', error)
return { success: false, error: error instanceof Error ? error.message : String(error) }
}
}
})
)
}

2
src/main/services/MsTTSService.ts
View File

@ -1,10 +1,10 @@
import fs from 'node:fs'
import path from 'node:path'
import { MsEdgeTTS, OUTPUT_FORMAT } from 'edge-tts-node' // 新版支持流式的TTS库
import { app } from 'electron'
import log from 'electron-log'
import { EdgeTTS } from 'node-edge-tts' // 旧版TTS库
import { MsEdgeTTS, OUTPUT_FORMAT } from 'edge-tts-node' // 新版支持流式的TTS库
// --- START OF HARDCODED VOICE LIST ---
// WARNING: This list is static and may become outdated.

2
src/renderer/src/components/ASRButton.tsx
View File

@ -33,7 +33,7 @@ const ASRButton: FC<Props> = ({ onTranscribed, disabled = false, style }) => {
try {
// 添加事件监听器，监听服务器发送的stopped消息
const originalCallback = ASRService.resultCallback
const stopCallback = (text: string, isFinal?: boolean) => {
const stopCallback = (text: string) => {
// 如果是空字符串，只重置状态，不调用原始回调
if (text === '') {
setIsProcessing(false)

108
src/renderer/src/components/DraggableVoiceCallWindow.tsx
View File

@ -289,7 +289,9 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
}
await VoiceCallService.startCall({
onTranscript: setTranscript,
onResponse: () => { /* 响应在聊天界面处理 */ },
onResponse: () => {
/* 响应在聊天界面处理 */
},
onListeningStateChange: setIsListening,
onSpeakingStateChange: setIsSpeaking
})
@ -402,7 +404,7 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
await VoiceCallService.cancelRecording()
setTranscript('')
} catch (error) {
console.error('取消录音出错:', error);
console.error('取消录音出错:', error)
} finally {
setTimeout(() => setIsProcessing(false), 500)
}
@ -412,23 +414,31 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
)
// --- 语音通话控制函数结束 ---
// --- 快捷键相关函数 ---
const getKeyDisplayName = (keyCode: string) => {
const keyMap: Record<string, string> = {
Space: '空格键', Enter: '回车键', ShiftLeft: '左Shift键', ShiftRight: '右Shift键',
ControlLeft: '左Ctrl键', ControlRight: '右Ctrl键', AltLeft: '左Alt键', AltRight: '右Alt键'
Space: '空格键',
Enter: '回车键',
ShiftLeft: '左Shift键',
ShiftRight: '右Shift键',
ControlLeft: '左Ctrl键',
ControlRight: '右Ctrl键',
AltLeft: '左Alt键',
AltRight: '右Alt键'
}
return keyMap[keyCode] || keyCode
}
const handleShortcutKeyChange = useCallback((e: KeyboardEvent) => {
e.preventDefault()
if (isRecordingShortcut) {
setTempShortcutKey(e.code)
setIsRecordingShortcut(false)
}
}, [isRecordingShortcut])
const handleShortcutKeyChange = useCallback(
(e: KeyboardEvent) => {
e.preventDefault()
if (isRecordingShortcut) {
setTempShortcutKey(e.code)
setIsRecordingShortcut(false)
}
},
[isRecordingShortcut]
)
const saveShortcutKey = useCallback(() => {
setShortcutKey(tempShortcutKey)
@ -437,31 +447,42 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
}, [tempShortcutKey])
// 现在可以安全地使用 handleRecordStart/End
const handleKeyDown = useCallback((e: KeyboardEvent) => {
if (isRecordingShortcut) {
handleShortcutKeyChange(e)
return
}
if (e.code === shortcutKey && !isProcessing && !isPaused && visible && !isShortcutPressed) {
e.preventDefault()
setIsShortcutPressed(true)
const mockEvent = new MouseEvent('mousedown') as unknown as React.MouseEvent // 类型断言
handleRecordStart(mockEvent) // 现在 handleRecordStart 已经定义
}
}, [
shortcutKey, isProcessing, isPaused, visible, isShortcutPressed,
handleRecordStart, // 依赖项
isRecordingShortcut, handleShortcutKeyChange
])
const handleKeyDown = useCallback(
(e: KeyboardEvent) => {
if (isRecordingShortcut) {
handleShortcutKeyChange(e)
return
}
if (e.code === shortcutKey && !isProcessing && !isPaused && visible && !isShortcutPressed) {
e.preventDefault()
setIsShortcutPressed(true)
const mockEvent = new MouseEvent('mousedown') as unknown as React.MouseEvent // 类型断言
handleRecordStart(mockEvent) // 现在 handleRecordStart 已经定义
}
},
[
shortcutKey,
isProcessing,
isPaused,
visible,
isShortcutPressed,
handleRecordStart, // 依赖项
isRecordingShortcut,
handleShortcutKeyChange
]
)
const handleKeyUp = useCallback((e: KeyboardEvent) => {
if (e.code === shortcutKey && isShortcutPressed && visible) {
e.preventDefault()
setIsShortcutPressed(false)
const mockEvent = new MouseEvent('mouseup') as unknown as React.MouseEvent // 类型断言
handleRecordEnd(mockEvent) // 现在 handleRecordEnd 已经定义
}
}, [shortcutKey, isShortcutPressed, visible, handleRecordEnd]) // 依赖项
const handleKeyUp = useCallback(
(e: KeyboardEvent) => {
if (e.code === shortcutKey && isShortcutPressed && visible) {
e.preventDefault()
setIsShortcutPressed(false)
const mockEvent = new MouseEvent('mouseup') as unknown as React.MouseEvent // 类型断言
handleRecordEnd(mockEvent) // 现在 handleRecordEnd 已经定义
}
},
[shortcutKey, isShortcutPressed, visible, handleRecordEnd]
) // 依赖项
useEffect(() => {
const savedShortcut = localStorage.getItem('voiceCallShortcutKey')
@ -483,7 +504,6 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
}, [visible, handleKeyDown, handleKeyUp])
// --- 快捷键相关函数结束 ---
// 如果不可见，直接返回 null
if (!visible) return null
@ -511,10 +531,14 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
<Content>
{isSettingsVisible && (
<SettingsPanel> {/* 使用 styled-component */}
<SettingsPanel>
{' '}
{/* 使用 styled-component */}
<SettingsTitle>{t('voice_call.shortcut_key_setting')}</SettingsTitle> {/* 使用 styled-component */}
<Space>
<ShortcutKeyButton onClick={() => setIsRecordingShortcut(true)}> {/* 使用 styled-component */}
<ShortcutKeyButton onClick={() => setIsRecordingShortcut(true)}>
{' '}
{/* 使用 styled-component */}
{isRecordingShortcut ? t('voice_call.press_any_key') : getKeyDisplayName(tempShortcutKey)}
</ShortcutKeyButton>
<Button type="primary" onClick={saveShortcutKey}>
@ -522,7 +546,9 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
</Button>
<Button onClick={() => setIsSettingsVisible(false)}>{t('voice_call.cancel')}</Button>
</Space>
<SettingsTip> {/* 使用 styled-component */}
<SettingsTip>
{' '}
{/* 使用 styled-component */}
{t('voice_call.shortcut_key_tip')}
</SettingsTip>
</SettingsPanel>
@ -579,4 +605,4 @@ const DraggableVoiceCallWindow: React.FC<Props> = ({
)
}
export default DraggableVoiceCallWindow
export default DraggableVoiceCallWindow

7
src/renderer/src/i18n/locales/en-us.json
View File

@ -15,7 +15,12 @@
"initialization_failed": "Failed to initialize voice call",
"error": "Voice call error",
"initializing": "Initializing voice call...",
"ready": "Voice call ready"
"ready": "Voice call ready",
"shortcut_key_setting": "[to be translated]:语音识别快捷键设置",
"press_any_key": "[to be translated]:请按任意键...",
"save": "[to be translated]:保存",
"cancel": "[to be translated]:取消",
"shortcut_key_tip": "[to be translated]:按下此快捷键开始录音，松开快捷键结束录音并发送"
},
"agents": {
"add.button": "Add to Assistant",

10
src/renderer/src/i18n/locales/ja-jp.json
View File

@ -1421,8 +1421,7 @@
"mstts.voice": "[to be translated]:免费在线 TTS音色",
"mstts.output_format": "[to be translated]:输出格式",
"mstts.info": "[to be translated]:免费在线TTS服务不需要API密钥，完全免费使用。",
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色",
"filter.emojis": "[to be translated]:过滤表情符号"
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色"
},
"asr": {
"title": "音声認識",
@ -1544,7 +1543,12 @@
"initialization_failed": "[to be translated]:初始化语音通话失败",
"error": "[to be translated]:语音通话出错",
"initializing": "[to be translated]:正在初始化语音通话...",
"ready": "[to be translated]:语音通话已就绪"
"ready": "[to be translated]:语音通话已就绪",
"shortcut_key_setting": "[to be translated]:语音识别快捷键设置",
"press_any_key": "[to be translated]:请按任意键...",
"save": "[to be translated]:保存",
"cancel": "[to be translated]:取消",
"shortcut_key_tip": "[to be translated]:按下此快捷键开始录音，松开快捷键结束录音并发送"
}
}
}

10
src/renderer/src/i18n/locales/ru-ru.json
View File

@ -1421,8 +1421,7 @@
"mstts.voice": "[to be translated]:免费在线 TTS音色",
"mstts.output_format": "[to be translated]:输出格式",
"mstts.info": "[to be translated]:免费在线TTS服务不需要API密钥，完全免费使用。",
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色",
"filter.emojis": "[to be translated]:过滤表情符号"
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色"
},
"voice": {
"title": "[to be translated]:语音功能",
@ -1544,7 +1543,12 @@
"initialization_failed": "[to be translated]:初始化语音通话失败",
"error": "[to be translated]:语音通话出错",
"initializing": "[to be translated]:正在初始化语音通话...",
"ready": "[to be translated]:语音通话已就绪"
"ready": "[to be translated]:语音通话已就绪",
"shortcut_key_setting": "[to be translated]:语音识别快捷键设置",
"press_any_key": "[to be translated]:请按任意键...",
"save": "[to be translated]:保存",
"cancel": "[to be translated]:取消",
"shortcut_key_tip": "[to be translated]:按下此快捷键开始录音，松开快捷键结束录音并发送"
}
}
}

10
src/renderer/src/i18n/locales/zh-tw.json
View File

@ -1421,8 +1421,7 @@
"mstts.voice": "[to be translated]:免费在线 TTS音色",
"mstts.output_format": "[to be translated]:输出格式",
"mstts.info": "[to be translated]:免费在线TTS服务不需要API密钥，完全免费使用。",
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色",
"filter.emojis": "[to be translated]:过滤表情符号"
"error.no_mstts_voice": "[to be translated]:未设置免费在线 TTS音色"
},
"voice": {
"title": "[to be translated]:语音功能",
@ -1544,7 +1543,12 @@
"initialization_failed": "[to be translated]:初始化语音通话失败",
"error": "[to be translated]:语音通话出错",
"initializing": "[to be translated]:正在初始化语音通话...",
"ready": "[to be translated]:语音通话已就绪"
"ready": "[to be translated]:语音通话已就绪",
"shortcut_key_setting": "[to be translated]:语音识别快捷键设置",
"press_any_key": "[to be translated]:请按任意键...",
"save": "[to be translated]:保存",
"cancel": "[to be translated]:取消",
"shortcut_key_tip": "[to be translated]:按下此快捷键开始录音，松开快捷键结束录音并发送"
}
}
}

3
src/renderer/src/pages/home/Inputbar/Inputbar.tsx
View File

@ -77,7 +77,8 @@ let _files: FileType[] = []
const Inputbar: FC<Props> = ({ assistant: _assistant, setActiveTopic, topic }) => {
const [text, setText] = useState(_text)
const [asrCurrentText, setAsrCurrentText] = useState('')
// 用于存储语音识别的中间结果，不直接显示在输入框中
const [, setAsrCurrentText] = useState('')
const [inputFocus, setInputFocus] = useState(false)
const { assistant, addTopic, model, setModel, updateAssistant } = useAssistant(_assistant.id)
const {

8
src/renderer/src/pages/home/Messages/Message.tsx
View File

@ -155,13 +155,7 @@ const MessageItem: FC<Props> = ({
useEffect(() => {
// 如果是最后一条助手消息，且消息状态为成功，且不是正在生成中，且TTS已启用
// 注意：只有在语音通话窗口打开时才自动播放TTS
if (
isLastMessage &&
isAssistantMessage &&
message.status === 'success' &&
!generating &&
ttsEnabled
) {
if (isLastMessage && isAssistantMessage && message.status === 'success' && !generating && ttsEnabled) {
// 如果语音通话窗口没有打开，则不自动播放TTS
if (!isVoiceCallActive) {
console.log('不自动播放TTS，因为语音通话窗口没有打开:', isVoiceCallActive)

3
src/renderer/src/services/ASRService.ts
View File

@ -170,7 +170,8 @@ class ASRService {
// 直接调用回调函数
this.resultCallback(data.data.text, true)
window.message.success({ content: i18n.t('settings.asr.success'), key: 'asr-processing' })
} else if (this.isRecording) { // 只在录音中才处理中间结果
} else if (this.isRecording) {
// 只在录音中才处理中间结果
// 非最终结果，也调用回调，但标记为非最终
console.log('[ASRService] 收到中间结果，调用回调函数，文本:', data.data.text)
this.resultCallback(data.data.text, false)

34
src/renderer/src/services/tts/AudioStreamProcessor.ts
View File

@ -6,10 +6,10 @@ export class AudioStreamProcessor {
private audioContext: AudioContext | null = null
private audioQueue: Uint8Array[] = []
private isProcessing: boolean = false
// 回调函数
public onAudioBuffer: ((buffer: AudioBuffer) => void) | null = null
/**
* 初始化音频处理器
*/
@ -19,7 +19,7 @@ export class AudioStreamProcessor {
this.audioQueue = []
this.isProcessing = false
}
/**
* 处理音频数据块
* @param chunk 音频数据块
@ -28,16 +28,16 @@ export class AudioStreamProcessor {
if (!this.audioContext) {
throw new Error('AudioStreamProcessor not initialized')
}
// 将数据块添加到队列
this.audioQueue.push(chunk)
// 如果没有正在处理，开始处理
if (!this.isProcessing) {
this.processQueue()
}
}
/**
* 处理队列中的音频数据
*/
@ -46,20 +46,18 @@ export class AudioStreamProcessor {
this.isProcessing = false
return
}
this.isProcessing = true
// 获取队列中的第一个数据块
const chunk = this.audioQueue.shift()!
try {
// 解码音频数据
// 将SharedArrayBuffer转换为ArrayBuffer
const arrayBuffer = chunk.buffer instanceof SharedArrayBuffer
? new Uint8Array(chunk.buffer).buffer
: chunk.buffer
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer)
const arrayBuffer = chunk.buffer instanceof SharedArrayBuffer ? new Uint8Array(chunk.buffer).buffer : chunk.buffer
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer as ArrayBuffer)
// 调用回调函数
if (this.onAudioBuffer) {
this.onAudioBuffer(audioBuffer)
@ -67,20 +65,20 @@ export class AudioStreamProcessor {
} catch (error) {
console.error('解码音频数据失败:', error)
}
// 继续处理队列中的下一个数据块
this.processQueue()
}
/**
* 完成处理
*/
public async finish(): Promise<void> {
// 等待队列处理完成
while (this.audioQueue.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
}
// 关闭音频上下文
if (this.audioContext) {
await this.audioContext.close()

14
src/renderer/src/services/tts/TTSService.ts
View File

@ -63,13 +63,13 @@ export class TTSService {
// 只有在使用EdgeTTS且标记为正在播放时才检查
if (this.isPlaying && this.playingServiceType === 'edge') {
// 检查是否还在播放
const isSpeaking = window.speechSynthesis.speaking;
const isSpeaking = window.speechSynthesis.speaking
if (!isSpeaking) {
console.log('检测到speechSynthesis不再播放，更新状态')
this.updatePlayingState(false);
this.updatePlayingState(false)
}
}
}, 500); // 每500毫秒检查一次
}, 500) // 每500毫秒检查一次
}
}
@ -109,7 +109,7 @@ export class TTSService {
private updatePlayingState(isPlaying: boolean): void {
// 只有状态变化时才更新和触发事件
if (this.isPlaying !== isPlaying) {
this.isPlaying = isPlaying;
this.isPlaying = isPlaying
console.log(`TTS播放状态更新: ${isPlaying ? '开始播放' : '停止播放'}`)
// 触发自定义事件，通知其他组件TTS状态变化
@ -118,11 +118,11 @@ export class TTSService {
// 如果停止播放，清除服务类型
if (!isPlaying) {
this.playingServiceType = null;
this.playingServiceType = null
// 确保Web Speech API也停止
if ('speechSynthesis' in window) {
window.speechSynthesis.cancel();
window.speechSynthesis.cancel()
}
}
}
@ -148,7 +148,7 @@ export class TTSService {
if (this.isPlaying) {
this.stop()
// 添加短暂延迟，确保上一个播放完全停止
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
}
// 确保文本不为空

8
src/renderer/src/services/tts/TTSServiceInterface.ts
View File

@ -19,5 +19,11 @@ export interface TTSServiceInterface {
* @param onError 错误回调
* @returns 返回请求ID
*/
synthesizeStream?(text: string, onStart: () => void, onData: (audioChunk: AudioBuffer) => void, onEnd: () => void, onError: (error: Error) => void): Promise<string>
synthesizeStream?(
text: string,
onStart: () => void,
onData: (audioChunk: AudioBuffer) => void,
onEnd: () => void,
onError: (error: Error) => void
): Promise<string>
}