refactor: reuse shiki highlighter utils (#6235)

* refactor: improve shiki highlighter utils and reuse it in ShikiStreamService

* refactor: reuse shiki highlighter and markdown-it renderer

* refactor: import shiki/markdown-it/core dynamically

* chore: update shiki
This commit is contained in:
one 2025-05-21 19:06:56 +08:00 committed by GitHub
parent d9661602b2
commit a9a0ae87d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 301 additions and 270 deletions

View File

@ -123,7 +123,7 @@
"@mozilla/readability": "^0.6.0", "@mozilla/readability": "^0.6.0",
"@notionhq/client": "^2.2.15", "@notionhq/client": "^2.2.15",
"@reduxjs/toolkit": "^2.2.5", "@reduxjs/toolkit": "^2.2.5",
"@shikijs/markdown-it": "^3.2.2", "@shikijs/markdown-it": "^3.4.2",
"@swc/plugin-styled-components": "^7.1.5", "@swc/plugin-styled-components": "^7.1.5",
"@tryfabric/martian": "^1.2.4", "@tryfabric/martian": "^1.2.4",
"@types/diff": "^7", "@types/diff": "^7",
@ -200,7 +200,7 @@
"remark-math": "^6.0.0", "remark-math": "^6.0.0",
"rollup-plugin-visualizer": "^5.12.0", "rollup-plugin-visualizer": "^5.12.0",
"sass": "^1.88.0", "sass": "^1.88.0",
"shiki": "^3.2.2", "shiki": "^3.4.2",
"string-width": "^7.2.0", "string-width": "^7.2.0",
"styled-components": "^6.1.11", "styled-components": "^6.1.11",
"tiny-pinyin": "^1.3.2", "tiny-pinyin": "^1.3.2",
@ -219,7 +219,6 @@
"openai@npm:^4.77.0": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch", "openai@npm:^4.77.0": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch",
"pkce-challenge@npm:^4.1.0": "patch:pkce-challenge@npm%3A4.1.0#~/.yarn/patches/pkce-challenge-npm-4.1.0-fbc51695a3.patch", "pkce-challenge@npm:^4.1.0": "patch:pkce-challenge@npm%3A4.1.0#~/.yarn/patches/pkce-challenge-npm-4.1.0-fbc51695a3.patch",
"app-builder-lib@npm:26.0.13": "patch:app-builder-lib@npm%3A26.0.13#~/.yarn/patches/app-builder-lib-npm-26.0.13-a064c9e1d0.patch", "app-builder-lib@npm:26.0.13": "patch:app-builder-lib@npm%3A26.0.13#~/.yarn/patches/app-builder-lib-npm-26.0.13-a064c9e1d0.patch",
"shiki": "3.2.2",
"openai@npm:^4.87.3": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch", "openai@npm:^4.87.3": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch",
"app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch" "app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch"
}, },

View File

@ -3,6 +3,7 @@ import { useMermaid } from '@renderer/hooks/useMermaid'
import { useSettings } from '@renderer/hooks/useSettings' import { useSettings } from '@renderer/hooks/useSettings'
import { HighlightChunkResult, ShikiPreProperties, shikiStreamService } from '@renderer/services/ShikiStreamService' import { HighlightChunkResult, ShikiPreProperties, shikiStreamService } from '@renderer/services/ShikiStreamService'
import { ThemeMode } from '@renderer/types' import { ThemeMode } from '@renderer/types'
import { getHighlighter, getMarkdownIt, getShiki, loadLanguageIfNeeded, loadThemeIfNeeded } from '@renderer/utils/shiki'
import * as cmThemes from '@uiw/codemirror-themes-all' import * as cmThemes from '@uiw/codemirror-themes-all'
import type React from 'react' import type React from 'react'
import { createContext, type PropsWithChildren, use, useCallback, useEffect, useMemo, useState } from 'react' import { createContext, type PropsWithChildren, use, useCallback, useEffect, useMemo, useState } from 'react'
@ -11,6 +12,8 @@ interface CodeStyleContextType {
highlightCodeChunk: (trunk: string, language: string, callerId: string) => Promise<HighlightChunkResult> highlightCodeChunk: (trunk: string, language: string, callerId: string) => Promise<HighlightChunkResult>
cleanupTokenizers: (callerId: string) => void cleanupTokenizers: (callerId: string) => void
getShikiPreProperties: (language: string) => Promise<ShikiPreProperties> getShikiPreProperties: (language: string) => Promise<ShikiPreProperties>
highlightCode: (code: string, language: string) => Promise<string>
shikiMarkdownIt: (code: string) => Promise<string>
themeNames: string[] themeNames: string[]
activeShikiTheme: string activeShikiTheme: string
activeCmTheme: any activeCmTheme: any
@ -21,6 +24,8 @@ const defaultCodeStyleContext: CodeStyleContextType = {
highlightCodeChunk: async () => ({ lines: [], recall: 0 }), highlightCodeChunk: async () => ({ lines: [], recall: 0 }),
cleanupTokenizers: () => {}, cleanupTokenizers: () => {},
getShikiPreProperties: async () => ({ class: '', style: '', tabindex: 0 }), getShikiPreProperties: async () => ({ class: '', style: '', tabindex: 0 }),
highlightCode: async () => '',
shikiMarkdownIt: async () => '',
themeNames: ['auto'], themeNames: ['auto'],
activeShikiTheme: 'auto', activeShikiTheme: 'auto',
activeCmTheme: null, activeCmTheme: null,
@ -37,7 +42,7 @@ export const CodeStyleProvider: React.FC<PropsWithChildren> = ({ children }) =>
useEffect(() => { useEffect(() => {
if (!codeEditor.enabled) { if (!codeEditor.enabled) {
import('shiki').then(({ bundledThemes }) => { getShiki().then(({ bundledThemes }) => {
setShikiThemes(bundledThemes) setShikiThemes(bundledThemes)
}) })
} }
@ -118,11 +123,35 @@ export const CodeStyleProvider: React.FC<PropsWithChildren> = ({ children }) =>
[activeShikiTheme, languageMap] [activeShikiTheme, languageMap]
) )
const highlightCode = useCallback(
async (code: string, language: string) => {
const highlighter = await getHighlighter()
await loadLanguageIfNeeded(highlighter, language)
await loadThemeIfNeeded(highlighter, activeShikiTheme)
return highlighter.codeToHtml(code, { lang: language, theme: activeShikiTheme })
},
[activeShikiTheme]
)
// 使用 Shiki 和 Markdown-it 渲染代码
const shikiMarkdownIt = useCallback(
async (code: string) => {
const renderer = await getMarkdownIt(activeShikiTheme)
if (!renderer) {
return code
}
return renderer.render(code)
},
[activeShikiTheme]
)
const contextValue = useMemo( const contextValue = useMemo(
() => ({ () => ({
highlightCodeChunk, highlightCodeChunk,
cleanupTokenizers, cleanupTokenizers,
getShikiPreProperties, getShikiPreProperties,
highlightCode,
shikiMarkdownIt,
themeNames, themeNames,
activeShikiTheme, activeShikiTheme,
activeCmTheme, activeCmTheme,
@ -132,6 +161,8 @@ export const CodeStyleProvider: React.FC<PropsWithChildren> = ({ children }) =>
highlightCodeChunk, highlightCodeChunk,
cleanupTokenizers, cleanupTokenizers,
getShikiPreProperties, getShikiPreProperties,
highlightCode,
shikiMarkdownIt,
themeNames, themeNames,
activeShikiTheme, activeShikiTheme,
activeCmTheme, activeCmTheme,

View File

@ -1306,6 +1306,7 @@
"dependenciesInstall": "Install Dependencies", "dependenciesInstall": "Install Dependencies",
"dependenciesInstalling": "Installing dependencies...", "dependenciesInstalling": "Installing dependencies...",
"description": "Description", "description": "Description",
"noDescriptionAvailable": "No description available",
"duplicateName": "A server with this name already exists", "duplicateName": "A server with this name already exists",
"editJson": "Edit JSON", "editJson": "Edit JSON",
"editServer": "Edit Server", "editServer": "Edit Server",

View File

@ -1303,6 +1303,7 @@
"dependenciesInstall": "依存関係をインストール", "dependenciesInstall": "依存関係をインストール",
"dependenciesInstalling": "依存関係をインストール中...", "dependenciesInstalling": "依存関係をインストール中...",
"description": "説明", "description": "説明",
"noDescriptionAvailable": "説明がありません",
"duplicateName": "同じ名前のサーバーが既に存在します", "duplicateName": "同じ名前のサーバーが既に存在します",
"editJson": "JSONを編集", "editJson": "JSONを編集",
"editServer": "サーバーを編集", "editServer": "サーバーを編集",

View File

@ -1303,6 +1303,7 @@
"dependenciesInstall": "Установить зависимости", "dependenciesInstall": "Установить зависимости",
"dependenciesInstalling": "Установка зависимостей...", "dependenciesInstalling": "Установка зависимостей...",
"description": "Описание", "description": "Описание",
"noDescriptionAvailable": "Описание отсутствует",
"duplicateName": "Сервер с таким именем уже существует", "duplicateName": "Сервер с таким именем уже существует",
"editJson": "Редактировать JSON", "editJson": "Редактировать JSON",
"editServer": "Редактировать сервер", "editServer": "Редактировать сервер",

View File

@ -1306,6 +1306,7 @@
"dependenciesInstall": "安装依赖项", "dependenciesInstall": "安装依赖项",
"dependenciesInstalling": "正在安装依赖项...", "dependenciesInstalling": "正在安装依赖项...",
"description": "描述", "description": "描述",
"noDescriptionAvailable": "暂无描述",
"duplicateName": "已存在同名服务器", "duplicateName": "已存在同名服务器",
"editJson": "编辑JSON", "editJson": "编辑JSON",
"editServer": "编辑服务器", "editServer": "编辑服务器",

View File

@ -1306,6 +1306,7 @@
"dependenciesInstall": "安裝相依套件", "dependenciesInstall": "安裝相依套件",
"dependenciesInstalling": "正在安裝相依套件...", "dependenciesInstalling": "正在安裝相依套件...",
"description": "描述", "description": "描述",
"noDescriptionAvailable": "描述不存在",
"duplicateName": "已存在相同名稱的伺服器", "duplicateName": "已存在相同名稱的伺服器",
"editJson": "編輯JSON", "editJson": "編輯JSON",
"editServer": "編輯伺服器", "editServer": "編輯伺服器",

View File

@ -1,9 +1,9 @@
import { CheckOutlined, ExpandOutlined, LoadingOutlined, WarningOutlined } from '@ant-design/icons' import { CheckOutlined, ExpandOutlined, LoadingOutlined, WarningOutlined } from '@ant-design/icons'
import { useCodeStyle } from '@renderer/context/CodeStyleProvider'
import { useSettings } from '@renderer/hooks/useSettings' import { useSettings } from '@renderer/hooks/useSettings'
import type { ToolMessageBlock } from '@renderer/types/newMessage' import type { ToolMessageBlock } from '@renderer/types/newMessage'
import { useShikiWithMarkdownIt } from '@renderer/utils/shiki'
import { Collapse, message as antdMessage, Modal, Tabs, Tooltip } from 'antd' import { Collapse, message as antdMessage, Modal, Tabs, Tooltip } from 'antd'
import { FC, useMemo, useState } from 'react' import { FC, memo, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
import styled from 'styled-components' import styled from 'styled-components'
@ -192,12 +192,7 @@ const MessageTools: FC<Props> = ({ blocks }) => {
{ {
key: 'preview', key: 'preview',
label: t('message.tools.preview'), label: t('message.tools.preview'),
children: ( children: <CollapsedContent isExpanded={true} resultString={resultString} />
<CollapsedContent
isExpanded={true}
resultString={resultString}
/>
)
}, },
{ {
key: 'raw', key: 'raw',
@ -215,9 +210,17 @@ const MessageTools: FC<Props> = ({ blocks }) => {
// New component to handle collapsed content // New component to handle collapsed content
const CollapsedContent: FC<{ isExpanded: boolean; resultString: string }> = ({ isExpanded, resultString }) => { const CollapsedContent: FC<{ isExpanded: boolean; resultString: string }> = ({ isExpanded, resultString }) => {
const { renderedMarkdown: styledResult } = useShikiWithMarkdownIt( const { highlightCode } = useCodeStyle()
isExpanded ? `\`\`\`json\n${resultString}\n\`\`\`` : '' const [styledResult, setStyledResult] = useState<string>('')
)
useEffect(() => {
const highlight = async () => {
const result = await highlightCode(isExpanded ? resultString : '', 'json')
setStyledResult(result)
}
setTimeout(highlight, 0)
}, [isExpanded, resultString, highlightCode])
if (!isExpanded) { if (!isExpanded) {
return null return null
@ -247,8 +250,11 @@ const CollapseContainer = styled(Collapse)`
` `
const MarkdownContainer = styled.div` const MarkdownContainer = styled.div`
& pre span { & pre {
white-space: pre-wrap; background: transparent !important;
span {
white-space: pre-wrap;
}
} }
` `
@ -371,4 +377,4 @@ const ExpandedResponseContainer = styled.div`
} }
` `
export default MessageTools export default memo(MessageTools)

View File

@ -1,47 +1,42 @@
import { useTheme } from '@renderer/context/ThemeProvider' import { useCodeStyle } from '@renderer/context/CodeStyleProvider'
import { runAsyncFunction } from '@renderer/utils'
import { getShikiInstance } from '@renderer/utils/shiki'
import { Card } from 'antd' import { Card } from 'antd'
import MarkdownIt from 'markdown-it'
import { npxFinder } from 'npx-scope-finder' import { npxFinder } from 'npx-scope-finder'
import { useCallback, useEffect, useRef, useState } from 'react' import { FC, memo, useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import styled from 'styled-components' import styled from 'styled-components'
interface McpDescriptionProps { interface McpDescriptionProps {
searchKey: string searchKey: string
} }
const MCPDescription = ({ searchKey }: McpDescriptionProps) => { const MCPDescription: FC<McpDescriptionProps> = ({ searchKey }) => {
const [renderedMarkdown, setRenderedMarkdown] = useState('') const { t } = useTranslation()
const { shikiMarkdownIt } = useCodeStyle()
const [loading, setLoading] = useState(false) const [loading, setLoading] = useState(false)
const [mcpInfo, setMcpInfo] = useState<string>('')
const md = useRef<MarkdownIt>(
new MarkdownIt({
linkify: true, // 自动转换 URL 为链接
typographer: true // 启用印刷格式优化
})
)
const { theme } = useTheme()
const getMcpInfo = useCallback(async () => {
setLoading(true)
const packages = await npxFinder(searchKey).finally(() => setLoading(false))
const readme = packages[0]?.original?.readme ?? '暂无描述'
setRenderedMarkdown(md.current.render(readme))
}, [md, searchKey])
useEffect(() => { useEffect(() => {
runAsyncFunction(async () => { let isMounted = true
const sk = await getShikiInstance(theme) setLoading(true)
md.current.use(sk) npxFinder(searchKey)
getMcpInfo() .then((packages) => {
}) const readme = packages[0]?.original?.readme ?? t('settings.mcp.noDescriptionAvailable')
}, [getMcpInfo, theme]) shikiMarkdownIt(readme).then((result) => {
if (isMounted) setMcpInfo(result)
})
})
.finally(() => {
if (isMounted) setLoading(false)
})
return () => {
isMounted = false
}
}, [shikiMarkdownIt, searchKey, t])
return ( return (
<Section> <Section>
<Card loading={loading}> <Card loading={loading}>
<div className="markdown" dangerouslySetInnerHTML={{ __html: renderedMarkdown }} /> <div className="markdown" dangerouslySetInnerHTML={{ __html: mcpInfo }} />
</Card> </Card>
</Section> </Section>
) )
@ -50,4 +45,4 @@ const Section = styled.div`
padding-top: 8px; padding-top: 8px;
` `
export default MCPDescription export default memo(MCPDescription)

View File

@ -1,5 +1,12 @@
import {
DEFAULT_LANGUAGES,
DEFAULT_THEMES,
getHighlighter,
loadLanguageIfNeeded,
loadThemeIfNeeded
} from '@renderer/utils/shiki'
import { LRUCache } from 'lru-cache' import { LRUCache } from 'lru-cache'
import type { HighlighterCore, SpecialLanguage, ThemedToken } from 'shiki/core' import type { HighlighterGeneric, ThemedToken } from 'shiki/core'
import { ShikiStreamTokenizer, ShikiStreamTokenizerOptions } from './ShikiStreamTokenizer' import { ShikiStreamTokenizer, ShikiStreamTokenizerOptions } from './ShikiStreamTokenizer'
@ -27,13 +34,8 @@ export interface HighlightChunkResult {
* - 使 Worker * - 使 Worker
*/ */
class ShikiStreamService { class ShikiStreamService {
// 默认配置
private static readonly DEFAULT_LANGUAGES = ['javascript', 'typescript', 'python', 'java', 'markdown']
private static readonly DEFAULT_THEMES = ['one-light', 'material-theme-darker']
// 主线程 highlighter 和 tokenizers // 主线程 highlighter 和 tokenizers
private highlighter: HighlighterCore | null = null private highlighter: HighlighterGeneric<any, any> | null = null
private highlighterInitPromise: Promise<void> | null = null
// 保存以 callerId-language-theme 为键的 tokenizer map // 保存以 callerId-language-theme 为键的 tokenizer map
private tokenizerCache = new LRUCache<string, ShikiStreamTokenizer>({ private tokenizerCache = new LRUCache<string, ShikiStreamTokenizer>({
@ -80,7 +82,7 @@ class ShikiStreamService {
* 使线 * 使线
*/ */
public hasMainHighlighter(): boolean { public hasMainHighlighter(): boolean {
return !!this.highlighter && !this.highlighterInitPromise return !!this.highlighter
} }
/** /**
@ -125,8 +127,8 @@ class ShikiStreamService {
// 初始化 worker // 初始化 worker
await this.sendWorkerMessage({ await this.sendWorkerMessage({
type: 'init', type: 'init',
languages: ShikiStreamService.DEFAULT_LANGUAGES, languages: DEFAULT_LANGUAGES,
themes: ShikiStreamService.DEFAULT_THEMES themes: DEFAULT_THEMES
}) })
this.workerInitRetryCount = 0 this.workerInitRetryCount = 0
} catch (error) { } catch (error) {
@ -215,28 +217,6 @@ class ShikiStreamService {
return promise return promise
} }
/**
* highlighter
*/
private async initHighlighter(): Promise<void> {
if (this.highlighterInitPromise) {
return this.highlighterInitPromise
} else if (this.highlighter) {
return Promise.resolve()
}
this.highlighterInitPromise = (async () => {
const { createHighlighter } = await import('shiki')
this.highlighter = await createHighlighter({
langs: ShikiStreamService.DEFAULT_LANGUAGES,
themes: ShikiStreamService.DEFAULT_THEMES
})
})()
return this.highlighterInitPromise
}
/** /**
* highlighter * highlighter
* @param language * @param language
@ -245,52 +225,15 @@ class ShikiStreamService {
private async ensureHighlighterConfigured( private async ensureHighlighterConfigured(
language: string, language: string,
theme: string theme: string
): Promise<{ actualLanguage: string; actualTheme: string }> { ): Promise<{ loadedLanguage: string; loadedTheme: string }> {
// 确保 highlighter 已初始化
if (!this.hasMainHighlighter()) {
await this.initHighlighter()
}
if (!this.highlighter) { if (!this.highlighter) {
throw new Error('Highlighter not initialized') this.highlighter = await getHighlighter()
} }
const shiki = await import('shiki') const loadedLanguage = await loadLanguageIfNeeded(this.highlighter, language)
let actualLanguage = language const loadedTheme = await loadThemeIfNeeded(this.highlighter, theme)
let actualTheme = theme
// 加载语言 return { loadedLanguage, loadedTheme }
if (!this.highlighter.getLoadedLanguages().includes(language)) {
try {
if (['text', 'ansi'].includes(language)) {
await this.highlighter.loadLanguage(language as SpecialLanguage)
} else {
const languageImportFn = shiki.bundledLanguages[language]
const langData = await languageImportFn()
await this.highlighter.loadLanguage(langData)
}
} catch (error) {
await this.highlighter.loadLanguage('text')
actualLanguage = 'text'
}
}
// 加载主题
if (!this.highlighter.getLoadedThemes().includes(theme)) {
try {
const themeImportFn = shiki.bundledThemes[theme]
const themeData = await themeImportFn()
await this.highlighter.loadTheme(themeData)
} catch (error) {
// 回退到 one-light
console.debug(`Failed to load theme '${theme}', falling back to 'one-light':`, error)
const oneLightTheme = await shiki.bundledThemes['one-light']()
await this.highlighter.loadTheme(oneLightTheme)
actualTheme = 'one-light'
}
}
return { actualLanguage, actualTheme }
} }
/** /**
@ -303,15 +246,15 @@ class ShikiStreamService {
* @returns pre * @returns pre
*/ */
async getShikiPreProperties(language: string, theme: string): Promise<ShikiPreProperties> { async getShikiPreProperties(language: string, theme: string): Promise<ShikiPreProperties> {
const { actualLanguage, actualTheme } = await this.ensureHighlighterConfigured(language, theme) const { loadedLanguage, loadedTheme } = await this.ensureHighlighterConfigured(language, theme)
if (!this.highlighter) { if (!this.highlighter) {
throw new Error('Highlighter not initialized') throw new Error('Highlighter not initialized')
} }
const hast = this.highlighter.codeToHast('1', { const hast = this.highlighter.codeToHast('1', {
lang: actualLanguage, lang: loadedLanguage,
theme: actualTheme theme: loadedTheme
}) })
// @ts-ignore hack // @ts-ignore hack
@ -428,7 +371,7 @@ class ShikiStreamService {
} }
// 确保 highlighter 已配置 // 确保 highlighter 已配置
const { actualLanguage, actualTheme } = await this.ensureHighlighterConfigured(language, theme) const { loadedLanguage, loadedTheme } = await this.ensureHighlighterConfigured(language, theme)
if (!this.highlighter) { if (!this.highlighter) {
throw new Error('Highlighter not initialized') throw new Error('Highlighter not initialized')
@ -437,8 +380,8 @@ class ShikiStreamService {
// 创建新的 tokenizer // 创建新的 tokenizer
const options: ShikiStreamTokenizerOptions = { const options: ShikiStreamTokenizerOptions = {
highlighter: this.highlighter, highlighter: this.highlighter,
lang: actualLanguage, lang: loadedLanguage,
theme: actualTheme theme: loadedTheme
} }
const tokenizer = new ShikiStreamTokenizer(options) const tokenizer = new ShikiStreamTokenizer(options)
@ -486,9 +429,7 @@ class ShikiStreamService {
this.workerDegradationCache.clear() this.workerDegradationCache.clear()
this.tokenizerCache.clear() this.tokenizerCache.clear()
this.highlighter?.dispose()
this.highlighter = null this.highlighter = null
this.highlighterInitPromise = null
this.workerInitPromise = null this.workerInitPromise = null
this.workerInitRetryCount = 0 this.workerInitRetryCount = 0
} }

View File

@ -256,7 +256,6 @@ describe('ShikiStreamService', () => {
expect((shikiStreamService as any).highlighter).toBeNull() expect((shikiStreamService as any).highlighter).toBeNull()
expect((shikiStreamService as any).tokenizerCache.size).toBe(0) expect((shikiStreamService as any).tokenizerCache.size).toBe(0)
expect((shikiStreamService as any).pendingRequests.size).toBe(0) expect((shikiStreamService as any).pendingRequests.size).toBe(0)
expect((shikiStreamService as any).highlighterInitPromise).toBeNull()
expect((shikiStreamService as any).workerInitPromise).toBeNull() expect((shikiStreamService as any).workerInitPromise).toBeNull()
expect((shikiStreamService as any).workerInitRetryCount).toBe(0) expect((shikiStreamService as any).workerInitRetryCount).toBe(0)
}) })

View File

@ -0,0 +1,16 @@
export class AsyncInitializer<T> {
private promise: Promise<T> | null = null
private factory: () => Promise<T>
constructor(factory: () => Promise<T>) {
this.factory = factory
}
async get(): Promise<T> {
if (!this.promise) {
this.promise = this.factory()
}
// 如果需要允许重试,可以重置 this.promise 为 null。
return this.promise
}
}

View File

@ -1,32 +0,0 @@
import { bundledLanguages, bundledThemes, createHighlighter, type Highlighter } from 'shiki'
let highlighterPromise: Promise<Highlighter> | null = null
export async function getHighlighter() {
if (!highlighterPromise) {
highlighterPromise = createHighlighter({
langs: ['javascript', 'typescript', 'python', 'java', 'markdown', 'json'],
themes: ['one-light', 'material-theme-darker']
})
}
return await highlighterPromise
}
export async function loadLanguageIfNeeded(highlighter: Highlighter, language: string) {
if (!highlighter.getLoadedLanguages().includes(language)) {
const languageImportFn = bundledLanguages[language]
if (languageImportFn) {
await highlighter.loadLanguage(await languageImportFn())
}
}
}
export async function loadThemeIfNeeded(highlighter: Highlighter, theme: string) {
if (!highlighter.getLoadedThemes().includes(theme)) {
const themeImportFn = bundledThemes[theme]
if (themeImportFn) {
await highlighter.loadTheme(await themeImportFn())
}
}
}

View File

@ -1,12 +1,100 @@
import { useTheme } from '@renderer/context/ThemeProvider' import { getTokenStyleObject, type HighlighterGeneric, SpecialLanguage, ThemedToken } from 'shiki/core'
import { ThemeMode } from '@renderer/types'
import { setupMarkdownIt } from '@shikijs/markdown-it'
import MarkdownIt from 'markdown-it'
import { useEffect, useRef, useState } from 'react'
import { getTokenStyleObject, ThemedToken } from 'shiki/core'
import { runAsyncFunction } from '.' import { AsyncInitializer } from './asyncInitializer'
import { getHighlighter } from './highlighter'
export const DEFAULT_LANGUAGES = ['javascript', 'typescript', 'python', 'java', 'markdown']
export const DEFAULT_THEMES = ['one-light', 'material-theme-darker']
/**
* shiki
*/
const shikiInitializer = new AsyncInitializer(async () => {
const shiki = await import('shiki')
return shiki
})
/**
* shiki package
*/
export async function getShiki() {
return shikiInitializer.get()
}
/**
* shiki highlighter
*/
const highlighterInitializer = new AsyncInitializer(async () => {
const shiki = await getShiki()
return shiki.createHighlighter({
langs: DEFAULT_LANGUAGES,
themes: DEFAULT_THEMES
})
})
/**
* shiki highlighter
*/
export async function getHighlighter() {
return highlighterInitializer.get()
}
/**
*
* @param highlighter - shiki highlighter
* @param language -
* @returns
*/
export async function loadLanguageIfNeeded(
highlighter: HighlighterGeneric<any, any>,
language: string
): Promise<string> {
const shiki = await getShiki()
let loadedLanguage = language
if (!highlighter.getLoadedLanguages().includes(language)) {
try {
if (['text', 'ansi'].includes(language)) {
await highlighter.loadLanguage(language as SpecialLanguage)
} else {
const languageImportFn = shiki.bundledLanguages[language]
const langData = await languageImportFn()
await highlighter.loadLanguage(langData)
}
} catch (error) {
await highlighter.loadLanguage('text')
loadedLanguage = 'text'
}
}
return loadedLanguage
}
/**
*
* @param highlighter - shiki highlighter
* @param theme -
* @returns
*/
export async function loadThemeIfNeeded(highlighter: HighlighterGeneric<any, any>, theme: string): Promise<string> {
const shiki = await getShiki()
let loadedTheme = theme
if (!highlighter.getLoadedThemes().includes(theme)) {
try {
const themeImportFn = shiki.bundledThemes[theme]
const themeData = await themeImportFn()
await highlighter.loadTheme(themeData)
} catch (error) {
// 回退到 one-light
console.debug(`Failed to load theme '${theme}', falling back to 'one-light':`, error)
const oneLightTheme = await shiki.bundledThemes['one-light']()
await highlighter.loadTheme(oneLightTheme)
loadedTheme = 'one-light'
}
}
return loadedTheme
}
/** /**
* Shiki token React * Shiki token React
@ -38,44 +126,35 @@ export function getReactStyleFromToken(token: ThemedToken): Record<string, strin
return reactStyle return reactStyle
} }
const defaultOptions = { /**
themes: { * markdown-it
light: 'one-light', */
dark: 'material-theme-darker' const mdInitializer = new AsyncInitializer(async () => {
}, const md = await import('markdown-it')
defaultColor: 'light' return md.default({
} linkify: true, // 自动转换 URL 为链接
typographer: true // 启用印刷格式优化
})
})
export async function getShikiInstance(theme: ThemeMode) { /**
* markdown-it
* @param theme -
*/
export async function getMarkdownIt(theme: string) {
const highlighter = await getHighlighter() const highlighter = await getHighlighter()
const md = await mdInitializer.get()
const { fromHighlighter } = await import('@shikijs/markdown-it/core')
const options = { md.use(
...defaultOptions, fromHighlighter(highlighter, {
defaultColor: theme themes: {
} light: 'one-light',
dark: 'material-theme-darker'
return function (markdownit: MarkdownIt) { },
setupMarkdownIt(markdownit, highlighter, options) defaultColor: theme
}
}
export function useShikiWithMarkdownIt(content: string) {
const [renderedMarkdown, setRenderedMarkdown] = useState('')
const md = useRef<MarkdownIt>(
new MarkdownIt({
linkify: true, // 自动转换 URL 为链接
typographer: true // 启用印刷格式优化
}) })
) )
const { theme } = useTheme()
useEffect(() => { return md
runAsyncFunction(async () => {
const sk = await getShikiInstance(theme)
md.current.use(sk)
setRenderedMarkdown(md.current.render(content))
})
}, [content, theme])
return {
renderedMarkdown
}
} }

120
yarn.lock
View File

@ -3986,79 +3986,79 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/core@npm:3.2.2": "@shikijs/core@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/core@npm:3.2.2" resolution: "@shikijs/core@npm:3.4.2"
dependencies: dependencies:
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
"@shikijs/vscode-textmate": "npm:^10.0.2" "@shikijs/vscode-textmate": "npm:^10.0.2"
"@types/hast": "npm:^3.0.4" "@types/hast": "npm:^3.0.4"
hast-util-to-html: "npm:^9.0.5" hast-util-to-html: "npm:^9.0.5"
checksum: 10c0/69afe788994653b69f1bafd4fd3c2143609b4b0c05e970c8dc8d82ec660d617850ad9eeb2b6aa5be2dd534cefa0213d577129cb9ae1070eb4890cbbf1ac0f63e checksum: 10c0/702469d9c80fc80e2b81dd10407cc946771dcf355d56048e1dab43e40d144395c14a6ecde92e03c70a35249ad6634ef4605bd17ad6974a2b4e04f9efccf24414
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/engine-javascript@npm:3.2.2": "@shikijs/engine-javascript@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/engine-javascript@npm:3.2.2" resolution: "@shikijs/engine-javascript@npm:3.4.2"
dependencies: dependencies:
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
"@shikijs/vscode-textmate": "npm:^10.0.2" "@shikijs/vscode-textmate": "npm:^10.0.2"
oniguruma-to-es: "npm:^4.1.0" oniguruma-to-es: "npm:^4.3.3"
checksum: 10c0/2db8f9c04cc8e40352eb69ddd4de81bda95c5318c897f43215622352c91591974522cefb3959bcfaa183fd36a18d1af9e704289ed7999273dcd763bfaa5a1827 checksum: 10c0/160056a6303978d4e40114fe0414acd5089ea39a55a3144b9cba5e50aa38c521948ee47a2edc5acda5fd3607e33b20539845cfd9ca3508163e989b8fb4220488
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/engine-oniguruma@npm:3.2.2": "@shikijs/engine-oniguruma@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/engine-oniguruma@npm:3.2.2" resolution: "@shikijs/engine-oniguruma@npm:3.4.2"
dependencies: dependencies:
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
"@shikijs/vscode-textmate": "npm:^10.0.2" "@shikijs/vscode-textmate": "npm:^10.0.2"
checksum: 10c0/b5eedfca26f7e1525fd079c1827ae9bdedafb574ce4eb535c54d484218b7428fb9ac93607f79a2adc1482483dd0366fdf07b0846403a76cd4767649adb8fa590 checksum: 10c0/b8a13123b8a41e1016b661c24b349163b5026841772c351aacddcdc724518a926a49065ac77e4a1d4bb94da12c6bf11e6b1c938ef881545064bb3b484223eba0
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/langs@npm:3.2.2": "@shikijs/langs@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/langs@npm:3.2.2" resolution: "@shikijs/langs@npm:3.4.2"
dependencies: dependencies:
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
checksum: 10c0/04b5c9b92de9070624d24e20a2b3607edcbe4894a1db8056927f0d0637f080e2eed4e54925f0ded36874361db14bab9e4d9c2d06614ddd733f3f314250eabaf8 checksum: 10c0/ca0260b00e32385db8db43d8dd147f480bc2ff699acaf6052ec3e421b1c6d27df6dfb0f69fadb673ef357333ba65fdce2fbcd8c31c7d245439756bfb3530eba4
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/markdown-it@npm:^3.2.2": "@shikijs/markdown-it@npm:^3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/markdown-it@npm:3.2.2" resolution: "@shikijs/markdown-it@npm:3.4.2"
dependencies: dependencies:
markdown-it: "npm:^14.1.0" markdown-it: "npm:^14.1.0"
shiki: "npm:3.2.2" shiki: "npm:3.4.2"
peerDependencies: peerDependencies:
markdown-it-async: ^2.2.0 markdown-it-async: ^2.2.0
peerDependenciesMeta: peerDependenciesMeta:
markdown-it-async: markdown-it-async:
optional: true optional: true
checksum: 10c0/37c98e45a0905ea58f605c7cd341c83f3289b6a37093862535c59f7cc178fe9bfb13413fea68d4d923341e51b2e5718fc5172147d15e07457a76aceed2ac1f95 checksum: 10c0/6bdb4fae6038867a370b454d39cb64ac78645385f3f0150e85fd68952965186a00ba87aa4ce1d0b280f599d6aaf78d30ee32b6ad7d92391091dd7af569e7cbdb
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/themes@npm:3.2.2": "@shikijs/themes@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/themes@npm:3.2.2" resolution: "@shikijs/themes@npm:3.4.2"
dependencies: dependencies:
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
checksum: 10c0/93745e76e7ed6cab1d797ec68b53a0a183d989201e5064b33a78b516e128848d2c9be194d29cf602d5017dc2a74013699c773d052aeb45593851ae35b035afaa checksum: 10c0/d50bca4384ccf88d68f007869e13bc7a9b55b16c40a3269fe120b2e5a2e882f6206ee0325f619bfa31ff00a0341452840d38f4ca2296dd3ba3e200e53445e22b
languageName: node languageName: node
linkType: hard linkType: hard
"@shikijs/types@npm:3.2.2": "@shikijs/types@npm:3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "@shikijs/types@npm:3.2.2" resolution: "@shikijs/types@npm:3.4.2"
dependencies: dependencies:
"@shikijs/vscode-textmate": "npm:^10.0.2" "@shikijs/vscode-textmate": "npm:^10.0.2"
"@types/hast": "npm:^3.0.4" "@types/hast": "npm:^3.0.4"
checksum: 10c0/aec3327d0cfc89af138ce195ac070ba62d8229864c079a3f06dff5a180036fdd963282068d67bd4c89a04ae688005c2b7c214c274ad0bb265f6f7ab6907a67a6 checksum: 10c0/a20d3535cc0d61a55d0c0d4dfcd33a52229ec8a4c650613cb0f424dcb499bcdf0230e007f70a18e12c102a04820557ff120f41f18b15a94f95f9ec343592906b
languageName: node languageName: node
linkType: hard linkType: hard
@ -5765,7 +5765,7 @@ __metadata:
"@mozilla/readability": "npm:^0.6.0" "@mozilla/readability": "npm:^0.6.0"
"@notionhq/client": "npm:^2.2.15" "@notionhq/client": "npm:^2.2.15"
"@reduxjs/toolkit": "npm:^2.2.5" "@reduxjs/toolkit": "npm:^2.2.5"
"@shikijs/markdown-it": "npm:^3.2.2" "@shikijs/markdown-it": "npm:^3.4.2"
"@strongtz/win32-arm64-msvc": "npm:^0.4.7" "@strongtz/win32-arm64-msvc": "npm:^0.4.7"
"@swc/plugin-styled-components": "npm:^7.1.5" "@swc/plugin-styled-components": "npm:^7.1.5"
"@tanstack/react-query": "npm:^5.27.0" "@tanstack/react-query": "npm:^5.27.0"
@ -5866,7 +5866,7 @@ __metadata:
remark-math: "npm:^6.0.0" remark-math: "npm:^6.0.0"
rollup-plugin-visualizer: "npm:^5.12.0" rollup-plugin-visualizer: "npm:^5.12.0"
sass: "npm:^1.88.0" sass: "npm:^1.88.0"
shiki: "npm:^3.2.2" shiki: "npm:^3.4.2"
string-width: "npm:^7.2.0" string-width: "npm:^7.2.0"
styled-components: "npm:^6.1.11" styled-components: "npm:^6.1.11"
tar: "npm:^7.4.3" tar: "npm:^7.4.3"
@ -8960,13 +8960,6 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"emoji-regex-xs@npm:^1.0.0":
version: 1.0.0
resolution: "emoji-regex-xs@npm:1.0.0"
checksum: 10c0/1082de006991eb05a3324ef0efe1950c7cdf66efc01d4578de82b0d0d62add4e55e97695a8a7eeda826c305081562dc79b477ddf18d886da77f3ba08c4b940a0
languageName: node
linkType: hard
"emoji-regex@npm:^10.3.0": "emoji-regex@npm:^10.3.0":
version: 10.4.0 version: 10.4.0
resolution: "emoji-regex@npm:10.4.0" resolution: "emoji-regex@npm:10.4.0"
@ -14793,22 +14786,21 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"oniguruma-parser@npm:^0.11.0": "oniguruma-parser@npm:^0.12.1":
version: 0.11.1 version: 0.12.1
resolution: "oniguruma-parser@npm:0.11.1" resolution: "oniguruma-parser@npm:0.12.1"
checksum: 10c0/d721cabe5632d0b772fec95dd6920cb6d6ba7a8e9b247dbb32a82b8a997137ecb62110d1788578dfd43596d4461a3319ca320d30aa2b6ebbaa19552a98e507ba checksum: 10c0/b843ea54cda833efb19f856314afcbd43e903ece3de489ab78c527ddec84859208052557daa9fad4bdba89ebdd15b0cc250de86b3daf8c7cbe37bac5a6a185d3
languageName: node languageName: node
linkType: hard linkType: hard
"oniguruma-to-es@npm:^4.1.0": "oniguruma-to-es@npm:^4.3.3":
version: 4.2.0 version: 4.3.3
resolution: "oniguruma-to-es@npm:4.2.0" resolution: "oniguruma-to-es@npm:4.3.3"
dependencies: dependencies:
emoji-regex-xs: "npm:^1.0.0" oniguruma-parser: "npm:^0.12.1"
oniguruma-parser: "npm:^0.11.0"
regex: "npm:^6.0.1" regex: "npm:^6.0.1"
regex-recursion: "npm:^6.0.2" regex-recursion: "npm:^6.0.2"
checksum: 10c0/a2c505ad9d4ccca9f71a5ea22dc68ab94f244e2fab5b04fea54f355411a2c13d65b5c28925af508ea3a694ce8cf9e86931681bfe3ea4a89722d9b50e24bf21fd checksum: 10c0/bc034e84dfee4dbc061cf6364023e66e1667fb8dc3afcad3b7d6a2c77e2d4a4809396ee2fb8c1fd3d6f00f76f7ca14b773586bf862c5f0c0074c059e2a219252
languageName: node languageName: node
linkType: hard linkType: hard
@ -17618,19 +17610,19 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"shiki@npm:3.2.2": "shiki@npm:3.4.2, shiki@npm:^3.4.2":
version: 3.2.2 version: 3.4.2
resolution: "shiki@npm:3.2.2" resolution: "shiki@npm:3.4.2"
dependencies: dependencies:
"@shikijs/core": "npm:3.2.2" "@shikijs/core": "npm:3.4.2"
"@shikijs/engine-javascript": "npm:3.2.2" "@shikijs/engine-javascript": "npm:3.4.2"
"@shikijs/engine-oniguruma": "npm:3.2.2" "@shikijs/engine-oniguruma": "npm:3.4.2"
"@shikijs/langs": "npm:3.2.2" "@shikijs/langs": "npm:3.4.2"
"@shikijs/themes": "npm:3.2.2" "@shikijs/themes": "npm:3.4.2"
"@shikijs/types": "npm:3.2.2" "@shikijs/types": "npm:3.4.2"
"@shikijs/vscode-textmate": "npm:^10.0.2" "@shikijs/vscode-textmate": "npm:^10.0.2"
"@types/hast": "npm:^3.0.4" "@types/hast": "npm:^3.0.4"
checksum: 10c0/0183f889029ff1d14f79aa34e36f1e5a67b667661422f8a7de8936164099827588df7b2b4ed6835ad2eb3efb11ea882b4cb8022550503108c958a796df01f35c checksum: 10c0/3cae825d8c341d7334e541efad30125fac0064db6004359e661a594782d59f93f66f2dcb5dbc1d8cb6508c43ccdd03ed6cf1d22306b382bc1f395a6130e5cbbb
languageName: node languageName: node
linkType: hard linkType: hard