From a9a0ae87d3fe214d99b03d710ea48f151f3cb9c3 Mon Sep 17 00:00:00 2001 From: one Date: Wed, 21 May 2025 19:06:56 +0800 Subject: [PATCH] refactor: reuse shiki highlighter utils (#6235) * refactor: improve shiki highlighter utils and reuse it in ShikiStreamService * refactor: reuse shiki highlighter and markdown-it renderer * refactor: import shiki/markdown-it/core dynamically * chore: update shiki --- package.json | 5 +- .../src/context/CodeStyleProvider.tsx | 33 +++- src/renderer/src/i18n/locales/en-us.json | 1 + src/renderer/src/i18n/locales/ja-jp.json | 1 + src/renderer/src/i18n/locales/ru-ru.json | 1 + src/renderer/src/i18n/locales/zh-cn.json | 1 + src/renderer/src/i18n/locales/zh-tw.json | 1 + .../src/pages/home/Messages/MessageTools.tsx | 34 ++-- .../settings/MCPSettings/McpDescription.tsx | 55 +++--- .../src/services/ShikiStreamService.ts | 105 +++-------- .../__tests__/ShikiStreamService.test.ts | 1 - src/renderer/src/utils/asyncInitializer.ts | 16 ++ src/renderer/src/utils/highlighter.ts | 32 ---- src/renderer/src/utils/shiki.ts | 165 +++++++++++++----- yarn.lock | 120 ++++++------- 15 files changed, 301 insertions(+), 270 deletions(-) create mode 100644 src/renderer/src/utils/asyncInitializer.ts delete mode 100644 src/renderer/src/utils/highlighter.ts diff --git a/package.json b/package.json index b32cde61b0..dd10d6c427 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,7 @@ "@mozilla/readability": "^0.6.0", "@notionhq/client": "^2.2.15", "@reduxjs/toolkit": "^2.2.5", - "@shikijs/markdown-it": "^3.2.2", + "@shikijs/markdown-it": "^3.4.2", "@swc/plugin-styled-components": "^7.1.5", "@tryfabric/martian": "^1.2.4", "@types/diff": "^7", @@ -200,7 +200,7 @@ "remark-math": "^6.0.0", "rollup-plugin-visualizer": "^5.12.0", "sass": "^1.88.0", - "shiki": "^3.2.2", + "shiki": "^3.4.2", "string-width": "^7.2.0", "styled-components": "^6.1.11", "tiny-pinyin": "^1.3.2", @@ -219,7 +219,6 @@ "openai@npm:^4.77.0": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch", "pkce-challenge@npm:^4.1.0": "patch:pkce-challenge@npm%3A4.1.0#~/.yarn/patches/pkce-challenge-npm-4.1.0-fbc51695a3.patch", "app-builder-lib@npm:26.0.13": "patch:app-builder-lib@npm%3A26.0.13#~/.yarn/patches/app-builder-lib-npm-26.0.13-a064c9e1d0.patch", - "shiki": "3.2.2", "openai@npm:^4.87.3": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch", "app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch" }, diff --git a/src/renderer/src/context/CodeStyleProvider.tsx b/src/renderer/src/context/CodeStyleProvider.tsx index 23e50f8deb..53f52888c4 100644 --- a/src/renderer/src/context/CodeStyleProvider.tsx +++ b/src/renderer/src/context/CodeStyleProvider.tsx @@ -3,6 +3,7 @@ import { useMermaid } from '@renderer/hooks/useMermaid' import { useSettings } from '@renderer/hooks/useSettings' import { HighlightChunkResult, ShikiPreProperties, shikiStreamService } from '@renderer/services/ShikiStreamService' import { ThemeMode } from '@renderer/types' +import { getHighlighter, getMarkdownIt, getShiki, loadLanguageIfNeeded, loadThemeIfNeeded } from '@renderer/utils/shiki' import * as cmThemes from '@uiw/codemirror-themes-all' import type React from 'react' import { createContext, type PropsWithChildren, use, useCallback, useEffect, useMemo, useState } from 'react' @@ -11,6 +12,8 @@ interface CodeStyleContextType { highlightCodeChunk: (trunk: string, language: string, callerId: string) => Promise cleanupTokenizers: (callerId: string) => void getShikiPreProperties: (language: string) => Promise + highlightCode: (code: string, language: string) => Promise + shikiMarkdownIt: (code: string) => Promise themeNames: string[] activeShikiTheme: string activeCmTheme: any @@ -21,6 +24,8 @@ const defaultCodeStyleContext: CodeStyleContextType = { highlightCodeChunk: async () => ({ lines: [], recall: 0 }), cleanupTokenizers: () => {}, getShikiPreProperties: async () => ({ class: '', style: '', tabindex: 0 }), + highlightCode: async () => '', + shikiMarkdownIt: async () => '', themeNames: ['auto'], activeShikiTheme: 'auto', activeCmTheme: null, @@ -37,7 +42,7 @@ export const CodeStyleProvider: React.FC = ({ children }) => useEffect(() => { if (!codeEditor.enabled) { - import('shiki').then(({ bundledThemes }) => { + getShiki().then(({ bundledThemes }) => { setShikiThemes(bundledThemes) }) } @@ -118,11 +123,35 @@ export const CodeStyleProvider: React.FC = ({ children }) => [activeShikiTheme, languageMap] ) + const highlightCode = useCallback( + async (code: string, language: string) => { + const highlighter = await getHighlighter() + await loadLanguageIfNeeded(highlighter, language) + await loadThemeIfNeeded(highlighter, activeShikiTheme) + return highlighter.codeToHtml(code, { lang: language, theme: activeShikiTheme }) + }, + [activeShikiTheme] + ) + + // 使用 Shiki 和 Markdown-it 渲染代码 + const shikiMarkdownIt = useCallback( + async (code: string) => { + const renderer = await getMarkdownIt(activeShikiTheme) + if (!renderer) { + return code + } + return renderer.render(code) + }, + [activeShikiTheme] + ) + const contextValue = useMemo( () => ({ highlightCodeChunk, cleanupTokenizers, getShikiPreProperties, + highlightCode, + shikiMarkdownIt, themeNames, activeShikiTheme, activeCmTheme, @@ -132,6 +161,8 @@ export const CodeStyleProvider: React.FC = ({ children }) => highlightCodeChunk, cleanupTokenizers, getShikiPreProperties, + highlightCode, + shikiMarkdownIt, themeNames, activeShikiTheme, activeCmTheme, diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index 84d17aeaaf..7be5836bfa 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -1306,6 +1306,7 @@ "dependenciesInstall": "Install Dependencies", "dependenciesInstalling": "Installing dependencies...", "description": "Description", + "noDescriptionAvailable": "No description available", "duplicateName": "A server with this name already exists", "editJson": "Edit JSON", "editServer": "Edit Server", diff --git a/src/renderer/src/i18n/locales/ja-jp.json b/src/renderer/src/i18n/locales/ja-jp.json index 6adbf75d39..2520a824f6 100644 --- a/src/renderer/src/i18n/locales/ja-jp.json +++ b/src/renderer/src/i18n/locales/ja-jp.json @@ -1303,6 +1303,7 @@ "dependenciesInstall": "依存関係をインストール", "dependenciesInstalling": "依存関係をインストール中...", "description": "説明", + "noDescriptionAvailable": "説明がありません", "duplicateName": "同じ名前のサーバーが既に存在します", "editJson": "JSONを編集", "editServer": "サーバーを編集", diff --git a/src/renderer/src/i18n/locales/ru-ru.json b/src/renderer/src/i18n/locales/ru-ru.json index 7903f23620..e2e27b73a5 100644 --- a/src/renderer/src/i18n/locales/ru-ru.json +++ b/src/renderer/src/i18n/locales/ru-ru.json @@ -1303,6 +1303,7 @@ "dependenciesInstall": "Установить зависимости", "dependenciesInstalling": "Установка зависимостей...", "description": "Описание", + "noDescriptionAvailable": "Описание отсутствует", "duplicateName": "Сервер с таким именем уже существует", "editJson": "Редактировать JSON", "editServer": "Редактировать сервер", diff --git a/src/renderer/src/i18n/locales/zh-cn.json b/src/renderer/src/i18n/locales/zh-cn.json index b01ebeccbb..0088532099 100644 --- a/src/renderer/src/i18n/locales/zh-cn.json +++ b/src/renderer/src/i18n/locales/zh-cn.json @@ -1306,6 +1306,7 @@ "dependenciesInstall": "安装依赖项", "dependenciesInstalling": "正在安装依赖项...", "description": "描述", + "noDescriptionAvailable": "暂无描述", "duplicateName": "已存在同名服务器", "editJson": "编辑JSON", "editServer": "编辑服务器", diff --git a/src/renderer/src/i18n/locales/zh-tw.json b/src/renderer/src/i18n/locales/zh-tw.json index b0b9444b3a..6ecd496108 100644 --- a/src/renderer/src/i18n/locales/zh-tw.json +++ b/src/renderer/src/i18n/locales/zh-tw.json @@ -1306,6 +1306,7 @@ "dependenciesInstall": "安裝相依套件", "dependenciesInstalling": "正在安裝相依套件...", "description": "描述", + "noDescriptionAvailable": "描述不存在", "duplicateName": "已存在相同名稱的伺服器", "editJson": "編輯JSON", "editServer": "編輯伺服器", diff --git a/src/renderer/src/pages/home/Messages/MessageTools.tsx b/src/renderer/src/pages/home/Messages/MessageTools.tsx index 39deeb323d..4490c3000c 100644 --- a/src/renderer/src/pages/home/Messages/MessageTools.tsx +++ b/src/renderer/src/pages/home/Messages/MessageTools.tsx @@ -1,9 +1,9 @@ import { CheckOutlined, ExpandOutlined, LoadingOutlined, WarningOutlined } from '@ant-design/icons' +import { useCodeStyle } from '@renderer/context/CodeStyleProvider' import { useSettings } from '@renderer/hooks/useSettings' import type { ToolMessageBlock } from '@renderer/types/newMessage' -import { useShikiWithMarkdownIt } from '@renderer/utils/shiki' import { Collapse, message as antdMessage, Modal, Tabs, Tooltip } from 'antd' -import { FC, useMemo, useState } from 'react' +import { FC, memo, useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import styled from 'styled-components' @@ -192,12 +192,7 @@ const MessageTools: FC = ({ blocks }) => { { key: 'preview', label: t('message.tools.preview'), - children: ( - - ) + children: }, { key: 'raw', @@ -215,9 +210,17 @@ const MessageTools: FC = ({ blocks }) => { // New component to handle collapsed content const CollapsedContent: FC<{ isExpanded: boolean; resultString: string }> = ({ isExpanded, resultString }) => { - const { renderedMarkdown: styledResult } = useShikiWithMarkdownIt( - isExpanded ? `\`\`\`json\n${resultString}\n\`\`\`` : '' - ) + const { highlightCode } = useCodeStyle() + const [styledResult, setStyledResult] = useState('') + + useEffect(() => { + const highlight = async () => { + const result = await highlightCode(isExpanded ? resultString : '', 'json') + setStyledResult(result) + } + + setTimeout(highlight, 0) + }, [isExpanded, resultString, highlightCode]) if (!isExpanded) { return null @@ -247,8 +250,11 @@ const CollapseContainer = styled(Collapse)` ` const MarkdownContainer = styled.div` - & pre span { - white-space: pre-wrap; + & pre { + background: transparent !important; + span { + white-space: pre-wrap; + } } ` @@ -371,4 +377,4 @@ const ExpandedResponseContainer = styled.div` } ` -export default MessageTools +export default memo(MessageTools) diff --git a/src/renderer/src/pages/settings/MCPSettings/McpDescription.tsx b/src/renderer/src/pages/settings/MCPSettings/McpDescription.tsx index 9877e0b989..0dda6abbdc 100644 --- a/src/renderer/src/pages/settings/MCPSettings/McpDescription.tsx +++ b/src/renderer/src/pages/settings/MCPSettings/McpDescription.tsx @@ -1,47 +1,42 @@ -import { useTheme } from '@renderer/context/ThemeProvider' -import { runAsyncFunction } from '@renderer/utils' -import { getShikiInstance } from '@renderer/utils/shiki' +import { useCodeStyle } from '@renderer/context/CodeStyleProvider' import { Card } from 'antd' -import MarkdownIt from 'markdown-it' import { npxFinder } from 'npx-scope-finder' -import { useCallback, useEffect, useRef, useState } from 'react' +import { FC, memo, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' import styled from 'styled-components' interface McpDescriptionProps { searchKey: string } -const MCPDescription = ({ searchKey }: McpDescriptionProps) => { - const [renderedMarkdown, setRenderedMarkdown] = useState('') +const MCPDescription: FC = ({ searchKey }) => { + const { t } = useTranslation() + const { shikiMarkdownIt } = useCodeStyle() const [loading, setLoading] = useState(false) - - const md = useRef( - new MarkdownIt({ - linkify: true, // 自动转换 URL 为链接 - typographer: true // 启用印刷格式优化 - }) - ) - const { theme } = useTheme() - - const getMcpInfo = useCallback(async () => { - setLoading(true) - const packages = await npxFinder(searchKey).finally(() => setLoading(false)) - const readme = packages[0]?.original?.readme ?? '暂无描述' - setRenderedMarkdown(md.current.render(readme)) - }, [md, searchKey]) + const [mcpInfo, setMcpInfo] = useState('') useEffect(() => { - runAsyncFunction(async () => { - const sk = await getShikiInstance(theme) - md.current.use(sk) - getMcpInfo() - }) - }, [getMcpInfo, theme]) + let isMounted = true + setLoading(true) + npxFinder(searchKey) + .then((packages) => { + const readme = packages[0]?.original?.readme ?? t('settings.mcp.noDescriptionAvailable') + shikiMarkdownIt(readme).then((result) => { + if (isMounted) setMcpInfo(result) + }) + }) + .finally(() => { + if (isMounted) setLoading(false) + }) + return () => { + isMounted = false + } + }, [shikiMarkdownIt, searchKey, t]) return (
-
+
) @@ -50,4 +45,4 @@ const Section = styled.div` padding-top: 8px; ` -export default MCPDescription +export default memo(MCPDescription) diff --git a/src/renderer/src/services/ShikiStreamService.ts b/src/renderer/src/services/ShikiStreamService.ts index c9a465b762..8ddc2a8be1 100644 --- a/src/renderer/src/services/ShikiStreamService.ts +++ b/src/renderer/src/services/ShikiStreamService.ts @@ -1,5 +1,12 @@ +import { + DEFAULT_LANGUAGES, + DEFAULT_THEMES, + getHighlighter, + loadLanguageIfNeeded, + loadThemeIfNeeded +} from '@renderer/utils/shiki' import { LRUCache } from 'lru-cache' -import type { HighlighterCore, SpecialLanguage, ThemedToken } from 'shiki/core' +import type { HighlighterGeneric, ThemedToken } from 'shiki/core' import { ShikiStreamTokenizer, ShikiStreamTokenizerOptions } from './ShikiStreamTokenizer' @@ -27,13 +34,8 @@ export interface HighlightChunkResult { * - 优先使用 Worker 处理高亮请求。 */ class ShikiStreamService { - // 默认配置 - private static readonly DEFAULT_LANGUAGES = ['javascript', 'typescript', 'python', 'java', 'markdown'] - private static readonly DEFAULT_THEMES = ['one-light', 'material-theme-darker'] - // 主线程 highlighter 和 tokenizers - private highlighter: HighlighterCore | null = null - private highlighterInitPromise: Promise | null = null + private highlighter: HighlighterGeneric | null = null // 保存以 callerId-language-theme 为键的 tokenizer map private tokenizerCache = new LRUCache({ @@ -80,7 +82,7 @@ class ShikiStreamService { * 判断是否正在使用主线程高亮。外部不要依赖这个方法来判断。 */ public hasMainHighlighter(): boolean { - return !!this.highlighter && !this.highlighterInitPromise + return !!this.highlighter } /** @@ -125,8 +127,8 @@ class ShikiStreamService { // 初始化 worker await this.sendWorkerMessage({ type: 'init', - languages: ShikiStreamService.DEFAULT_LANGUAGES, - themes: ShikiStreamService.DEFAULT_THEMES + languages: DEFAULT_LANGUAGES, + themes: DEFAULT_THEMES }) this.workerInitRetryCount = 0 } catch (error) { @@ -215,28 +217,6 @@ class ShikiStreamService { return promise } - /** - * 初始化 highlighter - */ - private async initHighlighter(): Promise { - if (this.highlighterInitPromise) { - return this.highlighterInitPromise - } else if (this.highlighter) { - return Promise.resolve() - } - - this.highlighterInitPromise = (async () => { - const { createHighlighter } = await import('shiki') - - this.highlighter = await createHighlighter({ - langs: ShikiStreamService.DEFAULT_LANGUAGES, - themes: ShikiStreamService.DEFAULT_THEMES - }) - })() - - return this.highlighterInitPromise - } - /** * 确保 highlighter 已配置 * @param language 语言 @@ -245,52 +225,15 @@ class ShikiStreamService { private async ensureHighlighterConfigured( language: string, theme: string - ): Promise<{ actualLanguage: string; actualTheme: string }> { - // 确保 highlighter 已初始化 - if (!this.hasMainHighlighter()) { - await this.initHighlighter() - } - + ): Promise<{ loadedLanguage: string; loadedTheme: string }> { if (!this.highlighter) { - throw new Error('Highlighter not initialized') + this.highlighter = await getHighlighter() } - const shiki = await import('shiki') - let actualLanguage = language - let actualTheme = theme + const loadedLanguage = await loadLanguageIfNeeded(this.highlighter, language) + const loadedTheme = await loadThemeIfNeeded(this.highlighter, theme) - // 加载语言 - if (!this.highlighter.getLoadedLanguages().includes(language)) { - try { - if (['text', 'ansi'].includes(language)) { - await this.highlighter.loadLanguage(language as SpecialLanguage) - } else { - const languageImportFn = shiki.bundledLanguages[language] - const langData = await languageImportFn() - await this.highlighter.loadLanguage(langData) - } - } catch (error) { - await this.highlighter.loadLanguage('text') - actualLanguage = 'text' - } - } - - // 加载主题 - if (!this.highlighter.getLoadedThemes().includes(theme)) { - try { - const themeImportFn = shiki.bundledThemes[theme] - const themeData = await themeImportFn() - await this.highlighter.loadTheme(themeData) - } catch (error) { - // 回退到 one-light - console.debug(`Failed to load theme '${theme}', falling back to 'one-light':`, error) - const oneLightTheme = await shiki.bundledThemes['one-light']() - await this.highlighter.loadTheme(oneLightTheme) - actualTheme = 'one-light' - } - } - - return { actualLanguage, actualTheme } + return { loadedLanguage, loadedTheme } } /** @@ -303,15 +246,15 @@ class ShikiStreamService { * @returns pre 标签属性 */ async getShikiPreProperties(language: string, theme: string): Promise { - const { actualLanguage, actualTheme } = await this.ensureHighlighterConfigured(language, theme) + const { loadedLanguage, loadedTheme } = await this.ensureHighlighterConfigured(language, theme) if (!this.highlighter) { throw new Error('Highlighter not initialized') } const hast = this.highlighter.codeToHast('1', { - lang: actualLanguage, - theme: actualTheme + lang: loadedLanguage, + theme: loadedTheme }) // @ts-ignore hack @@ -428,7 +371,7 @@ class ShikiStreamService { } // 确保 highlighter 已配置 - const { actualLanguage, actualTheme } = await this.ensureHighlighterConfigured(language, theme) + const { loadedLanguage, loadedTheme } = await this.ensureHighlighterConfigured(language, theme) if (!this.highlighter) { throw new Error('Highlighter not initialized') @@ -437,8 +380,8 @@ class ShikiStreamService { // 创建新的 tokenizer const options: ShikiStreamTokenizerOptions = { highlighter: this.highlighter, - lang: actualLanguage, - theme: actualTheme + lang: loadedLanguage, + theme: loadedTheme } const tokenizer = new ShikiStreamTokenizer(options) @@ -486,9 +429,7 @@ class ShikiStreamService { this.workerDegradationCache.clear() this.tokenizerCache.clear() - this.highlighter?.dispose() this.highlighter = null - this.highlighterInitPromise = null this.workerInitPromise = null this.workerInitRetryCount = 0 } diff --git a/src/renderer/src/services/__tests__/ShikiStreamService.test.ts b/src/renderer/src/services/__tests__/ShikiStreamService.test.ts index a4a1743fd5..260418fb6f 100644 --- a/src/renderer/src/services/__tests__/ShikiStreamService.test.ts +++ b/src/renderer/src/services/__tests__/ShikiStreamService.test.ts @@ -256,7 +256,6 @@ describe('ShikiStreamService', () => { expect((shikiStreamService as any).highlighter).toBeNull() expect((shikiStreamService as any).tokenizerCache.size).toBe(0) expect((shikiStreamService as any).pendingRequests.size).toBe(0) - expect((shikiStreamService as any).highlighterInitPromise).toBeNull() expect((shikiStreamService as any).workerInitPromise).toBeNull() expect((shikiStreamService as any).workerInitRetryCount).toBe(0) }) diff --git a/src/renderer/src/utils/asyncInitializer.ts b/src/renderer/src/utils/asyncInitializer.ts new file mode 100644 index 0000000000..9fdd3ea936 --- /dev/null +++ b/src/renderer/src/utils/asyncInitializer.ts @@ -0,0 +1,16 @@ +export class AsyncInitializer { + private promise: Promise | null = null + private factory: () => Promise + + constructor(factory: () => Promise) { + this.factory = factory + } + + async get(): Promise { + if (!this.promise) { + this.promise = this.factory() + } + // 如果需要允许重试，可以重置 this.promise 为 null。 + return this.promise + } +} diff --git a/src/renderer/src/utils/highlighter.ts b/src/renderer/src/utils/highlighter.ts deleted file mode 100644 index 6417c06104..0000000000 --- a/src/renderer/src/utils/highlighter.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { bundledLanguages, bundledThemes, createHighlighter, type Highlighter } from 'shiki' - -let highlighterPromise: Promise | null = null - -export async function getHighlighter() { - if (!highlighterPromise) { - highlighterPromise = createHighlighter({ - langs: ['javascript', 'typescript', 'python', 'java', 'markdown', 'json'], - themes: ['one-light', 'material-theme-darker'] - }) - } - - return await highlighterPromise -} - -export async function loadLanguageIfNeeded(highlighter: Highlighter, language: string) { - if (!highlighter.getLoadedLanguages().includes(language)) { - const languageImportFn = bundledLanguages[language] - if (languageImportFn) { - await highlighter.loadLanguage(await languageImportFn()) - } - } -} - -export async function loadThemeIfNeeded(highlighter: Highlighter, theme: string) { - if (!highlighter.getLoadedThemes().includes(theme)) { - const themeImportFn = bundledThemes[theme] - if (themeImportFn) { - await highlighter.loadTheme(await themeImportFn()) - } - } -} diff --git a/src/renderer/src/utils/shiki.ts b/src/renderer/src/utils/shiki.ts index 4eb1c71e92..111408ba1b 100644 --- a/src/renderer/src/utils/shiki.ts +++ b/src/renderer/src/utils/shiki.ts @@ -1,12 +1,100 @@ -import { useTheme } from '@renderer/context/ThemeProvider' -import { ThemeMode } from '@renderer/types' -import { setupMarkdownIt } from '@shikijs/markdown-it' -import MarkdownIt from 'markdown-it' -import { useEffect, useRef, useState } from 'react' -import { getTokenStyleObject, ThemedToken } from 'shiki/core' +import { getTokenStyleObject, type HighlighterGeneric, SpecialLanguage, ThemedToken } from 'shiki/core' -import { runAsyncFunction } from '.' -import { getHighlighter } from './highlighter' +import { AsyncInitializer } from './asyncInitializer' + +export const DEFAULT_LANGUAGES = ['javascript', 'typescript', 'python', 'java', 'markdown'] +export const DEFAULT_THEMES = ['one-light', 'material-theme-darker'] + +/** + * shiki 初始化器，避免并发问题 + */ +const shikiInitializer = new AsyncInitializer(async () => { + const shiki = await import('shiki') + return shiki +}) + +/** + * 获取 shiki package + */ +export async function getShiki() { + return shikiInitializer.get() +} + +/** + * shiki highlighter 初始化器，避免并发问题 + */ +const highlighterInitializer = new AsyncInitializer(async () => { + const shiki = await getShiki() + return shiki.createHighlighter({ + langs: DEFAULT_LANGUAGES, + themes: DEFAULT_THEMES + }) +}) + +/** + * 获取 shiki highlighter + */ +export async function getHighlighter() { + return highlighterInitializer.get() +} + +/** + * 加载语言 + * @param highlighter - shiki highlighter + * @param language - 语言 + * @returns 实际加载的语言 + */ +export async function loadLanguageIfNeeded( + highlighter: HighlighterGeneric, + language: string +): Promise { + const shiki = await getShiki() + + let loadedLanguage = language + if (!highlighter.getLoadedLanguages().includes(language)) { + try { + if (['text', 'ansi'].includes(language)) { + await highlighter.loadLanguage(language as SpecialLanguage) + } else { + const languageImportFn = shiki.bundledLanguages[language] + const langData = await languageImportFn() + await highlighter.loadLanguage(langData) + } + } catch (error) { + await highlighter.loadLanguage('text') + loadedLanguage = 'text' + } + } + + return loadedLanguage +} + +/** + * 加载主题 + * @param highlighter - shiki highlighter + * @param theme - 主题 + * @returns 实际加载的主题 + */ +export async function loadThemeIfNeeded(highlighter: HighlighterGeneric, theme: string): Promise { + const shiki = await getShiki() + + let loadedTheme = theme + if (!highlighter.getLoadedThemes().includes(theme)) { + try { + const themeImportFn = shiki.bundledThemes[theme] + const themeData = await themeImportFn() + await highlighter.loadTheme(themeData) + } catch (error) { + // 回退到 one-light + console.debug(`Failed to load theme '${theme}', falling back to 'one-light':`, error) + const oneLightTheme = await shiki.bundledThemes['one-light']() + await highlighter.loadTheme(oneLightTheme) + loadedTheme = 'one-light' + } + } + + return loadedTheme +} /** * Shiki token 样式转换为 React 样式对象 @@ -38,44 +126,35 @@ export function getReactStyleFromToken(token: ThemedToken): Record { + const md = await import('markdown-it') + return md.default({ + linkify: true, // 自动转换 URL 为链接 + typographer: true // 启用印刷格式优化 + }) +}) -export async function getShikiInstance(theme: ThemeMode) { +/** + * 获取 markdown-it 渲染器 + * @param theme - 主题 + */ +export async function getMarkdownIt(theme: string) { const highlighter = await getHighlighter() + const md = await mdInitializer.get() + const { fromHighlighter } = await import('@shikijs/markdown-it/core') - const options = { - ...defaultOptions, - defaultColor: theme - } - - return function (markdownit: MarkdownIt) { - setupMarkdownIt(markdownit, highlighter, options) - } -} - -export function useShikiWithMarkdownIt(content: string) { - const [renderedMarkdown, setRenderedMarkdown] = useState('') - const md = useRef( - new MarkdownIt({ - linkify: true, // 自动转换 URL 为链接 - typographer: true // 启用印刷格式优化 + md.use( + fromHighlighter(highlighter, { + themes: { + light: 'one-light', + dark: 'material-theme-darker' + }, + defaultColor: theme }) ) - const { theme } = useTheme() - useEffect(() => { - runAsyncFunction(async () => { - const sk = await getShikiInstance(theme) - md.current.use(sk) - setRenderedMarkdown(md.current.render(content)) - }) - }, [content, theme]) - return { - renderedMarkdown - } + + return md } diff --git a/yarn.lock b/yarn.lock index a05e585083..fba270ae73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3986,79 +3986,79 @@ __metadata: languageName: node linkType: hard -"@shikijs/core@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/core@npm:3.2.2" +"@shikijs/core@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/core@npm:3.4.2" dependencies: - "@shikijs/types": "npm:3.2.2" + "@shikijs/types": "npm:3.4.2" "@shikijs/vscode-textmate": "npm:^10.0.2" "@types/hast": "npm:^3.0.4" hast-util-to-html: "npm:^9.0.5" - checksum: 10c0/69afe788994653b69f1bafd4fd3c2143609b4b0c05e970c8dc8d82ec660d617850ad9eeb2b6aa5be2dd534cefa0213d577129cb9ae1070eb4890cbbf1ac0f63e + checksum: 10c0/702469d9c80fc80e2b81dd10407cc946771dcf355d56048e1dab43e40d144395c14a6ecde92e03c70a35249ad6634ef4605bd17ad6974a2b4e04f9efccf24414 languageName: node linkType: hard -"@shikijs/engine-javascript@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/engine-javascript@npm:3.2.2" +"@shikijs/engine-javascript@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/engine-javascript@npm:3.4.2" dependencies: - "@shikijs/types": "npm:3.2.2" + "@shikijs/types": "npm:3.4.2" "@shikijs/vscode-textmate": "npm:^10.0.2" - oniguruma-to-es: "npm:^4.1.0" - checksum: 10c0/2db8f9c04cc8e40352eb69ddd4de81bda95c5318c897f43215622352c91591974522cefb3959bcfaa183fd36a18d1af9e704289ed7999273dcd763bfaa5a1827 + oniguruma-to-es: "npm:^4.3.3" + checksum: 10c0/160056a6303978d4e40114fe0414acd5089ea39a55a3144b9cba5e50aa38c521948ee47a2edc5acda5fd3607e33b20539845cfd9ca3508163e989b8fb4220488 languageName: node linkType: hard -"@shikijs/engine-oniguruma@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/engine-oniguruma@npm:3.2.2" +"@shikijs/engine-oniguruma@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/engine-oniguruma@npm:3.4.2" dependencies: - "@shikijs/types": "npm:3.2.2" + "@shikijs/types": "npm:3.4.2" "@shikijs/vscode-textmate": "npm:^10.0.2" - checksum: 10c0/b5eedfca26f7e1525fd079c1827ae9bdedafb574ce4eb535c54d484218b7428fb9ac93607f79a2adc1482483dd0366fdf07b0846403a76cd4767649adb8fa590 + checksum: 10c0/b8a13123b8a41e1016b661c24b349163b5026841772c351aacddcdc724518a926a49065ac77e4a1d4bb94da12c6bf11e6b1c938ef881545064bb3b484223eba0 languageName: node linkType: hard -"@shikijs/langs@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/langs@npm:3.2.2" +"@shikijs/langs@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/langs@npm:3.4.2" dependencies: - "@shikijs/types": "npm:3.2.2" - checksum: 10c0/04b5c9b92de9070624d24e20a2b3607edcbe4894a1db8056927f0d0637f080e2eed4e54925f0ded36874361db14bab9e4d9c2d06614ddd733f3f314250eabaf8 + "@shikijs/types": "npm:3.4.2" + checksum: 10c0/ca0260b00e32385db8db43d8dd147f480bc2ff699acaf6052ec3e421b1c6d27df6dfb0f69fadb673ef357333ba65fdce2fbcd8c31c7d245439756bfb3530eba4 languageName: node linkType: hard -"@shikijs/markdown-it@npm:^3.2.2": - version: 3.2.2 - resolution: "@shikijs/markdown-it@npm:3.2.2" +"@shikijs/markdown-it@npm:^3.4.2": + version: 3.4.2 + resolution: "@shikijs/markdown-it@npm:3.4.2" dependencies: markdown-it: "npm:^14.1.0" - shiki: "npm:3.2.2" + shiki: "npm:3.4.2" peerDependencies: markdown-it-async: ^2.2.0 peerDependenciesMeta: markdown-it-async: optional: true - checksum: 10c0/37c98e45a0905ea58f605c7cd341c83f3289b6a37093862535c59f7cc178fe9bfb13413fea68d4d923341e51b2e5718fc5172147d15e07457a76aceed2ac1f95 + checksum: 10c0/6bdb4fae6038867a370b454d39cb64ac78645385f3f0150e85fd68952965186a00ba87aa4ce1d0b280f599d6aaf78d30ee32b6ad7d92391091dd7af569e7cbdb languageName: node linkType: hard -"@shikijs/themes@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/themes@npm:3.2.2" +"@shikijs/themes@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/themes@npm:3.4.2" dependencies: - "@shikijs/types": "npm:3.2.2" - checksum: 10c0/93745e76e7ed6cab1d797ec68b53a0a183d989201e5064b33a78b516e128848d2c9be194d29cf602d5017dc2a74013699c773d052aeb45593851ae35b035afaa + "@shikijs/types": "npm:3.4.2" + checksum: 10c0/d50bca4384ccf88d68f007869e13bc7a9b55b16c40a3269fe120b2e5a2e882f6206ee0325f619bfa31ff00a0341452840d38f4ca2296dd3ba3e200e53445e22b languageName: node linkType: hard -"@shikijs/types@npm:3.2.2": - version: 3.2.2 - resolution: "@shikijs/types@npm:3.2.2" +"@shikijs/types@npm:3.4.2": + version: 3.4.2 + resolution: "@shikijs/types@npm:3.4.2" dependencies: "@shikijs/vscode-textmate": "npm:^10.0.2" "@types/hast": "npm:^3.0.4" - checksum: 10c0/aec3327d0cfc89af138ce195ac070ba62d8229864c079a3f06dff5a180036fdd963282068d67bd4c89a04ae688005c2b7c214c274ad0bb265f6f7ab6907a67a6 + checksum: 10c0/a20d3535cc0d61a55d0c0d4dfcd33a52229ec8a4c650613cb0f424dcb499bcdf0230e007f70a18e12c102a04820557ff120f41f18b15a94f95f9ec343592906b languageName: node linkType: hard @@ -5765,7 +5765,7 @@ __metadata: "@mozilla/readability": "npm:^0.6.0" "@notionhq/client": "npm:^2.2.15" "@reduxjs/toolkit": "npm:^2.2.5" - "@shikijs/markdown-it": "npm:^3.2.2" + "@shikijs/markdown-it": "npm:^3.4.2" "@strongtz/win32-arm64-msvc": "npm:^0.4.7" "@swc/plugin-styled-components": "npm:^7.1.5" "@tanstack/react-query": "npm:^5.27.0" @@ -5866,7 +5866,7 @@ __metadata: remark-math: "npm:^6.0.0" rollup-plugin-visualizer: "npm:^5.12.0" sass: "npm:^1.88.0" - shiki: "npm:^3.2.2" + shiki: "npm:^3.4.2" string-width: "npm:^7.2.0" styled-components: "npm:^6.1.11" tar: "npm:^7.4.3" @@ -8960,13 +8960,6 @@ __metadata: languageName: node linkType: hard -"emoji-regex-xs@npm:^1.0.0": - version: 1.0.0 - resolution: "emoji-regex-xs@npm:1.0.0" - checksum: 10c0/1082de006991eb05a3324ef0efe1950c7cdf66efc01d4578de82b0d0d62add4e55e97695a8a7eeda826c305081562dc79b477ddf18d886da77f3ba08c4b940a0 - languageName: node - linkType: hard - "emoji-regex@npm:^10.3.0": version: 10.4.0 resolution: "emoji-regex@npm:10.4.0" @@ -14793,22 +14786,21 @@ __metadata: languageName: node linkType: hard -"oniguruma-parser@npm:^0.11.0": - version: 0.11.1 - resolution: "oniguruma-parser@npm:0.11.1" - checksum: 10c0/d721cabe5632d0b772fec95dd6920cb6d6ba7a8e9b247dbb32a82b8a997137ecb62110d1788578dfd43596d4461a3319ca320d30aa2b6ebbaa19552a98e507ba +"oniguruma-parser@npm:^0.12.1": + version: 0.12.1 + resolution: "oniguruma-parser@npm:0.12.1" + checksum: 10c0/b843ea54cda833efb19f856314afcbd43e903ece3de489ab78c527ddec84859208052557daa9fad4bdba89ebdd15b0cc250de86b3daf8c7cbe37bac5a6a185d3 languageName: node linkType: hard -"oniguruma-to-es@npm:^4.1.0": - version: 4.2.0 - resolution: "oniguruma-to-es@npm:4.2.0" +"oniguruma-to-es@npm:^4.3.3": + version: 4.3.3 + resolution: "oniguruma-to-es@npm:4.3.3" dependencies: - emoji-regex-xs: "npm:^1.0.0" - oniguruma-parser: "npm:^0.11.0" + oniguruma-parser: "npm:^0.12.1" regex: "npm:^6.0.1" regex-recursion: "npm:^6.0.2" - checksum: 10c0/a2c505ad9d4ccca9f71a5ea22dc68ab94f244e2fab5b04fea54f355411a2c13d65b5c28925af508ea3a694ce8cf9e86931681bfe3ea4a89722d9b50e24bf21fd + checksum: 10c0/bc034e84dfee4dbc061cf6364023e66e1667fb8dc3afcad3b7d6a2c77e2d4a4809396ee2fb8c1fd3d6f00f76f7ca14b773586bf862c5f0c0074c059e2a219252 languageName: node linkType: hard @@ -17618,19 +17610,19 @@ __metadata: languageName: node linkType: hard -"shiki@npm:3.2.2": - version: 3.2.2 - resolution: "shiki@npm:3.2.2" +"shiki@npm:3.4.2, shiki@npm:^3.4.2": + version: 3.4.2 + resolution: "shiki@npm:3.4.2" dependencies: - "@shikijs/core": "npm:3.2.2" - "@shikijs/engine-javascript": "npm:3.2.2" - "@shikijs/engine-oniguruma": "npm:3.2.2" - "@shikijs/langs": "npm:3.2.2" - "@shikijs/themes": "npm:3.2.2" - "@shikijs/types": "npm:3.2.2" + "@shikijs/core": "npm:3.4.2" + "@shikijs/engine-javascript": "npm:3.4.2" + "@shikijs/engine-oniguruma": "npm:3.4.2" + "@shikijs/langs": "npm:3.4.2" + "@shikijs/themes": "npm:3.4.2" + "@shikijs/types": "npm:3.4.2" "@shikijs/vscode-textmate": "npm:^10.0.2" "@types/hast": "npm:^3.0.4" - checksum: 10c0/0183f889029ff1d14f79aa34e36f1e5a67b667661422f8a7de8936164099827588df7b2b4ed6835ad2eb3efb11ea882b4cb8022550503108c958a796df01f35c + checksum: 10c0/3cae825d8c341d7334e541efad30125fac0064db6004359e661a594782d59f93f66f2dcb5dbc1d8cb6508c43ccdd03ed6cf1d22306b382bc1f395a6130e5cbbb languageName: node linkType: hard