chore(languages): update languages with a script (#8445)

* chore(languages): update languages with a script * refactor: update languages and merge it into constants * refactor: add usf and ush
2025-12-19 06:30:10 +08:00 · 2025-07-24 15:57:09 +08:00 · 2025-07-24 15:57:09 +08:00 · 0453402242
commit 0453402242
parent d3c348f8f2
10 changed files with 3627 additions and 2881 deletions
--- a/package.json
+++ b/package.json
@ -53,6 +53,7 @@
    "check:i18n": "tsx scripts/check-i18n.ts",
    "sync:i18n": "tsx scripts/sync-i18n.ts",
    "update:i18n": "dotenv -e .env -- tsx scripts/update-i18n.ts",
+    "update:languages": "tsx scripts/update-languages.ts",
    "test": "vitest run --silent",
    "test:main": "vitest run --project main",
    "test:renderer": "vitest run --project renderer",
@ -200,6 +201,7 @@
    "jaison": "^2.0.2",
    "jest-styled-components": "^7.2.0",
    "jschardet": "^3.1.4",
+    "linguist-languages": "^8.0.0",
    "lint-staged": "^15.5.0",
    "lodash": "^4.17.21",
    "lru-cache": "^11.1.0",
--- a/packages/shared/config/constant.ts
+++ b/packages/shared/config/constant.ts
@ -1,312 +1,127 @@
+import { languages } from './languages'
+
 export const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp']
 export const videoExts = ['.mp4', '.avi', '.mov', '.wmv', '.flv', '.mkv']
 export const audioExts = ['.mp3', '.wav', '.ogg', '.flac', '.aac']
 export const documentExts = ['.pdf', '.doc', '.docx', '.pptx', '.xlsx', '.odt', '.odp', '.ods']
 export const thirdPartyApplicationExts = ['.draftsExport']
 export const bookExts = ['.epub']
-const textExtsByCategory = new Map([
+
+/**
+ * A flat array of all file extensions known by the linguist database.
+ * This is the primary source for identifying code files.
+ */
+const linguistExtSet = new Set<string>()
+for (const lang of Object.values(languages)) {
+  if (lang.extensions) {
+    for (const ext of lang.extensions) {
+      linguistExtSet.add(ext)
+    }
+  }
+}
+export const codeLangExts = Array.from(linguistExtSet)
+
+/**
+ * A categorized map of custom text-based file extensions that are NOT included
+ * in the linguist database. This is for special cases or project-specific files.
+ */
+export const customTextExts = new Map([
  [
    'language',
    [
-      '.js',
-      '.mjs',
-      '.cjs',
-      '.ts',
-      '.jsx',
-      '.tsx', // JavaScript/TypeScript
-      '.py', // Python
-      '.java', // Java
-      '.cs', // C#
-      '.cpp',
-      '.c',
-      '.h',
-      '.hpp',
-      '.cc',
-      '.cxx',
-      '.cppm',
-      '.ipp',
-      '.ixx', // C/C++
-      '.php', // PHP
-      '.rb', // Ruby
-      '.pl', // Perl
-      '.go', // Go
-      '.rs', // Rust
-      '.swift', // Swift
-      '.kt',
-      '.kts', // Kotlin
-      '.scala', // Scala
-      '.lua', // Lua
-      '.groovy', // Groovy
-      '.dart', // Dart
-      '.hs', // Haskell
-      '.clj',
-      '.cljs', // Clojure
-      '.elm', // Elm
-      '.erl', // Erlang
-      '.ex',
-      '.exs', // Elixir
-      '.ml',
-      '.mli', // OCaml
-      '.fs', // F#
-      '.r',
      '.R', // R
-      '.sol', // Solidity
-      '.awk', // AWK
-      '.cob', // COBOL
-      '.asm',
-      '.s', // Assembly
-      '.lisp',
-      '.lsp', // Lisp
-      '.coffee', // CoffeeScript
-      '.ino', // Arduino
-      '.jl', // Julia
-      '.nim', // Nim
-      '.zig', // Zig
-      '.d', // D语言
-      '.pas', // Pascal
-      '.vb', // Visual Basic
-      '.rkt', // Racket
-      '.scm', // Scheme
-      '.hx', // Haxe
-      '.as', // ActionScript
-      '.pde', // Processing
-      '.f90',
-      '.f',
-      '.f03',
-      '.for',
-      '.f95', // Fortran
-      '.adb',
-      '.ads', // Ada
-      '.pro', // Prolog
-      '.m',
-      '.mm', // Objective-C/MATLAB
-      '.rpy', // Ren'Py
      '.ets', // OpenHarmony,
      '.uniswap', // DeFi
-      '.vy', // Vyper
-      '.shader',
-      '.glsl',
-      '.frag',
-      '.vert',
-      '.gd' // Godot
-    ]
-  ],
-  [
-    'script',
-    [
-      '.sh', // Shell
-      '.bat',
-      '.cmd', // Windows批处理
-      '.ps1', // PowerShell
-      '.tcl',
-      '.do', // Tcl
-      '.ahk', // AutoHotkey
-      '.zsh', // Zsh
-      '.fish', // Fish shell
-      '.csh', // C shell
-      '.vbs', // VBScript
-      '.applescript', // AppleScript
-      '.au3', // AutoIt
-      '.bash',
-      '.nu'
-    ]
-  ],
-  [
-    'style',
-    [
-      '.css', // CSS
-      '.less', // Less
-      '.scss',
-      '.sass', // Sass
-      '.styl', // Stylus
-      '.pcss', // PostCSS
-      '.postcss' // PostCSS
+      '.usf', // Unreal shader format
+      '.ush' // Unreal shader header
    ]
  ],
  [
    'template',
    [
-      '.vue', // Vue.js
-      '.pug',
-      '.jade', // Pug/Jade
-      '.haml', // Haml
-      '.slim', // Slim
-      '.tpl', // 通用模板
-      '.ejs', // EJS
-      '.hbs', // Handlebars
-      '.mustache', // Mustache
-      '.twig', // Twig
-      '.blade', // Blade (Laravel)
-      '.liquid', // Liquid
-      '.jinja',
-      '.jinja2',
-      '.j2', // Jinja
-      '.erb', // ERB
-      '.vm', // Velocity
-      '.ftl', // FreeMarker
-      '.svelte', // Svelte
-      '.astro' // Astro
+      '.vm' // Velocity
    ]
  ],
  [
    'config',
    [
-      '.ini', // INI配置
+      '.babelrc', // Babel
+      '.bashrc',
+      '.browserslistrc',
      '.conf',
      '.config', // 通用配置
-      '.env', // 环境变量
-      '.toml', // TOML
-      '.cfg', // 通用配置
-      '.properties', // Java属性
-      '.desktop', // Linux桌面文件
-      '.service', // systemd服务
-      '.rc',
-      '.bashrc',
-      '.zshrc', // Shell配置
-      '.fishrc', // Fish shell配置
-      '.vimrc', // Vim配置
-      '.htaccess', // Apache配置
-      '.robots', // robots.txt
-      '.editorconfig', // EditorConfig
-      '.eslintrc', // ESLint
-      '.prettierrc', // Prettier
-      '.babelrc', // Babel
-      '.npmrc', // npm
      '.dockerignore', // Docker ignore
-      '.npmignore',
-      '.yarnrc',
-      '.prettierignore',
      '.eslintignore',
-      '.browserslistrc',
-      '.json5',
-      '.tfvars'
+      '.eslintrc', // ESLint
+      '.fishrc', // Fish shell配置
+      '.htaccess', // Apache配置
+      '.npmignore',
+      '.npmrc', // npm
+      '.prettierignore',
+      '.prettierrc', // Prettier
+      '.rc',
+      '.robots', // robots.txt
+      '.yarnrc',
+      '.zshrc'
    ]
  ],
  [
    'document',
    [
-      '.txt',
-      '.text', // 纯文本
-      '.md',
-      '.mdx', // Markdown
-      '.html',
-      '.htm',
-      '.xhtml', // HTML
-      '.xml', // XML
-      '.fxml', // JavaFX XML
-      '.org', // Org-mode
-      '.wiki', // Wiki
-      '.tex',
-      '.bib', // LaTeX
-      '.rst', // reStructuredText
-      '.rtf', // 富文本
-      '.nfo', // 信息文件
-      '.adoc',
-      '.asciidoc', // AsciiDoc
-      '.pod', // Perl文档
-      '.1',
-      '.2',
-      '.3',
-      '.4',
-      '.5',
-      '.6',
-      '.7',
-      '.8',
-      '.9', // man页面
-      '.man', // man页面
-      '.texi',
-      '.texinfo', // Texinfo
-      '.readme',
-      '.me', // README
+      '.authors', // 作者文件
      '.changelog', // 变更日志
      '.license', // 许可证
-      '.authors', // 作者文件
-      '.po',
-      '.pot'
+      '.nfo', // 信息文件
+      '.readme',
+      '.text' // 纯文本
    ]
  ],
  [
    'data',
    [
-      '.json', // JSON
-      '.jsonc', // JSON with comments
-      '.yaml',
-      '.yml', // YAML
-      '.csv',
-      '.tsv', // 分隔值文件
-      '.edn', // Clojure数据
-      '.jsonl',
-      '.ndjson', // 换行分隔JSON
-      '.geojson', // GeoJSON
-      '.gpx', // GPS Exchange
-      '.kml', // Keyhole Markup
-      '.rss',
      '.atom', // Feed格式
-      '.vcf', // vCard
-      '.ics', // iCalendar
-      '.ldif', // LDAP数据交换
-      '.pbtxt',
-      '.map'
+      '.ldif',
+      '.map',
+      '.ndjson' // 换行分隔JSON
    ]
  ],
  [
    'build',
    [
-      '.gradle', // Gradle
-      '.make',
-      '.mk', // Make
-      '.cmake', // CMake
-      '.sbt', // SBT
-      '.rake', // Rake
-      '.spec', // RPM spec
-      '.pom',
+      '.bazel', // Bazel
      '.build', // Meson
-      '.bazel' // Bazel
+      '.pom'
    ]
  ],
  [
    'database',
    [
-      '.sql', // SQL
-      '.ddl',
      '.dml', // DDL/DML
-      '.plsql', // PL/SQL
-      '.psql', // PostgreSQL
-      '.cypher', // Cypher
-      '.sparql' // SPARQL
+      '.psql' // PostgreSQL
    ]
  ],
  [
    'web',
    [
-      '.graphql',
-      '.gql', // GraphQL
-      '.proto', // Protocol Buffers
-      '.thrift', // Thrift
-      '.wsdl', // WSDL
-      '.raml', // RAML
-      '.swagger',
-      '.openapi' // API文档
+      '.openapi', // API文档
+      '.swagger'
    ]
  ],
  [
    'version',
    [
-      '.gitignore', // Git ignore
-      '.gitattributes', // Git attributes
-      '.gitconfig', // Git config
-      '.hgignore', // Mercurial ignore
      '.bzrignore', // Bazaar ignore
-      '.svnignore', // SVN ignore
-      '.githistory' // Git history
+      '.gitattributes', // Git attributes
+      '.githistory', // Git history
+      '.hgignore', // Mercurial ignore
+      '.svnignore' // SVN ignore
    ]
  ],
  [
    'subtitle',
    [
-      '.srt',
-      '.sub',
-      '.ass' // 字幕格式
+      '.ass', // 字幕格式
+      '.sub'
    ]
  ],
  [
@ -319,55 +134,26 @@ const textExtsByCategory = new Map([
  [
    'eda',
    [
-      '.v',
-      '.sv',
-      '.svh', // Verilog/SystemVerilog
-      '.vhd',
-      '.vhdl', // VHDL
-      '.lef',
+      '.cir',
      '.def', // LEF/DEF
      '.edif', // EDIF
-      '.sdf', // SDF
-      '.sdc',
-      '.xdc', // 约束文件
-      '.sp',
-      '.spi',
-      '.cir',
-      '.net', // SPICE
-      '.scs', // Spectre
-      '.asc', // LTspice
-      '.tf', // Technology File
      '.il',
-      '.ils' // SKILL
-    ]
-  ],
-  [
-    'game',
-    [
-      '.mtl', // Material Template Library
-      '.x3d', // X3D文件
-      '.gltf', // glTF JSON
-      '.prefab', // Unity预制体 (YAML格式)
-      '.meta', // Unity元数据文件 (YAML格式)
-      '.tscn' // Godot场景文件
-    ]
-  ],
-  [
-    'other',
-    [
-      '.mcfunction', // Minecraft函数
-      '.jsp', // JSP
-      '.aspx', // ASP.NET
-      '.ipynb', // Jupyter Notebook
-      '.cake',
-      '.ctp', // CakePHP
-      '.cfm',
-      '.cfc' // ColdFusion
+      '.ils', // SKILL
+      '.lef',
+      '.net',
+      '.scs', // Spectre
+      '.sdf', // SDF
+      '.spi'
    ]
  ]
 ])

-export const textExts = Array.from(textExtsByCategory.values()).flat()
+/**
+ * A comprehensive list of all text-based file extensions, combining the
+ * extensive list from the linguist database with our custom additions.
+ * The Set ensures there are no duplicates.
+ */
+export const textExts = [...new Set([...Array.from(customTextExts.values()).flat(), ...codeLangExts])]

 export const ZOOM_LEVELS = [0.25, 0.33, 0.5, 0.67, 0.75, 0.8, 0.9, 1, 1.1, 1.25, 1.5, 1.75, 2, 2.5, 3, 4, 5]

--- a/packages/shared/config/languages.ts
+++ b/packages/shared/config/languages.ts
--- a/scripts/check-custom-exts.ts
+++ b/scripts/check-custom-exts.ts
@ -0,0 +1,45 @@
+import { codeLangExts, customTextExts } from '../packages/shared/config/constant'
+
+console.log('Running sanity check for custom extensions...')
+
+// Create a Set for efficient lookup of extensions from the linguist database.
+const linguistExtsSet = new Set(codeLangExts)
+
+const overlappingExtsByCategory = new Map<string, string[]>()
+let totalOverlaps = 0
+
+// Iterate over each category and its extensions in our custom map.
+for (const [category, exts] of customTextExts.entries()) {
+  const categoryOverlaps = exts.filter((ext) => linguistExtsSet.has(ext))
+
+  if (categoryOverlaps.length > 0) {
+    overlappingExtsByCategory.set(category, categoryOverlaps.sort())
+    totalOverlaps += categoryOverlaps.length
+  }
+}
+
+// Report the results.
+if (totalOverlaps === 0) {
+  console.log('\n✅ Check passed!')
+  console.log('The `customTextExts` map contains no extensions that are already in `codeLangExts`.')
+  console.log('\nCustom extensions checked:')
+  for (const [category, exts] of customTextExts.entries()) {
+    console.log(`  - Category '${category}' (${exts.length}):`)
+    console.log(`    ${exts.sort().join(', ')}`)
+  }
+  console.log('\n')
+} else {
+  console.error('\n⚠️ Check failed: Overlapping extensions found!')
+  console.error(
+    'The following extensions in `customTextExts` are already present in `codeLangExts` (from languages.ts).'
+  )
+  console.error('Please remove them from `customTextExts` in `packages/shared/config/constant.ts` to avoid redundancy.')
+  console.error(`\nFound ${totalOverlaps} overlapping extensions in ${overlappingExtsByCategory.size} categories:`)
+
+  for (const [category, exts] of overlappingExtsByCategory.entries()) {
+    console.error(`  - Category '${category}': ${exts.join(', ')}`)
+  }
+
+  console.error('\n')
+  process.exit(1) // Exit with an error code for CI/CD purposes.
+}
--- a/scripts/update-languages.ts
+++ b/scripts/update-languages.ts
@ -0,0 +1,135 @@
+import { exec } from 'child_process'
+import * as fs from 'fs/promises'
+import linguistLanguages from 'linguist-languages'
+import * as path from 'path'
+import { promisify } from 'util'
+
+const execAsync = promisify(exec)
+
+type LanguageData = {
+  type: string
+  aliases?: string[]
+  extensions?: string[]
+}
+
+const LANGUAGES_FILE_PATH = path.join(__dirname, '../packages/shared/config/languages.ts')
+
+/**
+ * Extracts and filters necessary language data from the linguist-languages package.
+ * @returns A record of language data.
+ */
+function extractAllLanguageData(): Record<string, LanguageData> {
+  console.log('🔍 Extracting language data from linguist-languages...')
+  const languages = Object.entries(linguistLanguages).reduce(
+    (acc, [name, langData]) => {
+      const { type, extensions, aliases } = langData as any
+
+      // Only include languages with extensions or aliases
+      if ((extensions && extensions.length > 0) || (aliases && aliases.length > 0)) {
+        acc[name] = {
+          type: type || 'programming',
+          ...(extensions && { extensions }),
+          ...(aliases && { aliases })
+        }
+      }
+      return acc
+    },
+    {} as Record<string, LanguageData>
+  )
+  console.log(`✅ Extracted ${Object.keys(languages).length} languages.`)
+  return languages
+}
+
+/**
+ * Generates the content for the languages.ts file.
+ * @param languages The language data to include in the file.
+ * @returns The generated file content as a string.
+ */
+function generateLanguagesFileContent(languages: Record<string, LanguageData>): string {
+  console.log('📝 Generating languages.ts file content...')
+  const sortedLanguages = Object.fromEntries(Object.entries(languages).sort(([a], [b]) => a.localeCompare(b)))
+
+  const languagesObjectString = JSON.stringify(sortedLanguages, null, 2)
+
+  const content = `/**
+ * Code language list.
+ * Data source: linguist-languages
+ *
+ * ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️
+ *   THIS FILE IS AUTOMATICALLY GENERATED BY A SCRIPT. DO NOT EDIT IT MANUALLY!
+ *   Run \`yarn update:languages\` to update this file.
+ * ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️
+ *
+ */
+
+type LanguageData = {
+  type: string;
+  aliases?: string[];
+  extensions?: string[];
+};
+
+export const languages: Record<string, LanguageData> = ${languagesObjectString};
+`
+  console.log('✅ File content generated.')
+  return content
+}
+
+/**
+ * Formats a file using Prettier.
+ * @param filePath The path to the file to format.
+ */
+async function formatWithPrettier(filePath: string): Promise<void> {
+  console.log('🎨 Formatting file with Prettier...')
+  try {
+    await execAsync(`yarn prettier --write ${filePath}`)
+    console.log('✅ Prettier formatting complete.')
+  } catch (e: any) {
+    console.error('❌ Prettier formatting failed:', e.stdout || e.stderr)
+    throw new Error('Prettier formatting failed.')
+  }
+}
+
+/**
+ * Checks a file with TypeScript compiler.
+ * @param filePath The path to the file to check.
+ */
+async function checkTypeScript(filePath: string): Promise<void> {
+  console.log('🧐 Checking file with TypeScript compiler...')
+  try {
+    await execAsync(`yarn tsc --noEmit --skipLibCheck ${filePath}`)
+    console.log('✅ TypeScript check passed.')
+  } catch (e: any) {
+    console.error('❌ TypeScript check failed:', e.stdout || e.stderr)
+    throw new Error('TypeScript check failed.')
+  }
+}
+
+/**
+ * Main function to update the languages.ts file.
+ */
+async function updateLanguagesFile(): Promise<void> {
+  console.log('🚀 Starting to update languages.ts...')
+  try {
+    const extractedLanguages = extractAllLanguageData()
+    const fileContent = generateLanguagesFileContent(extractedLanguages)
+
+    await fs.writeFile(LANGUAGES_FILE_PATH, fileContent, 'utf-8')
+    console.log(`✅ Successfully wrote to ${LANGUAGES_FILE_PATH}`)
+
+    await formatWithPrettier(LANGUAGES_FILE_PATH)
+    await checkTypeScript(LANGUAGES_FILE_PATH)
+
+    console.log('🎉 Successfully updated languages.ts file!')
+    console.log(`📊 Contains ${Object.keys(extractedLanguages).length} languages.`)
+  } catch (error) {
+    console.error('❌ An error occurred during the update process:', (error as Error).message)
+    // No need to restore backup as we write only at the end of successful generation.
+    process.exit(1)
+  }
+}
+
+if (require.main === module) {
+  updateLanguagesFile()
+}
+
+export { updateLanguagesFile }
--- a/src/renderer/src/i18n/translate/el-gr.json
+++ b/src/renderer/src/i18n/translate/el-gr.json
@ -2608,4 +2608,4 @@
      "visualization": "προβολή"
    }
  }
-}
+}
--- a/src/renderer/src/i18n/translate/es-es.json
+++ b/src/renderer/src/i18n/translate/es-es.json
@ -2608,4 +2608,4 @@
      "visualization": "Visualización"
    }
  }
-}
+}
--- a/src/renderer/src/i18n/translate/fr-fr.json
+++ b/src/renderer/src/i18n/translate/fr-fr.json
@ -2608,4 +2608,4 @@
      "visualization": "Visualisation"
    }
  }
-}
+}
--- a/src/renderer/src/i18n/translate/pt-pt.json
+++ b/src/renderer/src/i18n/translate/pt-pt.json
@ -2608,4 +2608,4 @@
      "visualization": "Visualização"
    }
  }
-}
+}
--- a/yarn.lock
+++ b/yarn.lock
@ -7252,6 +7252,7 @@ __metadata:
    jest-styled-components: "npm:^7.2.0"
    jschardet: "npm:^3.1.4"
    jsdom: "npm:26.1.0"
+    linguist-languages: "npm:^8.0.0"
    lint-staged: "npm:^15.5.0"
    lodash: "npm:^4.17.21"
    lru-cache: "npm:^11.1.0"
@ -13713,6 +13714,13 @@ __metadata:
  languageName: node
  linkType: hard

+"linguist-languages@npm:^8.0.0":
+  version: 8.0.0
+  resolution: "linguist-languages@npm:8.0.0"
+  checksum: 10c0/eaae46254247b9aa5b287ac98e062e7fe859314328ce305e34e152bc7bb172d69633999320cb47dc2a710388179712a76bb1ddd6e39e249af2684a4f0a66256c
+  languageName: node
+  linkType: hard
+
 "linkify-it@npm:^5.0.0":
  version: 5.0.0
  resolution: "linkify-it@npm:5.0.0"