Northern_Lights/cherry-studio
1
0
Fork 0
mirror of https://github.com/CherryHQ/cherry-studio.git synced 2025-12-25 19:30:17 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

fix: remove tiktoken

Browse Source
This commit is contained in:
kangfenmao 2025-05-27 16:57:18 +08:00
parent 55a5e7c3af
commit 96c1b3812b
3 changed files with 60 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch vendored Normal file
View File

@ -0,0 +1,37 @@
diff --git a/dist/utils/tiktoken.js b/dist/utils/tiktoken.js
index 8e41ee6f00f2f9c7fa2c59fa2b2f4297634b97aa..6708abfaabee108b8967c3fbd7fccc8dfb95e693 100644
--- a/dist/utils/tiktoken.js
+++ b/dist/utils/tiktoken.js
@@ -3,18 +3,20 @@ import { AsyncCaller } from "./async_caller.js";
const cache = {};
const caller = /* #__PURE__ */ new AsyncCaller({});
export async function getEncoding(encoding) {
- if (!(encoding in cache)) {
- cache[encoding] = caller
- .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
- .then((res) => res.json())
- .then((data) => new Tiktoken(data))
- .catch((e) => {
- delete cache[encoding];
- throw e;
- });
- }
- return await cache[encoding];
+ // if (!(encoding in cache)) {
+ // cache[encoding] = caller
+ // .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
+ // .then((res) => res.json())
+ // .then((data) => new Tiktoken(data))
+ // .catch((e) => {
+ // delete cache[encoding];
+ // throw e;
+ // });
+ // }
+ // return await cache[encoding];
+ throw new Error("TikToken Not implemented");
}
export async function encodingForModel(model) {
- return getEncoding(getEncodingNameForModel(model));
+ // return getEncoding(getEncodingNameForModel(model));
+ throw new Error("TikToken Not implemented");
}

3
package.json
View File

@ -223,7 +223,8 @@
"pkce-challenge@npm:^4.1.0": "patch:pkce-challenge@npm%3A4.1.0#~/.yarn/patches/pkce-challenge-npm-4.1.0-fbc51695a3.patch",
"app-builder-lib@npm:26.0.13": "patch:app-builder-lib@npm%3A26.0.13#~/.yarn/patches/app-builder-lib-npm-26.0.13-a064c9e1d0.patch",
"openai@npm:^4.87.3": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch",
"app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch"
"app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch",
"@langchain/core@npm:^0.3.26": "patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch"
},
"packageManager": "yarn@4.9.1",
"lint-staged": {

22
yarn.lock
View File

@ -3001,7 +3001,7 @@ __metadata:
languageName: node
linkType: hard
"@langchain/core@npm:^0.3.26":
"@langchain/core@npm:0.3.44":
version: 0.3.44
resolution: "@langchain/core@npm:0.3.44"
dependencies:
@ -3021,6 +3021,26 @@ __metadata:
languageName: node
linkType: hard
"@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch":
version: 0.3.44
resolution: "@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch::version=0.3.44&hash=c14420"
dependencies:
"@cfworker/json-schema": "npm:^4.0.2"
ansi-styles: "npm:^5.0.0"
camelcase: "npm:6"
decamelize: "npm:1.2.0"
js-tiktoken: "npm:^1.0.12"
langsmith: "npm:>=0.2.8 <0.4.0"
mustache: "npm:^4.2.0"
p-queue: "npm:^6.6.2"
p-retry: "npm:4"
uuid: "npm:^10.0.0"
zod: "npm:^3.22.4"
zod-to-json-schema: "npm:^3.22.3"
checksum: 10c0/184fa7720f49fe4619661bcc5f12ed2a778520dd9f0da3c2d97f1b404f0a8697fa4b3f5cf96652234ae96c29b07fb4f9ec48260c3d1233e92286c87bca56f5fd
languageName: node
linkType: hard
"@langchain/openai@npm:0.3.16":
version: 0.3.16
resolution: "@langchain/openai@npm:0.3.16"