mirror of
https://github.com/CherryHQ/cherry-studio.git
synced 2025-12-19 06:30:10 +08:00
1f0381aebe
* fix: update EmbeddingsFactory to use net.fetch and refactor KnowledgeService to use ModernAiProvider * fix: remove deprecated @langchain/community dependency from package.json * fix: add @langchain/community dependency to package.json and update yarn.lock
69 lines
2.7 KiB
Diff
Vendored
69 lines
2.7 KiB
Diff
Vendored
diff --git a/dist/utils/tiktoken.cjs b/dist/utils/tiktoken.cjs
|
|
index c5b41f121d2e3d24c3a4969e31fa1acffdcad3b9..ec724489dcae79ee6c61acf2d4d84bd19daef036 100644
|
|
--- a/dist/utils/tiktoken.cjs
|
|
+++ b/dist/utils/tiktoken.cjs
|
|
@@ -1,6 +1,5 @@
|
|
const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
|
|
const require_utils_async_caller = require('./async_caller.cjs');
|
|
-const js_tiktoken_lite = require_rolldown_runtime.__toESM(require("js-tiktoken/lite"));
|
|
|
|
//#region src/utils/tiktoken.ts
|
|
var tiktoken_exports = {};
|
|
@@ -11,14 +10,10 @@ require_rolldown_runtime.__export(tiktoken_exports, {
|
|
const cache = {};
|
|
const caller = /* @__PURE__ */ new require_utils_async_caller.AsyncCaller({});
|
|
async function getEncoding(encoding) {
|
|
- if (!(encoding in cache)) cache[encoding] = caller.fetch(`https://tiktoken.pages.dev/js/${encoding}.json`).then((res) => res.json()).then((data) => new js_tiktoken_lite.Tiktoken(data)).catch((e) => {
|
|
- delete cache[encoding];
|
|
- throw e;
|
|
- });
|
|
- return await cache[encoding];
|
|
+ throw new Error("TikToken Not implemented");
|
|
}
|
|
async function encodingForModel(model) {
|
|
- return getEncoding((0, js_tiktoken_lite.getEncodingNameForModel)(model));
|
|
+ throw new Error("TikToken Not implemented");
|
|
}
|
|
|
|
//#endregion
|
|
diff --git a/dist/utils/tiktoken.js b/dist/utils/tiktoken.js
|
|
index 641acca03cb92f04a6fa5c9c31f1880ce635572e..707389970ad957aa0ff20ef37fa8dd2875be737c 100644
|
|
--- a/dist/utils/tiktoken.js
|
|
+++ b/dist/utils/tiktoken.js
|
|
@@ -1,6 +1,5 @@
|
|
import { __export } from "../_virtual/rolldown_runtime.js";
|
|
import { AsyncCaller } from "./async_caller.js";
|
|
-import { Tiktoken, getEncodingNameForModel } from "js-tiktoken/lite";
|
|
|
|
//#region src/utils/tiktoken.ts
|
|
var tiktoken_exports = {};
|
|
@@ -11,14 +10,10 @@ __export(tiktoken_exports, {
|
|
const cache = {};
|
|
const caller = /* @__PURE__ */ new AsyncCaller({});
|
|
async function getEncoding(encoding) {
|
|
- if (!(encoding in cache)) cache[encoding] = caller.fetch(`https://tiktoken.pages.dev/js/${encoding}.json`).then((res) => res.json()).then((data) => new Tiktoken(data)).catch((e) => {
|
|
- delete cache[encoding];
|
|
- throw e;
|
|
- });
|
|
- return await cache[encoding];
|
|
+ throw new Error("TikToken Not implemented");
|
|
}
|
|
async function encodingForModel(model) {
|
|
- return getEncoding(getEncodingNameForModel(model));
|
|
+ throw new Error("TikToken Not implemented");
|
|
}
|
|
|
|
//#endregion
|
|
diff --git a/package.json b/package.json
|
|
index a24f8fc61de58526051999260f2ebee5f136354b..e885359e8966e7730c51772533ce37e01edb3046 100644
|
|
--- a/package.json
|
|
+++ b/package.json
|
|
@@ -20,7 +20,6 @@
|
|
"ansi-styles": "^5.0.0",
|
|
"camelcase": "6",
|
|
"decamelize": "1.2.0",
|
|
- "js-tiktoken": "^1.0.12",
|
|
"langsmith": "^0.3.64",
|
|
"mustache": "^4.2.0",
|
|
"p-queue": "^6.6.2",
|