From 96c1b3812b323d5b4675b0607211db06910fca75 Mon Sep 17 00:00:00 2001
From: kangfenmao <kangfenmao@qq.com>
Date: Tue, 27 May 2025 16:57:18 +0800
Subject: [PATCH] fix: remove tiktoken

---
 ...langchain-core-npm-0.3.44-41d5c3cb0a.patch | 37 +++++++++++++++++++
 package.json                                  |  3 +-
 yarn.lock                                     | 22 ++++++++++-
 3 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100644 .yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch

diff --git a/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch b/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch
new file mode 100644
index 0000000000..e1be294752
--- /dev/null
+++ b/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch
@@ -0,0 +1,37 @@
+diff --git a/dist/utils/tiktoken.js b/dist/utils/tiktoken.js
+index 8e41ee6f00f2f9c7fa2c59fa2b2f4297634b97aa..6708abfaabee108b8967c3fbd7fccc8dfb95e693 100644
+--- a/dist/utils/tiktoken.js
++++ b/dist/utils/tiktoken.js
+@@ -3,18 +3,20 @@ import { AsyncCaller } from "./async_caller.js";
+ const cache = {};
+ const caller = /* #__PURE__ */ new AsyncCaller({});
+ export async function getEncoding(encoding) {
+-    if (!(encoding in cache)) {
+-        cache[encoding] = caller
+-            .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
+-            .then((res) => res.json())
+-            .then((data) => new Tiktoken(data))
+-            .catch((e) => {
+-            delete cache[encoding];
+-            throw e;
+-        });
+-    }
+-    return await cache[encoding];
++    // if (!(encoding in cache)) {
++    //     cache[encoding] = caller
++    //         .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
++    //         .then((res) => res.json())
++    //         .then((data) => new Tiktoken(data))
++    //         .catch((e) => {
++    //         delete cache[encoding];
++    //         throw e;
++    //     });
++    // }
++    // return await cache[encoding];
++    throw new Error("TikToken Not implemented");
+ }
+ export async function encodingForModel(model) {
+-    return getEncoding(getEncodingNameForModel(model));
++    // return getEncoding(getEncodingNameForModel(model));
++    throw new Error("TikToken Not implemented");
+ }
diff --git a/package.json b/package.json
index 58d02378f6..e4f05157d0 100644
--- a/package.json
+++ b/package.json
@@ -223,7 +223,8 @@
     "pkce-challenge@npm:^4.1.0": "patch:pkce-challenge@npm%3A4.1.0#~/.yarn/patches/pkce-challenge-npm-4.1.0-fbc51695a3.patch",
     "app-builder-lib@npm:26.0.13": "patch:app-builder-lib@npm%3A26.0.13#~/.yarn/patches/app-builder-lib-npm-26.0.13-a064c9e1d0.patch",
     "openai@npm:^4.87.3": "patch:openai@npm%3A4.96.0#~/.yarn/patches/openai-npm-4.96.0-0665b05cb9.patch",
-    "app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch"
+    "app-builder-lib@npm:26.0.15": "patch:app-builder-lib@npm%3A26.0.15#~/.yarn/patches/app-builder-lib-npm-26.0.15-360e5b0476.patch",
+    "@langchain/core@npm:^0.3.26": "patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch"
   },
   "packageManager": "yarn@4.9.1",
   "lint-staged": {
diff --git a/yarn.lock b/yarn.lock
index d25e6a5b09..674a075f6a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3001,7 +3001,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@langchain/core@npm:^0.3.26":
+"@langchain/core@npm:0.3.44":
   version: 0.3.44
   resolution: "@langchain/core@npm:0.3.44"
   dependencies:
@@ -3021,6 +3021,26 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch":
+  version: 0.3.44
+  resolution: "@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch::version=0.3.44&hash=c14420"
+  dependencies:
+    "@cfworker/json-schema": "npm:^4.0.2"
+    ansi-styles: "npm:^5.0.0"
+    camelcase: "npm:6"
+    decamelize: "npm:1.2.0"
+    js-tiktoken: "npm:^1.0.12"
+    langsmith: "npm:>=0.2.8 <0.4.0"
+    mustache: "npm:^4.2.0"
+    p-queue: "npm:^6.6.2"
+    p-retry: "npm:4"
+    uuid: "npm:^10.0.0"
+    zod: "npm:^3.22.4"
+    zod-to-json-schema: "npm:^3.22.3"
+  checksum: 10c0/184fa7720f49fe4619661bcc5f12ed2a778520dd9f0da3c2d97f1b404f0a8697fa4b3f5cf96652234ae96c29b07fb4f9ec48260c3d1233e92286c87bca56f5fd
+  languageName: node
+  linkType: hard
+
 "@langchain/openai@npm:0.3.16":
   version: 0.3.16
   resolution: "@langchain/openai@npm:0.3.16"