fix: update TikToken implementation and remove js-tiktoken dependency

- Replaced the existing TikToken implementation with a placeholder error message indicating it is not implemented.
- Removed the js-tiktoken dependency from package.json to streamline the project.
- Updated yarn.lock to reflect changes in dependencies and checksums.

.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch

@@ -1,8 +1,42 @@
+diff --git a/dist/utils/tiktoken.cjs b/dist/utils/tiktoken.cjs
+index 973b0d0e75aeaf8de579419af31b879b32975413..f23c7caa8b9dc8bd404132725346a4786f6b278b 100644
+--- a/dist/utils/tiktoken.cjs
++++ b/dist/utils/tiktoken.cjs
+@@ -1,25 +1,14 @@
+ "use strict";
+ Object.defineProperty(exports, "__esModule", { value: true });
+ exports.encodingForModel = exports.getEncoding = void 0;
+-const lite_1 = require("js-tiktoken/lite");
+ const async_caller_js_1 = require("./async_caller.cjs");
+ const cache = {};
+ const caller = /* #__PURE__ */ new async_caller_js_1.AsyncCaller({});
+ async function getEncoding(encoding) {
+-    if (!(encoding in cache)) {
+-        cache[encoding] = caller
+-            .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
+-            .then((res) => res.json())
+-            .then((data) => new lite_1.Tiktoken(data))
+-            .catch((e) => {
+-            delete cache[encoding];
+-            throw e;
+-        });
+-    }
+-    return await cache[encoding];
++    throw new Error("TikToken Not implemented");
+ }
+ exports.getEncoding = getEncoding;
+ async function encodingForModel(model) {
+-    return getEncoding((0, lite_1.getEncodingNameForModel)(model));
++    throw new Error("TikToken Not implemented");
+ }
+ exports.encodingForModel = encodingForModel;
 diff --git a/dist/utils/tiktoken.js b/dist/utils/tiktoken.js
-index 8e41ee6f00f2f9c7fa2c59fa2b2f4297634b97aa..6708abfaabee108b8967c3fbd7fccc8dfb95e693 100644
+index 8e41ee6f00f2f9c7fa2c59fa2b2f4297634b97aa..aa5f314a6349ad0d1c5aea8631a56aad099176e0 100644
 --- a/dist/utils/tiktoken.js
 +++ b/dist/utils/tiktoken.js
-@@ -3,18 +3,20 @@ import { AsyncCaller } from "./async_caller.js";
+@@ -1,20 +1,9 @@
+-import { Tiktoken, getEncodingNameForModel, } from "js-tiktoken/lite";
+ import { AsyncCaller } from "./async_caller.js";
  const cache = {};
  const caller = /* #__PURE__ */ new AsyncCaller({});
  export async function getEncoding(encoding) {
@@ -17,21 +51,21 @@ index 8e41ee6f00f2f9c7fa2c59fa2b2f4297634b97aa..6708abfaabee108b8967c3fbd7fccc8d
 -        });
 -    }
 -    return await cache[encoding];
-+    // if (!(encoding in cache)) {
-+    //     cache[encoding] = caller
-+    //         .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)
-+    //         .then((res) => res.json())
-+    //         .then((data) => new Tiktoken(data))
-+    //         .catch((e) => {
-+    //         delete cache[encoding];
-+    //         throw e;
-+    //     });
-+    // }
-+    // return await cache[encoding];
 +    throw new Error("TikToken Not implemented");
  }
  export async function encodingForModel(model) {
 -    return getEncoding(getEncodingNameForModel(model));
-+    // return getEncoding(getEncodingNameForModel(model));
 +    throw new Error("TikToken Not implemented");
  }
+diff --git a/package.json b/package.json
+index 36072aecf700fca1bc49832a19be832eca726103..90b8922fba1c3d1b26f78477c891b07816d6238a 100644
+--- a/package.json
++++ b/package.json
+@@ -37,7 +37,6 @@
+     "ansi-styles": "^5.0.0",
+     "camelcase": "6",
+     "decamelize": "1.2.0",
+-    "js-tiktoken": "^1.0.12",
+     "langsmith": ">=0.2.8 <0.4.0",
+     "mustache": "^4.2.0",
+     "p-queue": "^6.6.2",

yarn.lock

@@ -2599,7 +2599,7 @@ __metadata:
 
 "@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch":
   version: 0.3.44
-  resolution: "@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch::version=0.3.44&hash=c14420"
+  resolution: "@langchain/core@patch:@langchain/core@npm%3A0.3.44#~/.yarn/patches/@langchain-core-npm-0.3.44-41d5c3cb0a.patch::version=0.3.44&hash=41dd7b"
   dependencies:
     "@cfworker/json-schema": "npm:^4.0.2"
     ansi-styles: "npm:^5.0.0"
@@ -2613,7 +2613,7 @@ __metadata:
     uuid: "npm:^10.0.0"
     zod: "npm:^3.22.4"
     zod-to-json-schema: "npm:^3.22.3"
-  checksum: 10c0/184fa7720f49fe4619661bcc5f12ed2a778520dd9f0da3c2d97f1b404f0a8697fa4b3f5cf96652234ae96c29b07fb4f9ec48260c3d1233e92286c87bca56f5fd
+  checksum: 10c0/7002581999230bc6fbdb6c2fad0eda2c1ca93a7629e28eadf1501dd0a7f785010d6b3e7841148ced186fc0fb4a84ec5f3b75850985579610e222943cc57e1649
   languageName: node
   linkType: hard