From 537957d5f50971c65e451823b32d83ce1b70c6ef Mon Sep 17 00:00:00 2001 From: Mckay Wrigley Date: Mon, 20 Mar 2023 22:02:24 -0600 Subject: [PATCH] Token based and model conditional limits (#36) * use tiktoken for api limit * model conditional char limits on frontend * adjust for completion tokens --------- Co-authored-by: Alan Pogrebinschi --- components/Chat/Chat.tsx | 34 ++++++++++++++++++++++++++++------ components/Chat/ChatInput.tsx | 19 ++++++++++++------- next.config.js | 13 +++++++++++-- package-lock.json | 11 +++++++++++ package.json | 1 + pages/api/chat.ts | 21 ++++++++++++++++----- 6 files changed, 79 insertions(+), 20 deletions(-) diff --git a/components/Chat/Chat.tsx b/components/Chat/Chat.tsx index 501291e..85ddd8b 100644 --- a/components/Chat/Chat.tsx +++ b/components/Chat/Chat.tsx @@ -18,7 +18,17 @@ interface Props { onModelChange: (conversation: Conversation, model: OpenAIModel) => void; } -export const Chat: FC = ({ conversation, models, messageIsStreaming, modelError, messageError, loading, lightMode, onSend, onModelChange }) => { +export const Chat: FC = ({ + conversation, + models, + messageIsStreaming, + modelError, + messageError, + loading, + lightMode, + onSend, + onModelChange, +}) => { const [currentMessage, setCurrentMessage] = useState(); const messagesEndRef = useRef(null); @@ -36,8 +46,13 @@ export const Chat: FC = ({ conversation, models, messageIsStreaming, mode {modelError ? (
Error fetching models.
-
Make sure your OpenAI API key is set in the bottom left of the sidebar or in a .env.local file and refresh.
-
If you completed this step, OpenAI may be experiencing issues.
+
+ Make sure your OpenAI API key is set in the bottom left of the + sidebar or in a .env.local file and refresh. +
+
+ If you completed this step, OpenAI may be experiencing issues. +
) : ( <> @@ -48,15 +63,21 @@ export const Chat: FC = ({ conversation, models, messageIsStreaming, mode onModelChange(conversation, model)} + onModelChange={(model) => + onModelChange(conversation, model) + } /> -
{models.length === 0 ? "Loading..." : "Chatbot UI"}
+
+ {models.length === 0 ? "Loading..." : "Chatbot UI"} +
) : ( <> -
Model: {conversation.model.name}
+
+ Model: {conversation.model.name} +
{conversation.messages.map((message, index) => ( = ({ conversation, models, messageIsStreaming, mode setCurrentMessage(message); onSend(message, false); }} + model={conversation.model} /> )} diff --git a/components/Chat/ChatInput.tsx b/components/Chat/ChatInput.tsx index f0e7cd0..ae91456 100644 --- a/components/Chat/ChatInput.tsx +++ b/components/Chat/ChatInput.tsx @@ -1,13 +1,14 @@ -import { Message } from "@/types"; +import { Message, OpenAIModel, OpenAIModelID } from "@/types"; import { IconSend } from "@tabler/icons-react"; import { FC, KeyboardEvent, useEffect, useRef, useState } from "react"; interface Props { messageIsStreaming: boolean; onSend: (message: Message) => void; + model: OpenAIModel; } -export const ChatInput: FC = ({ onSend, messageIsStreaming }) => { +export const ChatInput: FC = ({ onSend, messageIsStreaming, model }) => { const [content, setContent] = useState(); const [isTyping, setIsTyping] = useState(false); @@ -15,8 +16,10 @@ export const ChatInput: FC = ({ onSend, messageIsStreaming }) => { const handleChange = (e: React.ChangeEvent) => { const value = e.target.value; - if (value.length > 4000) { - alert("Message limit is 4000 characters"); + const maxLength = model.id === OpenAIModelID.GPT_3_5 ? 12000 : 24000; + + if (value.length > maxLength) { + alert(`Message limit is ${maxLength} characters`); return; } @@ -42,8 +45,10 @@ export const ChatInput: FC = ({ onSend, messageIsStreaming }) => { }; const isMobile = () => { - const userAgent = typeof window.navigator === "undefined" ? "" : navigator.userAgent; - const mobileRegex = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|mobile|CriOS/i; + const userAgent = + typeof window.navigator === "undefined" ? "" : navigator.userAgent; + const mobileRegex = + /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Mobile|mobile|CriOS/i; return mobileRegex.test(userAgent); }; @@ -72,7 +77,7 @@ export const ChatInput: FC = ({ onSend, messageIsStreaming }) => { resize: "none", bottom: `${textareaRef?.current?.scrollHeight}px`, maxHeight: "400px", - overflow: "auto" + overflow: "auto", }} placeholder="Type a message..." value={content} diff --git a/next.config.js b/next.config.js index a843cbe..7f56a4c 100644 --- a/next.config.js +++ b/next.config.js @@ -1,6 +1,15 @@ /** @type {import('next').NextConfig} */ const nextConfig = { reactStrictMode: true, -} -module.exports = nextConfig + webpack(config, { isServer, dev }) { + config.experiments = { + asyncWebAssembly: true, + layers: true, + }; + + return config; + }, +}; + +module.exports = nextConfig; diff --git a/package-lock.json b/package-lock.json index 3ee1aa6..50f0a4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "ai-chatbot-starter", "version": "0.1.0", "dependencies": { + "@dqbd/tiktoken": "^1.0.2", "@tabler/icons-react": "^2.9.0", "@types/node": "18.15.0", "@types/react": "18.0.28", @@ -43,6 +44,11 @@ "node": ">=6.9.0" } }, + "node_modules/@dqbd/tiktoken": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.2.tgz", + "integrity": "sha512-AjGTBRWsMoVmVeN55NLyupyM8TNamOUBl6tj5t/leLDVup3CFGO9tVagNL1jf3GyZLkWZSTmYVbPQ/M2LEcNzw==" + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.3.0.tgz", @@ -5543,6 +5549,11 @@ "regenerator-runtime": "^0.13.11" } }, + "@dqbd/tiktoken": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.2.tgz", + "integrity": "sha512-AjGTBRWsMoVmVeN55NLyupyM8TNamOUBl6tj5t/leLDVup3CFGO9tVagNL1jf3GyZLkWZSTmYVbPQ/M2LEcNzw==" + }, "@eslint-community/eslint-utils": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.3.0.tgz", diff --git a/package.json b/package.json index 0af3747..422f99a 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "lint": "next lint" }, "dependencies": { + "@dqbd/tiktoken": "^1.0.2", "@tabler/icons-react": "^2.9.0", "@types/node": "18.15.0", "@types/react": "18.0.28", diff --git a/pages/api/chat.ts b/pages/api/chat.ts index cf1fc01..c58e615 100644 --- a/pages/api/chat.ts +++ b/pages/api/chat.ts @@ -1,5 +1,9 @@ -import { Message, OpenAIModel } from "@/types"; +import { Message, OpenAIModel, OpenAIModelID } from "@/types"; import { OpenAIStream } from "@/utils/server"; +import tiktokenModel from "@dqbd/tiktoken/encoders/cl100k_base.json"; +import { init, Tiktoken } from "@dqbd/tiktoken/lite/init"; +// @ts-expect-error +import wasm from "../../node_modules/@dqbd/tiktoken/lite/tiktoken_bg.wasm?module"; export const config = { runtime: "edge" @@ -13,19 +17,26 @@ const handler = async (req: Request): Promise => { key: string; }; - const charLimit = 12000; - let charCount = 0; + await init((imports) => WebAssembly.instantiate(wasm, imports)); + const encoding = new Tiktoken(tiktokenModel.bpe_ranks, tiktokenModel.special_tokens, tiktokenModel.pat_str); + + const tokenLimit = model.id === OpenAIModelID.GPT_4 ? 6000 : 3000; + let tokenCount = 0; let messagesToSend: Message[] = []; for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; - if (charCount + message.content.length > charLimit) { + const tokens = encoding.encode(message.content); + + if (tokenCount + tokens.length > tokenLimit) { break; } - charCount += message.content.length; + tokenCount += tokens.length; messagesToSend = [message, ...messagesToSend]; } + encoding.free(); + const stream = await OpenAIStream(model, key, messagesToSend); return new Response(stream);