import { type GetServerSideProps, type InferGetServerSidePropsType, type NextPage, } from "next"; import Head from "next/head"; import { useMemo, useState } from "react"; import { Github, Twitter } from "lucide-react"; import { ChatGPTEditor } from "../sections/ChatGPTEditor"; import { EncoderSelect } from "~/sections/EncoderSelect"; import { TokenViewer } from "~/sections/TokenViewer"; import { TextArea } from "~/components/Input"; import { type AllOptions, isChatModel, isValidOption } from "~/models"; import { createTokenizer } from "~/models/tokenizer"; import { useQuery } from "@tanstack/react-query"; import { useRouter } from "next/router"; function useQueryParamsState() { const router = useRouter(); const params = useMemo((): AllOptions => { return isValidOption(router.query?.model) ? router.query.model : "gpt-4o"; }, [router.query]); const setParams = (model: AllOptions) => { router.push({ pathname: router.pathname, query: { model }, }); }; return [params, setParams] as const; } const Home: NextPage< InferGetServerSidePropsType > = () => { const [inputText, setInputText] = useState(""); const [model, setModel] = useQueryParamsState(); const tokenizer = useQuery({ queryKey: [model], queryFn: ({ queryKey: [model] }) => createTokenizer(model!), }); const tokens = tokenizer.data?.tokenize(inputText); return ( <> Tiktokenizer

Tiktokenizer

{ setModel(update); if (isChatModel(update) !== isChatModel(model)) { setInputText(""); } }} />
{isChatModel(model) && ( )}