import { Fragment, useState } from "react"; import { cn } from "~/utils/cn"; import BN from "bignumber.js"; import { Checkbox } from "~/components/Checkbox"; import { type TokenizerResult } from "~/models/tokenizer"; const COLORS = [ "bg-sky-200", "bg-amber-200", "bg-blue-200", "bg-green-200", "bg-orange-200", "bg-cyan-200", "bg-gray-200", "bg-purple-200", "bg-indigo-200", "bg-lime-200", "bg-rose-200", "bg-violet-200", "bg-yellow-200", "bg-emerald-200", "bg-zinc-200", "bg-red-200", "bg-fuchsia-200", "bg-pink-200", "bg-teal-200", ]; function encodeWhitespace(str: string) { let result = str; result = result.replaceAll(" ", "⋅"); result = result.replaceAll("\t", "→"); result = result.replaceAll("\f", "\\f\f"); result = result.replaceAll("\b", "\\b\b"); result = result.replaceAll("\v", "\\v\v"); result = result.replaceAll("\r", "\\r\r"); result = result.replaceAll("\n", "\\n\n"); result = result.replaceAll("\\r\r\\n\n", "\\r\\n\r\n"); return result; } export function TokenViewer(props: { isFetching: boolean; model: string | undefined; data: TokenizerResult | undefined; }) { const [indexHover, setIndexHover] = useState(null); const tokenCount = props.data?.segments?.reduce((memo, i) => memo + i.tokens.length, 0) ?? 0; const [showWhitespace, setShowWhitespace] = useState(false); return ( <>

Token count

{tokenCount}

        {props.data?.segments?.map(({ text }, idx) => (
           setIndexHover(idx)}
            onMouseLeave={() => setIndexHover(null)}
            className={cn(
              "transition-all",
              (indexHover == null || indexHover === idx) &&
                COLORS[idx % COLORS.length],
              props.isFetching && "opacity-50"
            )}
          >
            {showWhitespace || indexHover === idx
              ? encodeWhitespace(text)
              : text}
          
        ))}
      
        {props.data && tokenCount > 0 && (
          
            {props.data?.segments?.map((segment, segmentIdx) => (
              
                {segment.tokens.map((token) => (
                  
                     setIndexHover(segmentIdx)}
                      onMouseLeave={() => setIndexHover(null)}
                      className={cn(
                        "transition-colors",
                        indexHover === segmentIdx &&
                          COLORS[segmentIdx % COLORS.length]
                      )}
                    >
                      {token.id}
                    
                    {", "}
                  
                ))}
              
            ))}
          
        )}
      
setShowWhitespace((v) => !v)} />
); }