{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! pip install onnxruntime-gpu==1.19.2 -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! pip install onnxruntime-genai-cuda==0.4.0 -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnxruntime_genai as og"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = og.Model('Your Phi-3.5 Instruct ONNX Model Path')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = og.Tokenizer(model)\n",
    "tokenizer_stream = tokenizer.create_stream()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_options = {}\n",
    "search_options['max_length'] = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "chat_template = '<|user|>\\n{input} <|end|>\\n<|assistant|>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'can you introduce yourself?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = f'{chat_template.format(input=text)}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_tokens = tokenizer.encode(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "SsSparams = og.GeneratorParams(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "params = og.GeneratorParams(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "params.input_ids = input_tokens\n",
    "params.set_search_options(**search_options)\n",
    "generator = og.Generator(model, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Hello! I'm Phi, an AI digital assistant designed to help answer your questions and assist with a variety of tasks. I don't have personal experiences, but I'm here to provide information and support to the best of my abilities. How can I help you today?"
     ]
    }
   ],
   "source": [
    "while not generator.is_done():\n",
    "    generator.compute_logits()\n",
    "    generator.generate_next_token()\n",
    "\n",
    "    new_token = generator.get_next_tokens()[0]\n",
    "    print(tokenizer_stream.decode(new_token), end='', flush=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pydev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}