{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab-1-Introduce Phi-3.5 Instruct"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert Phi-3-mini-4k-instruct model to INT4 with optimum-cli"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "llm_model_id = \"microsoft/Phi-3.5-mini-instruct\"\n",
    "llm_model_path = \"../model/phi-3.5-mini-instruct-ov\"\n",
    "\n",
    "if not Path(llm_model_path).exists():\n",
    "    !optimum-cli export openvino --model {llm_model_id} --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.6  --sym  --trust-remote-code {llm_model_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoConfig, AutoTokenizer\n",
    "from optimum.intel.openvino import OVModelForCausalLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ov_config = {\"PERFORMANCE_HINT\": \"LATENCY\", \"NUM_STREAMS\": \"1\", \"CACHE_DIR\": \"\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ov_model = OVModelForCausalLM.from_pretrained(\n",
    "    llm_model_path,\n",
    "    device='GPU',\n",
    "    ov_config=ov_config,\n",
    "    config=AutoConfig.from_pretrained(llm_model_path, trust_remote_code=True),\n",
    "    trust_remote_code=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = AutoTokenizer.from_pretrained(llm_model_path, trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_kwargs =  {\"add_special_tokens\": False}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"<|user|>\\n你了解 .NET 吗?\\n<|end|><|assistant|>\\n\"\n",
    "input_tokens = tok(prompt, return_tensors=\"pt\", **tokenizer_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer = ov_model.generate(**input_tokens, max_new_tokens=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok.batch_decode(answer, skip_special_tokens=True)[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "openvinodev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}