{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! optimum-cli export openvino --model \"microsoft/Phi-3-mini-4k-instruct\" --task text-generation-with-past --weight-format fp16 --trust-remote-code ./openvinomodel/phi3/fp16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino\n",
      "\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   0% 0/128 • 0:00:00 • -:--:--\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:00 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:00 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 2/128 • 0:00:01 • 0:00:14\n",
      "Mixed-Precision assignment ---------------------   2% 3/128 • 0:00:01 • 0:00:55\n",
      "Mixed-Precision assignment ---------------------   2% 3/128 • 0:00:01 • 0:00:55\n",
      "Mixed-Precision assignment ---------------------   2% 3/128 • 0:00:01 • 0:00:55\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:01 • 0:00:50\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:01 • 0:00:50\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:02 • 0:00:50\n",
      "Mixed-Precision assignment  --------------------   3% 4/128 • 0:00:02 • 0:00:50\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment  --------------------   5% 6/128 • 0:00:02 • 0:00:43\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:03 • 0:00:49\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:03 • 0:00:49\n",
      "Mixed-Precision assignment - -------------------   5% 7/128 • 0:00:03 • 0:00:49\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:03 • 0:00:47\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:03 • 0:00:47\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:03 • 0:00:47\n",
      "Mixed-Precision assignment - -------------------   6% 8/128 • 0:00:03 • 0:00:47\n",
      "Mixed-Precision assignment - -------------------   7% 9/128 • 0:00:03 • 0:00:47\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:03 • 0:00:43\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:04 • 0:00:43\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:04 • 0:00:43\n",
      "Mixed-Precision assignment - ------------------   8% 10/128 • 0:00:04 • 0:00:43\n",
      "Mixed-Precision assignment - ------------------   9% 11/128 • 0:00:04 • 0:00:45\n",
      "Mixed-Precision assignment - ------------------   9% 11/128 • 0:00:04 • 0:00:45\n",
      "Mixed-Precision assignment - ------------------   9% 11/128 • 0:00:04 • 0:00:45\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:04 • 0:00:44\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:04 • 0:00:44\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:04 • 0:00:44\n",
      "Mixed-Precision assignment - ------------------   9% 12/128 • 0:00:05 • 0:00:44\n",
      "Mixed-Precision assignment -- -----------------  10% 13/128 • 0:00:05 • 0:00:44\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:05 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:05 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:05 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:05 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  11% 14/128 • 0:00:05 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  12% 15/128 • 0:00:05 • 0:00:43\n",
      "Mixed-Precision assignment -- -----------------  12% 15/128 • 0:00:05 • 0:00:43\n",
      "Mixed-Precision assignment -- -----------------  12% 16/128 • 0:00:06 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  12% 16/128 • 0:00:06 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  12% 16/128 • 0:00:06 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  13% 17/128 • 0:00:06 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:06 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:06 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:06 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:06 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:07 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  14% 18/128 • 0:00:07 • 0:00:39\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:07 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:07 • 0:00:41\n",
      "Mixed-Precision assignment -- -----------------  15% 19/128 • 0:00:07 • 0:00:41\n",
      "Mixed-Precision assignment --- ----------------  16% 20/128 • 0:00:07 • 0:00:40\n",
      "Mixed-Precision assignment --- ----------------  16% 20/128 • 0:00:07 • 0:00:40\n",
      "Mixed-Precision assignment --- ----------------  16% 21/128 • 0:00:07 • 0:00:39\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  17% 22/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  18% 23/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  19% 24/128 • 0:00:08 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  19% 24/128 • 0:00:09 • 0:00:38\n",
      "Mixed-Precision assignment --- ----------------  20% 25/128 • 0:00:09 • 0:00:37\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:09 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:09 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:09 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:09 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  20% 26/128 • 0:00:09 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:10 • 0:00:37\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:10 • 0:00:37\n",
      "Mixed-Precision assignment ---- ---------------  21% 27/128 • 0:00:10 • 0:00:37\n",
      "Mixed-Precision assignment ---- ---------------  22% 28/128 • 0:00:10 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  22% 28/128 • 0:00:10 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  23% 29/128 • 0:00:10 • 0:00:36\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:10 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:10 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:10 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:11 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  23% 30/128 • 0:00:11 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  24% 31/128 • 0:00:11 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  24% 31/128 • 0:00:11 • 0:00:35\n",
      "Mixed-Precision assignment ---- ---------------  24% 31/128 • 0:00:11 • 0:00:35\n",
      "Mixed-Precision assignment ----- --------------  25% 32/128 • 0:00:11 • 0:00:34\n",
      "Mixed-Precision assignment ----- --------------  25% 32/128 • 0:00:11 • 0:00:34\n",
      "Mixed-Precision assignment ----- --------------  25% 32/128 • 0:00:11 • 0:00:34\n",
      "Mixed-Precision assignment ----- --------------  26% 33/128 • 0:00:11 • 0:00:34\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 34/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 35/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  27% 35/128 • 0:00:12 • 0:00:33\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:12 • 0:00:32\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:12 • 0:00:32\n",
      "Mixed-Precision assignment ----- --------------  28% 36/128 • 0:00:12 • 0:00:32\n",
      "Mixed-Precision assignment ----- --------------  29% 37/128 • 0:00:13 • 0:00:32\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ----- --------------  30% 38/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ------ -------------  30% 39/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ------ -------------  30% 39/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ------ -------------  30% 39/128 • 0:00:13 • 0:00:31\n",
      "Mixed-Precision assignment ------ -------------  31% 40/128 • 0:00:14 • 0:00:31\n",
      "Mixed-Precision assignment ------ -------------  32% 41/128 • 0:00:14 • 0:00:30\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  33% 42/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:14 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 43/128 • 0:00:15 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 44/128 • 0:00:15 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 44/128 • 0:00:15 • 0:00:29\n",
      "Mixed-Precision assignment ------ -------------  34% 44/128 • 0:00:15 • 0:00:29\n",
      "Mixed-Precision assignment ------- ------------  35% 45/128 • 0:00:15 • 0:00:29\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:15 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:15 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:15 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:15 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  36% 46/128 • 0:00:16 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  37% 47/128 • 0:00:16 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  37% 47/128 • 0:00:16 • 0:00:28\n",
      "Mixed-Precision assignment ------- ------------  38% 48/128 • 0:00:16 • 0:00:27\n",
      "Mixed-Precision assignment ------- ------------  38% 48/128 • 0:00:16 • 0:00:27\n",
      "Mixed-Precision assignment ------- ------------  38% 49/128 • 0:00:16 • 0:00:27\n",
      "Mixed-Precision assignment ------- ------------  38% 49/128 • 0:00:16 • 0:00:27\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:16 • 0:00:26\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:16 • 0:00:26\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment ------- ------------  39% 50/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment ------- ------------  40% 51/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment ------- ------------  40% 51/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment -------- -----------  41% 52/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment -------- -----------  41% 52/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment -------- -----------  41% 52/128 • 0:00:17 • 0:00:26\n",
      "Mixed-Precision assignment -------- -----------  41% 53/128 • 0:00:17 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:17 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  42% 54/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  43% 55/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  43% 55/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  43% 55/128 • 0:00:18 • 0:00:25\n",
      "Mixed-Precision assignment -------- -----------  44% 56/128 • 0:00:18 • 0:00:24\n",
      "Mixed-Precision assignment -------- -----------  44% 56/128 • 0:00:18 • 0:00:24\n",
      "Mixed-Precision assignment -------- -----------  45% 57/128 • 0:00:19 • 0:00:24\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  45% 58/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  46% 59/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:19 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:20 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  47% 60/128 • 0:00:20 • 0:00:23\n",
      "Mixed-Precision assignment --------- ----------  48% 61/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  48% 62/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  49% 63/128 • 0:00:20 • 0:00:22\n",
      "Mixed-Precision assignment --------- ----------  49% 63/128 • 0:00:21 • 0:00:22\n",
      "Mixed-Precision assignment ---------- ---------  50% 64/128 • 0:00:21 • 0:00:21\n",
      "Mixed-Precision assignment ---------- ---------  50% 64/128 • 0:00:21 • 0:00:21\n",
      "Mixed-Precision assignment ---------- ---------  50% 64/128 • 0:00:21 • 0:00:21\n",
      "Mixed-Precision assignment ---------- ---------  51% 65/128 • 0:00:21 • 0:00:21\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:21 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:21 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:21 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:21 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 66/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 67/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 67/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  52% 67/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  53% 68/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  53% 68/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  54% 69/128 • 0:00:22 • 0:00:20\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:22 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:22 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:22 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ---------- ---------  55% 70/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ----------- --------  55% 71/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ----------- --------  55% 71/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ----------- --------  56% 72/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ----------- --------  56% 72/128 • 0:00:23 • 0:00:19\n",
      "Mixed-Precision assignment ----------- --------  57% 73/128 • 0:00:23 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:23 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  58% 74/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  59% 75/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  59% 75/128 • 0:00:24 • 0:00:18\n",
      "Mixed-Precision assignment ----------- --------  59% 76/128 • 0:00:24 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  59% 76/128 • 0:00:24 • 0:00:17\n",
      "Mixed-Precision assignment ----------- --------  59% 76/128 • 0:00:24 • 0:00:17\n",
      "Mixed-Precision assignment ------------ -------  60% 77/128 • 0:00:25 • 0:00:17\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  61% 78/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 79/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 79/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 79/128 • 0:00:25 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:26 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:26 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  62% 80/128 • 0:00:26 • 0:00:16\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  64% 82/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  65% 83/128 • 0:00:26 • 0:00:15\n",
      "Mixed-Precision assignment ------------ -------  65% 83/128 • 0:00:27 • 0:00:15\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:27 • 0:00:15\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:27 • 0:00:15\n",
      "Mixed-Precision assignment ------------- ------  66% 84/128 • 0:00:27 • 0:00:15\n",
      "Mixed-Precision assignment ------------- ------  66% 85/128 • 0:00:27 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:27 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:27 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:27 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:27 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  67% 86/128 • 0:00:28 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  68% 87/128 • 0:00:28 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  68% 87/128 • 0:00:28 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  68% 87/128 • 0:00:28 • 0:00:14\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:28 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:28 • 0:00:13\n",
      "Mixed-Precision assignment ------------- ------  69% 88/128 • 0:00:28 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:28 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:28 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:29 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:29 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  70% 90/128 • 0:00:29 • 0:00:13\n",
      "Mixed-Precision assignment -------------- -----  71% 91/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  71% 91/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  71% 91/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  72% 92/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  72% 92/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  73% 93/128 • 0:00:29 • 0:00:12\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  73% 94/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  74% 95/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment -------------- -----  74% 95/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:30 • 0:00:11\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:31 • 0:00:11\n",
      "Mixed-Precision assignment --------------- ----  75% 96/128 • 0:00:31 • 0:00:11\n",
      "Mixed-Precision assignment --------------- ----  76% 97/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 98/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:31 • 0:00:10\n",
      "Mixed-Precision assignment --------------- ----  77% 99/128 • 0:00:32 • 0:00:10\n",
      "Mixed-Precision assignment -------------- ----  78% 100/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  78% 100/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  79% 101/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment -------------- ----  79% 101/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:32 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 102/128 • 0:00:33 • 0:00:09\n",
      "Mixed-Precision assignment --------------- ---  80% 103/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  80% 103/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  81% 104/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  81% 104/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  81% 104/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  82% 105/128 • 0:00:33 • 0:00:08\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:33 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:33 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  83% 106/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  84% 107/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  84% 107/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment --------------- ---  84% 107/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment ---------------- --  84% 108/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment ---------------- --  84% 108/128 • 0:00:34 • 0:00:07\n",
      "Mixed-Precision assignment ---------------- --  85% 109/128 • 0:00:34 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:34 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  86% 110/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  87% 111/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  87% 111/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  87% 111/128 • 0:00:35 • 0:00:06\n",
      "Mixed-Precision assignment ---------------- --  88% 112/128 • 0:00:35 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 112/128 • 0:00:35 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  88% 113/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ---------------- --  89% 114/128 • 0:00:36 • 0:00:05\n",
      "Mixed-Precision assignment ----------------- -  90% 115/128 • 0:00:36 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  90% 115/128 • 0:00:36 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 116/128 • 0:00:36 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 116/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 116/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  91% 117/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  92% 118/128 • 0:00:37 • 0:00:04\n",
      "Mixed-Precision assignment ----------------- -  93% 119/128 • 0:00:37 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  93% 119/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  93% 119/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  94% 120/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ----------------- -  95% 121/128 • 0:00:38 • 0:00:03\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:38 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:38 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:38 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   95% 122/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   96% 123/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   96% 123/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   97% 124/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   97% 124/128 • 0:00:39 • 0:00:02\n",
      "Mixed-Precision assignment ------------------   98% 125/128 • 0:00:39 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:39 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:39 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:40 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:40 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   98% 126/128 • 0:00:40 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   99% 127/128 • 0:00:40 • 0:00:01\n",
      "Mixed-Precision assignment ------------------   99% 127/128 • 0:00:40 • 0:00:01\n",
      "Mixed-Precision assignment ------------------- 100% 128/128 • 0:00:40 • 0:00:00\n",
      "Mixed-Precision assignment ------------------- 100% 128/128 • 0:00:40 • 0:00:00\n",
      "INFO:nncf:Statistics of the bitwidth distribution:\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "|   Num bits (N) | % all parameters (layers)   | % ratio-defining parameters (layers)   |\n",
      "+================+=============================+========================================+\n",
      "|              8 | 44% (58 / 130)              | 40% (56 / 128)                         |\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "|              4 | 56% (72 / 130)              | 60% (72 / 128)                         |\n",
      "+----------------+-----------------------------+----------------------------------------+\n",
      "\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   0% 0/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   1% 1/130 • 0:00:00 • -:--:--\n",
      "Applying Weight Compression --------------------   2% 2/130 • 0:00:01 • 0:01:30\n",
      "Applying Weight Compression --------------------   2% 2/130 • 0:00:01 • 0:01:30\n",
      "Applying Weight Compression --------------------   2% 3/130 • 0:00:01 • 0:00:52\n",
      "Applying Weight Compression  -------------------   3% 4/130 • 0:00:01 • 0:00:44\n",
      "Applying Weight Compression  -------------------   3% 4/130 • 0:00:01 • 0:00:44\n",
      "Applying Weight Compression  -------------------   3% 4/130 • 0:00:02 • 0:00:44\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:02 • 0:00:50\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:02 • 0:00:50\n",
      "Applying Weight Compression  -------------------   4% 5/130 • 0:00:02 • 0:00:50\n",
      "Applying Weight Compression - ------------------   5% 7/130 • 0:00:02 • 0:00:45\n",
      "Applying Weight Compression - ------------------   5% 7/130 • 0:00:03 • 0:00:45\n",
      "Applying Weight Compression - ------------------   5% 7/130 • 0:00:03 • 0:00:45\n",
      "Applying Weight Compression - ------------------   6% 8/130 • 0:00:03 • 0:00:56\n",
      "Applying Weight Compression - ------------------   6% 8/130 • 0:00:04 • 0:00:56\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:04 • 0:00:57\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:04 • 0:00:57\n",
      "Applying Weight Compression - ------------------   7% 9/130 • 0:00:05 • 0:00:57\n",
      "Applying Weight Compression - -----------------   8% 11/130 • 0:00:05 • 0:00:53\n",
      "Applying Weight Compression - -----------------   8% 11/130 • 0:00:05 • 0:00:53\n",
      "Applying Weight Compression - -----------------   8% 11/130 • 0:00:05 • 0:00:53\n",
      "Applying Weight Compression - -----------------   9% 12/130 • 0:00:06 • 0:00:59\n",
      "Applying Weight Compression - -----------------   9% 12/130 • 0:00:06 • 0:00:59\n",
      "Applying Weight Compression - -----------------  10% 13/130 • 0:00:06 • 0:00:58\n",
      "Applying Weight Compression - -----------------  10% 13/130 • 0:00:06 • 0:00:58\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:07 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:07 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:07 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:07 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  12% 15/130 • 0:00:08 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  12% 16/130 • 0:00:08 • 0:00:58\n",
      "Applying Weight Compression -- ----------------  13% 17/130 • 0:00:08 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  13% 17/130 • 0:00:08 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  13% 17/130 • 0:00:08 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:09 • 0:00:51\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:09 • 0:00:51\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:09 • 0:00:51\n",
      "Applying Weight Compression -- ----------------  15% 19/130 • 0:00:09 • 0:00:51\n",
      "Applying Weight Compression -- ----------------  15% 20/130 • 0:00:10 • 0:00:54\n",
      "Applying Weight Compression -- ----------------  15% 20/130 • 0:00:10 • 0:00:54\n",
      "Applying Weight Compression --- ---------------  16% 21/130 • 0:00:10 • 0:00:54\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:10 • 0:00:51\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:11 • 0:00:51\n",
      "Applying Weight Compression --- ---------------  18% 23/130 • 0:00:11 • 0:00:51\n",
      "Applying Weight Compression --- ---------------  18% 24/130 • 0:00:11 • 0:00:50\n",
      "Applying Weight Compression --- ---------------  18% 24/130 • 0:00:11 • 0:00:50\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:11 • 0:00:49\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:11 • 0:00:49\n",
      "Applying Weight Compression --- ---------------  19% 25/130 • 0:00:12 • 0:00:49\n",
      "Applying Weight Compression --- ---------------  21% 27/130 • 0:00:12 • 0:00:47\n",
      "Applying Weight Compression --- ---------------  21% 27/130 • 0:00:12 • 0:00:47\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:12 • 0:00:46\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:12 • 0:00:46\n",
      "Applying Weight Compression ---- --------------  22% 28/130 • 0:00:13 • 0:00:46\n",
      "Applying Weight Compression ---- --------------  22% 29/130 • 0:00:13 • 0:00:46\n",
      "Applying Weight Compression ---- --------------  22% 29/130 • 0:00:13 • 0:00:46\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:13 • 0:00:44\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:13 • 0:00:44\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:14 • 0:00:44\n",
      "Applying Weight Compression ---- --------------  24% 31/130 • 0:00:14 • 0:00:44\n",
      "Applying Weight Compression ---- --------------  25% 32/130 • 0:00:14 • 0:00:45\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:15 • 0:00:45\n",
      "Applying Weight Compression ---- --------------  25% 33/130 • 0:00:15 • 0:00:45\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:15 • 0:00:43\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:16 • 0:00:43\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:16 • 0:00:43\n",
      "Applying Weight Compression ----- -------------  27% 35/130 • 0:00:16 • 0:00:43\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:16 • 0:00:44\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:17 • 0:00:44\n",
      "Applying Weight Compression ----- -------------  28% 36/130 • 0:00:17 • 0:00:44\n",
      "Applying Weight Compression ----- -------------  28% 37/130 • 0:00:17 • 0:00:44\n",
      "Applying Weight Compression ----- -------------  28% 37/130 • 0:00:18 • 0:00:44\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:18 • 0:00:42\n",
      "Applying Weight Compression ----- -------------  30% 39/130 • 0:00:18 • 0:00:42\n",
      "Applying Weight Compression ----- -------------  31% 40/130 • 0:00:18 • 0:00:41\n",
      "Applying Weight Compression ----- -------------  32% 41/130 • 0:00:18 • 0:00:40\n",
      "Applying Weight Compression ----- -------------  32% 41/130 • 0:00:18 • 0:00:40\n",
      "Applying Weight Compression ----- -------------  32% 41/130 • 0:00:19 • 0:00:40\n",
      "Applying Weight Compression ------ ------------  33% 43/130 • 0:00:19 • 0:00:39\n",
      "Applying Weight Compression ------ ------------  33% 43/130 • 0:00:19 • 0:00:39\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:19 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:19 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  34% 44/130 • 0:00:19 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  35% 45/130 • 0:00:20 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  35% 45/130 • 0:00:20 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  35% 45/130 • 0:00:20 • 0:00:38\n",
      "Applying Weight Compression ------ ------------  36% 47/130 • 0:00:20 • 0:00:36\n",
      "Applying Weight Compression ------ ------------  36% 47/130 • 0:00:20 • 0:00:36\n",
      "Applying Weight Compression ------- -----------  37% 48/130 • 0:00:20 • 0:00:36\n",
      "Applying Weight Compression ------- -----------  37% 48/130 • 0:00:20 • 0:00:36\n",
      "Applying Weight Compression ------- -----------  37% 48/130 • 0:00:21 • 0:00:36\n",
      "Applying Weight Compression ------- -----------  37% 48/130 • 0:00:21 • 0:00:36\n",
      "Applying Weight Compression ------- -----------  38% 49/130 • 0:00:21 • 0:00:35\n",
      "Applying Weight Compression ------- -----------  39% 51/130 • 0:00:21 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  39% 51/130 • 0:00:22 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  39% 51/130 • 0:00:22 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  40% 52/130 • 0:00:22 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  40% 52/130 • 0:00:22 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  40% 52/130 • 0:00:22 • 0:00:34\n",
      "Applying Weight Compression ------- -----------  41% 53/130 • 0:00:23 • 0:00:33\n",
      "Applying Weight Compression ------- -----------  42% 54/130 • 0:00:23 • 0:00:33\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:23 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:23 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:23 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:23 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  42% 55/130 • 0:00:24 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  43% 56/130 • 0:00:24 • 0:00:33\n",
      "Applying Weight Compression -------- ----------  43% 56/130 • 0:00:25 • 0:00:33\n",
      "Applying Weight Compression -------- ----------  44% 57/130 • 0:00:25 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  44% 57/130 • 0:00:25 • 0:00:32\n",
      "Applying Weight Compression -------- ----------  45% 59/130 • 0:00:25 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  45% 59/130 • 0:00:25 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  45% 59/130 • 0:00:26 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  46% 60/130 • 0:00:26 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  46% 60/130 • 0:00:26 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  46% 60/130 • 0:00:27 • 0:00:31\n",
      "Applying Weight Compression -------- ----------  47% 61/130 • 0:00:27 • 0:00:31\n",
      "Applying Weight Compression --------- ---------  48% 63/130 • 0:00:27 • 0:00:29\n",
      "Applying Weight Compression --------- ---------  48% 63/130 • 0:00:27 • 0:00:29\n",
      "Applying Weight Compression --------- ---------  48% 63/130 • 0:00:27 • 0:00:29\n",
      "Applying Weight Compression --------- ---------  49% 64/130 • 0:00:27 • 0:00:29\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:27 • 0:00:28\n",
      "Applying Weight Compression --------- ---------  50% 65/130 • 0:00:27 • 0:00:28\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:28 • 0:00:27\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:28 • 0:00:27\n",
      "Applying Weight Compression --------- ---------  52% 67/130 • 0:00:28 • 0:00:27\n",
      "Applying Weight Compression --------- ---------  52% 68/130 • 0:00:28 • 0:00:26\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:28 • 0:00:26\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:28 • 0:00:26\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:29 • 0:00:26\n",
      "Applying Weight Compression ---------- --------  53% 69/130 • 0:00:29 • 0:00:26\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:29 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:29 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:29 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:29 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  55% 71/130 • 0:00:30 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  55% 72/130 • 0:00:30 • 0:00:25\n",
      "Applying Weight Compression ---------- --------  56% 73/130 • 0:00:30 • 0:00:24\n",
      "Applying Weight Compression ---------- --------  56% 73/130 • 0:00:31 • 0:00:24\n",
      "Applying Weight Compression ---------- --------  58% 75/130 • 0:00:31 • 0:00:23\n",
      "Applying Weight Compression ---------- --------  58% 75/130 • 0:00:31 • 0:00:23\n",
      "Applying Weight Compression ---------- --------  58% 75/130 • 0:00:31 • 0:00:23\n",
      "Applying Weight Compression ----------- -------  58% 76/130 • 0:00:31 • 0:00:23\n",
      "Applying Weight Compression ----------- -------  59% 77/130 • 0:00:31 • 0:00:22\n",
      "Applying Weight Compression ----------- -------  59% 77/130 • 0:00:31 • 0:00:22\n",
      "Applying Weight Compression ----------- -------  59% 77/130 • 0:00:31 • 0:00:22\n",
      "Applying Weight Compression ----------- -------  59% 77/130 • 0:00:32 • 0:00:22\n",
      "Applying Weight Compression ----------- -------  61% 79/130 • 0:00:32 • 0:00:21\n",
      "Applying Weight Compression ----------- -------  61% 79/130 • 0:00:32 • 0:00:21\n",
      "Applying Weight Compression ----------- -------  62% 80/130 • 0:00:32 • 0:00:21\n",
      "Applying Weight Compression ----------- -------  62% 80/130 • 0:00:32 • 0:00:21\n",
      "Applying Weight Compression ----------- -------  62% 80/130 • 0:00:32 • 0:00:21\n",
      "Applying Weight Compression ----------- -------  62% 81/130 • 0:00:33 • 0:00:20\n",
      "Applying Weight Compression ----------- -------  62% 81/130 • 0:00:33 • 0:00:20\n",
      "Applying Weight Compression ------------ ------  64% 83/130 • 0:00:33 • 0:00:19\n",
      "Applying Weight Compression ------------ ------  64% 83/130 • 0:00:33 • 0:00:19\n",
      "Applying Weight Compression ------------ ------  64% 83/130 • 0:00:33 • 0:00:19\n",
      "Applying Weight Compression ------------ ------  65% 84/130 • 0:00:34 • 0:00:19\n",
      "Applying Weight Compression ------------ ------  65% 84/130 • 0:00:34 • 0:00:19\n",
      "Applying Weight Compression ------------ ------  65% 85/130 • 0:00:34 • 0:00:18\n",
      "Applying Weight Compression ------------ ------  65% 85/130 • 0:00:34 • 0:00:18\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  67% 87/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  68% 88/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  68% 88/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  68% 88/130 • 0:00:35 • 0:00:17\n",
      "Applying Weight Compression ------------ ------  68% 88/130 • 0:00:36 • 0:00:17\n",
      "Applying Weight Compression ------------- -----  68% 89/130 • 0:00:36 • 0:00:16\n",
      "Applying Weight Compression ------------- -----  68% 89/130 • 0:00:36 • 0:00:16\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:00:36 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:00:36 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  70% 91/130 • 0:00:36 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  71% 92/130 • 0:00:36 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  71% 92/130 • 0:00:37 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:00:37 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:00:37 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:00:37 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  72% 93/130 • 0:00:38 • 0:00:15\n",
      "Applying Weight Compression ------------- -----  73% 95/130 • 0:00:38 • 0:00:14\n",
      "Applying Weight Compression ------------- -----  73% 95/130 • 0:00:38 • 0:00:14\n",
      "Applying Weight Compression -------------- ----  74% 96/130 • 0:00:38 • 0:00:14\n",
      "Applying Weight Compression -------------- ----  74% 96/130 • 0:00:38 • 0:00:14\n",
      "Applying Weight Compression -------------- ----  74% 96/130 • 0:00:38 • 0:00:14\n",
      "Applying Weight Compression -------------- ----  75% 97/130 • 0:00:39 • 0:00:13\n",
      "Applying Weight Compression -------------- ----  75% 97/130 • 0:00:39 • 0:00:13\n",
      "Applying Weight Compression -------------- ----  76% 99/130 • 0:00:39 • 0:00:12\n",
      "Applying Weight Compression -------------- ----  76% 99/130 • 0:00:39 • 0:00:12\n",
      "Applying Weight Compression -------------- ----  76% 99/130 • 0:00:39 • 0:00:12\n",
      "Applying Weight Compression ------------- ----  78% 101/130 • 0:00:39 • 0:00:11\n",
      "Applying Weight Compression ------------- ----  78% 101/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression ------------- ----  78% 101/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression -------------- ---  79% 103/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression -------------- ---  79% 103/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression -------------- ---  79% 103/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression -------------- ---  79% 103/130 • 0:00:40 • 0:00:11\n",
      "Applying Weight Compression -------------- ---  80% 104/130 • 0:00:41 • 0:00:10\n",
      "Applying Weight Compression -------------- ---  81% 105/130 • 0:00:41 • 0:00:10\n",
      "Applying Weight Compression -------------- ---  82% 106/130 • 0:00:41 • 0:00:09\n",
      "Applying Weight Compression -------------- ---  82% 107/130 • 0:00:41 • 0:00:09\n",
      "Applying Weight Compression -------------- ---  82% 107/130 • 0:00:41 • 0:00:09\n",
      "Applying Weight Compression -------------- ---  82% 107/130 • 0:00:41 • 0:00:09\n",
      "Applying Weight Compression -------------- ---  82% 107/130 • 0:00:42 • 0:00:09\n",
      "Applying Weight Compression -------------- ---  83% 108/130 • 0:00:42 • 0:00:09\n",
      "Applying Weight Compression --------------- --  84% 109/130 • 0:00:42 • 0:00:08\n",
      "Applying Weight Compression --------------- --  84% 109/130 • 0:00:42 • 0:00:08\n",
      "Applying Weight Compression --------------- --  84% 109/130 • 0:00:43 • 0:00:08\n",
      "Applying Weight Compression --------------- --  84% 109/130 • 0:00:43 • 0:00:08\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:00:43 • 0:00:08\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:00:43 • 0:00:08\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:00:43 • 0:00:08\n",
      "Applying Weight Compression --------------- --  85% 111/130 • 0:00:44 • 0:00:08\n",
      "Applying Weight Compression --------------- --  86% 112/130 • 0:00:44 • 0:00:07\n",
      "Applying Weight Compression --------------- --  86% 112/130 • 0:00:45 • 0:00:07\n",
      "Applying Weight Compression --------------- --  87% 113/130 • 0:00:45 • 0:00:07\n",
      "Applying Weight Compression --------------- --  87% 113/130 • 0:00:45 • 0:00:07\n",
      "Applying Weight Compression --------------- --  88% 115/130 • 0:00:45 • 0:00:06\n",
      "Applying Weight Compression --------------- --  88% 115/130 • 0:00:45 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  89% 116/130 • 0:00:45 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  89% 116/130 • 0:00:46 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  89% 116/130 • 0:00:46 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  89% 116/130 • 0:00:46 • 0:00:06\n",
      "Applying Weight Compression ---------------- -  91% 118/130 • 0:00:46 • 0:00:05\n",
      "Applying Weight Compression ---------------- -  91% 118/130 • 0:00:46 • 0:00:05\n",
      "Applying Weight Compression ---------------- -  92% 119/130 • 0:00:46 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  92% 119/130 • 0:00:46 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  92% 119/130 • 0:00:47 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  92% 119/130 • 0:00:47 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  92% 120/130 • 0:00:47 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  92% 120/130 • 0:00:48 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  93% 121/130 • 0:00:48 • 0:00:04\n",
      "Applying Weight Compression ---------------- -  94% 122/130 • 0:00:48 • 0:00:03\n",
      "Applying Weight Compression ---------------- -  94% 122/130 • 0:00:48 • 0:00:03\n",
      "Applying Weight Compression -----------------   95% 123/130 • 0:00:48 • 0:00:03\n",
      "Applying Weight Compression -----------------   95% 123/130 • 0:00:48 • 0:00:03\n",
      "Applying Weight Compression -----------------   95% 123/130 • 0:00:49 • 0:00:03\n",
      "Applying Weight Compression -----------------   95% 123/130 • 0:00:49 • 0:00:03\n",
      "Applying Weight Compression -----------------   95% 124/130 • 0:00:49 • 0:00:03\n",
      "Applying Weight Compression -----------------   96% 125/130 • 0:00:50 • 0:00:02\n",
      "Applying Weight Compression -----------------   96% 125/130 • 0:00:50 • 0:00:02\n",
      "Applying Weight Compression -----------------   96% 125/130 • 0:00:50 • 0:00:02\n",
      "Applying Weight Compression -----------------   97% 126/130 • 0:00:50 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:00:51 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:00:51 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:00:51 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 127/130 • 0:00:51 • 0:00:02\n",
      "Applying Weight Compression -----------------   98% 128/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression -----------------   99% 129/130 • 0:00:52 • 0:00:01\n",
      "Applying Weight Compression ------------------ 100% 130/130 • 0:00:52 • 0:00:00\n",
      "Applying Weight Compression ------------------ 100% 130/130 • 0:00:52 • 0:00:00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Framework not specified. Using pt to export the model.\n",
      "`flash-attention` package not found, consider installing for better performance: No module named 'flash_attn'.\n",
      "Current `flash-attenton` does not support `window_size`. Either upgrade or use `attn_implementation='eager'`.\n",
      "\n",
      "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\n",
      "Loading checkpoint shards:  50%|█████     | 1/2 [00:03<00:03,  3.84s/it]\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  3.32s/it]\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  3.40s/it]\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "C:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Using framework PyTorch: 2.3.0+cpu\n",
      "WARNING:root:Cannot apply model.to_bettertransformer because of the exception:\n",
      "The model type phi3 is not yet supported to be used with BetterTransformer. Feel free to open an issue at https://github.com/huggingface/optimum/issues if you would like this model type to be supported. Currently supported models are: dict_keys(['albert', 'bark', 'bart', 'bert', 'bert-generation', 'blenderbot', 'bloom', 'camembert', 'blip-2', 'clip', 'codegen', 'data2vec-text', 'deit', 'distilbert', 'electra', 'ernie', 'fsmt', 'gpt2', 'gptj', 'gpt_neo', 'gpt_neox', 'hubert', 'layoutlm', 'm2m_100', 'marian', 'markuplm', 'mbart', 'opt', 'pegasus', 'rembert', 'prophetnet', 'roberta', 'roc_bert', 'roformer', 'splinter', 'tapas', 't5', 'vilt', 'vit', 'vit_mae', 'vit_msn', 'wav2vec2', 'xlm-roberta', 'yolos']).. Usage model with stateful=True may be non-effective if model does not contain torch.functional.scaled_dot_product_attention\n",
      "Overriding 1 configuration item(s)\n",
      "\t- use_cache -> True\n",
      "C:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\transformers\\modeling_utils.py:4371: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
      "  warnings.warn(\n",
      "C:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\transformers\\modeling_attn_mask_utils.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if (input_shape[-1] > 1 or self.sliding_window is not None) and self.is_causal:\n",
      "C:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\optimum\\exporters\\onnx\\model_patcher.py:300: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if past_key_values_length > 0:\n",
      "C:\\Users\\kinfeylo\\.cache\\huggingface\\modules\\transformers_modules\\microsoft\\Phi-3-mini-4k-instruct\\920b6cf52a79ecff578cc33f61922b23cbc88115\\modeling_phi3.py:391: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):\n",
      "C:\\Users\\kinfeylo\\.cache\\huggingface\\modules\\transformers_modules\\microsoft\\Phi-3-mini-4k-instruct\\920b6cf52a79ecff578cc33f61922b23cbc88115\\modeling_phi3.py:398: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):\n",
      "C:\\Users\\kinfeylo\\.cache\\huggingface\\modules\\transformers_modules\\microsoft\\Phi-3-mini-4k-instruct\\920b6cf52a79ecff578cc33f61922b23cbc88115\\modeling_phi3.py:410: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):\n",
      "OpenVINO Tokenizers is not available. To deploy models in production with C++ code, please follow installation instructions: https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n",
      "\n",
      "Tokenizer won't be converted.\n"
     ]
    }
   ],
   "source": [
    "! optimum-cli export openvino --model \"microsoft/Phi-3-mini-4k-instruct\" --task text-generation-with-past --weight-format int4 --group-size 128 --ratio 0.6  --sym  --trust-remote-code ./openvinomodel/phi3/int4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoConfig, AutoTokenizer\n",
    "from optimum.intel.openvino import OVModelForCausalLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dir = './openvinomodel/phi3/int4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ov_config = {\"PERFORMANCE_HINT\": \"LATENCY\", \"NUM_STREAMS\": \"1\", \"CACHE_DIR\": \"\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\kinfeylo\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "The argument `trust_remote_code` is to be used along with export=True. It will be ignored.\n",
      "Compiling the model to GPU.0 ...\n"
     ]
    }
   ],
   "source": [
    "\n",
    "ov_model = OVModelForCausalLM.from_pretrained(\n",
    "    model_dir,\n",
    "    device='GPU.0',\n",
    "    ov_config=ov_config,\n",
    "    config=AutoConfig.from_pretrained(model_dir, trust_remote_code=True),\n",
    "    trust_remote_code=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_kwargs =  {\"add_special_tokens\": False}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"<|system|>You are a helpful AI assistant.<|end|><|user|>can you introduce yourself?<|end|><|assistant|>\"\n",
    "input_tokens = tok(prompt, return_tensors=\"pt\", **tokenizer_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer = ov_model.generate(**input_tokens, max_new_tokens=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"You are a helpful AI assistant. can you introduce yourself? Hello! I am Phi, an advanced artificial intelligence developed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other subject, I'm here to help! How may I assist you today? Greetings! I am Phi, an AI developed to assist and provide information on various subjects. My goal is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and improving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Hello! I am Phi, an AI designed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other subject, I'm here to help! How may I assist you today? Hi there! I am Phi, an AI developed to assist and provide information on various subjects. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Greetings! I am Phi, an AI developed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other subject, I'm here to help! How may I assist you today? Hello! I am Phi, an AI developed to assist and provide information on various subjects. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Hi there! I am Phi, an AI developed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other subject, I'm here to help! How may I assist you today? Greetings! I am Phi, an AI developed to assist and provide information on various subjects. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Hello! I am Phi, an AI developed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other subject, I'm here to help! How may I assist you today? Hi there! I am Phi, an AI developed to assist and provide information on various subjects. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Greetings! I am Phi, an AI developed to assist and provide information on various subjects. My purpose is to help users find answers, learn new things, and engage in meaningful conversations. I am constantly learning and evolving to better serve your needs. Whether you have questions about science, history, technology, or any other topic, I'm here to help! How may I assist you today? Hello! I am Phi, an AI developed to assist and provide information on a wide range of topics. My purpose is to help users find answers, learn new things, and engage in\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok.batch_decode(answer, skip_special_tokens=True)[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}