{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"transformer_time_weight.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"aWjiMM8qbM__","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1599638558974,"user_tz":-480,"elapsed":2158,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}},"outputId":"1deb4a48-f322-40a7-d441-7fe2e49f7e22"},"source":["from google.colab import drive\n","drive.mount('/content/gdrive')\n","import os\n","os.chdir('/content/gdrive/My Drive/finch/tensorflow2/spoken_language_understanding/atis/main')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"jNTA2Vzj8j_V","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":52},"executionInfo":{"status":"ok","timestamp":1599638561940,"user_tz":-480,"elapsed":4781,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}},"outputId":"3ca84609-6816-4a29-9c58-b1ff9e2e9da3"},"source":["%tensorflow_version 2.x\n","!pip install tensorflow-addons"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)\n","Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"sRS_9RVFjoe1","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":104},"executionInfo":{"status":"ok","timestamp":1599638564564,"user_tz":-480,"elapsed":6386,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}},"outputId":"b15746ee-a16b-4483-ff92-18bcb03ba95b"},"source":["from tensorflow_addons.optimizers.cyclical_learning_rate import Triangular2CyclicalLearningRate\n","from sklearn.metrics import classification_report, f1_score, accuracy_score\n","\n","import tensorflow as tf\n","import pprint\n","import logging\n","import time\n","import numpy as np\n","\n","print(\"TensorFlow Version\", tf.__version__)\n","print('GPU Enabled:', tf.test.is_gpu_available())"],"execution_count":3,"outputs":[{"output_type":"stream","text":["TensorFlow Version 2.3.0\n","WARNING:tensorflow:From <ipython-input-3-27b53cc52b5d>:11: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `tf.config.list_physical_devices('GPU')` instead.\n","GPU Enabled: True\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SY89xO1qRPbQ","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638564567,"user_tz":-480,"elapsed":4975,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["def get_vocab(vocab_path):\n","  word2idx = {}\n","  with open(vocab_path) as f:\n","    for i, line in enumerate(f):\n","      line = line.rstrip()\n","      word2idx[line] = i\n","  return word2idx"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"id":"WszANU_obncn","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638564570,"user_tz":-480,"elapsed":3604,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["def data_generator(f_path, params):\n","  print('Reading', f_path)\n","  with open(f_path) as f:\n","    for line in f:\n","      line = line.rstrip()\n","      text, slot_intent = line.split('\\t')\n","      words = text.split()[1:-1]\n","      slot_intent = slot_intent.split()\n","      slots, intent = slot_intent[1:-1], slot_intent[-1]\n","      assert len(words) == len(slots)\n","      \n","      words = [params['word2idx'].get(w, len(params['word2idx'])) for w in words]\n","      intent = params['intent2idx'].get(intent, len(params['intent2idx']))\n","      slots = [params['slot2idx'].get(s, len(params['slot2idx'])) for s in slots]\n","      \n","      yield (words, (intent, slots))"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"pKnp6C-kf2lr","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638564573,"user_tz":-480,"elapsed":2395,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["def dataset(is_training, params):\n","  _shapes = ([None], ((), [None]))\n","  _types = (tf.int32, (tf.int32, tf.int32))\n","  _pads = (0, (-1, 0))\n","  \n","  if is_training:\n","    ds = tf.data.Dataset.from_generator(\n","      lambda: data_generator(params['train_path'], params),\n","      output_shapes = _shapes,\n","      output_types = _types,)\n","    ds = ds.shuffle(params['num_samples'])\n","    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)\n","    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)\n","  else:\n","    ds = tf.data.Dataset.from_generator(\n","      lambda: data_generator(params['test_path'], params),\n","      output_shapes = _shapes,\n","      output_types = _types,)\n","    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)\n","    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)\n","  \n","  return ds"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"uCS2VTlse11P","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638564574,"user_tz":-480,"elapsed":1018,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["def get_timing_signal_1d(length,\n","                         channels,\n","                         min_timescale=1.0,\n","                         max_timescale=1.0e4,\n","                         start_index=0):\n","  to_float = lambda x: tf.cast(x, tf.float32)\n","  position = to_float(tf.range(length) + start_index)\n","  num_timescales = channels // 2\n","  log_timescale_increment = (\n","      tf.math.log(float(max_timescale) / float(min_timescale)) /\n","      tf.maximum(to_float(num_timescales) - 1, 1))\n","  inv_timescales = min_timescale * tf.exp(\n","      to_float(tf.range(num_timescales)) * -log_timescale_increment)\n","  scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)\n","  signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)\n","  signal = tf.pad(signal, [[0, 0], [0, tf.compat.v1.mod(channels, 2)]])\n","  signal = tf.reshape(signal, [1, length, channels])\n","  return signal"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"0jbex2Vee21w","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638567292,"user_tz":-480,"elapsed":1600,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["class LayerNorm(tf.keras.layers.Layer):\n","  def __init__(self, params):\n","    super().__init__()\n","    self._epsilon = params['epsilon']\n","    self._hidden_units = params['global_units']\n","  \n","  def build(self, input_shape):\n","    self.scale = self.add_weight(name='scale',\n","                                 shape=[self._hidden_units],\n","                                 initializer=tf.ones_initializer(),\n","                                 trainable=True)\n","    self.bias = self.add_weight(name='bias',\n","                                shape=[self._hidden_units],\n","                                initializer=tf.zeros_initializer(),\n","                                trainable=True)\n","    super().build(input_shape)\n","  \n","  def call(self, inputs):\n","    mean, variance = tf.nn.moments(inputs, [-1], keepdims=True)\n","    norm_x = (inputs - mean) * tf.math.rsqrt(variance + self._epsilon)\n","    return norm_x * self.scale + self.bias\n","  \n","  def compute_output_shape(self, input_shape):\n","    return input_shape\n","\n","\n","class EncoderBlock(tf.keras.Model):\n","  def __init__(self, SubModel, params, name):\n","    super().__init__(name = name)\n","    self.layer_norm = LayerNorm(params)\n","    self.sub_model = SubModel(params)\n","    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])\n","  \n","  def call(self, inputs, training):\n","    inputs, masks = inputs\n","    x = self.layer_norm(inputs)\n","    x = self.sub_model((x, masks), training=training)\n","    x = self.dropout(x, training=training)\n","    x += inputs\n","    return x\n","\n","\n","class MultiheadSelfAttention(tf.keras.Model):\n","  def __init__(self, params):\n","    super().__init__()\n","    self.qkv_linear = tf.keras.layers.Dense(3*params['hidden_units'], name='qkv_linear')\n","    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])\n","    self.out_linear = tf.keras.layers.Dense(params['global_units'], params['activation'], name='out_linear')\n","    self.num_heads = params['num_heads']\n","    self.is_bidirectional = params['is_bidirectional']\n","    self.time_weight = self.add_weight(shape=(params['num_heads'], params['max_len'], params['max_len']), initializer='ones', trainable=True, name='time_weight')\n","  \n","  def call(self, inputs, training):\n","    x, masks = inputs\n","    batch_sz = tf.shape(x)[0]\n","    timesteps = tf.shape(x)[1]\n","    \n","    q_k_v = self.qkv_linear(x)\n","    q, k, v = tf.split(q_k_v, 3, axis=-1)\n","    \n","    if self.num_heads > 1:\n","      q = tf.concat(tf.split(q, self.num_heads, axis=2), axis=0)                        \n","      k = tf.concat(tf.split(k, self.num_heads, axis=2), axis=0)                        \n","      v = tf.concat(tf.split(v, self.num_heads, axis=2), axis=0)\n","    \n","    align = tf.matmul(q, k, transpose_b=True)\n","    align *= tf.math.rsqrt(tf.cast(k.shape[-1], tf.float32))\n","    \n","    if (masks is not None) or (not self.is_bidirectional):\n","      paddings = tf.fill(tf.shape(align), float('-inf'))\n","    \n","    if masks is not None:\n","      c_masks = tf.tile(masks, [params['num_heads'], 1])\n","      c_masks = tf.tile(tf.expand_dims(c_masks, 1), [1, timesteps, 1])\n","      align = tf.where(tf.equal(c_masks, 0), paddings, align)\n","    \n","    if not self.is_bidirectional:\n","      lower_tri = tf.ones((timesteps, timesteps))                                       \n","      lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()      \n","      t_masks = tf.tile(tf.expand_dims(lower_tri, 0), [tf.shape(align)[0], 1, 1])     \n","      align = tf.where(tf.equal(t_masks, 0), paddings, align)\n","    \n","    align = tf.nn.softmax(align)\n","    align *= tf.tile(self.time_weight[:, :timesteps, :timesteps], [batch_sz, 1, 1])\n","    align = self.dropout(align, training=training)\n","    \n","    if masks is not None:\n","      q_masks = tf.tile(masks, [params['num_heads'], 1])\n","      q_masks = tf.tile(tf.expand_dims(q_masks, 2), [1, 1, timesteps])\n","      align *= tf.cast(q_masks, tf.float32)\n","    \n","    x = tf.matmul(align, v)\n","    if self.num_heads > 1:\n","      x = tf.concat(tf.split(x, self.num_heads, axis=0), axis=2)\n","    x = self.out_linear(x)\n","    \n","    return x\n","  \n","\n","class PointwiseFFN(tf.keras.Model):\n","  def __init__(self, params):\n","    super().__init__()\n","    self.dense_1 = tf.keras.layers.Dense(params['multiplier']*params['global_units'], params['activation'], name='fc')\n","    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])\n","    self.dense_2 = tf.keras.layers.Dense(params['global_units'], name='linear')\n","  \n","  def call(self, inputs, training):\n","    x, masks = inputs\n","    return self.dense_2(self.dropout(self.dense_1(x), training=training))"],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"K1qU3IZFfD3n","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638570933,"user_tz":-480,"elapsed":2893,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["class Model(tf.keras.Model):\n","  def __init__(self, params: dict):\n","    super().__init__()\n","    self.embedding = tf.Variable(np.load(params['vocab_path']),\n","                                 dtype=tf.float32,\n","                                 name='pretrained_embedding')\n","    self.input_dropout = tf.keras.layers.Dropout(params['dropout_rate'])\n","    \n","    self.blocks = []\n","    for i in range(params['num_layers']):\n","      self.blocks.append(EncoderBlock(\n","          MultiheadSelfAttention, params, name='layer{}.1'.format(i+1)))\n","      self.blocks.append(EncoderBlock(\n","          PointwiseFFN, params, name='layer{}.2'.format(i+1)))\n","    \n","    self.intent_dropout = tf.keras.layers.Dropout(params['dropout_rate'])\n","    self.fc_intent = tf.keras.layers.Dense(params['global_units'], params['activation'], name='fc_intent')\n","    self.out_linear_intent = tf.keras.layers.Dense(params['intent_size'], name='output_intent')\n","    self.out_linear_slot = tf.keras.layers.Dense(params['slot_size'], name='output_slot')\n","    \n","  \n","  def call(self, inputs, training):\n","    if inputs.dtype != tf.int32:\n","      inputs = tf.cast(inputs, tf.int32)\n","    masks = tf.sign(inputs)\n","    \n","    x = tf.nn.embedding_lookup(self.embedding, inputs)\n","    if params['is_embedding_scaled']:\n","      x *= tf.sqrt(tf.cast(params['global_units'], tf.float32))\n","    x += get_timing_signal_1d(tf.shape(x)[1], params['global_units'])\n","    x = self.input_dropout(x, training=training)\n","    \n","    for block in self.blocks:\n","      x = block((x, masks), training=training)\n","    \n","    x_intent = tf.reduce_max(x, 1)\n","    x_intent = self.intent_dropout(x_intent, training=training)\n","    x_intent = self.out_linear_intent(self.fc_intent(x_intent))\n","    x_slot = self.out_linear_slot(x)\n","    return (x_intent, x_slot)"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"id":"GazxdEvTIblA","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638571689,"user_tz":-480,"elapsed":1793,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["params = {\n","  'train_path': '../data/atis.train.w-intent.iob',\n","  'test_path': '../data/atis.test.w-intent.iob',\n","  'word_path': '../vocab/word.txt',\n","  'vocab_path': '../vocab/word.npy',\n","  'intent_path': '../vocab/intent.txt',\n","  'slot_path': '../vocab/slot.txt',\n","  'batch_size': 16,\n","  'max_len': 50,\n","  'num_samples': 4978,\n","  'num_layers': 2,\n","  'global_units': 300,\n","  'hidden_units': 512,\n","  'activation': tf.nn.elu,\n","  'num_heads': 8,\n","  'multiplier': 2,\n","  'dropout_rate': .1,\n","  'epsilon': 1e-6,\n","  'is_bidirectional': True,\n","  'is_embedding_scaled': False,\n","  'clip_norm': 5.,\n","}"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"p3hzFqh5RLu1","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1599638573175,"user_tz":-480,"elapsed":1953,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}}},"source":["params['word2idx'] = get_vocab(params['word_path'])\n","params['intent2idx'] = get_vocab(params['intent_path'])\n","params['slot2idx'] = get_vocab(params['slot_path'])\n","\n","params['word_size'] = len(params['word2idx']) + 1\n","params['intent_size'] = len(params['intent2idx']) + 1\n","params['slot_size'] = len(params['slot2idx']) + 1"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"eGfxJdmzZPjs","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1599640683710,"user_tz":-480,"elapsed":2111614,"user":{"displayName":"如子","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64","userId":"01997730851420384589"}},"outputId":"4fe3a591-1588-4bc1-e933-1c740ad095ad"},"source":["model = Model(params)\n","model.build(input_shape=(None, None))\n","pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])\n","\n","decay_lr = Triangular2CyclicalLearningRate(\n","  initial_learning_rate = 1e-4,\n","  maximal_learning_rate = 8e-4,\n","  step_size = 8 * params['num_samples'] // params['batch_size'],\n",")\n","optim = tf.optimizers.Adam(1e-4)\n","global_step = 0\n","\n","slot_best_f1 = .0\n","intent_acc_with_that = .0\n","\n","t0 = time.time()\n","logger = logging.getLogger('tensorflow')\n","logger.setLevel(logging.INFO)\n","\n","for n_epoch in range(1, 64+1):\n","  # TRAINING\n","  for (words, (intent, slots)) in dataset(is_training=True, params=params):\n","    with tf.GradientTape() as tape:\n","      y_intent, y_slots = model(words, training=True)\n","      loss_intent = tf.compat.v1.losses.softmax_cross_entropy(\n","        onehot_labels = tf.one_hot(intent, len(params['intent2idx'])+1),\n","        logits = y_intent,\n","        label_smoothing = .2)\n","      # weight of 'O' is set to be small\n","      weights = tf.cast(tf.sign(slots), tf.float32)\n","      padding = tf.constant(1e-2, tf.float32, weights.shape)\n","      weights = tf.where(tf.equal(weights, 0.), padding, weights)\n","\n","      loss_slots = tf.compat.v1.losses.softmax_cross_entropy(\n","        onehot_labels = tf.one_hot(slots, len(params['slot2idx'])+1),\n","        logits = y_slots,\n","        weights = tf.cast(weights, tf.float32),\n","        label_smoothing = .2)\n","      # joint loss\n","      loss = loss_intent + loss_slots\n","    \n","    optim.lr.assign(decay_lr(global_step))\n","    grads = tape.gradient(loss, model.trainable_variables)\n","    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])\n","    optim.apply_gradients(zip(grads, model.trainable_variables))\n","\n","    if global_step % 50 == 0:\n","      logger.info(\"Step {} | Loss: {:.4f} | Loss_intent: {:.4f} | Loss_slots: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}\".format(\n","          global_step, loss.numpy().item(), loss_intent.numpy().item(), loss_slots.numpy().item(), time.time()-t0, optim.lr.numpy().item()))\n","      t0 = time.time()\n","    global_step += 1\n","    \n","  # EVALUATION\n","  intent_true = []\n","  intent_pred = []\n","  slot_true = []\n","  slot_pred = []\n","\n","  for (words, (intent, slots)) in dataset(is_training=False, params=params):\n","    y_intent, y_slots = model(words, training=False)\n","    y_intent = tf.argmax(y_intent, -1)\n","    y_slots = tf.argmax(y_slots, -1)\n","    \n","    intent_true += intent.numpy().flatten().tolist()\n","    intent_pred += y_intent.numpy().flatten().tolist()\n","    slot_true += slots.numpy().flatten().tolist()\n","    slot_pred += y_slots.numpy().flatten().tolist()\n","    \n","  f1_slots = f1_score(y_true = slot_true,\n","                      y_pred = slot_pred,\n","                      labels = list(params['slot2idx'].values()),\n","                      sample_weight = np.sign(slot_true),\n","                      average='micro',)\n","  \n","  acc_intent = accuracy_score(intent_true, intent_pred)\n","\n","  logger.info(\"Slot F1: {:.3f}, Intent Acc: {:.3f}\".format(f1_slots, acc_intent))\n","\n","  if n_epoch != 1 and n_epoch % 8 == 0:\n","    logger.info('\\n'+classification_report(y_true = intent_true,\n","                                          y_pred = intent_pred,\n","                                          labels = list(params['intent2idx'].values()),\n","                                          target_names = list(params['intent2idx'].keys()),\n","                                          digits=3))\n","    logger.info('\\n'+classification_report(y_true = slot_true,\n","                                          y_pred = slot_pred,\n","                                          labels = list(params['slot2idx'].values()),\n","                                          target_names = list(params['slot2idx'].keys()),\n","                                          sample_weight = np.sign(slot_true),\n","                                          digits=3))\n","  \n","  if f1_slots > slot_best_f1:\n","    slot_best_f1 = f1_slots\n","    intent_acc_with_that = acc_intent\n","    # you can save model here\n","  logger.info(\"Best Slot F1: {:.3f}, Intent Acc: {:.3f}\".format(slot_best_f1, intent_acc_with_that))"],"execution_count":12,"outputs":[{"output_type":"stream","text":["[('layer1.1/layer_norm/scale:0', TensorShape([300])),\n"," ('layer1.1/layer_norm/bias:0', TensorShape([300])),\n"," ('layer1.1/multihead_self_attention/qkv_linear/kernel:0',\n","  TensorShape([300, 1536])),\n"," ('layer1.1/multihead_self_attention/qkv_linear/bias:0', TensorShape([1536])),\n"," ('layer1.1/multihead_self_attention/out_linear/kernel:0',\n","  TensorShape([512, 300])),\n"," ('layer1.1/multihead_self_attention/out_linear/bias:0', TensorShape([300])),\n"," ('time_weight:0', TensorShape([8, 50, 50])),\n"," ('layer1.2/layer_norm_1/scale:0', TensorShape([300])),\n"," ('layer1.2/layer_norm_1/bias:0', TensorShape([300])),\n"," ('layer1.2/pointwise_ffn/fc/kernel:0', TensorShape([300, 600])),\n"," ('layer1.2/pointwise_ffn/fc/bias:0', TensorShape([600])),\n"," ('layer1.2/pointwise_ffn/linear/kernel:0', TensorShape([600, 300])),\n"," ('layer1.2/pointwise_ffn/linear/bias:0', TensorShape([300])),\n"," ('layer2.1/layer_norm_2/scale:0', TensorShape([300])),\n"," ('layer2.1/layer_norm_2/bias:0', TensorShape([300])),\n"," ('layer2.1/multihead_self_attention_1/qkv_linear/kernel:0',\n","  TensorShape([300, 1536])),\n"," ('layer2.1/multihead_self_attention_1/qkv_linear/bias:0', TensorShape([1536])),\n"," ('layer2.1/multihead_self_attention_1/out_linear/kernel:0',\n","  TensorShape([512, 300])),\n"," ('layer2.1/multihead_self_attention_1/out_linear/bias:0', TensorShape([300])),\n"," ('time_weight:0', TensorShape([8, 50, 50])),\n"," ('layer2.2/layer_norm_3/scale:0', TensorShape([300])),\n"," ('layer2.2/layer_norm_3/bias:0', TensorShape([300])),\n"," ('layer2.2/pointwise_ffn_1/fc/kernel:0', TensorShape([300, 600])),\n"," ('layer2.2/pointwise_ffn_1/fc/bias:0', TensorShape([600])),\n"," ('layer2.2/pointwise_ffn_1/linear/kernel:0', TensorShape([600, 300])),\n"," ('layer2.2/pointwise_ffn_1/linear/bias:0', TensorShape([300])),\n"," ('fc_intent/kernel:0', TensorShape([300, 300])),\n"," ('fc_intent/bias:0', TensorShape([300])),\n"," ('output_intent/kernel:0', TensorShape([300, 23])),\n"," ('output_intent/bias:0', TensorShape([23])),\n"," ('output_slot/kernel:0', TensorShape([300, 122])),\n"," ('output_slot/bias:0', TensorShape([122])),\n"," ('pretrained_embedding:0', TensorShape([750, 300]))]\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 0 | Loss: 5.8003 | Loss_intent: 4.3044 | Loss_slots: 1.4959 | Spent: 1.7 secs | LR: 0.000100\n","INFO:tensorflow:Step 50 | Loss: 2.9285 | Loss_intent: 2.2014 | Loss_slots: 0.7271 | Spent: 5.1 secs | LR: 0.000114\n","INFO:tensorflow:Step 100 | Loss: 2.5801 | Loss_intent: 1.7625 | Loss_slots: 0.8175 | Spent: 5.1 secs | LR: 0.000128\n","INFO:tensorflow:Step 150 | Loss: 1.8761 | Loss_intent: 1.3286 | Loss_slots: 0.5475 | Spent: 4.7 secs | LR: 0.000142\n","INFO:tensorflow:Step 200 | Loss: 2.3339 | Loss_intent: 1.5947 | Loss_slots: 0.7393 | Spent: 4.7 secs | LR: 0.000156\n","INFO:tensorflow:Step 250 | Loss: 2.0871 | Loss_intent: 1.3551 | Loss_slots: 0.7320 | Spent: 4.7 secs | LR: 0.000170\n","INFO:tensorflow:Step 300 | Loss: 2.5929 | Loss_intent: 2.0091 | Loss_slots: 0.5838 | Spent: 4.6 secs | LR: 0.000184\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.641, Intent Acc: 0.882\n","INFO:tensorflow:Best Slot F1: 0.641, Intent Acc: 0.882\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 350 | Loss: 2.0842 | Loss_intent: 1.4303 | Loss_slots: 0.6539 | Spent: 7.8 secs | LR: 0.000198\n","INFO:tensorflow:Step 400 | Loss: 1.8495 | Loss_intent: 1.2705 | Loss_slots: 0.5791 | Spent: 4.9 secs | LR: 0.000212\n","INFO:tensorflow:Step 450 | Loss: 1.5045 | Loss_intent: 1.1612 | Loss_slots: 0.3433 | Spent: 5.0 secs | LR: 0.000227\n","INFO:tensorflow:Step 500 | Loss: 1.8533 | Loss_intent: 1.1781 | Loss_slots: 0.6752 | Spent: 4.7 secs | LR: 0.000241\n","INFO:tensorflow:Step 550 | Loss: 1.8252 | Loss_intent: 1.3244 | Loss_slots: 0.5008 | Spent: 4.8 secs | LR: 0.000255\n","INFO:tensorflow:Step 600 | Loss: 1.7764 | Loss_intent: 1.1997 | Loss_slots: 0.5767 | Spent: 4.8 secs | LR: 0.000269\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.788, Intent Acc: 0.920\n","INFO:tensorflow:Best Slot F1: 0.788, Intent Acc: 0.920\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 650 | Loss: 1.9772 | Loss_intent: 1.4346 | Loss_slots: 0.5426 | Spent: 8.0 secs | LR: 0.000283\n","INFO:tensorflow:Step 700 | Loss: 1.8453 | Loss_intent: 1.4468 | Loss_slots: 0.3985 | Spent: 4.7 secs | LR: 0.000297\n","INFO:tensorflow:Step 750 | Loss: 1.5370 | Loss_intent: 1.1926 | Loss_slots: 0.3444 | Spent: 4.7 secs | LR: 0.000311\n","INFO:tensorflow:Step 800 | Loss: 1.7838 | Loss_intent: 1.2625 | Loss_slots: 0.5213 | Spent: 4.7 secs | LR: 0.000325\n","INFO:tensorflow:Step 850 | Loss: 1.8138 | Loss_intent: 1.2937 | Loss_slots: 0.5201 | Spent: 4.7 secs | LR: 0.000339\n","INFO:tensorflow:Step 900 | Loss: 1.5828 | Loss_intent: 1.1696 | Loss_slots: 0.4132 | Spent: 4.7 secs | LR: 0.000353\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.823, Intent Acc: 0.933\n","INFO:tensorflow:Best Slot F1: 0.823, Intent Acc: 0.933\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 950 | Loss: 1.6102 | Loss_intent: 1.1926 | Loss_slots: 0.4176 | Spent: 8.0 secs | LR: 0.000367\n","INFO:tensorflow:Step 1000 | Loss: 1.8253 | Loss_intent: 1.2865 | Loss_slots: 0.5388 | Spent: 4.8 secs | LR: 0.000381\n","INFO:tensorflow:Step 1050 | Loss: 1.7194 | Loss_intent: 1.1414 | Loss_slots: 0.5780 | Spent: 4.8 secs | LR: 0.000395\n","INFO:tensorflow:Step 1100 | Loss: 1.5349 | Loss_intent: 1.1191 | Loss_slots: 0.4158 | Spent: 4.7 secs | LR: 0.000409\n","INFO:tensorflow:Step 1150 | Loss: 1.5920 | Loss_intent: 1.2243 | Loss_slots: 0.3677 | Spent: 4.8 secs | LR: 0.000423\n","INFO:tensorflow:Step 1200 | Loss: 1.7069 | Loss_intent: 1.2514 | Loss_slots: 0.4555 | Spent: 4.7 secs | LR: 0.000437\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.873, Intent Acc: 0.965\n","INFO:tensorflow:Best Slot F1: 0.873, Intent Acc: 0.965\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 1250 | Loss: 1.6056 | Loss_intent: 1.1795 | Loss_slots: 0.4261 | Spent: 8.1 secs | LR: 0.000452\n","INFO:tensorflow:Step 1300 | Loss: 1.5142 | Loss_intent: 1.1934 | Loss_slots: 0.3208 | Spent: 4.8 secs | LR: 0.000466\n","INFO:tensorflow:Step 1350 | Loss: 1.7622 | Loss_intent: 1.3784 | Loss_slots: 0.3838 | Spent: 4.9 secs | LR: 0.000480\n","INFO:tensorflow:Step 1400 | Loss: 1.5714 | Loss_intent: 1.1541 | Loss_slots: 0.4173 | Spent: 4.8 secs | LR: 0.000494\n","INFO:tensorflow:Step 1450 | Loss: 1.5138 | Loss_intent: 1.1779 | Loss_slots: 0.3360 | Spent: 4.7 secs | LR: 0.000508\n","INFO:tensorflow:Step 1500 | Loss: 1.6333 | Loss_intent: 1.2284 | Loss_slots: 0.4050 | Spent: 4.8 secs | LR: 0.000522\n","INFO:tensorflow:Step 1550 | Loss: 1.6615 | Loss_intent: 1.1909 | Loss_slots: 0.4706 | Spent: 4.7 secs | LR: 0.000536\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.899, Intent Acc: 0.952\n","INFO:tensorflow:Best Slot F1: 0.899, Intent Acc: 0.952\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 1600 | Loss: 1.5846 | Loss_intent: 1.1094 | Loss_slots: 0.4753 | Spent: 8.0 secs | LR: 0.000550\n","INFO:tensorflow:Step 1650 | Loss: 1.5995 | Loss_intent: 1.1113 | Loss_slots: 0.4882 | Spent: 4.6 secs | LR: 0.000564\n","INFO:tensorflow:Step 1700 | Loss: 1.9264 | Loss_intent: 1.4512 | Loss_slots: 0.4752 | Spent: 4.6 secs | LR: 0.000578\n","INFO:tensorflow:Step 1750 | Loss: 1.4262 | Loss_intent: 1.1244 | Loss_slots: 0.3018 | Spent: 4.8 secs | LR: 0.000592\n","INFO:tensorflow:Step 1800 | Loss: 1.4406 | Loss_intent: 1.1166 | Loss_slots: 0.3240 | Spent: 4.8 secs | LR: 0.000606\n","INFO:tensorflow:Step 1850 | Loss: 1.6739 | Loss_intent: 1.1526 | Loss_slots: 0.5214 | Spent: 4.7 secs | LR: 0.000620\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.906, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.906, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 1900 | Loss: 1.6583 | Loss_intent: 1.1759 | Loss_slots: 0.4825 | Spent: 8.1 secs | LR: 0.000634\n","INFO:tensorflow:Step 1950 | Loss: 1.7890 | Loss_intent: 1.3581 | Loss_slots: 0.4308 | Spent: 4.7 secs | LR: 0.000648\n","INFO:tensorflow:Step 2000 | Loss: 1.5296 | Loss_intent: 1.1194 | Loss_slots: 0.4101 | Spent: 4.7 secs | LR: 0.000662\n","INFO:tensorflow:Step 2050 | Loss: 1.5767 | Loss_intent: 1.2121 | Loss_slots: 0.3645 | Spent: 4.7 secs | LR: 0.000677\n","INFO:tensorflow:Step 2100 | Loss: 1.4816 | Loss_intent: 1.1393 | Loss_slots: 0.3423 | Spent: 4.8 secs | LR: 0.000691\n","INFO:tensorflow:Step 2150 | Loss: 1.4625 | Loss_intent: 1.1343 | Loss_slots: 0.3282 | Spent: 4.6 secs | LR: 0.000705\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.918, Intent Acc: 0.959\n","INFO:tensorflow:Best Slot F1: 0.918, Intent Acc: 0.959\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 2200 | Loss: 1.5621 | Loss_intent: 1.1659 | Loss_slots: 0.3962 | Spent: 8.0 secs | LR: 0.000719\n","INFO:tensorflow:Step 2250 | Loss: 1.5604 | Loss_intent: 1.1142 | Loss_slots: 0.4461 | Spent: 4.6 secs | LR: 0.000733\n","INFO:tensorflow:Step 2300 | Loss: 1.5182 | Loss_intent: 1.1566 | Loss_slots: 0.3616 | Spent: 4.8 secs | LR: 0.000747\n","INFO:tensorflow:Step 2350 | Loss: 1.8201 | Loss_intent: 1.4034 | Loss_slots: 0.4168 | Spent: 4.8 secs | LR: 0.000761\n","INFO:tensorflow:Step 2400 | Loss: 1.5436 | Loss_intent: 1.1070 | Loss_slots: 0.4366 | Spent: 4.7 secs | LR: 0.000775\n","INFO:tensorflow:Step 2450 | Loss: 1.5229 | Loss_intent: 1.1360 | Loss_slots: 0.3869 | Spent: 4.7 secs | LR: 0.000789\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.916, Intent Acc: 0.961\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.970     0.987     0.979       632\n","                            atis_airfare      0.980     1.000     0.990        48\n","                     atis_ground_service      0.973     1.000     0.986        36\n","                            atis_airline      0.974     0.974     0.974        38\n","                       atis_abbreviation      0.939     0.939     0.939        33\n","                           atis_aircraft      1.000     0.889     0.941         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.375     1.000     0.545         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      0.947     1.000     0.973        18\n","                           atis_distance      1.000     0.900     0.947        10\n","                               atis_city      0.667     0.667     0.667         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     0.952     0.976        21\n","                          atis_flight_no      1.000     0.625     0.769         8\n","                               atis_meal      1.000     0.333     0.500         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.961     0.966     0.964       888\n","                               macro avg      0.666     0.622     0.625       888\n","                            weighted avg      0.966     0.966     0.963       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.969     0.996     0.982     716.0\n","         B-fromloc.city_name      0.978     0.994     0.986     704.0\n","           I-toloc.city_name      0.932     0.985     0.958     265.0\n","      B-depart_date.day_name      0.981     0.976     0.979     212.0\n","              B-airline_name      0.933     0.970     0.951     101.0\n","         I-fromloc.city_name      0.965     0.944     0.954     177.0\n"," B-depart_time.period_of_day      0.920     0.885     0.902     130.0\n","              I-airline_name      0.984     0.923     0.952      65.0\n","    B-depart_date.day_number      0.962     0.927     0.944      55.0\n","    B-depart_date.month_name      0.964     0.946     0.955      56.0\n","          B-depart_time.time      0.800     0.982     0.882      57.0\n","                B-round_trip      0.986     0.973     0.979      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      0.959     1.000     0.979      71.0\n","                B-flight_mod      1.000     0.875     0.933      24.0\n"," B-depart_time.time_relative      0.984     0.938     0.961      65.0\n","          I-depart_time.time      0.836     0.981     0.903      52.0\n","         B-stoploc.city_name      0.952     1.000     0.976      20.0\n","                 B-city_name      0.732     0.526     0.612      57.0\n","                B-class_type      0.774     1.000     0.873      24.0\n","          B-arrive_time.time      0.917     0.971     0.943      34.0\n"," B-arrive_time.time_relative      0.816     1.000     0.899      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      0.969     0.886     0.925      35.0\n","              B-airline_code      1.000     0.500     0.667      34.0\n","    I-depart_date.day_number      1.000     1.000     1.000      15.0\n","      I-fromloc.airport_name      0.311     0.933     0.467      15.0\n","      B-fromloc.airport_name      0.259     0.583     0.359      12.0\n","      B-arrive_date.day_name      0.600     0.818     0.692      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     0.889     0.941       9.0\n","             B-flight_number      0.500     0.909     0.645      11.0\n"," B-depart_date.date_relative      0.773     1.000     0.872      17.0\n","          B-toloc.state_name      0.929     0.929     0.929      28.0\n","           B-fare_basis_code      0.917     0.647     0.759      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.600     0.500     0.545       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.333     0.138     0.195      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      1.000     0.300     0.462      30.0\n","        I-toloc.airport_name      0.000     0.000     0.000       3.0\n","            B-transport_type      0.909     1.000     0.952      10.0\n","    B-arrive_date.month_name      0.571     0.667     0.615       6.0\n","    B-arrive_date.day_number      0.333     0.167     0.222       6.0\n","         I-stoploc.city_name      0.571     0.400     0.471      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.500     0.200     0.286       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     0.333     0.500       3.0\n","        B-fromloc.state_name      0.889     0.941     0.914      17.0\n","              B-airport_name      0.667     0.286     0.400      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      0.829     0.879     0.853      33.0\n","                       B-mod      0.000     0.000     0.000       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      0.000     0.000     0.000       3.0\n","      B-depart_time.end_time      1.000     0.333     0.500       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.000     0.000     0.000       1.0\n","          B-restriction_code      1.000     0.250     0.400       4.0\n","    B-arrive_time.start_time      0.533     1.000     0.696       8.0\n","        B-toloc.airport_code      0.667     0.500     0.571       4.0\n","      B-arrive_time.end_time      1.000     0.250     0.400       8.0\n","      I-arrive_time.end_time      0.615     1.000     0.762       8.0\n","      I-depart_time.end_time      0.000     0.000     0.000       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      1.000     0.600     0.750       5.0\n","          I-restriction_code      0.188     1.000     0.316       3.0\n","    I-depart_time.start_time      0.000     0.000     0.000       1.0\n","          I-toloc.state_name      0.500     1.000     0.667       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.000     0.000     0.000       2.0\n","                I-flight_mod      0.000     0.000     0.000       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      0.000     0.000     0.000       3.0\n","        I-fromloc.state_name      0.000     0.000     0.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      0.000     0.000     0.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      0.000     0.000     0.000       1.0\n"," I-depart_time.period_of_day      0.000     0.000     0.000       1.0\n","                  B-day_name      0.000     0.000     0.000       2.0\n","             B-period_of_day      0.000     0.000     0.000       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      0.000     0.000     0.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.000     0.000     0.000       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.915     0.917     0.916    3657.0\n","                   macro avg      0.502     0.486     0.474    3657.0\n","                weighted avg      0.916     0.917     0.909    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.918, Intent Acc: 0.959\n","Reading ../data/atis.train.w-intent.iob\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n","/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n","  _warn_prf(average, modifier, msg_start, len(result))\n"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:Step 2500 | Loss: 1.5931 | Loss_intent: 1.1421 | Loss_slots: 0.4510 | Spent: 8.1 secs | LR: 0.000797\n","INFO:tensorflow:Step 2550 | Loss: 1.7627 | Loss_intent: 1.3024 | Loss_slots: 0.4603 | Spent: 4.7 secs | LR: 0.000783\n","INFO:tensorflow:Step 2600 | Loss: 1.5325 | Loss_intent: 1.1022 | Loss_slots: 0.4303 | Spent: 4.8 secs | LR: 0.000769\n","INFO:tensorflow:Step 2650 | Loss: 1.3910 | Loss_intent: 1.1074 | Loss_slots: 0.2837 | Spent: 4.8 secs | LR: 0.000755\n","INFO:tensorflow:Step 2700 | Loss: 1.4958 | Loss_intent: 1.1179 | Loss_slots: 0.3780 | Spent: 4.7 secs | LR: 0.000741\n","INFO:tensorflow:Step 2750 | Loss: 1.4448 | Loss_intent: 1.0996 | Loss_slots: 0.3452 | Spent: 4.8 secs | LR: 0.000727\n","INFO:tensorflow:Step 2800 | Loss: 1.4941 | Loss_intent: 1.1431 | Loss_slots: 0.3510 | Spent: 4.7 secs | LR: 0.000713\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.921, Intent Acc: 0.959\n","INFO:tensorflow:Best Slot F1: 0.921, Intent Acc: 0.959\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 2850 | Loss: 1.5232 | Loss_intent: 1.0929 | Loss_slots: 0.4303 | Spent: 7.9 secs | LR: 0.000698\n","INFO:tensorflow:Step 2900 | Loss: 1.4229 | Loss_intent: 1.1089 | Loss_slots: 0.3141 | Spent: 4.8 secs | LR: 0.000684\n","INFO:tensorflow:Step 2950 | Loss: 1.4923 | Loss_intent: 1.1397 | Loss_slots: 0.3526 | Spent: 4.7 secs | LR: 0.000670\n","INFO:tensorflow:Step 3000 | Loss: 1.5198 | Loss_intent: 1.0961 | Loss_slots: 0.4237 | Spent: 5.1 secs | LR: 0.000656\n","INFO:tensorflow:Step 3050 | Loss: 1.4232 | Loss_intent: 1.1349 | Loss_slots: 0.2883 | Spent: 5.1 secs | LR: 0.000642\n","INFO:tensorflow:Step 3100 | Loss: 1.6216 | Loss_intent: 1.2299 | Loss_slots: 0.3916 | Spent: 4.9 secs | LR: 0.000628\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.931, Intent Acc: 0.959\n","INFO:tensorflow:Best Slot F1: 0.931, Intent Acc: 0.959\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 3150 | Loss: 1.6322 | Loss_intent: 1.2610 | Loss_slots: 0.3712 | Spent: 8.0 secs | LR: 0.000614\n","INFO:tensorflow:Step 3200 | Loss: 1.4461 | Loss_intent: 1.0988 | Loss_slots: 0.3473 | Spent: 4.7 secs | LR: 0.000600\n","INFO:tensorflow:Step 3250 | Loss: 1.5536 | Loss_intent: 1.0981 | Loss_slots: 0.4555 | Spent: 4.7 secs | LR: 0.000586\n","INFO:tensorflow:Step 3300 | Loss: 1.4453 | Loss_intent: 1.0849 | Loss_slots: 0.3604 | Spent: 4.8 secs | LR: 0.000572\n","INFO:tensorflow:Step 3350 | Loss: 1.6895 | Loss_intent: 1.2901 | Loss_slots: 0.3995 | Spent: 4.9 secs | LR: 0.000558\n","INFO:tensorflow:Step 3400 | Loss: 1.4462 | Loss_intent: 1.1180 | Loss_slots: 0.3282 | Spent: 4.7 secs | LR: 0.000544\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.937, Intent Acc: 0.964\n","INFO:tensorflow:Best Slot F1: 0.937, Intent Acc: 0.964\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 3450 | Loss: 1.4573 | Loss_intent: 1.0936 | Loss_slots: 0.3637 | Spent: 8.0 secs | LR: 0.000530\n","INFO:tensorflow:Step 3500 | Loss: 1.2953 | Loss_intent: 1.0920 | Loss_slots: 0.2033 | Spent: 4.6 secs | LR: 0.000516\n","INFO:tensorflow:Step 3550 | Loss: 1.4613 | Loss_intent: 1.0985 | Loss_slots: 0.3628 | Spent: 4.7 secs | LR: 0.000502\n","INFO:tensorflow:Step 3600 | Loss: 1.4778 | Loss_intent: 1.1022 | Loss_slots: 0.3756 | Spent: 4.7 secs | LR: 0.000488\n","INFO:tensorflow:Step 3650 | Loss: 1.3640 | Loss_intent: 1.0909 | Loss_slots: 0.2731 | Spent: 4.6 secs | LR: 0.000473\n","INFO:tensorflow:Step 3700 | Loss: 1.4402 | Loss_intent: 1.0889 | Loss_slots: 0.3513 | Spent: 4.7 secs | LR: 0.000459\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.940, Intent Acc: 0.963\n","INFO:tensorflow:Best Slot F1: 0.940, Intent Acc: 0.963\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 3750 | Loss: 1.3690 | Loss_intent: 1.0859 | Loss_slots: 0.2831 | Spent: 7.9 secs | LR: 0.000445\n","INFO:tensorflow:Step 3800 | Loss: 1.5451 | Loss_intent: 1.0914 | Loss_slots: 0.4538 | Spent: 4.8 secs | LR: 0.000431\n","INFO:tensorflow:Step 3850 | Loss: 1.5912 | Loss_intent: 1.1219 | Loss_slots: 0.4693 | Spent: 4.7 secs | LR: 0.000417\n","INFO:tensorflow:Step 3900 | Loss: 1.5872 | Loss_intent: 1.0849 | Loss_slots: 0.5023 | Spent: 4.7 secs | LR: 0.000403\n","INFO:tensorflow:Step 3950 | Loss: 1.4973 | Loss_intent: 1.0946 | Loss_slots: 0.4028 | Spent: 4.8 secs | LR: 0.000389\n","INFO:tensorflow:Step 4000 | Loss: 1.3221 | Loss_intent: 1.1290 | Loss_slots: 0.1931 | Spent: 4.7 secs | LR: 0.000375\n","INFO:tensorflow:Step 4050 | Loss: 1.4664 | Loss_intent: 1.0939 | Loss_slots: 0.3725 | Spent: 4.7 secs | LR: 0.000361\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.943, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.943, Intent Acc: 0.970\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 4100 | Loss: 1.5133 | Loss_intent: 1.0897 | Loss_slots: 0.4237 | Spent: 8.0 secs | LR: 0.000347\n","INFO:tensorflow:Step 4150 | Loss: 1.3161 | Loss_intent: 1.0820 | Loss_slots: 0.2341 | Spent: 4.7 secs | LR: 0.000333\n","INFO:tensorflow:Step 4200 | Loss: 1.4336 | Loss_intent: 1.0943 | Loss_slots: 0.3392 | Spent: 4.7 secs | LR: 0.000319\n","INFO:tensorflow:Step 4250 | Loss: 1.5562 | Loss_intent: 1.0869 | Loss_slots: 0.4693 | Spent: 4.7 secs | LR: 0.000305\n","INFO:tensorflow:Step 4300 | Loss: 1.5333 | Loss_intent: 1.0846 | Loss_slots: 0.4487 | Spent: 4.8 secs | LR: 0.000291\n","INFO:tensorflow:Step 4350 | Loss: 1.4166 | Loss_intent: 1.0865 | Loss_slots: 0.3301 | Spent: 4.7 secs | LR: 0.000277\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.947, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.947, Intent Acc: 0.970\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 4400 | Loss: 1.4318 | Loss_intent: 1.0869 | Loss_slots: 0.3449 | Spent: 7.9 secs | LR: 0.000263\n","INFO:tensorflow:Step 4450 | Loss: 1.4770 | Loss_intent: 1.0939 | Loss_slots: 0.3831 | Spent: 4.8 secs | LR: 0.000248\n","INFO:tensorflow:Step 4500 | Loss: 1.3891 | Loss_intent: 1.0909 | Loss_slots: 0.2982 | Spent: 4.8 secs | LR: 0.000234\n","INFO:tensorflow:Step 4550 | Loss: 1.5182 | Loss_intent: 1.0857 | Loss_slots: 0.4326 | Spent: 4.8 secs | LR: 0.000220\n","INFO:tensorflow:Step 4600 | Loss: 1.4884 | Loss_intent: 1.0861 | Loss_slots: 0.4023 | Spent: 4.7 secs | LR: 0.000206\n","INFO:tensorflow:Step 4650 | Loss: 1.3055 | Loss_intent: 1.0935 | Loss_slots: 0.2120 | Spent: 4.8 secs | LR: 0.000192\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.951, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 4700 | Loss: 1.4601 | Loss_intent: 1.0822 | Loss_slots: 0.3779 | Spent: 7.9 secs | LR: 0.000178\n","INFO:tensorflow:Step 4750 | Loss: 1.4245 | Loss_intent: 1.0825 | Loss_slots: 0.3421 | Spent: 4.7 secs | LR: 0.000164\n","INFO:tensorflow:Step 4800 | Loss: 1.4356 | Loss_intent: 1.0862 | Loss_slots: 0.3494 | Spent: 4.7 secs | LR: 0.000150\n","INFO:tensorflow:Step 4850 | Loss: 1.4593 | Loss_intent: 1.0939 | Loss_slots: 0.3654 | Spent: 4.7 secs | LR: 0.000136\n","INFO:tensorflow:Step 4900 | Loss: 1.3966 | Loss_intent: 1.0864 | Loss_slots: 0.3102 | Spent: 4.7 secs | LR: 0.000122\n","INFO:tensorflow:Step 4950 | Loss: 1.4716 | Loss_intent: 1.0849 | Loss_slots: 0.3867 | Spent: 4.7 secs | LR: 0.000108\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.968\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.980     0.986     0.983       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      0.974     0.974     0.974        38\n","                       atis_abbreviation      0.941     0.970     0.955        33\n","                           atis_aircraft      0.889     0.889     0.889         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.429     1.000     0.600         3\n","                atis_flight#atis_airfare      0.800     0.333     0.471        12\n","                            atis_airport      0.900     1.000     0.947        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      0.667     0.667     0.667         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     1.000     1.000        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.833     0.909         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.968     0.973     0.970       888\n","                               macro avg      0.657     0.665     0.651       888\n","                            weighted avg      0.970     0.973     0.970       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.977     0.996     0.986     716.0\n","         B-fromloc.city_name      0.987     0.996     0.992     704.0\n","           I-toloc.city_name      0.957     1.000     0.978     265.0\n","      B-depart_date.day_name      0.991     0.986     0.988     212.0\n","              B-airline_name      0.980     0.990     0.985     101.0\n","         I-fromloc.city_name      0.994     0.977     0.986     177.0\n"," B-depart_time.period_of_day      0.983     0.885     0.931     130.0\n","              I-airline_name      0.985     1.000     0.992      65.0\n","    B-depart_date.day_number      0.946     0.964     0.955      55.0\n","    B-depart_date.month_name      0.964     0.964     0.964      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      0.958     0.958     0.958      24.0\n"," B-depart_time.time_relative      0.969     0.969     0.969      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      1.000     1.000     1.000      20.0\n","                 B-city_name      0.756     0.596     0.667      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      1.000     0.971     0.986      35.0\n","              B-airline_code      0.967     0.853     0.906      34.0\n","    I-depart_date.day_number      1.000     1.000     1.000      15.0\n","      I-fromloc.airport_name      0.429     1.000     0.600      15.0\n","      B-fromloc.airport_name      0.355     0.917     0.512      12.0\n","      B-arrive_date.day_name      0.733     1.000     0.846      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.524     1.000     0.688      11.0\n"," B-depart_date.date_relative      0.810     1.000     0.895      17.0\n","          B-toloc.state_name      0.964     0.964     0.964      28.0\n","           B-fare_basis_code      0.941     0.941     0.941      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.818     0.310     0.450      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      0.950     0.633     0.760      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      0.909     1.000     0.952      10.0\n","    B-arrive_date.month_name      0.667     0.667     0.667       6.0\n","    B-arrive_date.day_number      0.667     0.667     0.667       6.0\n","         I-stoploc.city_name      1.000     0.600     0.750      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      1.000     1.000     1.000      17.0\n","              B-airport_name      0.750     0.286     0.414      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      1.000     0.939     0.969      33.0\n","                       B-mod      1.000     0.500     0.667       2.0\n","              B-airport_code      1.000     0.444     0.615       9.0\n","    B-depart_time.start_time      0.667     0.667     0.667       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.000     0.000     0.000       1.0\n","          B-restriction_code      1.000     1.000     1.000       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      0.833     1.000     0.909       5.0\n","          I-restriction_code      0.750     1.000     0.857       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      0.500     1.000     0.667       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.000     0.000     0.000       2.0\n","                I-flight_mod      0.000     0.000     0.000       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      0.500     0.333     0.400       3.0\n","        I-fromloc.state_name      0.000     0.000     0.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      0.000     0.000     0.000       1.0\n"," I-depart_time.period_of_day      0.000     0.000     0.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.250     0.400       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      0.000     0.000     0.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.000     0.000     0.000       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.949     0.951     0.950    3657.0\n","                   macro avg      0.614     0.603     0.595    3657.0\n","                weighted avg      0.951     0.951     0.947    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.951, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 5000 | Loss: 1.4893 | Loss_intent: 1.0912 | Loss_slots: 0.3980 | Spent: 8.2 secs | LR: 0.000103\n","INFO:tensorflow:Step 5050 | Loss: 1.3600 | Loss_intent: 1.0842 | Loss_slots: 0.2758 | Spent: 4.7 secs | LR: 0.000110\n","INFO:tensorflow:Step 5100 | Loss: 1.5184 | Loss_intent: 1.0944 | Loss_slots: 0.4240 | Spent: 4.8 secs | LR: 0.000117\n","INFO:tensorflow:Step 5150 | Loss: 1.4232 | Loss_intent: 1.0854 | Loss_slots: 0.3377 | Spent: 4.7 secs | LR: 0.000124\n","INFO:tensorflow:Step 5200 | Loss: 1.4912 | Loss_intent: 1.0822 | Loss_slots: 0.4090 | Spent: 4.8 secs | LR: 0.000131\n","INFO:tensorflow:Step 5250 | Loss: 1.5175 | Loss_intent: 1.0871 | Loss_slots: 0.4304 | Spent: 4.7 secs | LR: 0.000138\n","INFO:tensorflow:Step 5300 | Loss: 1.4504 | Loss_intent: 1.0839 | Loss_slots: 0.3665 | Spent: 4.7 secs | LR: 0.000145\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.975\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 5350 | Loss: 1.4438 | Loss_intent: 1.0839 | Loss_slots: 0.3599 | Spent: 8.1 secs | LR: 0.000152\n","INFO:tensorflow:Step 5400 | Loss: 1.3817 | Loss_intent: 1.0890 | Loss_slots: 0.2926 | Spent: 4.8 secs | LR: 0.000159\n","INFO:tensorflow:Step 5450 | Loss: 1.4792 | Loss_intent: 1.0867 | Loss_slots: 0.3925 | Spent: 4.8 secs | LR: 0.000166\n","INFO:tensorflow:Step 5500 | Loss: 1.5278 | Loss_intent: 1.0843 | Loss_slots: 0.4435 | Spent: 4.7 secs | LR: 0.000173\n","INFO:tensorflow:Step 5550 | Loss: 1.5056 | Loss_intent: 1.0930 | Loss_slots: 0.4127 | Spent: 4.7 secs | LR: 0.000180\n","INFO:tensorflow:Step 5600 | Loss: 1.4216 | Loss_intent: 1.0811 | Loss_slots: 0.3405 | Spent: 4.8 secs | LR: 0.000187\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 5650 | Loss: 1.3876 | Loss_intent: 1.0936 | Loss_slots: 0.2940 | Spent: 7.9 secs | LR: 0.000194\n","INFO:tensorflow:Step 5700 | Loss: 1.5398 | Loss_intent: 1.0950 | Loss_slots: 0.4448 | Spent: 4.8 secs | LR: 0.000202\n","INFO:tensorflow:Step 5750 | Loss: 1.2766 | Loss_intent: 1.0866 | Loss_slots: 0.1900 | Spent: 4.8 secs | LR: 0.000209\n","INFO:tensorflow:Step 5800 | Loss: 1.4626 | Loss_intent: 1.0913 | Loss_slots: 0.3714 | Spent: 4.8 secs | LR: 0.000216\n","INFO:tensorflow:Step 5850 | Loss: 1.4648 | Loss_intent: 1.0894 | Loss_slots: 0.3754 | Spent: 4.7 secs | LR: 0.000223\n","INFO:tensorflow:Step 5900 | Loss: 1.3968 | Loss_intent: 1.0877 | Loss_slots: 0.3091 | Spent: 4.7 secs | LR: 0.000230\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.947, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 5950 | Loss: 1.5204 | Loss_intent: 1.0959 | Loss_slots: 0.4245 | Spent: 8.3 secs | LR: 0.000237\n","INFO:tensorflow:Step 6000 | Loss: 1.2834 | Loss_intent: 1.0824 | Loss_slots: 0.2010 | Spent: 5.0 secs | LR: 0.000244\n","INFO:tensorflow:Step 6050 | Loss: 1.4331 | Loss_intent: 1.0836 | Loss_slots: 0.3495 | Spent: 4.7 secs | LR: 0.000251\n","INFO:tensorflow:Step 6100 | Loss: 1.4518 | Loss_intent: 1.1385 | Loss_slots: 0.3133 | Spent: 4.6 secs | LR: 0.000258\n","INFO:tensorflow:Step 6150 | Loss: 1.4332 | Loss_intent: 1.1888 | Loss_slots: 0.2444 | Spent: 4.8 secs | LR: 0.000265\n","INFO:tensorflow:Step 6200 | Loss: 1.4568 | Loss_intent: 1.0880 | Loss_slots: 0.3689 | Spent: 4.7 secs | LR: 0.000272\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.948, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 6250 | Loss: 1.4830 | Loss_intent: 1.0934 | Loss_slots: 0.3897 | Spent: 8.4 secs | LR: 0.000279\n","INFO:tensorflow:Step 6300 | Loss: 1.4336 | Loss_intent: 1.1119 | Loss_slots: 0.3217 | Spent: 4.6 secs | LR: 0.000286\n","INFO:tensorflow:Step 6350 | Loss: 1.4619 | Loss_intent: 1.0951 | Loss_slots: 0.3668 | Spent: 4.7 secs | LR: 0.000293\n","INFO:tensorflow:Step 6400 | Loss: 1.4387 | Loss_intent: 1.1054 | Loss_slots: 0.3333 | Spent: 4.7 secs | LR: 0.000300\n","INFO:tensorflow:Step 6450 | Loss: 1.4266 | Loss_intent: 1.0832 | Loss_slots: 0.3433 | Spent: 4.7 secs | LR: 0.000307\n","INFO:tensorflow:Step 6500 | Loss: 1.4417 | Loss_intent: 1.0894 | Loss_slots: 0.3523 | Spent: 4.7 secs | LR: 0.000314\n","INFO:tensorflow:Step 6550 | Loss: 1.3117 | Loss_intent: 1.0889 | Loss_slots: 0.2228 | Spent: 4.7 secs | LR: 0.000321\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.949, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 6600 | Loss: 1.5073 | Loss_intent: 1.1322 | Loss_slots: 0.3751 | Spent: 7.9 secs | LR: 0.000328\n","INFO:tensorflow:Step 6650 | Loss: 1.3934 | Loss_intent: 1.0863 | Loss_slots: 0.3071 | Spent: 4.6 secs | LR: 0.000335\n","INFO:tensorflow:Step 6700 | Loss: 1.4519 | Loss_intent: 1.0819 | Loss_slots: 0.3700 | Spent: 4.8 secs | LR: 0.000342\n","INFO:tensorflow:Step 6750 | Loss: 1.5057 | Loss_intent: 1.1120 | Loss_slots: 0.3937 | Spent: 4.6 secs | LR: 0.000349\n","INFO:tensorflow:Step 6800 | Loss: 1.4281 | Loss_intent: 1.1081 | Loss_slots: 0.3200 | Spent: 4.7 secs | LR: 0.000356\n","INFO:tensorflow:Step 6850 | Loss: 1.3523 | Loss_intent: 1.1023 | Loss_slots: 0.2500 | Spent: 4.6 secs | LR: 0.000363\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.942, Intent Acc: 0.968\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 6900 | Loss: 1.3474 | Loss_intent: 1.1122 | Loss_slots: 0.2353 | Spent: 8.0 secs | LR: 0.000370\n","INFO:tensorflow:Step 6950 | Loss: 1.4508 | Loss_intent: 1.0856 | Loss_slots: 0.3652 | Spent: 4.7 secs | LR: 0.000377\n","INFO:tensorflow:Step 7000 | Loss: 1.4633 | Loss_intent: 1.1221 | Loss_slots: 0.3412 | Spent: 4.8 secs | LR: 0.000384\n","INFO:tensorflow:Step 7050 | Loss: 1.4635 | Loss_intent: 1.0941 | Loss_slots: 0.3694 | Spent: 4.7 secs | LR: 0.000391\n","INFO:tensorflow:Step 7100 | Loss: 1.5028 | Loss_intent: 1.1054 | Loss_slots: 0.3974 | Spent: 4.7 secs | LR: 0.000398\n","INFO:tensorflow:Step 7150 | Loss: 1.4933 | Loss_intent: 1.1073 | Loss_slots: 0.3860 | Spent: 4.7 secs | LR: 0.000405\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.938, Intent Acc: 0.966\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 7200 | Loss: 1.4454 | Loss_intent: 1.0917 | Loss_slots: 0.3537 | Spent: 8.1 secs | LR: 0.000412\n","INFO:tensorflow:Step 7250 | Loss: 1.5850 | Loss_intent: 1.1280 | Loss_slots: 0.4570 | Spent: 4.7 secs | LR: 0.000419\n","INFO:tensorflow:Step 7300 | Loss: 1.4772 | Loss_intent: 1.1011 | Loss_slots: 0.3761 | Spent: 4.7 secs | LR: 0.000427\n","INFO:tensorflow:Step 7350 | Loss: 1.4631 | Loss_intent: 1.0984 | Loss_slots: 0.3648 | Spent: 4.8 secs | LR: 0.000434\n","INFO:tensorflow:Step 7400 | Loss: 1.4625 | Loss_intent: 1.1075 | Loss_slots: 0.3550 | Spent: 4.7 secs | LR: 0.000441\n","INFO:tensorflow:Step 7450 | Loss: 1.4168 | Loss_intent: 1.0864 | Loss_slots: 0.3305 | Spent: 4.8 secs | LR: 0.000448\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.942, Intent Acc: 0.962\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.978     0.987     0.983       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      0.947     1.000     0.973        36\n","                            atis_airline      1.000     0.974     0.987        38\n","                       atis_abbreviation      0.939     0.939     0.939        33\n","                           atis_aircraft      0.800     0.889     0.842         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.200     0.333     0.250         3\n","                atis_flight#atis_airfare      0.857     0.500     0.632        12\n","                            atis_airport      0.947     1.000     0.973        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      1.000     0.667     0.800         6\n","                        atis_ground_fare      1.000     0.714     0.833         7\n","                           atis_capacity      0.840     1.000     0.913        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.333     0.500         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.962     0.967     0.965       888\n","                               macro avg      0.654     0.605     0.616       888\n","                            weighted avg      0.966     0.967     0.964       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.971     0.996     0.983     716.0\n","         B-fromloc.city_name      0.986     0.996     0.991     704.0\n","           I-toloc.city_name      0.963     0.985     0.974     265.0\n","      B-depart_date.day_name      0.990     0.976     0.983     212.0\n","              B-airline_name      0.980     0.990     0.985     101.0\n","         I-fromloc.city_name      0.967     0.994     0.981     177.0\n"," B-depart_time.period_of_day      0.983     0.885     0.931     130.0\n","              I-airline_name      0.984     0.969     0.977      65.0\n","    B-depart_date.day_number      0.963     0.945     0.954      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.826     1.000     0.905      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      1.000     0.958     0.979      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      0.826     0.950     0.884      20.0\n","                 B-city_name      0.810     0.596     0.687      57.0\n","                B-class_type      0.923     1.000     0.960      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.909     0.968     0.937      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      1.000     0.971     0.986      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     1.000     1.000      15.0\n","      I-fromloc.airport_name      0.417     1.000     0.588      15.0\n","      B-fromloc.airport_name      0.344     0.917     0.500      12.0\n","      B-arrive_date.day_name      0.647     1.000     0.786      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.588     0.909     0.714      11.0\n"," B-depart_date.date_relative      0.895     1.000     0.944      17.0\n","          B-toloc.state_name      1.000     0.857     0.923      28.0\n","           B-fare_basis_code      0.909     0.588     0.714      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.769     0.345     0.476      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      0.900     0.600     0.720      30.0\n","        I-toloc.airport_name      1.000     0.667     0.800       3.0\n","            B-transport_type      1.000     1.000     1.000      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     0.200     0.333      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.800     0.800     0.800       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.938     0.882     0.909      17.0\n","              B-airport_name      0.400     0.286     0.333      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      0.964     0.818     0.885      33.0\n","                       B-mod      0.000     0.000     0.000       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      1.000     0.667     0.800       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.000     0.000     0.000       1.0\n","          B-restriction_code      0.364     1.000     0.533       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.500     0.667       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.800     1.000     0.889       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      1.000     1.000     1.000       5.0\n","          I-restriction_code      0.250     1.000     0.400       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      1.000     1.000     1.000       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      0.500     0.333     0.400       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      0.000     0.000     0.000       1.0\n","    I-arrive_time.start_time      0.000     0.000     0.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      0.000     0.000     0.000       1.0\n"," I-depart_time.period_of_day      0.000     0.000     0.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.500     0.667       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      0.000     0.000     0.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.000     0.000     0.000       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.941     0.943     0.942    3657.0\n","                   macro avg      0.601     0.584     0.575    3657.0\n","                weighted avg      0.946     0.943     0.939    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 7500 | Loss: 1.4883 | Loss_intent: 1.1024 | Loss_slots: 0.3858 | Spent: 8.1 secs | LR: 0.000445\n","INFO:tensorflow:Step 7550 | Loss: 1.3558 | Loss_intent: 1.1205 | Loss_slots: 0.2353 | Spent: 4.7 secs | LR: 0.000438\n","INFO:tensorflow:Step 7600 | Loss: 1.4268 | Loss_intent: 1.0897 | Loss_slots: 0.3371 | Spent: 4.7 secs | LR: 0.000431\n","INFO:tensorflow:Step 7650 | Loss: 1.3612 | Loss_intent: 1.0827 | Loss_slots: 0.2785 | Spent: 4.8 secs | LR: 0.000424\n","INFO:tensorflow:Step 7700 | Loss: 1.3352 | Loss_intent: 1.0838 | Loss_slots: 0.2514 | Spent: 4.7 secs | LR: 0.000417\n","INFO:tensorflow:Step 7750 | Loss: 1.3588 | Loss_intent: 1.0866 | Loss_slots: 0.2722 | Spent: 4.7 secs | LR: 0.000410\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.946, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 7800 | Loss: 1.2966 | Loss_intent: 1.0836 | Loss_slots: 0.2130 | Spent: 8.0 secs | LR: 0.000403\n","INFO:tensorflow:Step 7850 | Loss: 1.4240 | Loss_intent: 1.0885 | Loss_slots: 0.3355 | Spent: 4.7 secs | LR: 0.000396\n","INFO:tensorflow:Step 7900 | Loss: 1.5369 | Loss_intent: 1.0853 | Loss_slots: 0.4516 | Spent: 4.8 secs | LR: 0.000389\n","INFO:tensorflow:Step 7950 | Loss: 1.4682 | Loss_intent: 1.0872 | Loss_slots: 0.3809 | Spent: 4.7 secs | LR: 0.000382\n","INFO:tensorflow:Step 8000 | Loss: 1.3944 | Loss_intent: 1.0886 | Loss_slots: 0.3058 | Spent: 4.8 secs | LR: 0.000375\n","INFO:tensorflow:Step 8050 | Loss: 1.4542 | Loss_intent: 1.0814 | Loss_slots: 0.3728 | Spent: 4.7 secs | LR: 0.000368\n","INFO:tensorflow:Step 8100 | Loss: 1.5440 | Loss_intent: 1.0864 | Loss_slots: 0.4575 | Spent: 4.7 secs | LR: 0.000361\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.951, Intent Acc: 0.964\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 8150 | Loss: 1.3299 | Loss_intent: 1.0997 | Loss_slots: 0.2302 | Spent: 8.1 secs | LR: 0.000354\n","INFO:tensorflow:Step 8200 | Loss: 1.2739 | Loss_intent: 1.0865 | Loss_slots: 0.1874 | Spent: 4.7 secs | LR: 0.000347\n","INFO:tensorflow:Step 8250 | Loss: 1.4467 | Loss_intent: 1.0862 | Loss_slots: 0.3605 | Spent: 4.7 secs | LR: 0.000340\n","INFO:tensorflow:Step 8300 | Loss: 1.4709 | Loss_intent: 1.0819 | Loss_slots: 0.3890 | Spent: 4.7 secs | LR: 0.000333\n","INFO:tensorflow:Step 8350 | Loss: 1.4901 | Loss_intent: 1.0880 | Loss_slots: 0.4022 | Spent: 4.7 secs | LR: 0.000326\n","INFO:tensorflow:Step 8400 | Loss: 1.5162 | Loss_intent: 1.1091 | Loss_slots: 0.4071 | Spent: 4.7 secs | LR: 0.000319\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.965\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 8450 | Loss: 1.3222 | Loss_intent: 1.0825 | Loss_slots: 0.2396 | Spent: 8.0 secs | LR: 0.000312\n","INFO:tensorflow:Step 8500 | Loss: 1.3320 | Loss_intent: 1.0859 | Loss_slots: 0.2461 | Spent: 4.8 secs | LR: 0.000305\n","INFO:tensorflow:Step 8550 | Loss: 1.4828 | Loss_intent: 1.1211 | Loss_slots: 0.3617 | Spent: 4.7 secs | LR: 0.000298\n","INFO:tensorflow:Step 8600 | Loss: 1.4953 | Loss_intent: 1.0841 | Loss_slots: 0.4112 | Spent: 4.7 secs | LR: 0.000291\n","INFO:tensorflow:Step 8650 | Loss: 1.3724 | Loss_intent: 1.0856 | Loss_slots: 0.2868 | Spent: 4.7 secs | LR: 0.000284\n","INFO:tensorflow:Step 8700 | Loss: 1.3699 | Loss_intent: 1.0827 | Loss_slots: 0.2872 | Spent: 4.6 secs | LR: 0.000277\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.975\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 8750 | Loss: 1.3995 | Loss_intent: 1.0865 | Loss_slots: 0.3130 | Spent: 8.0 secs | LR: 0.000270\n","INFO:tensorflow:Step 8800 | Loss: 1.3126 | Loss_intent: 1.0849 | Loss_slots: 0.2276 | Spent: 4.8 secs | LR: 0.000263\n","INFO:tensorflow:Step 8850 | Loss: 1.5089 | Loss_intent: 1.0875 | Loss_slots: 0.4214 | Spent: 4.8 secs | LR: 0.000256\n","INFO:tensorflow:Step 8900 | Loss: 1.5273 | Loss_intent: 1.0859 | Loss_slots: 0.4414 | Spent: 5.0 secs | LR: 0.000248\n","INFO:tensorflow:Step 8950 | Loss: 1.4085 | Loss_intent: 1.0826 | Loss_slots: 0.3259 | Spent: 5.2 secs | LR: 0.000241\n","INFO:tensorflow:Step 9000 | Loss: 1.2778 | Loss_intent: 1.0859 | Loss_slots: 0.1919 | Spent: 4.9 secs | LR: 0.000234\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.965\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.965\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 9050 | Loss: 1.2899 | Loss_intent: 1.0830 | Loss_slots: 0.2069 | Spent: 8.0 secs | LR: 0.000227\n","INFO:tensorflow:Step 9100 | Loss: 1.4770 | Loss_intent: 1.0821 | Loss_slots: 0.3948 | Spent: 4.9 secs | LR: 0.000220\n","INFO:tensorflow:Step 9150 | Loss: 1.4177 | Loss_intent: 1.0933 | Loss_slots: 0.3244 | Spent: 4.9 secs | LR: 0.000213\n","INFO:tensorflow:Step 9200 | Loss: 1.4197 | Loss_intent: 1.0920 | Loss_slots: 0.3277 | Spent: 4.7 secs | LR: 0.000206\n","INFO:tensorflow:Step 9250 | Loss: 1.3984 | Loss_intent: 1.1080 | Loss_slots: 0.2904 | Spent: 4.7 secs | LR: 0.000199\n","INFO:tensorflow:Step 9300 | Loss: 1.4309 | Loss_intent: 1.0819 | Loss_slots: 0.3490 | Spent: 4.7 secs | LR: 0.000192\n","INFO:tensorflow:Step 9350 | Loss: 1.5417 | Loss_intent: 1.0825 | Loss_slots: 0.4592 | Spent: 4.7 secs | LR: 0.000185\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.952, Intent Acc: 0.965\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 9400 | Loss: 1.3817 | Loss_intent: 1.1009 | Loss_slots: 0.2808 | Spent: 7.9 secs | LR: 0.000178\n","INFO:tensorflow:Step 9450 | Loss: 1.5014 | Loss_intent: 1.0971 | Loss_slots: 0.4043 | Spent: 4.7 secs | LR: 0.000171\n","INFO:tensorflow:Step 9500 | Loss: 1.4281 | Loss_intent: 1.0916 | Loss_slots: 0.3365 | Spent: 4.7 secs | LR: 0.000164\n","INFO:tensorflow:Step 9550 | Loss: 1.4252 | Loss_intent: 1.0816 | Loss_slots: 0.3436 | Spent: 4.7 secs | LR: 0.000157\n","INFO:tensorflow:Step 9600 | Loss: 1.4387 | Loss_intent: 1.0816 | Loss_slots: 0.3571 | Spent: 4.7 secs | LR: 0.000150\n","INFO:tensorflow:Step 9650 | Loss: 1.4663 | Loss_intent: 1.0856 | Loss_slots: 0.3807 | Spent: 4.7 secs | LR: 0.000143\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 9700 | Loss: 1.3539 | Loss_intent: 1.0818 | Loss_slots: 0.2721 | Spent: 8.0 secs | LR: 0.000136\n","INFO:tensorflow:Step 9750 | Loss: 1.3213 | Loss_intent: 1.0842 | Loss_slots: 0.2371 | Spent: 4.7 secs | LR: 0.000129\n","INFO:tensorflow:Step 9800 | Loss: 1.2536 | Loss_intent: 1.0883 | Loss_slots: 0.1653 | Spent: 4.7 secs | LR: 0.000122\n","INFO:tensorflow:Step 9850 | Loss: 1.4604 | Loss_intent: 1.0830 | Loss_slots: 0.3775 | Spent: 4.7 secs | LR: 0.000115\n","INFO:tensorflow:Step 9900 | Loss: 1.3347 | Loss_intent: 1.0846 | Loss_slots: 0.2501 | Spent: 4.8 secs | LR: 0.000108\n","INFO:tensorflow:Step 9950 | Loss: 1.4614 | Loss_intent: 1.0819 | Loss_slots: 0.3795 | Spent: 4.8 secs | LR: 0.000101\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.968\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.981     0.986     0.983       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      1.000     0.974     0.987        38\n","                       atis_abbreviation      0.939     0.939     0.939        33\n","                           atis_aircraft      0.889     0.889     0.889         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.375     1.000     0.545         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      0.947     1.000     0.973        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      0.667     0.667     0.667         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      0.955     1.000     0.977        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.833     0.909         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.968     0.973     0.970       888\n","                               macro avg      0.657     0.667     0.652       888\n","                            weighted avg      0.972     0.973     0.971       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.975     0.999     0.987     716.0\n","         B-fromloc.city_name      0.990     0.997     0.994     704.0\n","           I-toloc.city_name      0.978     0.996     0.987     265.0\n","      B-depart_date.day_name      0.990     0.981     0.986     212.0\n","              B-airline_name      0.981     1.000     0.990     101.0\n","         I-fromloc.city_name      0.967     0.989     0.978     177.0\n"," B-depart_time.period_of_day      1.000     0.885     0.939     130.0\n","              I-airline_name      1.000     1.000     1.000      65.0\n","    B-depart_date.day_number      0.964     0.964     0.964      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      1.000     0.958     0.979      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      1.000     1.000     1.000      20.0\n","                 B-city_name      0.829     0.596     0.694      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      1.000     0.971     0.986      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     0.933     0.966      15.0\n","      I-fromloc.airport_name      0.405     1.000     0.577      15.0\n","      B-fromloc.airport_name      0.423     0.917     0.579      12.0\n","      B-arrive_date.day_name      0.688     1.000     0.815      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.524     1.000     0.688      11.0\n"," B-depart_date.date_relative      0.889     0.941     0.914      17.0\n","          B-toloc.state_name      0.933     1.000     0.966      28.0\n","           B-fare_basis_code      0.944     1.000     0.971      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.909     0.345     0.500      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      1.000     0.500     0.667      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      0.909     1.000     0.952      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     1.000     1.000      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.944     1.000     0.971      17.0\n","              B-airport_name      0.778     0.333     0.467      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      0.969     0.939     0.954      33.0\n","                       B-mod      1.000     0.500     0.667       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      0.667     0.667     0.667       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.000     0.000     0.000       1.0\n","          B-restriction_code      0.444     1.000     0.615       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      0.833     1.000     0.909       5.0\n","          I-restriction_code      1.000     1.000     1.000       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.500     0.500     0.500       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      0.667     0.667     0.667       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      0.000     0.000     0.000       1.0\n"," I-depart_time.period_of_day      1.000     1.000     1.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.500     0.667       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      1.000     1.000     1.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      1.000     0.667     0.800       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.953     0.955     0.954    3657.0\n","                   macro avg      0.658     0.645     0.638    3657.0\n","                weighted avg      0.957     0.955     0.951    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 10000 | Loss: 1.4548 | Loss_intent: 1.0850 | Loss_slots: 0.3698 | Spent: 8.1 secs | LR: 0.000103\n","INFO:tensorflow:Step 10050 | Loss: 1.3330 | Loss_intent: 1.0825 | Loss_slots: 0.2505 | Spent: 4.6 secs | LR: 0.000107\n","INFO:tensorflow:Step 10100 | Loss: 1.4070 | Loss_intent: 1.0810 | Loss_slots: 0.3260 | Spent: 4.8 secs | LR: 0.000110\n","INFO:tensorflow:Step 10150 | Loss: 1.3685 | Loss_intent: 1.0840 | Loss_slots: 0.2845 | Spent: 4.7 secs | LR: 0.000114\n","INFO:tensorflow:Step 10200 | Loss: 1.4771 | Loss_intent: 1.0820 | Loss_slots: 0.3951 | Spent: 4.7 secs | LR: 0.000117\n","INFO:tensorflow:Step 10250 | Loss: 1.3770 | Loss_intent: 1.0845 | Loss_slots: 0.2924 | Spent: 4.7 secs | LR: 0.000121\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 10300 | Loss: 1.4075 | Loss_intent: 1.0831 | Loss_slots: 0.3244 | Spent: 7.9 secs | LR: 0.000124\n","INFO:tensorflow:Step 10350 | Loss: 1.4358 | Loss_intent: 1.0842 | Loss_slots: 0.3516 | Spent: 4.8 secs | LR: 0.000128\n","INFO:tensorflow:Step 10400 | Loss: 1.3873 | Loss_intent: 1.0805 | Loss_slots: 0.3069 | Spent: 4.8 secs | LR: 0.000131\n","INFO:tensorflow:Step 10450 | Loss: 1.4018 | Loss_intent: 1.0810 | Loss_slots: 0.3208 | Spent: 4.7 secs | LR: 0.000135\n","INFO:tensorflow:Step 10500 | Loss: 1.3858 | Loss_intent: 1.0812 | Loss_slots: 0.3046 | Spent: 4.8 secs | LR: 0.000138\n","INFO:tensorflow:Step 10550 | Loss: 1.4216 | Loss_intent: 1.0815 | Loss_slots: 0.3400 | Spent: 4.7 secs | LR: 0.000142\n","INFO:tensorflow:Step 10600 | Loss: 1.4958 | Loss_intent: 1.0809 | Loss_slots: 0.4149 | Spent: 4.6 secs | LR: 0.000145\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 10650 | Loss: 1.4597 | Loss_intent: 1.0813 | Loss_slots: 0.3784 | Spent: 7.9 secs | LR: 0.000149\n","INFO:tensorflow:Step 10700 | Loss: 1.3856 | Loss_intent: 1.0828 | Loss_slots: 0.3028 | Spent: 4.8 secs | LR: 0.000152\n","INFO:tensorflow:Step 10750 | Loss: 1.4340 | Loss_intent: 1.0810 | Loss_slots: 0.3529 | Spent: 4.7 secs | LR: 0.000156\n","INFO:tensorflow:Step 10800 | Loss: 1.4103 | Loss_intent: 1.0810 | Loss_slots: 0.3293 | Spent: 4.7 secs | LR: 0.000159\n","INFO:tensorflow:Step 10850 | Loss: 1.2689 | Loss_intent: 1.0812 | Loss_slots: 0.1877 | Spent: 4.7 secs | LR: 0.000163\n","INFO:tensorflow:Step 10900 | Loss: 1.3389 | Loss_intent: 1.0823 | Loss_slots: 0.2566 | Spent: 4.7 secs | LR: 0.000166\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 10950 | Loss: 1.4134 | Loss_intent: 1.0833 | Loss_slots: 0.3301 | Spent: 8.1 secs | LR: 0.000170\n","INFO:tensorflow:Step 11000 | Loss: 1.3613 | Loss_intent: 1.0813 | Loss_slots: 0.2800 | Spent: 4.6 secs | LR: 0.000173\n","INFO:tensorflow:Step 11050 | Loss: 1.4827 | Loss_intent: 1.0881 | Loss_slots: 0.3945 | Spent: 4.7 secs | LR: 0.000177\n","INFO:tensorflow:Step 11100 | Loss: 1.5191 | Loss_intent: 1.0838 | Loss_slots: 0.4353 | Spent: 4.8 secs | LR: 0.000180\n","INFO:tensorflow:Step 11150 | Loss: 1.4613 | Loss_intent: 1.0819 | Loss_slots: 0.3794 | Spent: 4.7 secs | LR: 0.000184\n","INFO:tensorflow:Step 11200 | Loss: 1.3288 | Loss_intent: 1.0807 | Loss_slots: 0.2481 | Spent: 4.7 secs | LR: 0.000187\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 11250 | Loss: 1.3408 | Loss_intent: 1.0861 | Loss_slots: 0.2548 | Spent: 7.9 secs | LR: 0.000191\n","INFO:tensorflow:Step 11300 | Loss: 1.3869 | Loss_intent: 1.0815 | Loss_slots: 0.3054 | Spent: 4.7 secs | LR: 0.000194\n","INFO:tensorflow:Step 11350 | Loss: 1.3046 | Loss_intent: 1.0831 | Loss_slots: 0.2215 | Spent: 4.7 secs | LR: 0.000198\n","INFO:tensorflow:Step 11400 | Loss: 1.4395 | Loss_intent: 1.0823 | Loss_slots: 0.3572 | Spent: 4.6 secs | LR: 0.000202\n","INFO:tensorflow:Step 11450 | Loss: 1.2477 | Loss_intent: 1.0827 | Loss_slots: 0.1650 | Spent: 4.7 secs | LR: 0.000205\n","INFO:tensorflow:Step 11500 | Loss: 1.5072 | Loss_intent: 1.0814 | Loss_slots: 0.4258 | Spent: 4.7 secs | LR: 0.000209\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 11550 | Loss: 1.3389 | Loss_intent: 1.0845 | Loss_slots: 0.2543 | Spent: 7.8 secs | LR: 0.000212\n","INFO:tensorflow:Step 11600 | Loss: 1.5272 | Loss_intent: 1.0830 | Loss_slots: 0.4443 | Spent: 4.7 secs | LR: 0.000216\n","INFO:tensorflow:Step 11650 | Loss: 1.3690 | Loss_intent: 1.0815 | Loss_slots: 0.2875 | Spent: 4.7 secs | LR: 0.000219\n","INFO:tensorflow:Step 11700 | Loss: 1.3340 | Loss_intent: 1.0900 | Loss_slots: 0.2440 | Spent: 4.7 secs | LR: 0.000223\n","INFO:tensorflow:Step 11750 | Loss: 1.4563 | Loss_intent: 1.0880 | Loss_slots: 0.3683 | Spent: 4.8 secs | LR: 0.000226\n","INFO:tensorflow:Step 11800 | Loss: 1.3797 | Loss_intent: 1.1214 | Loss_slots: 0.2584 | Spent: 4.7 secs | LR: 0.000230\n","INFO:tensorflow:Step 11850 | Loss: 1.4445 | Loss_intent: 1.0813 | Loss_slots: 0.3632 | Spent: 4.8 secs | LR: 0.000233\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 11900 | Loss: 1.4229 | Loss_intent: 1.0829 | Loss_slots: 0.3399 | Spent: 8.7 secs | LR: 0.000237\n","INFO:tensorflow:Step 11950 | Loss: 1.4080 | Loss_intent: 1.0875 | Loss_slots: 0.3205 | Spent: 4.7 secs | LR: 0.000240\n","INFO:tensorflow:Step 12000 | Loss: 1.4120 | Loss_intent: 1.0837 | Loss_slots: 0.3283 | Spent: 4.8 secs | LR: 0.000244\n","INFO:tensorflow:Step 12050 | Loss: 1.4031 | Loss_intent: 1.0861 | Loss_slots: 0.3170 | Spent: 5.0 secs | LR: 0.000247\n","INFO:tensorflow:Step 12100 | Loss: 1.4875 | Loss_intent: 1.0831 | Loss_slots: 0.4044 | Spent: 4.8 secs | LR: 0.000251\n","INFO:tensorflow:Step 12150 | Loss: 1.4649 | Loss_intent: 1.0804 | Loss_slots: 0.3845 | Spent: 4.8 secs | LR: 0.000254\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 12200 | Loss: 1.3736 | Loss_intent: 1.0819 | Loss_slots: 0.2916 | Spent: 8.1 secs | LR: 0.000258\n","INFO:tensorflow:Step 12250 | Loss: 1.4720 | Loss_intent: 1.0844 | Loss_slots: 0.3876 | Spent: 4.8 secs | LR: 0.000261\n","INFO:tensorflow:Step 12300 | Loss: 1.4430 | Loss_intent: 1.1105 | Loss_slots: 0.3325 | Spent: 4.7 secs | LR: 0.000265\n","INFO:tensorflow:Step 12350 | Loss: 1.4522 | Loss_intent: 1.0804 | Loss_slots: 0.3718 | Spent: 4.7 secs | LR: 0.000268\n","INFO:tensorflow:Step 12400 | Loss: 1.4260 | Loss_intent: 1.0817 | Loss_slots: 0.3443 | Spent: 4.8 secs | LR: 0.000272\n","INFO:tensorflow:Step 12450 | Loss: 1.4276 | Loss_intent: 1.0881 | Loss_slots: 0.3395 | Spent: 4.7 secs | LR: 0.000275\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.986     0.987     0.987       632\n","                            atis_airfare      0.959     0.979     0.969        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      0.950     1.000     0.974        38\n","                       atis_abbreviation      0.941     0.970     0.955        33\n","                           atis_aircraft      0.900     1.000     0.947         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.429     1.000     0.600         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      0.947     1.000     0.973        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      1.000     0.667     0.800         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     1.000     1.000        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      0.833     0.833     0.833         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.972     0.977     0.975       888\n","                               macro avg      0.667     0.675     0.661       888\n","                            weighted avg      0.975     0.977     0.975       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.978     0.997     0.988     716.0\n","         B-fromloc.city_name      0.993     0.999     0.996     704.0\n","           I-toloc.city_name      0.974     1.000     0.987     265.0\n","      B-depart_date.day_name      0.990     0.981     0.986     212.0\n","              B-airline_name      0.981     1.000     0.990     101.0\n","         I-fromloc.city_name      0.978     0.989     0.983     177.0\n"," B-depart_time.period_of_day      1.000     0.885     0.939     130.0\n","              I-airline_name      0.985     1.000     0.992      65.0\n","    B-depart_date.day_number      0.964     0.964     0.964      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      1.000     0.958     0.979      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      0.952     1.000     0.976      20.0\n","                 B-city_name      0.778     0.614     0.686      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      1.000     0.971     0.986      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     0.933     0.966      15.0\n","      I-fromloc.airport_name      0.405     1.000     0.577      15.0\n","      B-fromloc.airport_name      0.440     0.917     0.595      12.0\n","      B-arrive_date.day_name      0.688     1.000     0.815      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.524     1.000     0.688      11.0\n"," B-depart_date.date_relative      0.938     0.882     0.909      17.0\n","          B-toloc.state_name      0.966     1.000     0.982      28.0\n","           B-fare_basis_code      0.944     1.000     0.971      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.900     0.310     0.462      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      0.938     0.500     0.652      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      1.000     0.600     0.750      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     1.000     1.000      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.850     1.000     0.919      17.0\n","              B-airport_name      0.750     0.286     0.414      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      1.000     0.909     0.952      33.0\n","                       B-mod      0.500     0.500     0.500       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      1.000     1.000     1.000       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.200     1.000     0.333       1.0\n","          B-restriction_code      0.500     1.000     0.667       4.0\n","    B-arrive_time.start_time      1.000     1.000     1.000       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      0.833     1.000     0.909       5.0\n","          I-restriction_code      0.750     1.000     0.857       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.500     1.000     0.667       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      1.000     0.667     0.800       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      1.000     1.000     1.000       1.0\n"," I-depart_time.period_of_day      1.000     1.000     1.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.250     0.400       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      0.000     0.000     0.000       1.0\n","                 B-days_code      0.333     1.000     0.500       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      1.000     1.000     1.000       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.952     0.954     0.953    3657.0\n","                   macro avg      0.655     0.657     0.636    3657.0\n","                weighted avg      0.957     0.954     0.950    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 12500 | Loss: 1.4581 | Loss_intent: 1.0948 | Loss_slots: 0.3633 | Spent: 8.0 secs | LR: 0.000271\n","INFO:tensorflow:Step 12550 | Loss: 1.3473 | Loss_intent: 1.0843 | Loss_slots: 0.2630 | Spent: 4.7 secs | LR: 0.000268\n","INFO:tensorflow:Step 12600 | Loss: 1.4641 | Loss_intent: 1.0900 | Loss_slots: 0.3741 | Spent: 4.8 secs | LR: 0.000264\n","INFO:tensorflow:Step 12650 | Loss: 1.4392 | Loss_intent: 1.0814 | Loss_slots: 0.3577 | Spent: 4.7 secs | LR: 0.000261\n","INFO:tensorflow:Step 12700 | Loss: 1.3957 | Loss_intent: 1.0812 | Loss_slots: 0.3145 | Spent: 4.8 secs | LR: 0.000257\n","INFO:tensorflow:Step 12750 | Loss: 1.4211 | Loss_intent: 1.0835 | Loss_slots: 0.3375 | Spent: 4.7 secs | LR: 0.000254\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.950, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 12800 | Loss: 1.4258 | Loss_intent: 1.0884 | Loss_slots: 0.3374 | Spent: 7.9 secs | LR: 0.000250\n","INFO:tensorflow:Step 12850 | Loss: 1.4560 | Loss_intent: 1.0822 | Loss_slots: 0.3738 | Spent: 4.6 secs | LR: 0.000247\n","INFO:tensorflow:Step 12900 | Loss: 1.3639 | Loss_intent: 1.0914 | Loss_slots: 0.2724 | Spent: 4.7 secs | LR: 0.000243\n","INFO:tensorflow:Step 12950 | Loss: 1.4441 | Loss_intent: 1.0833 | Loss_slots: 0.3608 | Spent: 4.7 secs | LR: 0.000239\n","INFO:tensorflow:Step 13000 | Loss: 1.4268 | Loss_intent: 1.0832 | Loss_slots: 0.3435 | Spent: 4.7 secs | LR: 0.000236\n","INFO:tensorflow:Step 13050 | Loss: 1.4460 | Loss_intent: 1.0830 | Loss_slots: 0.3630 | Spent: 4.8 secs | LR: 0.000232\n","INFO:tensorflow:Step 13100 | Loss: 1.4478 | Loss_intent: 1.0817 | Loss_slots: 0.3660 | Spent: 4.7 secs | LR: 0.000229\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 13150 | Loss: 1.3371 | Loss_intent: 1.0817 | Loss_slots: 0.2555 | Spent: 7.9 secs | LR: 0.000225\n","INFO:tensorflow:Step 13200 | Loss: 1.3906 | Loss_intent: 1.0832 | Loss_slots: 0.3073 | Spent: 4.7 secs | LR: 0.000222\n","INFO:tensorflow:Step 13250 | Loss: 1.3094 | Loss_intent: 1.0829 | Loss_slots: 0.2264 | Spent: 4.7 secs | LR: 0.000218\n","INFO:tensorflow:Step 13300 | Loss: 1.4608 | Loss_intent: 1.0832 | Loss_slots: 0.3776 | Spent: 4.7 secs | LR: 0.000215\n","INFO:tensorflow:Step 13350 | Loss: 1.4010 | Loss_intent: 1.0821 | Loss_slots: 0.3189 | Spent: 4.7 secs | LR: 0.000211\n","INFO:tensorflow:Step 13400 | Loss: 1.4238 | Loss_intent: 1.0856 | Loss_slots: 0.3382 | Spent: 4.7 secs | LR: 0.000208\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 13450 | Loss: 1.3966 | Loss_intent: 1.0828 | Loss_slots: 0.3137 | Spent: 7.9 secs | LR: 0.000204\n","INFO:tensorflow:Step 13500 | Loss: 1.3811 | Loss_intent: 1.0878 | Loss_slots: 0.2932 | Spent: 4.7 secs | LR: 0.000201\n","INFO:tensorflow:Step 13550 | Loss: 1.4100 | Loss_intent: 1.0876 | Loss_slots: 0.3224 | Spent: 4.8 secs | LR: 0.000197\n","INFO:tensorflow:Step 13600 | Loss: 1.3831 | Loss_intent: 1.0873 | Loss_slots: 0.2958 | Spent: 4.7 secs | LR: 0.000194\n","INFO:tensorflow:Step 13650 | Loss: 1.2847 | Loss_intent: 1.0814 | Loss_slots: 0.2033 | Spent: 4.7 secs | LR: 0.000190\n","INFO:tensorflow:Step 13700 | Loss: 1.3939 | Loss_intent: 1.0855 | Loss_slots: 0.3084 | Spent: 4.7 secs | LR: 0.000187\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.954, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 13750 | Loss: 1.4284 | Loss_intent: 1.0817 | Loss_slots: 0.3468 | Spent: 8.0 secs | LR: 0.000183\n","INFO:tensorflow:Step 13800 | Loss: 1.4559 | Loss_intent: 1.1113 | Loss_slots: 0.3446 | Spent: 4.6 secs | LR: 0.000180\n","INFO:tensorflow:Step 13850 | Loss: 1.3284 | Loss_intent: 1.0938 | Loss_slots: 0.2346 | Spent: 4.8 secs | LR: 0.000176\n","INFO:tensorflow:Step 13900 | Loss: 1.3488 | Loss_intent: 1.0873 | Loss_slots: 0.2615 | Spent: 4.7 secs | LR: 0.000173\n","INFO:tensorflow:Step 13950 | Loss: 1.3428 | Loss_intent: 1.0826 | Loss_slots: 0.2602 | Spent: 4.7 secs | LR: 0.000169\n","INFO:tensorflow:Step 14000 | Loss: 1.3915 | Loss_intent: 1.0823 | Loss_slots: 0.3092 | Spent: 4.8 secs | LR: 0.000166\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 14050 | Loss: 1.4730 | Loss_intent: 1.0811 | Loss_slots: 0.3918 | Spent: 8.0 secs | LR: 0.000162\n","INFO:tensorflow:Step 14100 | Loss: 1.4370 | Loss_intent: 1.0849 | Loss_slots: 0.3521 | Spent: 4.7 secs | LR: 0.000159\n","INFO:tensorflow:Step 14150 | Loss: 1.4315 | Loss_intent: 1.0827 | Loss_slots: 0.3488 | Spent: 4.7 secs | LR: 0.000155\n","INFO:tensorflow:Step 14200 | Loss: 1.4573 | Loss_intent: 1.0830 | Loss_slots: 0.3743 | Spent: 4.8 secs | LR: 0.000152\n","INFO:tensorflow:Step 14250 | Loss: 1.4152 | Loss_intent: 1.0811 | Loss_slots: 0.3341 | Spent: 4.8 secs | LR: 0.000148\n","INFO:tensorflow:Step 14300 | Loss: 1.3214 | Loss_intent: 1.0805 | Loss_slots: 0.2409 | Spent: 4.9 secs | LR: 0.000145\n","INFO:tensorflow:Step 14350 | Loss: 1.4241 | Loss_intent: 1.0927 | Loss_slots: 0.3314 | Spent: 4.8 secs | LR: 0.000141\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 14400 | Loss: 1.4230 | Loss_intent: 1.0863 | Loss_slots: 0.3368 | Spent: 8.0 secs | LR: 0.000138\n","INFO:tensorflow:Step 14450 | Loss: 1.3571 | Loss_intent: 1.0810 | Loss_slots: 0.2761 | Spent: 4.8 secs | LR: 0.000134\n","INFO:tensorflow:Step 14500 | Loss: 1.4090 | Loss_intent: 1.0824 | Loss_slots: 0.3266 | Spent: 4.8 secs | LR: 0.000131\n","INFO:tensorflow:Step 14550 | Loss: 1.4100 | Loss_intent: 1.0871 | Loss_slots: 0.3229 | Spent: 4.8 secs | LR: 0.000127\n","INFO:tensorflow:Step 14600 | Loss: 1.5322 | Loss_intent: 1.0811 | Loss_slots: 0.4511 | Spent: 4.8 secs | LR: 0.000123\n","INFO:tensorflow:Step 14650 | Loss: 1.3940 | Loss_intent: 1.0812 | Loss_slots: 0.3128 | Spent: 4.8 secs | LR: 0.000120\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 14700 | Loss: 1.3372 | Loss_intent: 1.0811 | Loss_slots: 0.2561 | Spent: 8.0 secs | LR: 0.000116\n","INFO:tensorflow:Step 14750 | Loss: 1.4222 | Loss_intent: 1.0854 | Loss_slots: 0.3367 | Spent: 4.7 secs | LR: 0.000113\n","INFO:tensorflow:Step 14800 | Loss: 1.2573 | Loss_intent: 1.0826 | Loss_slots: 0.1748 | Spent: 5.0 secs | LR: 0.000109\n","INFO:tensorflow:Step 14850 | Loss: 1.6202 | Loss_intent: 1.2968 | Loss_slots: 0.3234 | Spent: 5.1 secs | LR: 0.000106\n","INFO:tensorflow:Step 14900 | Loss: 1.3431 | Loss_intent: 1.0821 | Loss_slots: 0.2610 | Spent: 4.9 secs | LR: 0.000102\n","INFO:tensorflow:Step 14950 | Loss: 1.2714 | Loss_intent: 1.0809 | Loss_slots: 0.1905 | Spent: 5.0 secs | LR: 0.000101\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.981     0.989     0.985       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      0.974     0.974     0.974        38\n","                       atis_abbreviation      0.939     0.939     0.939        33\n","                           atis_aircraft      0.889     0.889     0.889         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.429     1.000     0.600         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      0.900     1.000     0.947        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      1.000     0.667     0.800         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     1.000     1.000        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.833     0.909         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.970     0.975     0.972       888\n","                               macro avg      0.673     0.668     0.660       888\n","                            weighted avg      0.974     0.975     0.973       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.978     0.997     0.988     716.0\n","         B-fromloc.city_name      0.989     0.999     0.994     704.0\n","           I-toloc.city_name      0.981     0.985     0.983     265.0\n","      B-depart_date.day_name      0.990     0.981     0.986     212.0\n","              B-airline_name      0.981     1.000     0.990     101.0\n","         I-fromloc.city_name      0.957     0.994     0.975     177.0\n"," B-depart_time.period_of_day      1.000     0.885     0.939     130.0\n","              I-airline_name      1.000     1.000     1.000      65.0\n","    B-depart_date.day_number      0.964     0.964     0.964      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      0.958     0.958     0.958      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      1.000     1.000     1.000      20.0\n","                 B-city_name      0.854     0.614     0.714      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      0.971     0.971     0.971      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     0.933     0.966      15.0\n","      I-fromloc.airport_name      0.405     1.000     0.577      15.0\n","      B-fromloc.airport_name      0.440     0.917     0.595      12.0\n","      B-arrive_date.day_name      0.688     1.000     0.815      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.458     1.000     0.629      11.0\n"," B-depart_date.date_relative      0.938     0.882     0.909      17.0\n","          B-toloc.state_name      0.903     1.000     0.949      28.0\n","           B-fare_basis_code      0.944     1.000     0.971      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.909     0.345     0.500      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      1.000     0.533     0.696      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      1.000     1.000     1.000      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     1.000     1.000      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.889     0.941     0.914      17.0\n","              B-airport_name      0.800     0.381     0.516      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      1.000     0.909     0.952      33.0\n","                       B-mod      1.000     0.500     0.667       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      1.000     0.667     0.800       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      1.000     1.000     1.000       1.0\n","          B-restriction_code      0.444     1.000     0.615       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.875     0.875     0.875       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      0.833     1.000     0.909       5.0\n","          I-restriction_code      1.000     1.000     1.000       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.500     1.000     0.667       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      1.000     0.333     0.500       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      0.000     0.000     0.000       1.0\n"," I-depart_time.period_of_day      1.000     1.000     1.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.500     0.667       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      1.000     1.000     1.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.750     1.000     0.857       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.953     0.954     0.954    3657.0\n","                   macro avg      0.670     0.656     0.647    3657.0\n","                weighted avg      0.958     0.954     0.951    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 15000 | Loss: 1.4595 | Loss_intent: 1.0812 | Loss_slots: 0.3783 | Spent: 8.0 secs | LR: 0.000102\n","INFO:tensorflow:Step 15050 | Loss: 1.4029 | Loss_intent: 1.0840 | Loss_slots: 0.3188 | Spent: 4.7 secs | LR: 0.000104\n","INFO:tensorflow:Step 15100 | Loss: 1.3603 | Loss_intent: 1.0854 | Loss_slots: 0.2749 | Spent: 4.7 secs | LR: 0.000106\n","INFO:tensorflow:Step 15150 | Loss: 1.4142 | Loss_intent: 1.0798 | Loss_slots: 0.3343 | Spent: 4.7 secs | LR: 0.000108\n","INFO:tensorflow:Step 15200 | Loss: 1.3723 | Loss_intent: 1.0810 | Loss_slots: 0.2913 | Spent: 4.6 secs | LR: 0.000109\n","INFO:tensorflow:Step 15250 | Loss: 1.4568 | Loss_intent: 1.0814 | Loss_slots: 0.3754 | Spent: 4.7 secs | LR: 0.000111\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 15300 | Loss: 1.4010 | Loss_intent: 1.0835 | Loss_slots: 0.3175 | Spent: 7.9 secs | LR: 0.000113\n","INFO:tensorflow:Step 15350 | Loss: 1.4397 | Loss_intent: 1.0813 | Loss_slots: 0.3584 | Spent: 4.8 secs | LR: 0.000115\n","INFO:tensorflow:Step 15400 | Loss: 1.4215 | Loss_intent: 1.0822 | Loss_slots: 0.3393 | Spent: 4.8 secs | LR: 0.000116\n","INFO:tensorflow:Step 15450 | Loss: 1.4770 | Loss_intent: 1.0804 | Loss_slots: 0.3966 | Spent: 4.9 secs | LR: 0.000118\n","INFO:tensorflow:Step 15500 | Loss: 1.3459 | Loss_intent: 1.0816 | Loss_slots: 0.2643 | Spent: 4.8 secs | LR: 0.000120\n","INFO:tensorflow:Step 15550 | Loss: 1.4422 | Loss_intent: 1.0804 | Loss_slots: 0.3618 | Spent: 4.7 secs | LR: 0.000122\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.968\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 15600 | Loss: 1.4625 | Loss_intent: 1.0817 | Loss_slots: 0.3808 | Spent: 8.1 secs | LR: 0.000123\n","INFO:tensorflow:Step 15650 | Loss: 1.4260 | Loss_intent: 1.0814 | Loss_slots: 0.3446 | Spent: 4.7 secs | LR: 0.000125\n","INFO:tensorflow:Step 15700 | Loss: 1.3429 | Loss_intent: 1.0818 | Loss_slots: 0.2611 | Spent: 4.8 secs | LR: 0.000127\n","INFO:tensorflow:Step 15750 | Loss: 1.4888 | Loss_intent: 1.0812 | Loss_slots: 0.4076 | Spent: 4.8 secs | LR: 0.000129\n","INFO:tensorflow:Step 15800 | Loss: 1.4167 | Loss_intent: 1.0820 | Loss_slots: 0.3347 | Spent: 4.8 secs | LR: 0.000130\n","INFO:tensorflow:Step 15850 | Loss: 1.4556 | Loss_intent: 1.0826 | Loss_slots: 0.3730 | Spent: 4.8 secs | LR: 0.000132\n","INFO:tensorflow:Step 15900 | Loss: 1.5058 | Loss_intent: 1.0814 | Loss_slots: 0.4244 | Spent: 4.8 secs | LR: 0.000134\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 15950 | Loss: 1.3838 | Loss_intent: 1.0800 | Loss_slots: 0.3038 | Spent: 8.1 secs | LR: 0.000136\n","INFO:tensorflow:Step 16000 | Loss: 1.4580 | Loss_intent: 1.0804 | Loss_slots: 0.3776 | Spent: 4.8 secs | LR: 0.000137\n","INFO:tensorflow:Step 16050 | Loss: 1.4191 | Loss_intent: 1.0834 | Loss_slots: 0.3357 | Spent: 4.8 secs | LR: 0.000139\n","INFO:tensorflow:Step 16100 | Loss: 1.3804 | Loss_intent: 1.0804 | Loss_slots: 0.3000 | Spent: 4.7 secs | LR: 0.000141\n","INFO:tensorflow:Step 16150 | Loss: 1.4176 | Loss_intent: 1.0817 | Loss_slots: 0.3359 | Spent: 4.8 secs | LR: 0.000143\n","INFO:tensorflow:Step 16200 | Loss: 1.3426 | Loss_intent: 1.0806 | Loss_slots: 0.2621 | Spent: 4.7 secs | LR: 0.000145\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 16250 | Loss: 1.4096 | Loss_intent: 1.0818 | Loss_slots: 0.3278 | Spent: 8.1 secs | LR: 0.000146\n","INFO:tensorflow:Step 16300 | Loss: 1.3719 | Loss_intent: 1.0802 | Loss_slots: 0.2917 | Spent: 4.8 secs | LR: 0.000148\n","INFO:tensorflow:Step 16350 | Loss: 1.4664 | Loss_intent: 1.0806 | Loss_slots: 0.3858 | Spent: 4.9 secs | LR: 0.000150\n","INFO:tensorflow:Step 16400 | Loss: 1.5193 | Loss_intent: 1.0819 | Loss_slots: 0.4374 | Spent: 4.8 secs | LR: 0.000152\n","INFO:tensorflow:Step 16450 | Loss: 1.3834 | Loss_intent: 1.0810 | Loss_slots: 0.3023 | Spent: 4.8 secs | LR: 0.000153\n","INFO:tensorflow:Step 16500 | Loss: 1.4058 | Loss_intent: 1.0800 | Loss_slots: 0.3257 | Spent: 4.7 secs | LR: 0.000155\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.953, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 16550 | Loss: 1.4549 | Loss_intent: 1.0805 | Loss_slots: 0.3744 | Spent: 8.1 secs | LR: 0.000157\n","INFO:tensorflow:Step 16600 | Loss: 1.4239 | Loss_intent: 1.0812 | Loss_slots: 0.3427 | Spent: 4.9 secs | LR: 0.000159\n","INFO:tensorflow:Step 16650 | Loss: 1.2875 | Loss_intent: 1.0856 | Loss_slots: 0.2019 | Spent: 4.7 secs | LR: 0.000160\n","INFO:tensorflow:Step 16700 | Loss: 1.3806 | Loss_intent: 1.0836 | Loss_slots: 0.2970 | Spent: 4.8 secs | LR: 0.000162\n","INFO:tensorflow:Step 16750 | Loss: 1.4031 | Loss_intent: 1.0804 | Loss_slots: 0.3227 | Spent: 4.8 secs | LR: 0.000164\n","INFO:tensorflow:Step 16800 | Loss: 1.4628 | Loss_intent: 1.0806 | Loss_slots: 0.3822 | Spent: 4.7 secs | LR: 0.000166\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.970\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 16850 | Loss: 1.2979 | Loss_intent: 1.0804 | Loss_slots: 0.2175 | Spent: 8.0 secs | LR: 0.000167\n","INFO:tensorflow:Step 16900 | Loss: 1.3848 | Loss_intent: 1.0866 | Loss_slots: 0.2982 | Spent: 4.8 secs | LR: 0.000169\n","INFO:tensorflow:Step 16950 | Loss: 1.3684 | Loss_intent: 1.0811 | Loss_slots: 0.2873 | Spent: 4.8 secs | LR: 0.000171\n","INFO:tensorflow:Step 17000 | Loss: 1.4044 | Loss_intent: 1.0810 | Loss_slots: 0.3234 | Spent: 4.8 secs | LR: 0.000173\n","INFO:tensorflow:Step 17050 | Loss: 1.3777 | Loss_intent: 1.0817 | Loss_slots: 0.2959 | Spent: 4.8 secs | LR: 0.000174\n","INFO:tensorflow:Step 17100 | Loss: 1.4589 | Loss_intent: 1.0977 | Loss_slots: 0.3612 | Spent: 4.8 secs | LR: 0.000176\n","INFO:tensorflow:Step 17150 | Loss: 1.4194 | Loss_intent: 1.0808 | Loss_slots: 0.3386 | Spent: 4.7 secs | LR: 0.000178\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.969\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 17200 | Loss: 1.3481 | Loss_intent: 1.1033 | Loss_slots: 0.2448 | Spent: 8.2 secs | LR: 0.000180\n","INFO:tensorflow:Step 17250 | Loss: 1.3269 | Loss_intent: 1.0830 | Loss_slots: 0.2439 | Spent: 4.9 secs | LR: 0.000181\n","INFO:tensorflow:Step 17300 | Loss: 1.3988 | Loss_intent: 1.1282 | Loss_slots: 0.2706 | Spent: 4.8 secs | LR: 0.000183\n","INFO:tensorflow:Step 17350 | Loss: 1.4494 | Loss_intent: 1.0824 | Loss_slots: 0.3669 | Spent: 4.8 secs | LR: 0.000185\n","INFO:tensorflow:Step 17400 | Loss: 1.3703 | Loss_intent: 1.0826 | Loss_slots: 0.2877 | Spent: 4.8 secs | LR: 0.000187\n","INFO:tensorflow:Step 17450 | Loss: 1.3108 | Loss_intent: 1.0828 | Loss_slots: 0.2280 | Spent: 4.9 secs | LR: 0.000187\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.952, Intent Acc: 0.973\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.983     0.991     0.987       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      0.974     1.000     0.987        38\n","                       atis_abbreviation      0.941     0.970     0.955        33\n","                           atis_aircraft      0.889     0.889     0.889         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.375     1.000     0.545         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      1.000     1.000     1.000        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      1.000     0.667     0.800         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     1.000     1.000        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.833     0.909         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.973     0.979     0.976       888\n","                               macro avg      0.676     0.670     0.661       888\n","                            weighted avg      0.977     0.979     0.976       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.975     0.997     0.986     716.0\n","         B-fromloc.city_name      0.989     0.999     0.994     704.0\n","           I-toloc.city_name      0.964     1.000     0.981     265.0\n","      B-depart_date.day_name      0.990     0.976     0.983     212.0\n","              B-airline_name      0.981     1.000     0.990     101.0\n","         I-fromloc.city_name      0.978     0.989     0.983     177.0\n"," B-depart_time.period_of_day      1.000     0.885     0.939     130.0\n","              I-airline_name      1.000     1.000     1.000      65.0\n","    B-depart_date.day_number      0.981     0.964     0.972      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      0.960     1.000     0.980      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.881     1.000     0.937      52.0\n","         B-stoploc.city_name      1.000     1.000     1.000      20.0\n","                 B-city_name      0.868     0.579     0.695      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.943     0.971     0.957      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      0.971     0.971     0.971      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     1.000     1.000      15.0\n","      I-fromloc.airport_name      0.405     1.000     0.577      15.0\n","      B-fromloc.airport_name      0.423     0.917     0.579      12.0\n","      B-arrive_date.day_name      0.647     1.000     0.786      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.440     1.000     0.611      11.0\n"," B-depart_date.date_relative      0.938     0.882     0.909      17.0\n","          B-toloc.state_name      0.848     1.000     0.918      28.0\n","           B-fare_basis_code      0.944     1.000     0.971      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.909     0.345     0.500      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      1.000     0.467     0.636      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      1.000     0.900     0.947      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     0.800     0.889      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.941     0.941     0.941      17.0\n","              B-airport_name      0.778     0.333     0.467      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      1.000     0.909     0.952      33.0\n","                       B-mod      1.000     0.500     0.667       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      1.000     0.667     0.800       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      0.500     1.000     0.667       1.0\n","          B-restriction_code      0.500     1.000     0.667       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.875     0.875     0.875       8.0\n","      I-arrive_time.end_time      0.875     0.875     0.875       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      0.833     1.000     0.909       5.0\n","          I-restriction_code      1.000     1.000     1.000       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.500     1.000     0.667       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      1.000     0.333     0.500       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      1.000     1.000     1.000       1.0\n"," I-depart_time.period_of_day      0.500     1.000     0.667       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.250     0.400       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      1.000     1.000     1.000       1.0\n","                 B-days_code      1.000     1.000     1.000       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.750     1.000     0.857       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.951     0.953     0.952    3657.0\n","                   macro avg      0.670     0.659     0.645    3657.0\n","                weighted avg      0.957     0.953     0.949    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 17500 | Loss: 1.4108 | Loss_intent: 1.0814 | Loss_slots: 0.3294 | Spent: 8.1 secs | LR: 0.000185\n","INFO:tensorflow:Step 17550 | Loss: 1.3992 | Loss_intent: 1.0825 | Loss_slots: 0.3167 | Spent: 4.7 secs | LR: 0.000183\n","INFO:tensorflow:Step 17600 | Loss: 1.4882 | Loss_intent: 1.0837 | Loss_slots: 0.4045 | Spent: 4.8 secs | LR: 0.000181\n","INFO:tensorflow:Step 17650 | Loss: 1.4163 | Loss_intent: 1.0810 | Loss_slots: 0.3353 | Spent: 4.7 secs | LR: 0.000180\n","INFO:tensorflow:Step 17700 | Loss: 1.4864 | Loss_intent: 1.0949 | Loss_slots: 0.3915 | Spent: 4.7 secs | LR: 0.000178\n","INFO:tensorflow:Step 17750 | Loss: 1.4250 | Loss_intent: 1.0808 | Loss_slots: 0.3443 | Spent: 5.1 secs | LR: 0.000176\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.969\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 17800 | Loss: 1.3835 | Loss_intent: 1.0813 | Loss_slots: 0.3021 | Spent: 8.7 secs | LR: 0.000174\n","INFO:tensorflow:Step 17850 | Loss: 1.3915 | Loss_intent: 1.0812 | Loss_slots: 0.3103 | Spent: 4.8 secs | LR: 0.000172\n","INFO:tensorflow:Step 17900 | Loss: 1.4693 | Loss_intent: 1.0818 | Loss_slots: 0.3875 | Spent: 4.8 secs | LR: 0.000171\n","INFO:tensorflow:Step 17950 | Loss: 1.4395 | Loss_intent: 1.0820 | Loss_slots: 0.3575 | Spent: 4.8 secs | LR: 0.000169\n","INFO:tensorflow:Step 18000 | Loss: 1.3556 | Loss_intent: 1.0810 | Loss_slots: 0.2747 | Spent: 4.8 secs | LR: 0.000167\n","INFO:tensorflow:Step 18050 | Loss: 1.3581 | Loss_intent: 1.0814 | Loss_slots: 0.2766 | Spent: 4.7 secs | LR: 0.000165\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 18100 | Loss: 1.2628 | Loss_intent: 1.0805 | Loss_slots: 0.1822 | Spent: 8.1 secs | LR: 0.000164\n","INFO:tensorflow:Step 18150 | Loss: 1.4208 | Loss_intent: 1.0804 | Loss_slots: 0.3404 | Spent: 4.8 secs | LR: 0.000162\n","INFO:tensorflow:Step 18200 | Loss: 1.4115 | Loss_intent: 1.0874 | Loss_slots: 0.3241 | Spent: 4.7 secs | LR: 0.000160\n","INFO:tensorflow:Step 18250 | Loss: 1.4096 | Loss_intent: 1.0804 | Loss_slots: 0.3292 | Spent: 4.7 secs | LR: 0.000158\n","INFO:tensorflow:Step 18300 | Loss: 1.2829 | Loss_intent: 1.0820 | Loss_slots: 0.2009 | Spent: 4.8 secs | LR: 0.000157\n","INFO:tensorflow:Step 18350 | Loss: 1.3815 | Loss_intent: 1.0821 | Loss_slots: 0.2995 | Spent: 4.7 secs | LR: 0.000155\n","INFO:tensorflow:Step 18400 | Loss: 1.3523 | Loss_intent: 1.0808 | Loss_slots: 0.2714 | Spent: 4.8 secs | LR: 0.000153\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.954, Intent Acc: 0.972\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 18450 | Loss: 1.3020 | Loss_intent: 1.0840 | Loss_slots: 0.2180 | Spent: 8.1 secs | LR: 0.000151\n","INFO:tensorflow:Step 18500 | Loss: 1.4251 | Loss_intent: 1.0833 | Loss_slots: 0.3417 | Spent: 4.7 secs | LR: 0.000150\n","INFO:tensorflow:Step 18550 | Loss: 1.3623 | Loss_intent: 1.0822 | Loss_slots: 0.2801 | Spent: 4.6 secs | LR: 0.000148\n","INFO:tensorflow:Step 18600 | Loss: 1.3192 | Loss_intent: 1.0808 | Loss_slots: 0.2384 | Spent: 4.7 secs | LR: 0.000146\n","INFO:tensorflow:Step 18650 | Loss: 1.3293 | Loss_intent: 1.0837 | Loss_slots: 0.2456 | Spent: 4.7 secs | LR: 0.000144\n","INFO:tensorflow:Step 18700 | Loss: 1.4442 | Loss_intent: 1.0888 | Loss_slots: 0.3554 | Spent: 4.8 secs | LR: 0.000143\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.975\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 18750 | Loss: 1.4954 | Loss_intent: 1.0813 | Loss_slots: 0.4142 | Spent: 7.9 secs | LR: 0.000141\n","INFO:tensorflow:Step 18800 | Loss: 1.4497 | Loss_intent: 1.0826 | Loss_slots: 0.3671 | Spent: 4.7 secs | LR: 0.000139\n","INFO:tensorflow:Step 18850 | Loss: 1.3393 | Loss_intent: 1.0818 | Loss_slots: 0.2575 | Spent: 4.7 secs | LR: 0.000137\n","INFO:tensorflow:Step 18900 | Loss: 1.4523 | Loss_intent: 1.0816 | Loss_slots: 0.3708 | Spent: 4.8 secs | LR: 0.000136\n","INFO:tensorflow:Step 18950 | Loss: 1.3207 | Loss_intent: 1.0814 | Loss_slots: 0.2392 | Spent: 4.7 secs | LR: 0.000134\n","INFO:tensorflow:Step 19000 | Loss: 1.4440 | Loss_intent: 1.0832 | Loss_slots: 0.3607 | Spent: 4.8 secs | LR: 0.000132\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 19050 | Loss: 1.4943 | Loss_intent: 1.0809 | Loss_slots: 0.4134 | Spent: 8.0 secs | LR: 0.000130\n","INFO:tensorflow:Step 19100 | Loss: 1.4131 | Loss_intent: 1.0840 | Loss_slots: 0.3291 | Spent: 4.7 secs | LR: 0.000129\n","INFO:tensorflow:Step 19150 | Loss: 1.5775 | Loss_intent: 1.0841 | Loss_slots: 0.4934 | Spent: 4.7 secs | LR: 0.000127\n","INFO:tensorflow:Step 19200 | Loss: 1.4280 | Loss_intent: 1.0914 | Loss_slots: 0.3366 | Spent: 4.7 secs | LR: 0.000125\n","INFO:tensorflow:Step 19250 | Loss: 1.4192 | Loss_intent: 1.0805 | Loss_slots: 0.3387 | Spent: 4.7 secs | LR: 0.000123\n","INFO:tensorflow:Step 19300 | Loss: 1.4016 | Loss_intent: 1.0832 | Loss_slots: 0.3185 | Spent: 4.7 secs | LR: 0.000122\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.971\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 19350 | Loss: 1.4909 | Loss_intent: 1.0814 | Loss_slots: 0.4095 | Spent: 8.0 secs | LR: 0.000120\n","INFO:tensorflow:Step 19400 | Loss: 1.4052 | Loss_intent: 1.0819 | Loss_slots: 0.3234 | Spent: 4.7 secs | LR: 0.000118\n","INFO:tensorflow:Step 19450 | Loss: 1.5072 | Loss_intent: 1.0808 | Loss_slots: 0.4264 | Spent: 4.8 secs | LR: 0.000116\n","INFO:tensorflow:Step 19500 | Loss: 1.3607 | Loss_intent: 1.0808 | Loss_slots: 0.2799 | Spent: 4.9 secs | LR: 0.000114\n","INFO:tensorflow:Step 19550 | Loss: 1.4114 | Loss_intent: 1.0811 | Loss_slots: 0.3303 | Spent: 4.8 secs | LR: 0.000113\n","INFO:tensorflow:Step 19600 | Loss: 1.3814 | Loss_intent: 1.0834 | Loss_slots: 0.2979 | Spent: 4.7 secs | LR: 0.000111\n","INFO:tensorflow:Step 19650 | Loss: 1.4371 | Loss_intent: 1.0817 | Loss_slots: 0.3554 | Spent: 4.8 secs | LR: 0.000109\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.955, Intent Acc: 0.973\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n","Reading ../data/atis.train.w-intent.iob\n","INFO:tensorflow:Step 19700 | Loss: 1.3759 | Loss_intent: 1.0806 | Loss_slots: 0.2953 | Spent: 8.0 secs | LR: 0.000107\n","INFO:tensorflow:Step 19750 | Loss: 1.3666 | Loss_intent: 1.0816 | Loss_slots: 0.2850 | Spent: 4.6 secs | LR: 0.000106\n","INFO:tensorflow:Step 19800 | Loss: 1.3372 | Loss_intent: 1.0805 | Loss_slots: 0.2568 | Spent: 4.8 secs | LR: 0.000104\n","INFO:tensorflow:Step 19850 | Loss: 1.4057 | Loss_intent: 1.0800 | Loss_slots: 0.3258 | Spent: 4.8 secs | LR: 0.000102\n","INFO:tensorflow:Step 19900 | Loss: 1.3969 | Loss_intent: 1.0977 | Loss_slots: 0.2992 | Spent: 4.7 secs | LR: 0.000100\n","INFO:tensorflow:Step 19950 | Loss: 1.3106 | Loss_intent: 1.0830 | Loss_slots: 0.2276 | Spent: 4.7 secs | LR: 0.000101\n","Reading ../data/atis.test.w-intent.iob\n","INFO:tensorflow:Slot F1: 0.956, Intent Acc: 0.971\n","INFO:tensorflow:\n","                                          precision    recall  f1-score   support\n","\n","                             atis_flight      0.980     0.991     0.985       632\n","                            atis_airfare      0.979     0.979     0.979        48\n","                     atis_ground_service      1.000     1.000     1.000        36\n","                            atis_airline      1.000     0.974     0.987        38\n","                       atis_abbreviation      0.939     0.939     0.939        33\n","                           atis_aircraft      0.889     0.889     0.889         9\n","                        atis_flight_time      1.000     1.000     1.000         1\n","                           atis_quantity      0.375     1.000     0.545         3\n","                atis_flight#atis_airfare      0.833     0.417     0.556        12\n","                            atis_airport      1.000     1.000     1.000        18\n","                           atis_distance      1.000     1.000     1.000        10\n","                               atis_city      1.000     0.667     0.800         6\n","                        atis_ground_fare      1.000     1.000     1.000         7\n","                           atis_capacity      1.000     1.000     1.000        21\n","                          atis_flight_no      0.889     1.000     0.941         8\n","                               atis_meal      1.000     0.833     0.909         6\n","                        atis_restriction      0.000     0.000     0.000         0\n","             atis_airline#atis_flight_no      0.000     0.000     0.000         0\n","    atis_ground_service#atis_ground_fare      0.000     0.000     0.000         0\n","           atis_airfare#atis_flight_time      0.000     0.000     0.000         0\n","                           atis_cheapest      0.000     0.000     0.000         0\n","atis_aircraft#atis_flight#atis_flight_no      0.000     0.000     0.000         0\n","\n","                               micro avg      0.971     0.976     0.974       888\n","                               macro avg      0.677     0.668     0.660       888\n","                            weighted avg      0.976     0.976     0.974       888\n","\n","INFO:tensorflow:\n","                              precision    recall  f1-score   support\n","\n","                           O      0.000     0.000     0.000       0.0\n","           B-toloc.city_name      0.973     0.996     0.984     716.0\n","         B-fromloc.city_name      0.990     0.999     0.994     704.0\n","           I-toloc.city_name      0.978     1.000     0.989     265.0\n","      B-depart_date.day_name      0.990     0.981     0.986     212.0\n","              B-airline_name      0.981     1.000     0.990     101.0\n","         I-fromloc.city_name      0.983     0.994     0.989     177.0\n"," B-depart_time.period_of_day      1.000     0.908     0.952     130.0\n","              I-airline_name      1.000     1.000     1.000      65.0\n","    B-depart_date.day_number      0.981     0.964     0.972      55.0\n","    B-depart_date.month_name      0.982     0.964     0.973      56.0\n","          B-depart_time.time      0.851     1.000     0.919      57.0\n","                B-round_trip      1.000     0.986     0.993      73.0\n","             B-cost_relative      1.000     0.973     0.986      37.0\n","                I-round_trip      1.000     1.000     1.000      71.0\n","                B-flight_mod      0.960     1.000     0.980      24.0\n"," B-depart_time.time_relative      0.984     0.969     0.977      65.0\n","          I-depart_time.time      0.929     1.000     0.963      52.0\n","         B-stoploc.city_name      1.000     1.000     1.000      20.0\n","                 B-city_name      0.842     0.561     0.674      57.0\n","                B-class_type      0.960     1.000     0.980      24.0\n","          B-arrive_time.time      0.971     0.971     0.971      34.0\n"," B-arrive_time.time_relative      0.857     0.968     0.909      31.0\n","                I-class_type      1.000     1.000     1.000      17.0\n","               B-flight_stop      1.000     1.000     1.000      21.0\n","          I-arrive_time.time      1.000     0.971     0.986      35.0\n","              B-airline_code      1.000     0.853     0.921      34.0\n","    I-depart_date.day_number      1.000     0.933     0.966      15.0\n","      I-fromloc.airport_name      0.417     1.000     0.588      15.0\n","      B-fromloc.airport_name      0.429     1.000     0.600      12.0\n","      B-arrive_date.day_name      0.688     1.000     0.815      11.0\n","          B-toloc.state_code      1.000     1.000     1.000      18.0\n","B-depart_date.today_relative      1.000     1.000     1.000       9.0\n","             B-flight_number      0.524     1.000     0.688      11.0\n"," B-depart_date.date_relative      0.938     0.882     0.909      17.0\n","          B-toloc.state_name      0.933     1.000     0.966      28.0\n","           B-fare_basis_code      0.944     1.000     0.971      17.0\n","               B-flight_time      1.000     1.000     1.000       1.0\n","                        B-or      1.000     1.000     1.000       3.0\n"," B-arrive_time.period_of_day      0.750     1.000     0.857       6.0\n","          B-meal_description      1.000     0.900     0.947      10.0\n","             I-cost_relative      1.000     0.667     0.800       3.0\n","              I-airport_name      0.917     0.379     0.537      29.0\n","               B-fare_amount      0.400     1.000     0.571       2.0\n","               I-fare_amount      1.000     1.000     1.000       2.0\n","                 I-city_name      1.000     0.533     0.696      30.0\n","        I-toloc.airport_name      1.000     1.000     1.000       3.0\n","            B-transport_type      1.000     1.000     1.000      10.0\n","    B-arrive_date.month_name      0.714     0.833     0.769       6.0\n","    B-arrive_date.day_number      0.714     0.833     0.769       6.0\n","         I-stoploc.city_name      1.000     1.000     1.000      10.0\n","                      B-meal      1.000     1.000     1.000      16.0\n","        B-fromloc.state_code      1.000     1.000     1.000      23.0\n","    B-depart_time.period_mod      0.833     1.000     0.909       5.0\n","                   B-connect      1.000     1.000     1.000       6.0\n","               B-flight_days      1.000     1.000     1.000      10.0\n","        B-toloc.airport_name      1.000     1.000     1.000       3.0\n","        B-fromloc.state_name      0.895     1.000     0.944      17.0\n","              B-airport_name      0.800     0.381     0.516      21.0\n","                   B-economy      1.000     1.000     1.000       6.0\n","               I-flight_time      1.000     1.000     1.000       1.0\n","             B-aircraft_code      1.000     0.909     0.952      33.0\n","                       B-mod      0.500     0.500     0.500       2.0\n","              B-airport_code      0.000     0.000     0.000       9.0\n","    B-depart_time.start_time      1.000     0.667     0.800       3.0\n","      B-depart_time.end_time      1.000     0.667     0.800       3.0\n","          B-depart_date.year      1.000     0.667     0.800       3.0\n","            I-transport_type      1.000     1.000     1.000       1.0\n","          B-restriction_code      0.500     1.000     0.667       4.0\n","    B-arrive_time.start_time      0.889     1.000     0.941       8.0\n","        B-toloc.airport_code      1.000     0.750     0.857       4.0\n","      B-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-arrive_time.end_time      0.889     1.000     0.941       8.0\n","      I-depart_time.end_time      1.000     0.667     0.800       3.0\n","               I-flight_stop      0.000     0.000     0.000       0.0\n","      B-fromloc.airport_code      1.000     1.000     1.000       5.0\n","          I-restriction_code      1.000     1.000     1.000       3.0\n","    I-depart_time.start_time      1.000     1.000     1.000       1.0\n","          I-toloc.state_name      1.000     1.000     1.000       1.0\n","I-depart_date.today_relative      0.000     0.000     0.000       0.0\n"," B-arrive_date.date_relative      0.500     1.000     0.667       2.0\n","                I-flight_mod      1.000     0.167     0.286       6.0\n","                   I-economy      0.000     0.000     0.000       0.0\n"," B-return_date.date_relative      1.000     0.333     0.500       3.0\n","        I-fromloc.state_name      1.000     1.000     1.000       1.0\n","                B-state_code      1.000     1.000     1.000       1.0\n","    I-arrive_time.start_time      1.000     1.000     1.000       1.0\n","    I-arrive_date.day_number      0.000     0.000     0.000       0.0\n","                 B-meal_code      1.000     1.000     1.000       1.0\n"," I-depart_time.period_of_day      1.000     1.000     1.000       1.0\n","                  B-day_name      1.000     0.500     0.667       2.0\n","             B-period_of_day      1.000     0.500     0.667       4.0\n","        B-stoploc.state_code      0.000     0.000     0.000       0.0\n","    B-return_date.month_name      0.000     0.000     0.000       0.0\n","    B-return_date.day_number      0.000     0.000     0.000       0.0\n","    B-arrive_time.period_mod      0.000     0.000     0.000       0.0\n","                 I-meal_code      0.000     0.000     0.000       0.0\n","        B-toloc.country_name      1.000     1.000     1.000       1.0\n","                 B-days_code      0.333     1.000     0.500       1.0\n"," I-arrive_time.period_of_day      0.000     0.000     0.000       0.0\n","            I-today_relative      0.000     0.000     0.000       0.0\n"," B-return_time.period_of_day      0.000     0.000     0.000       0.0\n","                      B-time      0.000     0.000     0.000       0.0\n","           I-fare_basis_code      0.000     0.000     0.000       0.0\n"," I-arrive_time.time_relative      0.000     0.000     0.000       4.0\n"," I-depart_time.time_relative      0.000     0.000     0.000       1.0\n","            B-today_relative      0.000     0.000     0.000       0.0\n","                B-state_name      0.000     0.000     0.000       9.0\n","B-arrive_date.today_relative      0.000     0.000     0.000       0.0\n","    B-return_time.period_mod      0.000     0.000     0.000       0.0\n","                B-month_name      0.000     0.000     0.000       0.0\n","                B-day_number      0.000     0.000     0.000       0.0\n"," I-return_date.date_relative      0.750     1.000     0.857       3.0\n","I-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-stoploc.airport_name      0.000     0.000     0.000       0.0\n","             B-time_relative      0.000     0.000     0.000       0.0\n","                      I-time      0.000     0.000     0.000       0.0\n","    I-return_date.day_number      0.000     0.000     0.000       0.0\n","          I-meal_description      0.000     0.000     0.000       0.0\n","B-return_date.today_relative      0.000     0.000     0.000       0.0\n","      B-return_date.day_name      0.000     0.000     0.000       2.0\n","\n","                   micro avg      0.955     0.957     0.956    3657.0\n","                   macro avg      0.672     0.667     0.653    3657.0\n","                weighted avg      0.960     0.957     0.953    3657.0\n","\n","INFO:tensorflow:Best Slot F1: 0.956, Intent Acc: 0.972\n"],"name":"stdout"}]}]}