Toxic_BERT_(TPU).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Toxic BERT (TPU).ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "sjQ5cGnOOryq"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ostamand/bert-classifier/blob/master/Toxic_BERT_(TPU).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsIyvG30p9Y5",
        "colab_type": "text"
      },
      "source": [
        "# Toxic BERT (TPU)\n",
        "Reference: \n",
        "\n",
        "- https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb\n",
        "- https://github.com/google-research/bert"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hRfZbnCPqdyR",
        "colab_type": "text"
      },
      "source": [
        "## Install"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYCrx9J4sWGt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python --version"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbDoct7BYqfw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip uninstall -y tensorflow"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JwHBHnZ3Yuru",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorflow-gpu>=1.11.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5MpVCqmOp3Ju",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/google-research/bert bert"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_iOIU5jmPYju",
        "colab_type": "text"
      },
      "source": [
        "## Setup TensorBoard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pP38eqLXcd62",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip ngrok-stable-linux-amd64.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GCPDGT_AWk8P",
        "colab_type": "text"
      },
      "source": [
        "Start tensorboad"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n5r3Pu_hPcCi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "get_ipython().system_raw(\n",
        "    f'tensorboard --logdir {config.output_dir} --host 0.0.0.0 --port 6006 &'\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bAsMOdnPrDS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "get_ipython().system_raw('./ngrok http 6006 &')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYd-bheVPs9o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8WQsktGWYrO",
        "colab_type": "text"
      },
      "source": [
        "To restart tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xnsWPrAPWCzu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ps"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V1EaHXWKYJMe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!kill 3025"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CfmL7HGWfyJh",
        "colab_type": "text"
      },
      "source": [
        "Tensorboard debugging commands:\n",
        "\n",
        "`!find ./tmp | grep tfevents`\n",
        "\n",
        "`!tensorboard --inspect --logdir tmp`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPWtREoNbHLs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext tensorboard"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oWzrrULDXord",
        "colab_type": "text"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGuyv85aPzov",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QiTyD0nqqhgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sys.path.insert(0, 'bert')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d_7AsRiYqwBk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import json\n",
        "import pprint\n",
        "import pdb\n",
        "\n",
        "import modeling\n",
        "import optimization\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow as tf\n",
        "\n",
        "from run_classifier_with_tfhub import create_tokenizer_from_hub_module\n",
        "from run_classifier import convert_examples_to_features, DataProcessor, InputExample\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jWNYN3dDNnnQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.VERSION"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUTTjJajpZ3l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.logging.set_verbosity(tf.logging.INFO)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CWT_jGyxrtz1",
        "colab_type": "text"
      },
      "source": [
        "## Setup gdrive\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdMXm-9jD6V5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bg2whUzQgv63",
        "colab_type": "text"
      },
      "source": [
        "## Config"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMmpeEjVgvim",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Config:\n",
        "  def __init__(self, *args, **kwargs):\n",
        "    self.lr = 2e-5\n",
        "    self.aux_targets=6\n",
        "    self.module_handle = 'https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1'\n",
        "    self.maxlen = 220 \n",
        "    self.bs = 8\n",
        "    self.epochs = 1\n",
        "    self.output_dir = 'gs://bert-train-logs/toxic'\n",
        "    self.train_batch_size = 32\n",
        "    self.eval_batch_size = 8\n",
        "    self.predict_batch_size = 8\n",
        "    self.save_checkpoints_steps = 10 \n",
        "    self.save_summary_steps = 10 \n",
        "    \n",
        "    # tpu stuff\n",
        "    self.num_tpu_cores = 8\n",
        "    self.iterations_per_loop = 1000\n",
        "    \n",
        "config = Config()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5TrXKYzOrid3",
        "colab_type": "text"
      },
      "source": [
        "## Setup TPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GNbIVLf1rl-3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "assert 'COLAB_TPU_ADDR' in os.environ, 'ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!'\n",
        "TPU_ADDRESS = 'grpc://' + os.environ['COLAB_TPU_ADDR']\n",
        "print('TPU address is', TPU_ADDRESS)\n",
        "\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "with tf.Session(TPU_ADDRESS) as session:\n",
        "  print('TPU devices:')\n",
        "  pprint.pprint(session.list_devices())\n",
        "\n",
        "  # Upload credentials to TPU.\n",
        "  with open('/content/adc.json', 'r') as f:\n",
        "    auth_info = json.load(f)\n",
        "  tf.contrib.cloud.configure_gcs(session, credentials=auth_info)\n",
        "  # Now credentials are set for all future sessions on this TPU."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Myfcd5mmtNyn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Setup TPU related config\n",
        "tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(TPU_ADDRESS)\n",
        "\n",
        "def get_run_config(config):\n",
        "  return tf.contrib.tpu.RunConfig(\n",
        "    cluster=tpu_cluster_resolver,\n",
        "    model_dir=config.output_dir,\n",
        "    save_checkpoints_steps=config.save_checkpoints_steps,\n",
        "    tpu_config=tf.contrib.tpu.TPUConfig(\n",
        "        iterations_per_loop=config.iterations_per_loop,\n",
        "        num_shards=config.num_tpu_cores,\n",
        "        per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4zUwgLZGr3nh",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jogaq2-01OUc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ToxicProcessor(DataProcessor):\n",
        "  \n",
        "  def __init__(self, num_to_load=None):\n",
        "    self.num_to_load = num_to_load\n",
        "    self.label_list = ['0', '1']\n",
        "  \n",
        "  def get_train_examples(self, data_dir):\n",
        "    df = self._read_csv(os.path.join(data_dir, 'train.csv'))\n",
        "    if self.num_to_load:\n",
        "      df = df.iloc[:self.num_to_load]\n",
        "    df['comment_text'].astype(str)\n",
        "    examples = []\n",
        "    for i, row in df.iterrows():\n",
        "      guid = f\"train-{i}\"\n",
        "      label = '1' if row['target'] >= 0.5 else '0'\n",
        "      text_a = str(row['comment_text'])\n",
        "      examples.append(\n",
        "          InputExample(guid=guid, text_a=text_a, text_b=None, label=label)\n",
        "      )\n",
        "    return examples\n",
        "  \n",
        "  @classmethod\n",
        "  def _read_csv(cls, file_path):\n",
        "    df = pd.read_csv(file_path)\n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sR5ZK8-C1IgN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_model(config, input_ids, input_mask, segment_ids, label_ids):\n",
        "  tags = set()\n",
        "  tags.add('train')\n",
        "  \n",
        "  with tf.variable_scope('bert'):\n",
        "    bert_module = hub.Module(config.module_handle, tags=tags, trainable=True)\n",
        "    bert_inputs = dict(\n",
        "          input_ids=input_ids,\n",
        "          input_mask=input_mask,\n",
        "          segment_ids=segment_ids\n",
        "    )\n",
        "    bert_outputs = bert_module(\n",
        "          inputs=bert_inputs,\n",
        "          signature=\"tokens\",\n",
        "          as_dict=True\n",
        "    )\n",
        "    output_layer = bert_outputs[\"pooled_output\"]\n",
        "    hidden_size = output_layer.shape[-1].value\n",
        "  \n",
        "  # classifier layers on top of BERT\n",
        "  \n",
        "  with tf.variable_scope(\"classifier\"):\n",
        "    fc_weights = tf.get_variable(\n",
        "        \"fc_weights\", [1, hidden_size],\n",
        "        initializer=tf.truncated_normal_initializer(stddev=0.02)\n",
        "    )\n",
        "\n",
        "    fc_aux_weights = tf.get_variable(\n",
        "        \"fc_aux_weights\", [6, hidden_size],\n",
        "        initializer=tf.truncated_normal_initializer(stddev=0.02)\n",
        "    )\n",
        "\n",
        "    fc_bias = tf.get_variable(\n",
        "        \"fc_bias\", [1], initializer=tf.zeros_initializer()\n",
        "    )\n",
        "\n",
        "    fc_aux_bias = tf.get_variable(\n",
        "        \"fc_aux_bias\", [6], initializer=tf.zeros_initializer()\n",
        "    )\n",
        "    \n",
        "    output_layer = tf.nn.dropout(output_layer, rate=0.4)\n",
        "    logits_fc = tf.matmul(output_layer, fc_weights, transpose_b=True)\n",
        "    logits_fc = tf.nn.bias_add(logits_fc, fc_bias)\n",
        "    \n",
        "    # not used for now, for custom loss calculation \n",
        "    logits_fc_aux = tf.matmul(output_layer, fc_aux_weights, transpose_b=True)\n",
        "    logits_fc_aux = tf.nn.bias_add(logits_fc_aux, fc_aux_bias)\n",
        "    \n",
        "    logits = tf.concat([logits_fc, logits_fc_aux], axis=1, name='logits')\n",
        "    \n",
        "    # for now, use only labels \n",
        "    probs = tf.nn.sigmoid(logits_fc, name='probs')\n",
        "    \n",
        "    preds = tf.squeeze(tf.cast((probs >= 0.5), tf.float32), name='preds')\n",
        "    \n",
        "    logits_for_loss = tf.reshape(tf.slice(logits, [0, 0] , [-1, 1]), [-1])\n",
        "    \n",
        "    labels = tf.cast(label_ids, tf.float32, name='labels')\n",
        "    \n",
        "    eval_op = tf.metrics.accuracy(labels=labels, predictions=preds)\n",
        "    accuracy = tf.reduce_mean(tf.cast(tf.math.equal(preds, labels), tf.float32), name='accuracy')\n",
        "    #tf.summary.scalar('accuracy', accuracy)\n",
        "    \n",
        "  with tf.variable_scope(\"optimizer\"):\n",
        "    loss = tf.nn.sigmoid_cross_entropy_with_logits(\n",
        "      labels=labels,\n",
        "      logits=logits_for_loss\n",
        "    )\n",
        "    loss = tf.reduce_mean(loss, name='loss')\n",
        "    train_op = optimization.create_optimizer(loss, config.lr, config.num_train_steps, config.num_warmup_steps, True) # use tpu\n",
        "    \n",
        "  return loss, train_op, eval_op, accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-y_NC6U61WdI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_input_fn(features, seq_length, batch_size, mode):\n",
        "\n",
        "  def input_fn(params):\n",
        "    all_input_ids = []\n",
        "    all_input_mask = []\n",
        "    all_segment_ids = []\n",
        "    all_label_ids = []\n",
        "\n",
        "    for feature in features:\n",
        "      all_input_ids.append(feature.input_ids)\n",
        "      all_input_mask.append(feature.input_mask)\n",
        "      all_segment_ids.append(feature.segment_ids)\n",
        "      all_label_ids.append(feature.label_id)\n",
        "\n",
        "    num_examples = len(features)\n",
        "\n",
        "    dataset = tf.data.Dataset.from_tensor_slices({\n",
        "          \"input_ids\":\n",
        "              tf.constant(\n",
        "                  all_input_ids, shape=[num_examples, seq_length],\n",
        "                  dtype=tf.int32),\n",
        "          \"input_mask\":\n",
        "              tf.constant(\n",
        "                  all_input_mask,\n",
        "                  shape=[num_examples, seq_length],\n",
        "                  dtype=tf.int32),\n",
        "          \"segment_ids\":\n",
        "              tf.constant(\n",
        "                  all_segment_ids,\n",
        "                  shape=[num_examples, seq_length],\n",
        "                  dtype=tf.int32),\n",
        "          \"label_ids\":\n",
        "              tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32),\n",
        "      })\n",
        "\n",
        "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
        "      dataset = dataset.repeat()\n",
        "      dataset = dataset.shuffle(buffer_size=100)\n",
        "      dataset = dataset.batch(batch_size=batch_size, drop_remainder=True)\n",
        "    elif mode == tf.estimator.ModeKeys.EVAL:\n",
        "      dataset = dataset.batch(batch_size=batch_size, drop_remainder=True)\n",
        "\n",
        "    return dataset\n",
        "  \n",
        "  return input_fn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYfiCBnY1XgY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ref: https://www.tensorflow.org/guide/custom_estimators\n",
        "\n",
        "def model_fn(features, labels, mode, params):\n",
        "  input_ids = features[\"input_ids\"]\n",
        "  input_mask = features[\"input_mask\"]\n",
        "  segment_ids = features[\"segment_ids\"]\n",
        "  label_ids = features[\"label_ids\"]\n",
        "    \n",
        "  loss, train_op, eval_op, accuracy = build_model(\n",
        "      params['config'],\n",
        "      input_ids, \n",
        "      input_mask,\n",
        "      segment_ids,\n",
        "      label_ids\n",
        "  )\n",
        "    \n",
        "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
        "    spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
        "        mode=mode,\n",
        "        loss=loss,\n",
        "        train_op=train_op\n",
        "    )\n",
        "  elif mode == tf.estimator.ModeKeys.EVAL:\n",
        "    \n",
        "    def metric_fn(loss, eval_op):\n",
        "      return {\n",
        "          \"eval_accuracy\": eval_op,\n",
        "          \"eval_loss\": loss\n",
        "      }\n",
        "    eval_metrics = (metric_fn, [loss, eval_op])\n",
        "      \n",
        "    spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
        "        mode=mode,\n",
        "        loss=loss,\n",
        "        eval_metrics=eval_metrics\n",
        "    )\n",
        "\n",
        "  return spec  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "anfsDZ8h1sYe",
        "colab_type": "text"
      },
      "source": [
        "## Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2jbL5j8eP_Gu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "processor = ToxicProcessor(num_to_load=100000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tl724M9WHhNg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "examples = processor.get_train_examples('gdrive/My Drive/code/jigsaw-bias-toxicity/data/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3p-lae7Iwc79",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = create_tokenizer_from_hub_module(config.module_handle)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WzwqqyCz4M3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "features = convert_examples_to_features(examples, processor.label_list, config.maxlen, tokenizer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9LlztNGiOo_k",
        "colab_type": "text"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tZNfIFX0uWC9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "config.num_train_steps =  int(len(features) / config.train_batch_size * config.epochs)\n",
        "config.num_eval_steps = int(len(features) / config.eval_batch_size)\n",
        "config.num_warmup_steps = int(config.num_train_steps * 0.05)\n",
        "\n",
        "# Force TF Hub writes to the GS bucket we provide.\n",
        "# To run with TPU all files need to be accessible on Google Storage\n",
        "os.environ['TFHUB_CACHE_DIR'] = config.output_dir\n",
        "\n",
        "classifier = tf.contrib.tpu.TPUEstimator(\n",
        "  use_tpu=True,\n",
        "  model_fn=model_fn,\n",
        "  config=get_run_config(config),\n",
        "  train_batch_size=config.train_batch_size,\n",
        "  eval_batch_size=config.eval_batch_size,\n",
        "  predict_batch_size=config.predict_batch_size,\n",
        "  params = {'config': config }\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WSO4xNxiU7ir",
        "colab_type": "text"
      },
      "source": [
        "`%tensorboard --logdir tmp/`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x0Yk8TcHZ-VU",
        "colab_type": "text"
      },
      "source": [
        "Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9SCra-nofb6a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time \n",
        "classifier.train(\n",
        "    input_fn= build_input_fn(features, config.maxlen, config.bs, tf.estimator.ModeKeys.TRAIN),\n",
        "    max_steps=config.num_train_steps\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YQpJGcdPHoHr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eval_result = classifier.evaluate(\n",
        "    input_fn=build_input_fn(features, config.maxlen, config.bs, tf.estimator.ModeKeys.EVAL),\n",
        "    steps=config.num_eval_steps\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QrQWn2eLXGpQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eval_result"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}