intent_recognition_with_bert.py

# -*- coding: utf-8 -*-
"""Intent Recognition with BERT

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1VE2z-4jtPXVlb2_HwZCCLhArkMO4v1u8

# BERT Embeddings with TensorFlow 2.0

- See BERT on paper: https://arxiv.org/pdf/1810.04805.pdf
- See BERT on GitHub: https://github.com/google-research/bert
- See BERT on TensorHub: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1
- See 'old' use of BERT for comparison: https://colab.research.google.com/github/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb
"""

!pip install tf-nightly

import tensorflow as tf

from tensorflow.python.client import device_lib
device_lib.list_local_devices()

import numpy as np
import tensorflow as tf
import os

import distutils
if distutils.version.LooseVersion(tf.__version__) < '2.0':
    raise Exception('This notebook is compatible with TensorFlow 2.0 or higher.')

"""# TPU Testing

Testing that either tensorflow geting access to TPU or not.
"""

tf.keras.backend.clear_session()

resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
print("All devices: ", tf.config.list_logical_devices('TPU'))

"""# Installing Packages"""

# Instaling Packages
!pip install tensorflow_hub
!pip install bert-for-tf2
!pip install sentencepiece
!pip install tf-hub-nightly

import tensorflow_hub as hub
print("TF version: ", tf.__version__)
print("Hub version: ", hub.__version__)

hub.__version__

"""## Importing Modules"""

import os
import math
import datetime

from tqdm import tqdm

import pandas as pd
import numpy as np

from tensorflow import keras

import bert # Importing BERT
from bert import BertModelLayer
from bert.loader import StockBertConfig, map_stock_config_to_params, load_stock_weights
from bert.tokenization.bert_tokenization import FullTokenizer

import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
from matplotlib import rc

from sklearn.metrics import confusion_matrix, classification_report

"""## Configurations"""

# Commented out IPython magic to ensure Python compatibility.
# COnfiguration 
# %matplotlib inline
# %config InlineBackend.figure_format='retina'

sns.set(style='whitegrid', palette='muted', font_scale=1.2)

HAPPY_COLORS_PALETTE = ["#01BEFE", "#FFDD00", "#FF7D00", "#FF006D", "#ADFF02", "#8F00FF"]

sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))

rcParams['figure.figsize'] = 12, 8

RANDOM_SEED = 42

np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)

"""## Downloading Data Set"""

!gdown --id 1OlcvGWReJMuyYQuOZm149vHWwPtlboR6 --output train.csv
!gdown --id 1Oi5cRlTybuIF2Fl5Bfsr-KkqrXrdt77w --output valid.csv
!gdown --id 1ep9H6-HvhB4utJRLVcLzieWNUSG3P_uF --output test.csv

# Reading Data from csvfiles
train = pd.read_csv("train.csv")
valid = pd.read_csv("valid.csv")
test = pd.read_csv("test.csv")

# merging train and validation data
train = train.append(valid).reset_index(drop=True)

train.head()

train.shape

# Visulising the data
chart = sns.countplot(train.intent, palette=HAPPY_COLORS_PALETTE)
plt.title("Number of texts per intent")
chart.set_xticklabels(chart.get_xticklabels(), rotation=30, horizontalalignment='right');

"""# Intent Recognition


---

### Downloading pretrained BERT model
"""

# Download the Pre trained BERT Model
!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip

# Unziping the model zip file
!unzip uncased_L-12_H-768_A-12.zip

# Making 'model' folder
os.makedirs("model", exist_ok=True)

# Move all the files of model into 'model' folder
!mv uncased_L-12_H-768_A-12/ model

# Accessing the files of model
bert_model_name="uncased_L-12_H-768_A-12"

bert_ckpt_dir = os.path.join("model/", bert_model_name)
bert_ckpt_file = os.path.join(bert_ckpt_dir, "bert_model.ckpt")
bert_config_file = os.path.join(bert_ckpt_dir, "bert_config.json")

print(bert_ckpt_dir)
print(bert_ckpt_file)
print(bert_config_file)

"""## Data Preprocesing

Converting to tokens and then tokenids
"""

class IntentDetectionData:
  DATA_COLUMN = "text"
  LABEL_COLUMN = "intent"

  def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192):
    self.tokenizer = tokenizer
    self.max_seq_len = 0
    self.classes = classes
    
    train, test = map(lambda df: df.reindex(df[IntentDetectionData.DATA_COLUMN].str.len().sort_values().index), [train, test])
    
    ((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self._prepare, [train, test])

    print("max seq_len", self.max_seq_len)
    self.max_seq_len = min(self.max_seq_len, max_seq_len)
    self.train_x, self.test_x = map(self._pad, [self.train_x, self.test_x])

  def _prepare(self, df):
    x, y = [], []
    
    for _, row in tqdm(df.iterrows()):
      text, label = row[IntentDetectionData.DATA_COLUMN], row[IntentDetectionData.LABEL_COLUMN]
      tokens = self.tokenizer.tokenize(text)
      tokens = ["[CLS]"] + tokens + ["[SEP]"]
      token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
      self.max_seq_len = max(self.max_seq_len, len(token_ids))
      x.append(token_ids)
      y.append(self.classes.index(label))

    return np.array(x), np.array(y)

  def _pad(self, ids):
    x = []
    for input_ids in ids:
      input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)]
      input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids))
      x.append(np.array(input_ids))
    return np.array(x)

# Tokenization of vacablery
tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))

tokenizer.vocab.keys()

"""## Creating Model

Use BERT pretrained model and fine tune it for our problem by adding some layers.
"""

def Create_Modle(max_seq_len, bert_ckpt_file):
  with tf.io.gfile.GFile(bert_config_file,'r') as reader:
    bc = StockBertConfig.from_json_string(reader.read())
    bert_params = map_stock_config_to_params(bc)
    bert_params.adapter_size = None
    # Creating Model
    bert = BertModelLayer.from_params(bert_params, name='bert')
  # Keras Input Layer
  input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32', name='input_ids')
  bert_output = bert(input_ids)
  # pirnting bert shape
  print('Bert Shape: ', bert_output.shape)

  cls_out = keras.layers.Lambda(lambda seq: seq[:,0,:])(bert_output)
  cls_out = keras.layers.Dropout(0.5)(cls_out)

  logits = keras.layers.Dense(units=768, activation='tanh')(cls_out)
  logits = keras.layers.Dropout(0.5)(logits)
  logits = keras.layers.Dense(units=len(classes), activation='softmax')(logits)

  model = keras.Model(inputs=input_ids, outputs=logits)
  model.build(input_shape = (None, max_seq_len))
  load_stock_weights(bert, bert_ckpt_file) # loading weights

  return model # returning model

"""## Traing Model"""

classes = train.intent.unique().tolist()
data = IntentDetectionData(train, test, tokenizer, classes, max_seq_len=128)

data.train_x.shape

data.test_x.shape

model = Create_Modle(data.max_seq_len, bert_ckpt_file)

model.summary()

"""## Compiling Model"""

model.compile(
    optimizer = keras.optimizers.Adam(1e-5),
    loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    # matrics = [keras.metrics.SparseCategoricalAccuracy(name='acc')]
)

"""## Fitting Model"""

# Commented out IPython magic to ensure Python compatibility.
# %load_ext tensorboard

# loging
log_dir = 'log/intent_detection/'+datetime.datetime.now().strftime('%Y%m%d-%H%M%s')
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=log_dir)

history = model.fit(
    x = data.train_x,
    y = data.train_y,
    validation_split = 0.1,
    batch_size = 16,
    shuffle = True,
    epochs = 4
)

# evaluate the model
scores = model.evaluate(data.test_x, data.test_y, verbose=0)
print(scores)

"""## Saving Model"""

model.save('./Intent_Trained_Bert-Model.h5')  # creates a HDF5 file 'my_model.h5'

"""## Evaluating Model"""

_, train_acc = model.evaluate(data.train_x, data.train_y)
_, test_acc = model.evaluate(data.test_x, data.test_y)

print('Training Accuracy: ',train_acc)
print('Testing Accuracy: ',test_acc)



"""# Prediting"""

# predictions by model over test_x data
y_pred = model.predict(data.test_x).argmax(axis=-1)

# printing precision, recall, f1-score, support
print(classification_report(data.test_y,y_pred, target_names=classes))

# Confusion Matrix
cm = confusion_matrix(data.test_y, y_pred)
df_cm = pd.DataFrame(cm, index=classes, columns=classes) # converting to panda dataframe

hmap = sns.heatmap(df_cm, annot=True, fmt='d')
hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')
hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')
plt.ylabel('True label')
plt.xlabel('Predicted label');

def prepare_Sentence(sentences):
  # preproceing on these sentences
  pred_tokens = map(tokenizer.tokenize, sentences) # tokenization
  pred_tokens = map(lambda tok: ["[CLS]"] + tok + ["[SEP]"], pred_tokens) # appending sperator
  pred_token_ids = list(map(tokenizer.convert_tokens_to_ids, pred_tokens)) # converting tokens to their ids

  pred_token_ids = map(lambda tids: tids+[0]*(data.max_seq_len - len(tids)),  pred_token_ids)
  pred_token_ids = np.array(list(pred_token_ids))
  return pred_token_ids

# Predicting Sentences
def predict_Sentence(sentence):
  pred_token_ids = prepare_Sentence(sentence)
  predictions = model.predict(pred_token_ids).argmax(axis = -1)
  for text, label in zip(sentence, predictions):
    print('Intent:', classes[label])
    print()

for cls in data.classes:
  print('> ',cls)

text = input('Enter Text: ')
predict_Sentence([text])

text = input('Enter Text: ')
predict_Sentence([text])

text = input('Enter Text: ')
predict_Sentence([text])