deep_lstm.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"8TFX7wSAmg9y"},"outputs":[{"name":"stdout","output_type":"stream","text":["WARNING:tensorflow:From c:\\Users\\quand\\anaconda3\\Lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n","\n"]}],"source":["import tensorflow as tf\n","import pandas as pd\n","import numpy as np\n","from string import digits\n","from collections import Counter\n","from pyvi import ViTokenizer\n","from gensim.models.word2vec import Word2Vec\n","from tensorflow.keras.utils import to_categorical"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"a7lMy03omg93","scrolled":true},"outputs":[],"source":["data_train = pd.read_csv(\"vlsp_sentiment_train.csv\", sep='\\t')\n","data_train.columns =['Class', 'Data']\n","data_test = pd.read_csv(\"vlsp_sentiment_test.csv\", sep='\\t')\n","data_test.columns =['Class', 'Data']"]},{"cell_type":"code","execution_count":3,"metadata":{"id":"4HR1jAzImg94"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Class</th>\n","      <th>Data</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>-1</td>\n","      <td>Mình đã dùng anywhere thế hệ đầu, quả là đầy t...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>-1</td>\n","      <td>Quan tâm nhất là độ trễ có cao không, dùng thi...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>-1</td>\n","      <td>dag xài con cùi bắp 98k....pin trâu, mỗi tội đ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>-1</td>\n","      <td>logitech chắc hàng phải tiền triệu trở lên dùn...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>-1</td>\n","      <td>Đang xài con m175 cùi mía , nhà xài nhiều chuộ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Class                                               Data\n","0     -1  Mình đã dùng anywhere thế hệ đầu, quả là đầy t...\n","1     -1  Quan tâm nhất là độ trễ có cao không, dùng thi...\n","2     -1  dag xài con cùi bắp 98k....pin trâu, mỗi tội đ...\n","3     -1  logitech chắc hàng phải tiền triệu trở lên dùn...\n","4     -1  Đang xài con m175 cùi mía , nhà xài nhiều chuộ..."]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["data_train.head()"]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":["data_train = data_train.sample(frac=1, random_state=42)"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Class</th>\n","      <th>Data</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4039</th>\n","      <td>0</td>\n","      <td>tuy có sự sáng tạo , nhưng cần phải có phong c...</td>\n","    </tr>\n","    <tr>\n","      <th>3815</th>\n","      <td>0</td>\n","      <td>khoảng 3-4s j đó</td>\n","    </tr>\n","    <tr>\n","      <th>848</th>\n","      <td>-1</td>\n","      <td>Chiều dài 45cm :( bỏ vào túi kiểu gì</td>\n","    </tr>\n","    <tr>\n","      <th>4863</th>\n","      <td>0</td>\n","      <td>không , không nên mua . mua samsung ngon hơn .</td>\n","    </tr>\n","    <tr>\n","      <th>79</th>\n","      <td>-1</td>\n","      <td>thế thì quất thôi, chứ con miband 1s của e bên...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      Class                                               Data\n","4039      0  tuy có sự sáng tạo , nhưng cần phải có phong c...\n","3815      0                                   khoảng 3-4s j đó\n","848      -1               Chiều dài 45cm :( bỏ vào túi kiểu gì\n","4863      0     không , không nên mua . mua samsung ngon hơn .\n","79       -1  thế thì quất thôi, chứ con miband 1s của e bên..."]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["data_train.head()"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"jvrbwPfZmg95"},"outputs":[],"source":["labels = data_train.iloc[:, 0].values\n","reviews = data_train.iloc[:, 1].values"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"3HlbVeHimg95"},"outputs":[],"source":["encoded_labels = []\n","\n","for label in labels:\n","    if label == -1:\n","        encoded_labels.append([1,0,0])\n","    elif label == 0:\n","        encoded_labels.append([0,1,0])\n","    else:\n","        encoded_labels.append([0,0,1])\n","\n","encoded_labels = np.array(encoded_labels)"]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["[0 1 0]\n","[0 1 0]\n","[1 0 0]\n","[0 1 0]\n","[1 0 0]\n"]}],"source":["for i in range(5):\n","    print(encoded_labels[i])"]},{"cell_type":"code","execution_count":9,"metadata":{"id":"Lm4OCwxXmg96"},"outputs":[],"source":["reviews_processed = []\n","unlabeled_processed = []\n","for review in reviews:\n","    review_cool_one = ''.join([char for char in review if char not in digits])\n","    reviews_processed.append(review_cool_one)"]},{"cell_type":"code","execution_count":10,"metadata":{"id":"nW2OZgkgmg97"},"outputs":[],"source":["#Use PyVi for Vietnamese word tokenizer\n","word_reviews = []\n","all_words = []\n","for review in reviews_processed:\n","    review = ViTokenizer.tokenize(review.lower())\n","    word_reviews.append(review.split())\n"]},{"cell_type":"code","execution_count":11,"metadata":{"id":"t-8cJ7z4z2C0"},"outputs":[{"data":{"text/plain":["['tuy',\n"," 'có',\n"," 'sự',\n"," 'sáng_tạo',\n"," ',',\n"," 'nhưng',\n"," 'cần',\n"," 'phải',\n"," 'có',\n"," 'phong_cách',\n"," 'riêng',\n"," ',',\n"," 'đừng',\n"," 'chạy',\n"," 'theo',\n"," 'iphone',\n"," ',',\n"," 'samsung',\n"," 'rất',\n"," 'cố_gắng',\n"," ',',\n"," 'biết',\n"," 'nắm_bắt',\n"," 'nhu_cầu',\n"," 'khách_hàng',\n"," '(',\n"," 's',\n"," 's',\n"," 'edge',\n"," 'người',\n"," 'châu',\n"," 'á',\n"," 'rất',\n"," 'chuộng',\n"," ',',\n"," 'nhưng',\n"," 'ko',\n"," 'thấy',\n"," 'phát_triển',\n"," 'nữa',\n"," ')',\n"," 's',\n"," 'là',\n"," 'sự',\n"," 'hoàn_thiện',\n"," 'của',\n"," 's',\n"," ',',\n"," 'nhưng',\n"," 'sfan',\n"," 'thì',\n"," 'luôn',\n"," 'gato',\n"," ',',\n"," 'vì',\n"," 'các',\n"," 'bạn',\n"," 'ấy',\n"," 'thấy',\n"," 'iphone',\n"," 'quá',\n"," 'đắt',\n"," 'và',\n"," 'các',\n"," 'bạn',\n"," 'ấy',\n"," 'chuẩn_bị',\n"," 'lên_tiếng',\n"," '.']"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["word_reviews[0]"]},{"cell_type":"code","execution_count":12,"metadata":{"id":"pTb0MeDRmg98"},"outputs":[],"source":["EMBEDDING_DIM = 400 # how big is each word vector\n","MAX_VOCAB_SIZE = 10000 # how many unique words to use (i.e num rows in embedding vector)\n","MAX_SEQUENCE_LENGTH = 300 # max number of words in a comment to use"]},{"cell_type":"code","execution_count":13,"metadata":{"id":"jW-7mKtWmg9-"},"outputs":[],"source":["from tensorflow.keras.preprocessing.text import Tokenizer\n","from tensorflow.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.keras.utils import to_categorical"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"-BHpPSLTmg9_"},"outputs":[],"source":["tokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE, lower=True, char_level=False)\n","tokenizer.fit_on_texts(word_reviews)\n","sequences_train = tokenizer.texts_to_sequences(word_reviews)\n","word_index = tokenizer.word_index\n"]},{"cell_type":"code","execution_count":15,"metadata":{"id":"fX8asUU4MxMK"},"outputs":[],"source":["# word_index[1]"]},{"cell_type":"code","execution_count":16,"metadata":{"id":"LlV3M2dimg9_"},"outputs":[],"source":["data = pad_sequences(sequences_train, maxlen=MAX_SEQUENCE_LENGTH)\n","labels = encoded_labels"]},{"cell_type":"code","execution_count":17,"metadata":{"id":"PTfd3gXdNoU4"},"outputs":[{"data":{"text/plain":["array([[   0,    0,    0, ...,  950, 2022,    1],\n","       [   0,    0,    0, ...,   60,  309,   62],\n","       [   0,    0,    0, ...,  726,  310,   43],\n","       ...,\n","       [   0,    0,    0, ...,   11,  434, 1036],\n","       [   0,    0,    0, ..., 4142,    6,  158],\n","       [   0,    0,    0, ...,   33,   33,    1]])"]},"execution_count":17,"metadata":{},"output_type":"execute_result"}],"source":["data"]},{"cell_type":"code","execution_count":18,"metadata":{"id":"4dl9VZ3Rmg-A"},"outputs":[{"name":"stdout","output_type":"stream","text":["Shape of X train and X validation tensor: (5100, 300)\n","Shape of label train and validation tensor: (5100, 3)\n"]}],"source":["print('Shape of X train and X validation tensor:',data.shape)\n","print('Shape of label train and validation tensor:', labels.shape)"]},{"cell_type":"code","execution_count":19,"metadata":{"id":"-KKSjJdJmg-A"},"outputs":[{"name":"stdout","output_type":"stream","text":["WARNING:tensorflow:From c:\\Users\\quand\\anaconda3\\Lib\\site-packages\\keras\\src\\backend.py:873: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n","\n"]}],"source":["import gensim\n","from gensim.models import Word2Vec\n","from gensim.utils import simple_preprocess\n","\n","from gensim.models.keyedvectors import KeyedVectors\n","\n","word_vectors = KeyedVectors.load_word2vec_format('vi-model-CBOW.bin', binary=True)\n","\n","\n","vocabulary_size=min(len(word_index)+1,MAX_VOCAB_SIZE)\n","embedding_matrix = np.zeros((vocabulary_size, EMBEDDING_DIM))\n","for word, i in word_index.items():\n","    if i>=MAX_VOCAB_SIZE:\n","        continue\n","    try:\n","        embedding_vector = word_vectors[word]\n","        embedding_matrix[i] = embedding_vector\n","    except KeyError:\n","        embedding_matrix[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)\n","\n","del(word_vectors)\n","\n","from keras.layers import Embedding\n","embedding_layer = Embedding(vocabulary_size,\n","                            EMBEDDING_DIM,\n","                            weights=[embedding_matrix],\n","                            trainable=True)"]},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[],"source":["from tensorflow.keras.layers import Dense, Input, GlobalMaxPooling1D\n","from tensorflow.keras.layers import Conv1D, MaxPooling1D, Embedding, BatchNormalization, AveragePooling1D, LSTM, Bidirectional\n","from tensorflow.keras.models import Model, load_model\n","from tensorflow.keras.layers import Input, Dense, Embedding, Dropout,concatenate\n","from tensorflow.keras.layers import Reshape, Flatten\n","from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n","from tensorflow.keras.optimizers import Adam, SGD\n","from tensorflow.keras.models import Model\n","from tensorflow.keras import regularizers"]},{"cell_type":"code","execution_count":21,"metadata":{"id":"njBANdn5mg-B"},"outputs":[{"name":"stdout","output_type":"stream","text":["<keras.src.engine.functional.Functional object at 0x000001F6F632B8D0>\n","Model: \"model\"\n","_________________________________________________________________\n"," Layer (type)                Output Shape              Param #   \n","=================================================================\n"," input_1 (InputLayer)        [(None, 300)]             0         \n","                                                                 \n"," embedding (Embedding)       (None, 300, 400)          3167600   \n","                                                                 \n"," lstm (LSTM)                 (None, 300, 512)          1869824   \n","                                                                 \n"," lstm_1 (LSTM)               (None, 300, 256)          787456    \n","                                                                 \n"," lstm_2 (LSTM)               (None, 128)               197120    \n","                                                                 \n"," dropout (Dropout)           (None, 128)               0         \n","                                                                 \n"," dense (Dense)               (None, 3)                 387       \n","                                                                 \n","=================================================================\n","Total params: 6022387 (22.97 MB)\n","Trainable params: 6022387 (22.97 MB)\n","Non-trainable params: 0 (0.00 Byte)\n","_________________________________________________________________\n"]}],"source":["sequence_length = data.shape[1]\n","filter_sizes = [3,4,5]\n","num_filters = 100\n","drop = 0.5\n","\n","inputs = Input(shape=(sequence_length,))\n","embedding = embedding_layer(inputs)\n","\n","lstm_2 = LSTM(512, return_sequences=True)(embedding)\n","lstm_1 = LSTM(256, return_sequences=True)(lstm_2)\n","lstm_0 = LSTM(128)(lstm_1)\n","\n","dropout1 = Dropout(drop)(lstm_0)\n","output = Dense(units=3, activation='softmax', kernel_regularizer=regularizers.l2(0.01))(dropout1)\n","\n","model = Model(inputs, output)\n","print(model)\n","\n","adam = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n","model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n","model.summary()\n","\n","early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=10, verbose=1)"]},{"cell_type":"code","execution_count":22,"metadata":{"colab":{"background_save":true},"id":"Jn0dBlzjmg-D"},"outputs":[{"name":"stdout","output_type":"stream","text":["Epoch 1/50\n","WARNING:tensorflow:From c:\\Users\\quand\\anaconda3\\Lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n","\n","WARNING:tensorflow:From c:\\Users\\quand\\anaconda3\\Lib\\site-packages\\keras\\src\\engine\\base_layer_utils.py:384: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n","\n","16/16 [==============================] - 332s 17s/step - loss: 1.1304 - accuracy: 0.4294 - val_loss: 0.9924 - val_accuracy: 0.5490\n","Epoch 2/50\n","16/16 [==============================] - 202s 12s/step - loss: 0.9719 - accuracy: 0.5686 - val_loss: 0.9577 - val_accuracy: 0.5696\n","Epoch 3/50\n","16/16 [==============================] - 162s 10s/step - loss: 0.8513 - accuracy: 0.6485 - val_loss: 0.8712 - val_accuracy: 0.6422\n","Epoch 4/50\n","16/16 [==============================] - 177s 11s/step - loss: 0.7121 - accuracy: 0.7260 - val_loss: 0.8936 - val_accuracy: 0.6402\n","Epoch 5/50\n","16/16 [==============================] - 161s 10s/step - loss: 0.5841 - accuracy: 0.7924 - val_loss: 0.8926 - val_accuracy: 0.6529\n","Epoch 6/50\n","16/16 [==============================] - 146s 9s/step - loss: 0.4676 - accuracy: 0.8400 - val_loss: 0.9493 - val_accuracy: 0.6608\n","Epoch 7/50\n","16/16 [==============================] - 103s 6s/step - loss: 0.3441 - accuracy: 0.8924 - val_loss: 0.9997 - val_accuracy: 0.6402\n","Epoch 8/50\n","16/16 [==============================] - 121s 8s/step - loss: 0.2285 - accuracy: 0.9350 - val_loss: 1.2930 - val_accuracy: 0.6549\n","Epoch 9/50\n","16/16 [==============================] - 155s 10s/step - loss: 0.1766 - accuracy: 0.9495 - val_loss: 1.3562 - val_accuracy: 0.6529\n","Epoch 10/50\n","16/16 [==============================] - 186s 12s/step - loss: 0.5356 - accuracy: 0.8431 - val_loss: 0.9693 - val_accuracy: 0.6422\n","Epoch 11/50\n","16/16 [==============================] - 249s 16s/step - loss: 0.2545 - accuracy: 0.9299 - val_loss: 1.1526 - val_accuracy: 0.6814\n","Epoch 12/50\n","16/16 [==============================] - 246s 16s/step - loss: 0.1188 - accuracy: 0.9743 - val_loss: 1.2831 - val_accuracy: 0.6833\n","Epoch 13/50\n","16/16 [==============================] - 291s 17s/step - loss: 0.0723 - accuracy: 0.9902 - val_loss: 1.5041 - val_accuracy: 0.6686\n","Epoch 13: early stopping\n"]},{"data":{"text/plain":["<keras.src.callbacks.History at 0x1f6f6fb5650>"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["checkpoint = ModelCheckpoint('deep_lstm.keras',\n","                             monitor='val_accuracy',\n","                             save_best_only=True, verbose=False, mode='max')\n","callbacks_list = [checkpoint, early_stopping]\n","\n","model.fit(data, labels, validation_split=0.2,\n","          epochs=50 , batch_size=256, callbacks=callbacks_list, shuffle=True)"]},{"cell_type":"code","execution_count":23,"metadata":{"colab":{"background_save":true},"id":"8XoN2UOamg-D"},"outputs":[],"source":["labels_test = data_test.iloc[:, 0].values\n","reviews_test = data_test.iloc[:, 1].values"]},{"cell_type":"code","execution_count":24,"metadata":{"colab":{"background_save":true},"id":"PwiYb3Ohmg-E"},"outputs":[],"source":["encoded_labels_test = []\n","\n","for label_test in labels_test:\n","    if label_test == -1:\n","        encoded_labels_test.append([1,0,0])\n","    elif label_test == 0:\n","        encoded_labels_test.append([0,1,0])\n","    else:\n","        encoded_labels_test.append([0,0,1])\n","\n","encoded_labels_test = np.array(encoded_labels_test)"]},{"cell_type":"code","execution_count":25,"metadata":{"colab":{"background_save":true},"id":"E08tBw9img-E"},"outputs":[],"source":["reviews_processed_test = []\n","unlabeled_processed_test = []\n","for review_test in reviews_test:\n","    review_cool_one = ''.join([char for char in review_test])\n","    reviews_processed_test.append(review_cool_one)"]},{"cell_type":"code","execution_count":26,"metadata":{"colab":{"background_save":true},"id":"OwgI9Xywmg-E"},"outputs":[],"source":["#Use PyVi for Vietnamese word tokenizer\n","word_reviews_test = []\n","all_words = []\n","for review_test in reviews_processed_test:\n","    review_test = ViTokenizer.tokenize(review_test.lower())\n","    word_reviews_test.append(review_test.split())"]},{"cell_type":"code","execution_count":27,"metadata":{"colab":{"background_save":true},"id":"p02GxCh6mg-F"},"outputs":[],"source":["sequences_test = tokenizer.texts_to_sequences(word_reviews_test)\n","data_test = pad_sequences(sequences_test, maxlen=MAX_SEQUENCE_LENGTH)\n","labels_test = encoded_labels_test"]},{"cell_type":"code","execution_count":28,"metadata":{"colab":{"background_save":true},"id":"jAqUMGInmg-F"},"outputs":[{"name":"stdout","output_type":"stream","text":["Shape of X train and X validation tensor: (1050, 300)\n","Shape of label train and validation tensor: (1050, 3)\n"]}],"source":["print('Shape of X train and X validation tensor:',data_test.shape)\n","print('Shape of label train and validation tensor:', labels_test.shape)"]},{"cell_type":"code","execution_count":29,"metadata":{"colab":{"background_save":true},"id":"LKclttiOmg-F"},"outputs":[{"name":"stdout","output_type":"stream","text":["33/33 [==============================] - 34s 967ms/step - loss: 1.5719 - accuracy: 0.6514\n"]}],"source":["# model = load_model('best_model.keras')\n","score = model.evaluate(data_test, labels_test)"]},{"cell_type":"code","execution_count":30,"metadata":{"colab":{"background_save":true},"id":"r31_uxxgmg-G"},"outputs":[{"name":"stdout","output_type":"stream","text":["loss: 157.19%\n","accuracy: 65.14%\n"]}],"source":["print(\"%s: %.2f%%\" % (model.metrics_names[0], score[0]*100))\n","print(\"%s: %.2f%%\" % (model.metrics_names[1], score[1]*100))\n"]}],"metadata":{"accelerator":"GPU","colab":{"gpuType":"T4","provenance":[{"file_id":"1rFZZf9ECknkLNDv8so_kQNPhqain0RqJ","timestamp":1653989613942}]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.4"}},"nbformat":4,"nbformat_minor":0}