create_specific_user_ds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yhschan/miniforge3/envs/nlp/lib/python3.9/site-packages/jax/_src/lib/__init__.py:33: UserWarning: JAX on Mac ARM machines is experimental and minimally tested. Please see https://github.com/google/jax/issues/5501 in the event of problems.\n",
      "  warnings.warn(\"JAX on Mac ARM machines is experimental and minimally tested. \"\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "from data import Dataset\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 55966/55966 [00:00<00:00, 2351805.57it/s]\n",
      "  1%|          | 418/55966 [00:00<00:27, 2048.59it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (1072 > 1024). Running this sequence through the model will result in indexing errors\n",
      "100%|██████████| 55966/55966 [00:28<00:00, 1934.74it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset = Dataset(dataset = 'toronto', model_type = 'gpt2', csv_file = \"dataset/yelp_toronto_fix.csv\", masking=False, max_len=400, reindex=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique, counts = np.unique(np.concatenate([dataset.user_labels_train, dataset.user_labels_eval, dataset.user_labels_test]), return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Top 3 user index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[125  69 160]\n"
     ]
    }
   ],
   "source": [
    "top_3_indices = counts.argsort()[::-1][:3]\n",
    "print(top_3_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(536, 206, 201)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts[top_3_indices[0]], counts[top_3_indices[1]], counts[top_3_indices[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "427 49 60\n",
      "170 21 15\n",
      "162 20 19\n"
     ]
    }
   ],
   "source": [
    "# Top 3: 125, 69, 160\n",
    "for user_index in top_3_indices:\n",
    "    print(np.sum(dataset.user_labels_train == user_index), np.sum(dataset.user_labels_eval == user_index), np.sum(dataset.user_labels_test == user_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CxDOIDnH8gp9KXzpBHJYXw eZeBuiVZWT7u3SktO7mv9w JrXC_MDp38BWwLn2SFdNsA\n"
     ]
    }
   ],
   "source": [
    "selected_user1 = dataset.user_id_list[top_3_indices[0]]\n",
    "selected_user2 = dataset.user_id_list[top_3_indices[1]]\n",
    "selected_user3 = dataset.user_id_list[top_3_indices[2]]\n",
    "print(selected_user1, selected_user2, selected_user3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "536\n",
      "206\n",
      "201\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset.df.query(\"user_id == @selected_user1\")))\n",
    "print(len(dataset.df.query(\"user_id == @selected_user2\")))\n",
    "print(len(dataset.df.query(\"user_id == @selected_user3\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.df.query(\"user_id == @selected_user1\").to_csv(\"dataset/toronto/top3/selected_user1.csv\")\n",
    "dataset.df.query(\"user_id == @selected_user2\").to_csv(\"dataset/toronto/top3/selected_user2.csv\")\n",
    "dataset.df.query(\"user_id == @selected_user3\").to_csv(\"dataset/toronto/top3/selected_user3.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Last 3 user index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1061 1072 1071]\n"
     ]
    }
   ],
   "source": [
    "last_3_indices = counts.argsort()[:3]\n",
    "print(last_3_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 8, 8)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts[last_3_indices[0]], counts[last_3_indices[1]], counts[last_3_indices[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 1 1\n",
      "4 2 2\n",
      "4 2 2\n"
     ]
    }
   ],
   "source": [
    "for user_index in last_3_indices:\n",
    "    print(np.sum(dataset.user_labels_train == user_index), np.sum(dataset.user_labels_eval == user_index), np.sum(dataset.user_labels_test == user_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UhlXS6NdA5k4SRunpcKHJw qoS4nIc5MlfF_j8DkYHBgw xHMq2fBArXBp881TVlRt5g\n"
     ]
    }
   ],
   "source": [
    "selected_user1 = dataset.user_id_list[last_3_indices[0]]\n",
    "selected_user2 = dataset.user_id_list[last_3_indices[1]]\n",
    "selected_user3 = dataset.user_id_list[last_3_indices[2]]\n",
    "print(selected_user1, selected_user2, selected_user3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "8\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset.df.query(\"user_id == @selected_user1\")))\n",
    "print(len(dataset.df.query(\"user_id == @selected_user2\")))\n",
    "print(len(dataset.df.query(\"user_id == @selected_user3\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.df.query(\"user_id == @selected_user1\").to_csv(\"dataset/toronto/last3/selected_user1.csv\")\n",
    "dataset.df.query(\"user_id == @selected_user2\").to_csv(\"dataset/toronto/last3/selected_user2.csv\")\n",
    "dataset.df.query(\"user_id == @selected_user3\").to_csv(\"dataset/toronto/last3/selected_user3.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 536/536 [00:00<00:00, 2075851.29it/s]\n",
      "  0%|          | 0/536 [00:00<?, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (539 > 512). Running this sequence through the model will result in indexing errors\n",
      "100%|██████████| 536/536 [00:00<00:00, 1957.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "536"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Debug cell for resize function\n",
    "dataset_user = Dataset(dataset = 'toronto', model_type = 'bert', csv_file = \"dataset/toronto/top3/selected_user1.csv\", masking=False, max_len=400, reindex=False)\n",
    "# print((len(dataset_user.user_labels_train), len(dataset_user.user_labels_eval), len(dataset_user.user_labels_test)))\n",
    "dataset_user.resize(427,49)\n",
    "# len(dataset_user.user_labels_train), len(dataset_user.user_labels_eval), len(dataset_user.user_labels_test)\n",
    "len(dataset_user.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the exact same selected user dataset class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "user_indices = list(top_3_indices) + list(last_3_indices)\n",
    "i = 0\n",
    "for dir in ['dataset/toronto/top3', 'dataset/toronto/last3']:\n",
    "    for selected_user in ['selected_user1', 'selected_user2', 'selected_user3']:\n",
    "        for model_type in ['bert', 'bart', 'gpt2']:\n",
    "            csv_file = os.path.join(dir, selected_user) + '.csv'\n",
    "            print(csv_file, model_type)\n",
    "            dataset_user = Dataset(dataset = 'toronto', model_type = model_type, csv_file = csv_file, masking=False, max_len=400, reindex=False)\n",
    "\n",
    "            old_train_len, old_eval_len, old_test_len = np.sum(dataset.user_labels_train == user_indices[i]), np.sum(dataset.user_labels_eval == user_indices[i]), np.sum(dataset.user_labels_test == user_indices[i])\n",
    "            dataset_user.resize(old_train_len, old_eval_len)\n",
    "            new_train_len, new_eval_len, new_test_len = len(dataset_user.user_labels_train), len(dataset_user.user_labels_eval), len(dataset_user.user_labels_test)\n",
    "\n",
    "            assert (new_train_len, new_eval_len, new_test_len) == (old_train_len, old_eval_len, old_test_len)\n",
    "\n",
    "            pickle_file = os.path.join(dir, selected_user) + f\"_{model_type}.pkl\" \n",
    "            with open(pickle_file, 'wb') as fout:\n",
    "                pickle.dump(dataset_user, fout)\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8e2dd819c5cb01a5132ccedcc67e995c99b04b604f12c7cfb35359fd74a719ef"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('nlp')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}