kaggle-multi-vectorizers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from lightgbm import LGBMClassifier, early_stopping\n",
    "from catboost import CatBoostClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "from tokenizers import (\n",
    "    decoders,\n",
    "    models,\n",
    "    normalizers,\n",
    "    pre_tokenizers,\n",
    "    processors,\n",
    "    trainers,\n",
    "    Tokenizer\n",
    ")\n",
    "\n",
    "from datasets import Dataset\n",
    "from tqdm.auto import tqdm\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB, GaussianNB\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(44868, 5)\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(\"fake_test_essays.csv\")\n",
    "sub = pd.read_csv(\"sample_submission.csv\")\n",
    "org_train = pd.read_csv(\"train_essays.csv\")\n",
    "train = pd.read_csv(\"train_v2_drcat_02.csv\", sep=\",\")\n",
    "train = train.dropna()\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CFG:\n",
    "    IS_TRAIN_ON_FULL = True\n",
    "    HALF_TRAIN_SAMPLE = 22500\n",
    "    RANDOM_STATE = 42\n",
    "    LOWER_CASE = False\n",
    "    VOCAB_SIZE = 32000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----Using full training data-----\n",
      "The shape of training dataset is: (44868, 5)\n",
      "                                                text  label  \\\n",
      "0  My argument about Lukes point of view is that ...      0   \n",
      "1  Dear Principal,\\n\\nI'm writing to express my t...      1   \n",
      "2  Dear principle.\\n\\nI strongly think that you s...      0   \n",
      "3  Winston Churchill is a well-known figure in hi...      1   \n",
      "4  With the growing popularity of social media pl...      1   \n",
      "\n",
      "                             prompt_name                source  RDizzl3_seven  \n",
      "0          \"A Cowboy Who Rode the Waves\"       persuade_corpus           True  \n",
      "1  Grades for extracurricular activities           llama2_chat          False  \n",
      "2                  Cell phones at school       persuade_corpus          False  \n",
      "3              Seeking multiple opinions  mistral7binstruct_v1          False  \n",
      "4                     Phones and driving        falcon_180b_v1          False  \n"
     ]
    }
   ],
   "source": [
    "if CFG.IS_TRAIN_ON_FULL:\n",
    "    print(\"-----Using full training data-----\")\n",
    "    train = train.drop_duplicates(subset=[\"text\"])\n",
    "    train = train.sample(len(train))\n",
    "    print(\"The shape of training dataset is:\", train.shape)\n",
    "    train.reset_index(drop=True, inplace=True)\n",
    "    print(train.head())\n",
    "else:\n",
    "    print(\"-----Using partial training data-----\")\n",
    "    train = train.drop_duplicates(subset=[\"text\"])\n",
    "    train_label_0 = train[train[\"label\"] == 0]\n",
    "    train_label_1 = train[train[\"label\"] == 1]\n",
    "    train_label_0 = train_label_0.sample(\n",
    "        CFG.HALF_TRAIN_SAMPLE, random_state=CFG.RANDOM_STATE\n",
    "    )\n",
    "    train_label_1 = train_label_1.sample(\n",
    "        CFG.HALF_TRAIN_SAMPLE, random_state=CFG.RANDOM_STATE\n",
    "    )\n",
    "    train = pd.concat([train_label_0, train_label_1])\n",
    "    train = train.sample(len(train))\n",
    "    print(\"The shape of training dataset is:\", train.shape)\n",
    "    train.reset_index(drop=True, inplace=True)\n",
    "    print(train.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用bpe_trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f159cb3061af4c2085c4a9a769b8afe0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae8940eea5b94a28b5ed3e405ab307ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "447bd207ce454d0d9acf042fdf6378b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/44868 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "raw_tokenizer = Tokenizer(models.BPE(unk_token=\"[UNK]\"))\n",
    "raw_tokenizer.normalizer = normalizers.Sequence(\n",
    "    [normalizers.NFC()] + [normalizers.Lowercase()] if CFG.LOWER_CASE else []\n",
    ")\n",
    "raw_tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()\n",
    "\n",
    "special_tokens = [\"[UNK]\", \"[PAD]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]\n",
    "trainer = trainers.BpeTrainer(\n",
    "    vocab_size=CFG.VOCAB_SIZE,\n",
    "    special_tokens=special_tokens\n",
    ")\n",
    "\n",
    "dataset = Dataset.from_pandas(test[[\"text\"]])\n",
    "\n",
    "\n",
    "def train_corpus():\n",
    "    for i in tqdm(range(0, len(dataset), 100)):\n",
    "        yield dataset[i:i + 100][\"text\"]\n",
    "\n",
    "\n",
    "raw_tokenizer.train_from_iterator(train_corpus(), trainer=trainer)\n",
    "tokenizer = PreTrainedTokenizerFast(\n",
    "    tokenizer_object=raw_tokenizer,\n",
    "    unk_token=\"[UNK]\",\n",
    "    pad_token=\"[PAD]\",\n",
    "    cls_token=\"[CLS]\",\n",
    "    sep_token=\"[SEP]\",\n",
    "    mask_token=\"[MASK]\",\n",
    ")\n",
    "\n",
    "tokenized_texts_test = []\n",
    "for text in tqdm(test[\"text\"].tolist()):\n",
    "    tokenized_texts_test.append(tokenizer.tokenize(text))\n",
    "\n",
    "tokenized_texts_train = []\n",
    "for text in tqdm(train[\"text\"].tolist()):\n",
    "    tokenized_texts_train.append(tokenizer.tokenize(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练第一种vectorizer，其中min_df=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1300\n",
      "The shape of X_train is: (44868, 1300)\n",
      "The shape of y_train is: (44868,)\n",
      "The shape of X_test is: (100, 1300)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def dummy(text):\n",
    "    return text\n",
    "\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    ngram_range=(3, 5),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\",\n",
    "    min_df=2\n",
    ")\n",
    "\n",
    "vectorizer.fit(tokenized_texts_test)\n",
    "vocab = vectorizer.vocabulary_\n",
    "print(len(vocab))\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    ngram_range=(3, 5),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    vocabulary=vocab,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "X_train = vectorizer.fit_transform(tokenized_texts_train)\n",
    "# print(sys.getsizeof(X_train) / 1024 ** 2, \"MB\")\n",
    "# X_train = X_train.astype(\"float16\")\n",
    "# print(sys.getsizeof(X_train) / 1024 ** 2, \"MB\")\n",
    "y_train = train[\"label\"].values\n",
    "# y_train = y_train.astype(\"float16\")\n",
    "X_test = vectorizer.transform(tokenized_texts_test)\n",
    "# X_test = X_test.astype(\"float16\")\n",
    "print(\"The shape of X_train is:\", X_train.shape)\n",
    "print(\"The shape of y_train is:\", y_train.shape)\n",
    "print(\"The shape of X_test is:\", X_test.shape)\n",
    "\n",
    "del vectorizer\n",
    "gc.collect()\n",
    "\n",
    "# num_features = X_train.shape[1]\n",
    "\n",
    "# # 使用卡方检验选择特征\n",
    "# k = int(num_features / 4)\n",
    "# chi2_selector = SelectKBest(chi2, k=k)\n",
    "# X_train_chi2_selected = chi2_selector.fit_transform(X_train, y_train)\n",
    "# print(sys.getsizeof(X_train_chi2_selected) / 1024 ** 2, \"MB\")\n",
    "# X_test_chi2_selected = chi2_selector.transform(X_test)\n",
    "\n",
    "# X_train = X_train_chi2_selected\n",
    "# X_test = X_test_chi2_selected\n",
    "# print(\"The shape of X_train is:\", X_train.shape)\n",
    "# print(\"The shape of y_train is:\", y_train.shape)\n",
    "# print(\"The shape of X_test is:\", X_test.shape)\n",
    "\n",
    "# # 使用SVD进行降维\n",
    "# n_components = int(num_features / 4)\n",
    "# svd = TruncatedSVD(n_components=n_components)\n",
    "# X_train_svd = svd.fit_transform(X_train)\n",
    "# X_test_svd = svd.transform(X_test)\n",
    "\n",
    "# X_train = hstack([X_train_chi2_selected, X_train_svd])\n",
    "# X_train = X_train.toarray()\n",
    "# X_test = hstack([X_test_chi2_selected, X_test_svd])\n",
    "# X_test = X_test.toarray()\n",
    "# print(\"The shape of X_train is:\", X_train.shape)\n",
    "# print(\"The shape of y_train is:\", y_train.shape)\n",
    "# print(\"The shape of X_test is:\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.54291002 0.55637294 0.6347229  0.59797108 0.527079   0.63198155\n",
      " 0.62015758 0.57555722 0.5988353  0.57504675 0.57388124 0.58702718\n",
      " 0.62123113 0.61658833 0.59907856 0.54438224 0.60998554 0.63955693\n",
      " 0.60674103 0.63735445 0.670477   0.61250974 0.5922073  0.70250016\n",
      " 0.59531311 0.61463299 0.61950628 0.63628117 0.62172642 0.64291536\n",
      " 0.61540233 0.59996019 0.63624397 0.58798119 0.53836085 0.58111776\n",
      " 0.59803494 0.61696371 0.57783281 0.62621101 0.68290536 0.66835786\n",
      " 0.65123665 0.62247822 0.58187826 0.66612308 0.65301556 0.56523692\n",
      " 0.60966133 0.55008195 0.63267952 0.57593313 0.70959765 0.66357961\n",
      " 0.6112364  0.61158293 0.63166446 0.62714442 0.67722951 0.63997403\n",
      " 0.61324349 0.62620266 0.52328662 0.50371779 0.59142339 0.65735504\n",
      " 0.61187449 0.621005   0.56201223 0.6237869  0.62712722 0.62649503\n",
      " 0.61058237 0.66892744 0.60874938 0.65104842 0.61402145 0.5523276\n",
      " 0.59716871 0.54619571 0.63909745 0.68620415 0.5933087  0.63483033\n",
      " 0.62877053 0.61708232 0.57919023 0.60577338 0.62802357 0.65638919\n",
      " 0.62010565 0.69352367 0.58644882 0.62012962 0.55282668 0.60575802\n",
      " 0.53512074 0.68038852 0.65826198 0.67567526]\n"
     ]
    }
   ],
   "source": [
    "if len(test.text.values) <= 2:\n",
    "    sub.to_csv(\"submission.csv\", index=False)\n",
    "else:\n",
    "    mnb = MultinomialNB(alpha=0.0225)\n",
    "\n",
    "    # gnb = GaussianNB()\n",
    "\n",
    "    sgd_model = SGDClassifier(max_iter=9000, tol=1e-4, loss=\"modified_huber\")\n",
    "\n",
    "    weights = [0.10, 0.31]\n",
    "\n",
    "    ensemble = VotingClassifier(\n",
    "        estimators=[\n",
    "            (\"mnb\", mnb),\n",
    "            # (\"gnb\", gnb),\n",
    "            (\"sgd\", sgd_model)\n",
    "        ],\n",
    "        weights=weights,\n",
    "        voting=\"soft\",\n",
    "        n_jobs=-1\n",
    "    )\n",
    "    ensemble.fit(X_train, y_train)\n",
    "    gc.collect()\n",
    "    final_preds1 = ensemble.predict_proba(X_test)[:, 1]\n",
    "    print(final_preds1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练第二种tokenizer，不使用min_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "632\n",
      "The shape of X_train3 is: (44868, 632)\n",
      "The shape of y_train3 is: (44868,)\n",
      "The shape of X_test3 is: (100, 632)\n",
      "The shape of X_train3 is: (44868, 316)\n",
      "The shape of y_train3 is: (44868,)\n",
      "The shape of X_test3 is: (100, 316)\n"
     ]
    }
   ],
   "source": [
    "def dummy(text):\n",
    "    return text\n",
    "\n",
    "\n",
    "vectorizer3 = TfidfVectorizer(\n",
    "    ngram_range=(3, 3),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "vectorizer3.fit(tokenized_texts_test)\n",
    "vocab = vectorizer3.vocabulary_\n",
    "print(len(vocab))\n",
    "\n",
    "vectorizer3 = TfidfVectorizer(\n",
    "    ngram_range=(3, 3),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    vocabulary=vocab,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "X_train3 = vectorizer3.fit_transform(tokenized_texts_train)\n",
    "y_train3 = train[\"label\"].values\n",
    "X_test3 = vectorizer3.transform(tokenized_texts_test)\n",
    "print(\"The shape of X_train3 is:\", X_train3.shape)\n",
    "print(\"The shape of y_train3 is:\", y_train3.shape)\n",
    "print(\"The shape of X_test3 is:\", X_test3.shape)\n",
    "\n",
    "del vectorizer3\n",
    "gc.collect()\n",
    "\n",
    "num_features = X_train3.shape[1]\n",
    "k = int(num_features / 2)\n",
    "chi2_selector = SelectKBest(chi2, k=k)\n",
    "X_train3 = chi2_selector.fit_transform(X_train3, y_train3)\n",
    "X_test3 = chi2_selector.transform(X_test3)\n",
    "print(\"The shape of X_train3 is:\", X_train3.shape)\n",
    "print(\"The shape of y_train3 is:\", y_train3.shape)\n",
    "print(\"The shape of X_test3 is:\", X_test3.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "910\n",
      "The shape of X_train4 is: (44868, 910)\n",
      "The shape of y_train4 is: (44868,)\n",
      "The shape of X_test4 is: (100, 910)\n",
      "The shape of X_train4 is: (44868, 455)\n",
      "The shape of y_train4 is: (44868,)\n",
      "The shape of X_test4 is: (100, 455)\n"
     ]
    }
   ],
   "source": [
    "def dummy(text):\n",
    "    return text\n",
    "\n",
    "\n",
    "vectorizer4 = TfidfVectorizer(\n",
    "    ngram_range=(4, 4),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "vectorizer4.fit(tokenized_texts_test)\n",
    "vocab = vectorizer4.vocabulary_\n",
    "print(len(vocab))\n",
    "\n",
    "vectorizer4 = TfidfVectorizer(\n",
    "    ngram_range=(4, 4),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    vocabulary=vocab,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "X_train4 = vectorizer4.fit_transform(tokenized_texts_train)\n",
    "y_train4 = train[\"label\"].values\n",
    "X_test4 = vectorizer4.transform(tokenized_texts_test)\n",
    "print(\"The shape of X_train4 is:\", X_train4.shape)\n",
    "print(\"The shape of y_train4 is:\", y_train4.shape)\n",
    "print(\"The shape of X_test4 is:\", X_test4.shape)\n",
    "\n",
    "del vectorizer4\n",
    "gc.collect()\n",
    "\n",
    "num_features = X_train4.shape[1]\n",
    "k = int(num_features / 2)\n",
    "chi2_selector = SelectKBest(chi2, k=k)\n",
    "X_train4 = chi2_selector.fit_transform(X_train4, y_train4)\n",
    "X_test4 = chi2_selector.transform(X_test4)\n",
    "print(\"The shape of X_train4 is:\", X_train4.shape)\n",
    "print(\"The shape of y_train4 is:\", y_train4.shape)\n",
    "print(\"The shape of X_test4 is:\", X_test4.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1106\n",
      "The shape of X_train5 is: (44868, 1106)\n",
      "The shape of y_train5 is: (44868,)\n",
      "The shape of X_test5 is: (100, 1106)\n",
      "The shape of X_train5 is: (44868, 553)\n",
      "The shape of y_train5 is: (44868,)\n",
      "The shape of X_test5 is: (100, 553)\n"
     ]
    }
   ],
   "source": [
    "def dummy(text):\n",
    "    return text\n",
    "\n",
    "\n",
    "vectorizer5 = TfidfVectorizer(\n",
    "    ngram_range=(5, 5),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "vectorizer5.fit(tokenized_texts_test)\n",
    "vocab = vectorizer5.vocabulary_\n",
    "print(len(vocab))\n",
    "\n",
    "vectorizer5 = TfidfVectorizer(\n",
    "    ngram_range=(5, 5),\n",
    "    lowercase=False,\n",
    "    sublinear_tf=True,\n",
    "    vocabulary=vocab,\n",
    "    analyzer=\"word\",\n",
    "    tokenizer=dummy,\n",
    "    preprocessor=dummy,\n",
    "    token_pattern=None,\n",
    "    strip_accents=\"unicode\"\n",
    ")\n",
    "\n",
    "X_train5 = vectorizer5.fit_transform(tokenized_texts_train)\n",
    "y_train5 = train[\"label\"].values\n",
    "X_test5 = vectorizer5.transform(tokenized_texts_test)\n",
    "print(\"The shape of X_train5 is:\", X_train5.shape)\n",
    "print(\"The shape of y_train5 is:\", y_train5.shape)\n",
    "print(\"The shape of X_test5 is:\", X_test5.shape)\n",
    "\n",
    "del vectorizer5\n",
    "gc.collect()\n",
    "\n",
    "num_features = X_train5.shape[1]\n",
    "k = int(num_features / 2)\n",
    "chi2_selector = SelectKBest(chi2, k=k)\n",
    "X_train5 = chi2_selector.fit_transform(X_train5, y_train5)\n",
    "X_test5 = chi2_selector.transform(X_test5)\n",
    "print(\"The shape of X_train5 is:\", X_train5.shape)\n",
    "print(\"The shape of y_train5 is:\", y_train5.shape)\n",
    "print(\"The shape of X_test5 is:\", X_test5.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of X_train is: (44868, 1324)\n",
      "The shape of y_train is: (44868,)\n",
      "The shape of X_test is: (100, 1324)\n"
     ]
    }
   ],
   "source": [
    "X_train = hstack([X_train3, X_train4, X_train5])\n",
    "X_test = hstack([X_test3, X_test4, X_test5])\n",
    "y_train = train[\"label\"].values\n",
    "print(\"The shape of X_train is:\", X_train.shape)\n",
    "print(\"The shape of y_train is:\", y_train.shape)\n",
    "print(\"The shape of X_test is:\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.56038317 0.55045213 0.56038317 0.56038317 0.56038317 0.56038317\n",
      " 0.55822735 0.56038317 0.55822735 0.55822735 0.55045213 0.56038317\n",
      " 0.56038317 0.56090667 0.55822735 0.56038317 0.56038317 0.56038317\n",
      " 0.56038317 0.56038317 0.55077559 0.56038317 0.56038317 0.56376587\n",
      " 0.56090667 0.56038317 0.56038317 0.56038317 0.56038317 0.56038317\n",
      " 0.56038317 0.56038317 0.56038317 0.55822735 0.56038317 0.56038317\n",
      " 0.56038317 0.55822735 0.56038317 0.56038317 0.56090667 0.56376587\n",
      " 0.56038317 0.55045213 0.56038317 0.55045213 0.56038317 0.56038317\n",
      " 0.56038317 0.56038317 0.56038317 0.56090667 0.56376587 0.56376587\n",
      " 0.56090667 0.56038317 0.56376587 0.55822735 0.56376587 0.56038317\n",
      " 0.56038317 0.56038317 0.55822735 0.56038317 0.55441531 0.56376587\n",
      " 0.55822735 0.56038317 0.56038317 0.56038317 0.56038317 0.56038317\n",
      " 0.56038317 0.55045213 0.56038317 0.56376587 0.55045213 0.56038317\n",
      " 0.55822735 0.56038317 0.56038317 0.54051749 0.56038317 0.56038317\n",
      " 0.55822735 0.56038317 0.56038317 0.56038317 0.56038317 0.55077559\n",
      " 0.56038317 0.56376587 0.55822735 0.56038317 0.55045213 0.56038317\n",
      " 0.55045213 0.56412123 0.56376587 0.54051749]\n"
     ]
    }
   ],
   "source": [
    "if len(test.text.values) <= 2:\n",
    "    sub.to_csv(\"submission.csv\", index=False)\n",
    "else:\n",
    "    lgb_params = {\n",
    "        \"n_iter\": 3000,\n",
    "        \"verbose\": 1,\n",
    "        \"objective\": \"cross_entropy\",\n",
    "        \"metric\": \"auc\",\n",
    "        \"learning_rate\": 0.0056,\n",
    "        \"colsample_bytree\": 0.7,\n",
    "        \"colsample_bynode\": 0.8\n",
    "    }\n",
    "    lgb = LGBMClassifier(**lgb_params)\n",
    "\n",
    "    cat = CatBoostClassifier(\n",
    "        iterations=3000,\n",
    "        verbose=1,\n",
    "        learning_rate=0.0056,\n",
    "        subsample=0.4,\n",
    "        allow_const_label=True,\n",
    "        loss_function=\"CrossEntropy\"\n",
    "    )\n",
    "\n",
    "    xgb_params = {\n",
    "        \"n_estimators\": 3000,\n",
    "        \"verbosity\": 1,\n",
    "        \"objective\": \"binary:logistic\",\n",
    "        \"eval_metric\": \"auc\",\n",
    "        \"learning_rate\": 0.005,\n",
    "        \"colsample_bytree\": 0.6,\n",
    "        \"random_state\": 42\n",
    "    }\n",
    "    xgb = XGBClassifier(**xgb_params)\n",
    "\n",
    "    weights = [0.28, 0.67]\n",
    "\n",
    "    ensemble = VotingClassifier(\n",
    "        estimators=[\n",
    "            (\"lgb\", lgb),\n",
    "            (\"cat\", cat)\n",
    "        ],\n",
    "        weights=weights,\n",
    "        voting=\"soft\",\n",
    "        n_jobs=-1\n",
    "    )\n",
    "    ensemble.fit(X_train, y_train)\n",
    "    gc.collect()\n",
    "    final_preds2 = ensemble.predict_proba(X_test)[:, 1]\n",
    "    print(final_preds2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Length of values (100) does not match length of index (3)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m sub[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerated\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m final_preds1 \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m0.30\u001b[39m \u001b[38;5;241m+\u001b[39m final_preds2 \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m0.70\u001b[39m\n\u001b[0;32m      2\u001b[0m sub\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msubmission.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m      3\u001b[0m sub\n",
      "File \u001b[1;32mc:\\Users\\86183\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4091\u001b[0m, in \u001b[0;36mDataFrame.__setitem__\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   4088\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_setitem_array([key], value)\n\u001b[0;32m   4089\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   4090\u001b[0m     \u001b[38;5;66;03m# set column\u001b[39;00m\n\u001b[1;32m-> 4091\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_set_item\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\Users\\86183\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4300\u001b[0m, in \u001b[0;36mDataFrame._set_item\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   4290\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_set_item\u001b[39m(\u001b[38;5;28mself\u001b[39m, key, value) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   4291\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   4292\u001b[0m \u001b[38;5;124;03m    Add series to DataFrame in specified column.\u001b[39;00m\n\u001b[0;32m   4293\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4298\u001b[0m \u001b[38;5;124;03m    ensure homogeneity.\u001b[39;00m\n\u001b[0;32m   4299\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 4300\u001b[0m     value, refs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sanitize_column\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4302\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[0;32m   4303\u001b[0m         key \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\n\u001b[0;32m   4304\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m value\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m   4305\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value\u001b[38;5;241m.\u001b[39mdtype, ExtensionDtype)\n\u001b[0;32m   4306\u001b[0m     ):\n\u001b[0;32m   4307\u001b[0m         \u001b[38;5;66;03m# broadcast across multiple columns if necessary\u001b[39;00m\n\u001b[0;32m   4308\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mis_unique \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns, MultiIndex):\n",
      "File \u001b[1;32mc:\\Users\\86183\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:5039\u001b[0m, in \u001b[0;36mDataFrame._sanitize_column\u001b[1;34m(self, value)\u001b[0m\n\u001b[0;32m   5036\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _reindex_for_setitem(value, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindex)\n\u001b[0;32m   5038\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_list_like(value):\n\u001b[1;32m-> 5039\u001b[0m     \u001b[43mcom\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequire_length_match\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   5040\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m sanitize_array(value, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindex, copy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, allow_2d\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m), \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\86183\\anaconda3\\lib\\site-packages\\pandas\\core\\common.py:561\u001b[0m, in \u001b[0;36mrequire_length_match\u001b[1;34m(data, index)\u001b[0m\n\u001b[0;32m    557\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    558\u001b[0m \u001b[38;5;124;03mCheck the length of data matches the length of the index.\u001b[39;00m\n\u001b[0;32m    559\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    560\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(data) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(index):\n\u001b[1;32m--> 561\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    562\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLength of values \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    563\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(data)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    564\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdoes not match length of index \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    565\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m(\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(index)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    566\u001b[0m     )\n",
      "\u001b[1;31mValueError\u001b[0m: Length of values (100) does not match length of index (3)"
     ]
    }
   ],
   "source": [
    "sub[\"generated\"] = final_preds1 * 0.30 + final_preds2 * 0.70\n",
    "sub.to_csv(\"submission.csv\", index=False)\n",
    "sub"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}