03_inference_google_2022_both_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7f9806d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.model_selection as ms\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn import metrics\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from joblib import dump, load\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9a7d2573",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input\n",
    "\n",
    "# This is the output table from the part of the data-post-production repo that merges\n",
    "# preprocessed results. \n",
    "# Source: data-post-production/01-merge-results/01_merge_preprocessed_results\n",
    "path_inference_data = \"g2022_adid_01062021_11082022_text.csv\"\n",
    "\n",
    "# This is the output table from the part of the data-post-production repo that merges\n",
    "# preprocessed results. \n",
    "# Source: data-post-production/01-merge-results/01_merge_preprocessed_results\n",
    "path_inference_data_vars = \"g2022_adid_01062021_11082022_text.csv.gz\"\n",
    "\n",
    "path_model = \"models/party_clf_facebook_and_google_2020.joblib\"\n",
    "path_model_smooth = \"models/party_clf_facebook_and_google_2020_smooth.joblib\"\n",
    "\n",
    "# Output\n",
    "path_predictions = \"data/google/party_predictions_google_2022_both_model.csv.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "400b9aff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['ad_id', 'wmp_creative_id', 'ad_type', 'ad_title', 'google_asr_text',\n",
      "       'aws_ocr_video_text', 'aws_ocr_img_text', 'csum_agg', 'advertiser_id',\n",
      "       'advertiser_name', 'ad_text'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Inference dataset\n",
    "df = pd.read_csv(path_inference_data, encoding='UTF-8', keep_default_na = False, dtype = 'str')\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c42fb291",
   "metadata": {},
   "outputs": [],
   "source": [
    "# All fields\n",
    "cols = ['advertiser_name', 'ad_title', 'google_asr_text', 'aws_ocr_video_text', 'aws_ocr_img_text', 'ad_text']\n",
    "# Combine and clean up\n",
    "df['combined'] = df[cols].apply(lambda row: ' '.join(row.values.astype(str)), axis=1)\n",
    "df['combined'] = df['combined'].str.strip() # Remove whitesaces at end of string\n",
    "df['combined'] = df['combined'].str.replace(' +', ' ', regex = True) # Remove double (and triple etc.) whitespaces inside\n",
    "df = df[['ad_id', 'combined']]\n",
    "# Deduplicate by text to save time during inference\n",
    "df = df.groupby(['combined'])['ad_id'].apply(list)\n",
    "df = df.to_frame().reset_index()\n",
    "# Remove empty ads\n",
    "df = df[df['combined'] != \"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "966eb0a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading model: 100%|█████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.42it/s]\n",
      "Predicting probabilities: 100%|████████████████████████████████████████████████| 58297/58297 [00:09<00:00, 5892.50it/s]\n"
     ]
    }
   ],
   "source": [
    "# Regular model\n",
    "# Load model\n",
    "with tqdm(total=1, desc='Loading model') as pbar:\n",
    "    clf = load(path_model)\n",
    "    pbar.update(1)\n",
    "\n",
    "# Predicted probabilities\n",
    "with tqdm(total=len(df), desc='Predicting probabilities') as pbar:\n",
    "    pp = clf.predict_proba(df['combined'])\n",
    "    df['prob_dem'] = pp[:, 0]\n",
    "    df['prob_rep'] = pp[:, 1]\n",
    "    pbar.update(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "29a42ea3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading smooth model: 100%|██████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.34it/s]\n",
      "Predicting smooth probabilities: 100%|█████████████████████████████████████████| 58297/58297 [00:09<00:00, 5893.12it/s]\n"
     ]
    }
   ],
   "source": [
    "# Smooth model\n",
    "# Load model\n",
    "with tqdm(total=1, desc='Loading smooth model') as pbar:\n",
    "    clf_smooth = load(path_model_smooth)\n",
    "    pbar.update(1)\n",
    "\n",
    "# Predicted probabilities\n",
    "with tqdm(total=len(df), desc='Predicting smooth probabilities') as pbar:\n",
    "    pp = clf_smooth.predict_proba(df['combined'])\n",
    "    df['prob_dem_smooth'] = pp[:, 0]\n",
    "    df['prob_rep_smooth'] = pp[:, 1]\n",
    "    pbar.update(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "40daa678",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.explode('ad_id')\n",
    "\n",
    "# Keep only the relevant variables\n",
    "df = df[['ad_id', 'prob_dem', 'prob_rep', 'prob_dem_smooth', 'prob_rep_smooth']]\n",
    "\n",
    "# Save\n",
    "df.to_csv(path_predictions, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d48c9b5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}