03_inference_google_2022.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd4c2fe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.model_selection as ms\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn import metrics\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from joblib import dump, load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c112c985",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input\n",
    "# This is the output table from the part of the data-post-production repo that merges\n",
    "# preprocessed results. \n",
    "# Source: data-post-production/01-merge-results/01_merge_preprocessed_results\n",
    "path_inference_data = \"g2022_adid_01062021_11082022_text.csv\"\n",
    "\n",
    "# This is the output table from the part of the data-post-production repo that merges\n",
    "# preprocessed results. \n",
    "# Source: data-post-production/01-merge-results/01_merge_preprocessed_results\n",
    "path_inference_data_vars = \"g2022_adid_01062021_11082022_text.csv.gz\"\n",
    "\n",
    "path_model = \"models/party_clf.joblib\"\n",
    "\n",
    "# Output\n",
    "path_predictions = \"data/google/party_predictions_google_2022.csv.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "668555f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['ad_id', 'wmp_creative_id', 'ad_type', 'ad_title', 'google_asr_text',\n",
      "       'aws_ocr_video_text', 'aws_ocr_img_text', 'csum_agg', 'advertiser_id',\n",
      "       'advertiser_name', 'ad_text'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Inference dataset\n",
    "df = pd.read_csv(path_inference_data, encoding='UTF-8', keep_default_na = False, dtype = 'str')\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c76e8dab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# All fields\n",
    "cols = ['advertiser_name', 'ad_title', 'google_asr_text', 'aws_ocr_video_text', 'aws_ocr_img_text', 'ad_text']\n",
    "# Combine and clean up\n",
    "df['combined'] = df[cols].apply(lambda row: ' '.join(row.values.astype(str)), axis=1)\n",
    "df['combined'] = df['combined'].str.strip() # Remove whitesaces at end of string\n",
    "df['combined'] = df['combined'].str.replace(' +', ' ', regex = True) # Remove double (and triple etc.) whitespaces inside\n",
    "df = df[['ad_id', 'combined']]\n",
    "# Deduplicate by text to save time during inference\n",
    "df = df.groupby(['combined'])['ad_id'].apply(list)\n",
    "df = df.to_frame().reset_index()\n",
    "# Remove empty ads\n",
    "df = df[df['combined'] != \"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d70dfceb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "clf = load(path_model)\n",
    "\n",
    "# Predicted probabilities\n",
    "pp = clf.predict_proba(df['combined'])\n",
    "df['prob_dem'] = pp[:,0]\n",
    "df['prob_other'] = pp[:,1]\n",
    "df['prob_rep'] = pp[:,2]\n",
    "df['predicted_party_all'] = clf.classes_[np.argmax(pp, axis = 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ba0910b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.explode('ad_id')\n",
    "\n",
    "# Merge in advertiser_id\n",
    "df_vars = pd.read_csv(path_inference_data_vars, encoding='UTF-8')\n",
    "df_vars = df_vars[['ad_id', 'advertiser_id']]\n",
    "df = df.merge(df_vars, on = 'ad_id')\n",
    "# Create a variable where all party classifications of a pd-id\n",
    "# are assigned by majority vote\n",
    "dft = df[['advertiser_id', 'predicted_party_all']]\n",
    "maj_vote = dft.groupby(['advertiser_id'])['predicted_party_all'].agg(pd.Series.mode)\n",
    "maj_vote = pd.DataFrame(maj_vote)\n",
    "maj_vote = maj_vote.reset_index()\n",
    "# In case of ties, make it OTHER\n",
    "maj_vote['predicted_party_all'][[f is not str for f in maj_vote['predicted_party_all'].apply(type)]] = 'OTHER'\n",
    "maj_vote.columns = ['advertiser_id', 'predicted_party_all_majvote']\n",
    "df = df.merge(maj_vote, how = 'left', on = 'advertiser_id')\n",
    "\n",
    "# Keep only the relevant variables\n",
    "df = df[['ad_id', 'prob_dem', 'prob_other', 'prob_rep', 'predicted_party_all', 'predicted_party_all_majvote']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "df4993c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save\n",
    "df.to_csv(path_predictions, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c9c7c98",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}