difference_significance_testing_bootstrap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "from scipy.io import loadmat\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.linear_model import Lasso, Ridge\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import r2_score\n",
    "from scipy import stats\n",
    "from sklearn.decomposition import PCA\n",
    "from IPython.core.debugger import set_trace\n",
    "import pickle\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "random.seed(97)\n",
    "np.random.seed(97)\n",
    "\n",
    "root_save_dir = \"predictions/\"\n",
    "sub_data = \"sub_space_data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def R2(Real,Pred):\n",
    "    SSres = np.mean((Real-Pred)**2,0)\n",
    "    SStot = np.var(Real,0)\n",
    "    return np.nan_to_num(1-SSres/SStot)\n",
    "\n",
    "\n",
    "def compute_sig(sub, feat1, feat2):\n",
    "    print(\"Subject {} Feature 1 : {} Feature 2: {}\".format(sub,feat1,feat2))\n",
    "        \n",
    "    save_dir = root_save_dir + \"{}_diff_{}/\".format(feat1, feat2)\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.mkdir(save_dir)\n",
    "    \n",
    "    feat1_dir = root_save_dir + \"{}/\".format(feat1)\n",
    "    feat2_dir = root_save_dir + \"{}/\".format(feat2)\n",
    "    \n",
    "    if os.path.exists(save_dir + \"{}_sig_boot.npy\".format(sub)):\n",
    "        return 0.0002\n",
    "    \n",
    "    reals = []\n",
    "    for split_num in range(4):\n",
    "        reals.append(np.load(feat2_dir + sub + \"_y_test_{}.npy\".format(split_num)))\n",
    "    all_reals = np.vstack(reals)\n",
    "\n",
    "    preds_feat1 = []\n",
    "    for split_num in range(4):\n",
    "        preds_feat1.append(np.load(feat1_dir + sub + \"_y_pred_{}.npy\".format(split_num)))\n",
    "    all_preds_feat1 = np.vstack(preds_feat1)\n",
    "    \n",
    "    preds_feat2 = []\n",
    "    for split_num in range(4):\n",
    "        preds_feat2.append(np.load(feat2_dir + sub + \"_y_pred_{}.npy\".format(split_num)))\n",
    "    all_preds_feat2 = np.vstack(preds_feat2)\n",
    "\n",
    "    real_r2_feat1 = np.load(feat1_dir + sub + \"_r2s.npy\")\n",
    "    real_r2_feat2 = np.load(feat2_dir + sub + \"_r2s.npy\")\n",
    "    \n",
    "    real_diff = real_r2_feat1 - real_r2_feat2\n",
    "    \n",
    "    greater = np.zeros(real_diff.shape[0])\n",
    "\n",
    "    blocks_feat1 = []\n",
    "    for i in range(4):\n",
    "        p = np.array([preds_feat1[i][j:j+blocksize] for j in range(0,len(preds_feat1[i]),blocksize)])\n",
    "        blocks_feat1.append(p)\n",
    "        \n",
    "    blocks_feat2 = []\n",
    "    for i in range(4):\n",
    "        p = np.array([preds_feat2[i][j:j+blocksize] for j in range(0,len(preds_feat2[i]),blocksize)])\n",
    "        blocks_feat2.append(p)\n",
    "        \n",
    "    blocks_real = []\n",
    "    for i in range(4):\n",
    "        p = np.array([reals[i][j:j+blocksize] for j in range(0,len(reals[i]),blocksize)])\n",
    "        blocks_real.append(p)\n",
    "    \n",
    "    for s in tqdm(range(iters)):\n",
    "        new_preds_feat1 = []\n",
    "        new_preds_feat2 = []\n",
    "        new_reals = []\n",
    "        for i in range(4):\n",
    "            sample = np.random.choice(len(blocks_feat1[i]),len(blocks_feat1[i]))\n",
    "            \n",
    "            bl1 = blocks_feat1[i][sample]\n",
    "            shuffled_preds_feat1 = np.vstack(bl1)\n",
    "            new_preds_feat1.append(shuffled_preds_feat1)\n",
    "            \n",
    "            bl2 = blocks_feat2[i][sample]\n",
    "            shuffled_preds_feat2 = np.vstack(bl2)\n",
    "            new_preds_feat2.append(shuffled_preds_feat2)\n",
    "            \n",
    "            blr = blocks_real[i][sample]\n",
    "            shuffled_reals = np.vstack(blr)\n",
    "            new_reals.append(shuffled_reals)\n",
    "            \n",
    "        new_preds_feat1 = np.vstack(new_preds_feat1)\n",
    "        new_preds_feat2 = np.vstack(new_preds_feat2)\n",
    "        new_reals = np.vstack(new_reals)\n",
    "        \n",
    "        shuffled_r2_feat1 = R2(new_reals,new_preds_feat1)\n",
    "        shuffled_r2_feat1[real_r2_feat1 == 0] = 0\n",
    "        \n",
    "        shuffled_r2_feat2 = R2(new_reals,new_preds_feat2)\n",
    "        shuffled_r2_feat2[real_r2_feat2 == 0] = 0\n",
    "        \n",
    "        shuffled_diff = shuffled_r2_feat1 - shuffled_r2_feat2        \n",
    "        greater[shuffled_diff <= 0] += 1            \n",
    "    sig = greater/iters\n",
    "    sig[sig == 0] = 1/iters\n",
    "    np.save(save_dir + \"{}_sig_boot.npy\".format(sub),sig)\n",
    "    return np.min(sig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('node_count_punct', 'punct_final'), ('syntactic_surprisal_punct', 'punct_final'), ('word_frequency_punct', 'punct_final'), ('word_length_punct', 'punct_final'), ('all_effort_based_metrics_punct', 'punct_final'), ('pos_dep_tags_all_effort_based_metrics', 'all_effort_based_metrics_punct'), ('aggregated_contrege_pos_dep_tags_all_effort_based_metrics', 'pos_dep_tags_all_effort_based_metrics'), ('aggregated_contrege_comp_pos_dep_tags_all_effort_based_metrics', 'pos_dep_tags_all_effort_based_metrics'), ('aggregated_beam_incontrege_pos_dep_tags_all_effort_based_metrics', 'pos_dep_tags_all_effort_based_metrics'), ('aggregated_bert_PCA_dims_15_contrege_pos_dep_tags_all_effort_based_metrics', 'aggregated_contrege_pos_dep_tags_all_effort_based_metrics')]\n"
     ]
    }
   ],
   "source": [
    "all_feature_pairs = [\n",
    "                     (\"node_count_punct\", \"punct_final\"), # (NC + PU) - (PU)\n",
    "                     (\"syntactic_surprisal_punct\", \"punct_final\"), # (SS + PU) - (PU)                 \n",
    "                     (\"word_frequency_punct\", \"punct_final\"), # (WF + PU) - (PU)\n",
    "                     (\"word_length_punct\", \"punct_final\"), # (WL + PU) - (PU)\n",
    "                     (\"all_complexity_metrics_punct\", \"punct_final\"), # (CM + PU) - (PU)\n",
    "                     (\"pos_dep_tags_all_complexity_metrics\", \"all_complexity_metrics_punct\"), # since PD already contains PU, this tests (PD + CM + PU) - (CM + PU)\n",
    "                     (\"aggregated_contrege_comp_pos_dep_tags_all_complexity_metrics\", \"pos_dep_tags_all_complexity_metrics\"), # (CC + PD + CM + PU) - (PD + CM + PU)\n",
    "                     (\"aggregated_contrege_incomp_pos_dep_tags_all_complexity_metrics\", \"pos_dep_tags_all_complexity_metrics\"), # (CI + PD + CM + PU) - (PD + CM + PU)\n",
    "                     (\"aggregated_incontrege_pos_dep_tags_all_complexity_metrics\", \"pos_dep_tags_all_complexity_metrics\"), # (INC + PD + CM + PU) - (PD + CM + PU)\n",
    "                     (\"aggregated_bert_PCA_dims_15_contrege_incomp_pos_dep_tags_all_complexity_metrics\", \"aggregated_contrege_incomp_pos_dep_tags_all_complexity_metrics\") # (BERT + CI + PD + CM + PU) - (CI + PD + CM + PU)\n",
    "                    ]\n",
    "\n",
    "print(all_feature_pairs)\n",
    "\n",
    "all_subjects = [\"F\",\"G\",\"H\",\"I\",\"J\",\"K\",\"L\",\"M\",\"N\"]\n",
    "\n",
    "blocksize = 10\n",
    "iters = 5000\n",
    "\n",
    "Parallel(n_jobs=18)(delayed(compute_sig)(sub,feat[0],feat[1]) for sub in all_subjects for feat in all_feature_pairs)\n",
    "# for sub in all_subjects:\n",
    "#     for feat in all_feature_pairs:\n",
    "#         compute_sig(sub,feat[0],feat[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}