mutual_info_and_conquest_epoch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "Mutual information functions\n",
    "* `sklearn.metrics.mutual_info_score`: [link](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mutual_info_score.html)\n",
    "* `sklearn.metrics.adjusted_mutual_info_score`: [link](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html#sklearn.metrics.adjusted_mutual_info_score)\n",
    "* `sklearn.metrics.normalized_mutual_info_score`: [link](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.normalized_mutual_info_score.html#sklearn.metrics.normalized_mutual_info_score)\n",
    "\n",
    "Conquest epoch order\n",
    "* div < sub < add < mod < mul\n",
    "\n",
    "Mutual information order\n",
    "* sub < add < div < mod < mul\n",
    "\n",
    "Conclusion\n",
    "* Mutual information does not provide a good explanation for coquest epoch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "import data_utils\n",
    "import config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "operand_digits = 4\n",
    "(train_ratio, dev_ratio, test_ratio) = (1.0, 0.0, 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sum_mi(np_input, np_target, mi_type='mutual_info_score'):\n",
    "    '''\n",
    "    Parameters\n",
    "    - np_input  : np.ndarray. shape == (examples, input_dimension).\n",
    "    - np_target : np.ndarray. shape == (examples, target_dimension).\n",
    "    Returns\n",
    "    - \n",
    "    '''\n",
    "    sum_mi = 0\n",
    "    for i_x in range(input_train.shape[1]):\n",
    "        for i_y in range(target_train.shape[1]):\n",
    "            labels_x = input_train[:,i_x]\n",
    "            labels_y = target_train[:,i_y]\n",
    "            if mi_type == 'mutual_info_score':\n",
    "                mi = metrics.mutual_info_score(labels_x, labels_y)\n",
    "            if mi_type == 'adjusted_mutual_info_score':\n",
    "                mi = metrics.adjusted_mutual_info_score(labels_x, labels_y)\n",
    "            if mi_type == 'normalized_mutual_info_score':\n",
    "                mi = metrics.normalized_mutual_info_score(labels_x, labels_y)\n",
    "            sum_mi += mi\n",
    "    \n",
    "    return sum_mi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mi_digits(np_input, np_target, mi_type='mutual_info_score'):\n",
    "    '''\n",
    "    Parameters\n",
    "    - np_input  : np.ndarray. shape == (examples, input_dimension).\n",
    "    - np_target : np.ndarray. shape == (examples, target_dimension).\n",
    "    Returns\n",
    "    - \n",
    "    '''\n",
    "    mi_digits = np.zeros((target_train.shape[1]))\n",
    "    for i_y in range(target_train.shape[1]):\n",
    "        sum_mi = 0\n",
    "        for i_x in range(input_train.shape[1]):\n",
    "            labels_x = input_train[:,i_x]\n",
    "            labels_y = target_train[:,i_y]\n",
    "            if mi_type == 'mutual_info_score':\n",
    "                mi = metrics.mutual_info_score(labels_x, labels_y)\n",
    "            if mi_type == 'adjusted_mutual_info_score':\n",
    "                mi = metrics.adjusted_mutual_info_score(labels_x, labels_y)\n",
    "            if mi_type == 'normalized_mutual_info_score':\n",
    "                mi = metrics.normalized_mutual_info_score(labels_x, labels_y)\n",
    "            sum_mi += mi\n",
    "        if sum_mi > 10**(-10):\n",
    "            mi_digits[i_y] = sum_mi\n",
    "    \n",
    "    return mi_digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "operator = 'subtract'\n",
    "(input_train, input_dev, input_test,\n",
    "     target_train, target_dev, target_test) = data_utils.import_op_dataset(operator, operand_digits, train_ratio, dev_ratio, test_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operator : add\n",
      "Sum mutual information : 0.3460160260244719\n",
      "MI digits : [0.0, 0.0, 0.0, 0.3460160260244719, 0.0, 0.0, 0.0, 0.0]\n",
      "Operator : subtract\n",
      "Sum mutual information : 0.26562494360035976\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.2171237149423389, 0.03301521707312155, 0.01033439657406815, 0.005151615010813404]\n",
      "Operator : multiply\n",
      "Sum mutual information : 1.1346354359805524\n",
      "MI digits : [0.23962584694241074, 0.13876432162009497, 0.07455803563045837, 0.0514251286505476, 0.015696704679991402, 0.04775398750496784, 0.13528830227441468, 0.4315231086776665]\n",
      "Operator : divide\n",
      "Sum mutual information : 0.6559750247709321\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.12332373524069233, 0.17222253024847134, 0.21211283373214948, 0.14831592554961892]\n",
      "Operator : modulo\n",
      "Sum mutual information : 0.8489121337956735\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.21987701311600533, 0.20673753741571102, 0.17729538768794284, 0.24500219557601427]\n"
     ]
    }
   ],
   "source": [
    "for operator in config.operators_list():\n",
    "    (input_train, input_dev, input_test,\n",
    "     target_train, target_dev, target_test) = data_utils.import_op_dataset(operator, operand_digits, train_ratio, dev_ratio, test_ratio)\n",
    "    mi_type = 'mutual_info_score'\n",
    "    sum_mi = get_sum_mi(input_train, target_train, mi_type)\n",
    "    mi_digits = get_mi_digits(input_train, target_train, mi_type)\n",
    "    print('Operator : {}'.format(operator))\n",
    "    print('Sum mutual information : {}'.format(sum_mi))\n",
    "    print('MI digits : {}'.format(list(mi_digits)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operator : add\n",
      "Sum mutual information : 0.38691601197480746\n",
      "MI digits : [0.0, 0.0, 0.0, 0.4778743007225715, 0.0, 0.0, 0.0, 0.0]\n",
      "Operator : subtract\n",
      "Sum mutual information : 0.26793481816386333\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.32660983585731085, 0.005567250983015102, 0.0, 0.0]\n",
      "Operator : multiply\n",
      "Sum mutual information : 1.4592838497805312\n",
      "MI digits : [0.3237197012766313, 0.1779693483933377, 0.08511121952095642, 0.05165302458502611, 0.0, 0.046347834802115725, 0.17298313341177807, 0.6015299443910096]\n",
      "Operator : divide\n",
      "Sum mutual information : 0.851551937046836\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.15283205474807068, 0.22506400095689216, 0.2832212152581411, 0.19043466608373186]\n",
      "Operator : modulo\n",
      "Sum mutual information : 1.1317248523606454\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.2940479425061816, 0.27501472023649737, 0.23234948602545286, 0.3303127035925129]\n"
     ]
    }
   ],
   "source": [
    "for operator in config.operators_list():\n",
    "    (input_train, input_dev, input_test,\n",
    "     target_train, target_dev, target_test) = data_utils.import_op_dataset(operator, operand_digits, train_ratio, dev_ratio, test_ratio)\n",
    "    mi_type = 'adjusted_mutual_info_score'\n",
    "    sum_mi = get_sum_mi(input_train, target_train, mi_type)\n",
    "    mi_digits = get_mi_digits(input_train, target_train, mi_type)\n",
    "    print('Operator : {}'.format(operator))\n",
    "    print('Sum mutual information : {}'.format(sum_mi))\n",
    "    print('MI digits : {}'.format(list(mi_digits)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operator : add\n",
      "Sum mutual information : 0.4999008628138723\n",
      "MI digits : [0.0, 0.0, 0.0, 0.4999008628138723, 0.0, 0.0, 0.0, 0.0]\n",
      "Operator : subtract\n",
      "Sum mutual information : 0.44782044419706\n",
      "MI digits : [4.440892098500626e-05, 4.440892098500626e-05, 4.440892098500626e-05, 4.440892098500626e-05, 0.3729593972174917, 0.05148849849057989, 0.015603880307116761, 0.007591032497931641]\n",
      "Operator : multiply\n",
      "Sum mutual information : 1.859607065502502\n",
      "MI digits : [0.46890264209373833, 0.22061736353805023, 0.1116375511835406, 0.07551633197769882, 0.02267755139268401, 0.06928704558011928, 0.1997845198482558, 0.6911840598884149]\n",
      "Operator : divide\n",
      "Sum mutual information : 1.3699109401734066\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.3879772347874971, 0.3867382270929076, 0.37457825649898324, 0.22061722179401852]\n",
      "Operator : modulo\n",
      "Sum mutual information : 1.3899278277711125\n",
      "MI digits : [0.0, 0.0, 0.0, 0.0, 0.44035178369156314, 0.3299350570895878, 0.26321709204056337, 0.35642389494939886]\n"
     ]
    }
   ],
   "source": [
    "for operator in config.operators_list():\n",
    "    (input_train, input_dev, input_test,\n",
    "     target_train, target_dev, target_test) = data_utils.import_op_dataset(operator, operand_digits, train_ratio, dev_ratio, test_ratio)\n",
    "    mi_type = 'normalized_mutual_info_score'\n",
    "    sum_mi = get_sum_mi(input_train, target_train, mi_type)\n",
    "    mi_digits = get_mi_digits(input_train, target_train, mi_type)\n",
    "    print('Operator : {}'.format(operator))\n",
    "    print('Sum mutual information : {}'.format(sum_mi))\n",
    "    print('MI digits : {}'.format(list(mi_digits)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}