CH4/templates/08_stat.two_proportions_diff_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Z-критерий для двух долей "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import scipy\n",
    "from statsmodels.stats.weightstats import *\n",
    "from statsmodels.stats.proportion import proportion_confint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Загрузка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('banner_click_stat.txt', header = None, sep = '\\t')\n",
    "data.columns = ['banner_a', 'banner_b']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>banner_a</th>\n",
       "      <th>banner_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   banner_a  banner_b\n",
       "0         0         0\n",
       "1         1         1\n",
       "2         0         0\n",
       "3         0         0\n",
       "4         0         0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>banner_a</th>\n",
       "      <th>banner_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.037000</td>\n",
       "      <td>0.053000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.188856</td>\n",
       "      <td>0.224146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          banner_a     banner_b\n",
       "count  1000.000000  1000.000000\n",
       "mean      0.037000     0.053000\n",
       "std       0.188856     0.224146\n",
       "min       0.000000     0.000000\n",
       "25%       0.000000     0.000000\n",
       "50%       0.000000     0.000000\n",
       "75%       0.000000     0.000000\n",
       "max       1.000000     1.000000"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Интервальные оценки долей"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\frac1{ 1 + \\frac{z^2}{n} } \\left( \\hat{p} + \\frac{z^2}{2n} \\pm z \\sqrt{ \\frac{ \\hat{p}\\left(1-\\hat{p}\\right)}{n} + \\frac{z^2}{4n^2} } \\right), \\;\\; z \\equiv z_{1-\\frac{\\alpha}{2}}$$ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "conf_interval_banner_a = proportion_confint(sum(data.banner_a), \n",
    "                                            data.shape[0],\n",
    "                                            method = 'wilson')\n",
    "conf_interval_banner_b = proportion_confint(sum(data.banner_b), \n",
    "                                            data.shape[0],\n",
    "                                            method = 'wilson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% confidence interval for a click probability, banner a: [0.026961, 0.050582]\n",
      "95% confidence interval for a click probability, banner b [0.040747, 0.068675]\n"
     ]
    }
   ],
   "source": [
    "print '95%% confidence interval for a click probability, banner a: [%f, %f]' % conf_interval_banner_a\n",
    "print '95%% confidence interval for a click probability, banner b [%f, %f]' % conf_interval_banner_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Z-критерий для разности долей (независимые выборки)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   | $X_1$ | $X_2$  \n",
    "  ------------- | -------------|\n",
    "  1  | a | b \n",
    "  0  | c | d \n",
    "  $\\sum$ | $n_1$| $n_2$\n",
    "  \n",
    "$$ \\hat{p}_1 = \\frac{a}{n_1}$$\n",
    "\n",
    "$$ \\hat{p}_2 = \\frac{b}{n_2}$$\n",
    "\n",
    "\n",
    "$$\\text{Доверительный интервал для }p_1 - p_2\\colon \\;\\; \\hat{p}_1 - \\hat{p}_2 \\pm z_{1-\\frac{\\alpha}{2}}\\sqrt{\\frac{\\hat{p}_1(1 - \\hat{p}_1)}{n_1} + \\frac{\\hat{p}_2(1 - \\hat{p}_2)}{n_2}}$$\n",
    "\n",
    "$$Z-статистика: Z({X_1, X_2}) =  \\frac{\\hat{p}_1 - \\hat{p}_2}{\\sqrt{P(1 - P)(\\frac{1}{n_1} + \\frac{1}{n_2})}}$$\n",
    "$$P = \\frac{\\hat{p}_1{n_1} + \\hat{p}_2{n_2}}{{n_1} + {n_2}} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proportions_diff_confint_ind(sample1, sample2, alpha = 0.05):    \n",
    "    z = scipy.stats.norm.ppf(1 - alpha / 2.)\n",
    "    \n",
    "    p1 = float(sum(sample1)) / len(sample1)\n",
    "    p2 = float(sum(sample2)) / len(sample2)\n",
    "    \n",
    "    left_boundary = (p1 - p2) - z * np.sqrt(p1 * (1 - p1)/ len(sample1) + p2 * (1 - p2)/ len(sample2))\n",
    "    right_boundary = (p1 - p2) + z * np.sqrt(p1 * (1 - p1)/ len(sample1) + p2 * (1 - p2)/ len(sample2))\n",
    "    \n",
    "    return (left_boundary, right_boundary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proportions_diff_z_stat_ind(sample1, sample2):\n",
    "    n1 = len(sample1)\n",
    "    n2 = len(sample2)\n",
    "    \n",
    "    p1 = float(sum(sample1)) / n1\n",
    "    p2 = float(sum(sample2)) / n2 \n",
    "    P = float(p1*n1 + p2*n2) / (n1 + n2)\n",
    "    \n",
    "    return (p1 - p2) / np.sqrt(P * (1 - P) * (1. / n1 + 1. / n2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proportions_diff_z_test(z_stat, alternative = 'two-sided'):\n",
    "    if alternative not in ('two-sided', 'less', 'greater'):\n",
    "        raise ValueError(\"alternative not recognized\\n\"\n",
    "                         \"should be 'two-sided', 'less' or 'greater'\")\n",
    "    \n",
    "    if alternative == 'two-sided':\n",
    "        return 2 * (1 - scipy.stats.norm.cdf(np.abs(z_stat)))\n",
    "    \n",
    "    if alternative == 'less':\n",
    "        return scipy.stats.norm.cdf(z_stat)\n",
    "\n",
    "    if alternative == 'greater':\n",
    "        return 1 - scipy.stats.norm.cdf(z_stat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% confidence interval for a difference between proportions: [-0.034157, 0.002157]\n"
     ]
    }
   ],
   "source": [
    "print \"95%% confidence interval for a difference between proportions: [%f, %f]\" %\\\n",
    "      proportions_diff_confint_ind(data.banner_a, data.banner_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.084379\n"
     ]
    }
   ],
   "source": [
    "print \"p-value: %f\" % proportions_diff_z_test(proportions_diff_z_stat_ind(data.banner_a, data.banner_b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.042189\n"
     ]
    }
   ],
   "source": [
    "print \"p-value: %f\" % proportions_diff_z_test(proportions_diff_z_stat_ind(data.banner_a, data.banner_b), 'less')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Z-критерий для разности долей (связанные выборки)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "  $X_1$ \\ $X_2$ | 1| 0 | $\\sum$\n",
    "  ------------- | -------------|\n",
    "  1  | e | f | e + f\n",
    "  0  | g | h | g + h\n",
    "  $\\sum$ | e + g| f + h | n  \n",
    "  \n",
    "$$ \\hat{p}_1 = \\frac{e + f}{n}$$\n",
    "\n",
    "$$ \\hat{p}_2 = \\frac{e + g}{n}$$\n",
    "\n",
    "$$ \\hat{p}_1 - \\hat{p}_2 = \\frac{f - g}{n}$$\n",
    "\n",
    "\n",
    "$$\\text{Доверительный интервал для }p_1 - p_2\\colon \\;\\;  \\frac{f - g}{n} \\pm z_{1-\\frac{\\alpha}{2}}\\sqrt{\\frac{f + g}{n^2} - \\frac{(f - g)^2}{n^3}}$$\n",
    "\n",
    "$$Z-статистика: Z({X_1, X_2}) = \\frac{f - g}{\\sqrt{f + g - \\frac{(f-g)^2}{n}}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def proportions_diff_confint_rel(sample1, sample2, alpha = 0.05):\n",
    "    z = scipy.stats.norm.ppf(1 - alpha / 2.)\n",
    "    sample = zip(sample1, sample2)\n",
    "    n = len(sample)\n",
    "        \n",
    "    f = sum([1 if (x[0] == 1 and x[1] == 0) else 0 for x in sample])\n",
    "    g = sum([1 if (x[0] == 0 and x[1] == 1) else 0 for x in sample])\n",
    "    \n",
    "    left_boundary = float(f - g) / n  - z * np.sqrt(float((f + g)) / n**2 - float((f - g)**2) / n**3)\n",
    "    right_boundary = float(f - g) / n  + z * np.sqrt(float((f + g)) / n**2 - float((f - g)**2) / n**3)\n",
    "    return (left_boundary, right_boundary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def proportions_diff_z_stat_rel(sample1, sample2):\n",
    "    sample = zip(sample1, sample2)\n",
    "    n = len(sample)\n",
    "    \n",
    "    f = sum([1 if (x[0] == 1 and x[1] == 0) else 0 for x in sample])\n",
    "    g = sum([1 if (x[0] == 0 and x[1] == 1) else 0 for x in sample])\n",
    "    \n",
    "    return float(f - g) / np.sqrt(f + g - float((f - g)**2) / n )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% confidence interval for a difference between proportions: [-0.026689, -0.005311]\n"
     ]
    }
   ],
   "source": [
    "print \"95%% confidence interval for a difference between proportions: [%f, %f]\" \\\n",
    "      % proportions_diff_confint_rel(data.banner_a, data.banner_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.003349\n"
     ]
    }
   ],
   "source": [
    "print \"p-value: %f\" % proportions_diff_z_test(proportions_diff_z_stat_rel(data.banner_a, data.banner_b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.001675\n"
     ]
    }
   ],
   "source": [
    "print \"p-value: %f\" % proportions_diff_z_test(proportions_diff_z_stat_rel(data.banner_a, data.banner_b), 'less')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}