AMNH_Data_Munging.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:24bb5608579bfae04c58ed6c9bebea2c5891c588a0f6f38813e82ca11d1863ac"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "#import matplotlib.pyplot as plt\n",
      "#import matplotlib.dates as dates"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 406
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mw_data = pd.DataFrame.from_csv('/Users/williamliu/Desktop/amnh/mw_expl_new.csv', sep=',')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 407
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mw_data.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>numplanets</th>\n",
        "      <th>distance</th>\n",
        "      <th>texture1</th>\n",
        "      <th>texture2</th>\n",
        "      <th>targetblue</th>\n",
        "      <th>extra</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>x</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>-2.2931 </th>\n",
        "      <td>-22.3478</td>\n",
        "      <td> 108.2944</td>\n",
        "      <td> 1</td>\n",
        "      <td> 360.6</td>\n",
        "      <td> 1</td>\n",
        "      <td> 11</td>\n",
        "      <td> Com</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-28.9180</th>\n",
        "      <td> 85.3411</td>\n",
        "      <td>  78.4522</td>\n",
        "      <td> 1</td>\n",
        "      <td> 389.5</td>\n",
        "      <td> 1</td>\n",
        "      <td> 11</td>\n",
        "      <td> UMi</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-20.1427</th>\n",
        "      <td> 68.3916</td>\n",
        "      <td> -27.4397</td>\n",
        "      <td> 1</td>\n",
        "      <td> 249.1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 14</td>\n",
        "      <td> And</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th> 4.4048 </th>\n",
        "      <td> 11.5783</td>\n",
        "      <td>  13.2590</td>\n",
        "      <td> 1</td>\n",
        "      <td>  59.1</td>\n",
        "      <td> 1</td>\n",
        "      <td> 14</td>\n",
        "      <td> Her</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th> 2.4412 </th>\n",
        "      <td> 20.9070</td>\n",
        "      <td>   4.9427</td>\n",
        "      <td> 1</td>\n",
        "      <td>  70.5</td>\n",
        "      <td> 1</td>\n",
        "      <td> 16</td>\n",
        "      <td> Cyg</td>\n",
        "      <td>   B</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 408,
       "text": [
        "                y         z  numplanets  distance  texture1 texture2  \\\n",
        "x                                                                      \n",
        "-2.2931  -22.3478  108.2944           1     360.6         1       11   \n",
        "-28.9180  85.3411   78.4522           1     389.5         1       11   \n",
        "-20.1427  68.3916  -27.4397           1     249.1         1       14   \n",
        " 4.4048   11.5783   13.2590           1      59.1         1       14   \n",
        " 2.4412   20.9070    4.9427           1      70.5         1       16   \n",
        "\n",
        "         targetblue extra  \n",
        "x                          \n",
        "-2.2931         Com   NaN  \n",
        "-28.9180        UMi   NaN  \n",
        "-20.1427        And   NaN  \n",
        " 4.4048         Her   NaN  \n",
        " 2.4412         Cyg     B  "
       ]
      }
     ],
     "prompt_number": 408
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "del mw_data['extra']  # Some columns have two extra at end, we don't need"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 409
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mw_data.tail()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>numplanets</th>\n",
        "      <th>distance</th>\n",
        "      <th>texture1</th>\n",
        "      <th>texture2</th>\n",
        "      <th>targetblue</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>x</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th> 93.1725 </th>\n",
        "      <td>  95.9901</td>\n",
        "      <td> 148.6768</td>\n",
        "      <td> 1</td>\n",
        "      <td> 652.0</td>\n",
        "      <td> 1</td>\n",
        "      <td> XO-1</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-127.1962</th>\n",
        "      <td>  26.3658</td>\n",
        "      <td>  72.9862</td>\n",
        "      <td> 1</td>\n",
        "      <td> 485.7</td>\n",
        "      <td> 1</td>\n",
        "      <td> XO-2</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-219.6632</th>\n",
        "      <td> 136.7396</td>\n",
        "      <td>  25.5017</td>\n",
        "      <td> 1</td>\n",
        "      <td> 847.6</td>\n",
        "      <td> 1</td>\n",
        "      <td> XO-3</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-243.7934</th>\n",
        "      <td>  95.3681</td>\n",
        "      <td> 131.6006</td>\n",
        "      <td> 1</td>\n",
        "      <td> 955.2</td>\n",
        "      <td> 1</td>\n",
        "      <td> XO-4</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>-227.3245</th>\n",
        "      <td>  -2.4818</td>\n",
        "      <td> 115.5094</td>\n",
        "      <td> 1</td>\n",
        "      <td> 831.3</td>\n",
        "      <td> 1</td>\n",
        "      <td> XO-5</td>\n",
        "      <td> NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 410,
       "text": [
        "                  y         z  numplanets  distance  texture1 texture2  \\\n",
        "x                                                                        \n",
        " 93.1725    95.9901  148.6768           1     652.0         1     XO-1   \n",
        "-127.1962   26.3658   72.9862           1     485.7         1     XO-2   \n",
        "-219.6632  136.7396   25.5017           1     847.6         1     XO-3   \n",
        "-243.7934   95.3681  131.6006           1     955.2         1     XO-4   \n",
        "-227.3245   -2.4818  115.5094           1     831.3         1     XO-5   \n",
        "\n",
        "          targetblue  \n",
        "x                     \n",
        " 93.1725         NaN  \n",
        "-127.1962        NaN  \n",
        "-219.6632        NaN  \n",
        "-243.7934        NaN  \n",
        "-227.3245        NaN  "
       ]
      }
     ],
     "prompt_number": 410
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#star_data = pd.DataFrame.from_csv('/Users/williamliu/Desktop/amnh/stars_new.csv', sep=',')\n",
      "star_data_small = pd.DataFrame.from_csv('/Users/williamliu/Desktop/amnh/stars_new_small.csv', sep=',')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 411
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#star_data_small.head()\n",
      "star_data_small['date']  = pd.to_datetime(star_data_small['date'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 412
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "star_data_small['YearMonth'] = star_data_small['date'].map(lambda x: 100*x.year + x.month)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 413
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#star_data_small.set_index(star_data_small.YearMonth, inplace=True)\n",
      "#star_data_small.drop('YearMonth', axis=1, inplace=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 414
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "star_data_small.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>x</th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>color</th>\n",
        "      <th>lum</th>\n",
        "      <th>absmag</th>\n",
        "      <th>appmag</th>\n",
        "      <th>distly</th>\n",
        "      <th>distly_datetime</th>\n",
        "      <th>date</th>\n",
        "      <th>YearMonth</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>index</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1    </th>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.650</td>\n",
        "      <td> 0.89130</td>\n",
        "      <td>  4.85</td>\n",
        "      <td>-26.72</td>\n",
        "      <td> 0.00</td>\n",
        "      <td>    0.00</td>\n",
        "      <td>2014-11-08</td>\n",
        "      <td> 201411</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>60456</th>\n",
        "      <td> 0.8981</td>\n",
        "      <td>-0.9317</td>\n",
        "      <td>-0.0437</td>\n",
        "      <td> 1.748</td>\n",
        "      <td> 0.01000</td>\n",
        "      <td> 15.45</td>\n",
        "      <td> 11.01</td>\n",
        "      <td> 4.22</td>\n",
        "      <td> 1540.30</td>\n",
        "      <td>2010-08-20</td>\n",
        "      <td> 201008</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>61108</th>\n",
        "      <td> 0.9649</td>\n",
        "      <td>-0.9404</td>\n",
        "      <td>-0.0162</td>\n",
        "      <td> 0.946</td>\n",
        "      <td> 0.40649</td>\n",
        "      <td>  5.70</td>\n",
        "      <td>  1.35</td>\n",
        "      <td> 4.39</td>\n",
        "      <td> 1602.35</td>\n",
        "      <td>2010-06-19</td>\n",
        "      <td> 201006</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>61111</th>\n",
        "      <td> 0.9650</td>\n",
        "      <td>-0.9403</td>\n",
        "      <td>-0.0161</td>\n",
        "      <td> 0.603</td>\n",
        "      <td> 1.42250</td>\n",
        "      <td>  4.34</td>\n",
        "      <td> -0.01</td>\n",
        "      <td> 4.39</td>\n",
        "      <td> 1602.35</td>\n",
        "      <td>2010-06-19</td>\n",
        "      <td> 201006</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>74940</th>\n",
        "      <td> 1.5148</td>\n",
        "      <td> 0.9097</td>\n",
        "      <td> 0.4422</td>\n",
        "      <td>-0.630</td>\n",
        "      <td> 0.00039</td>\n",
        "      <td> 13.24</td>\n",
        "      <td>  9.54</td>\n",
        "      <td> 5.94</td>\n",
        "      <td> 2168.10</td>\n",
        "      <td>2008-11-30</td>\n",
        "      <td> 200811</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 415,
       "text": [
        "            x       y       z  color      lum  absmag  appmag  distly  \\\n",
        "index                                                                   \n",
        "1      0.0000  0.0000  0.0000  0.650  0.89130    4.85  -26.72    0.00   \n",
        "60456  0.8981 -0.9317 -0.0437  1.748  0.01000   15.45   11.01    4.22   \n",
        "61108  0.9649 -0.9404 -0.0162  0.946  0.40649    5.70    1.35    4.39   \n",
        "61111  0.9650 -0.9403 -0.0161  0.603  1.42250    4.34   -0.01    4.39   \n",
        "74940  1.5148  0.9097  0.4422 -0.630  0.00039   13.24    9.54    5.94   \n",
        "\n",
        "       distly_datetime       date  YearMonth  \n",
        "index                                         \n",
        "1                 0.00 2014-11-08     201411  \n",
        "60456          1540.30 2010-08-20     201008  \n",
        "61108          1602.35 2010-06-19     201006  \n",
        "61111          1602.35 2010-06-19     201006  \n",
        "74940          2168.10 2008-11-30     200811  "
       ]
      }
     ],
     "prompt_number": 415
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "star_data_no_dupes = star_data_small.drop_duplicates(cols='YearMonth')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 416
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "star_data_no_dupes.head(10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>x</th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>color</th>\n",
        "      <th>lum</th>\n",
        "      <th>absmag</th>\n",
        "      <th>appmag</th>\n",
        "      <th>distly</th>\n",
        "      <th>distly_datetime</th>\n",
        "      <th>date</th>\n",
        "      <th>YearMonth</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>index</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1    </th>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.0000</td>\n",
        "      <td> 0.650</td>\n",
        "      <td>  0.89130</td>\n",
        "      <td>  4.85</td>\n",
        "      <td>-26.72</td>\n",
        "      <td>  0.00</td>\n",
        "      <td>    0.00</td>\n",
        "      <td>2014-11-08</td>\n",
        "      <td> 201411</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>60456</th>\n",
        "      <td> 0.8981</td>\n",
        "      <td>-0.9317</td>\n",
        "      <td>-0.0437</td>\n",
        "      <td> 1.748</td>\n",
        "      <td>  0.01000</td>\n",
        "      <td> 15.45</td>\n",
        "      <td> 11.01</td>\n",
        "      <td>  4.22</td>\n",
        "      <td> 1540.30</td>\n",
        "      <td>2010-08-20</td>\n",
        "      <td> 201008</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>61108</th>\n",
        "      <td> 0.9649</td>\n",
        "      <td>-0.9404</td>\n",
        "      <td>-0.0162</td>\n",
        "      <td> 0.946</td>\n",
        "      <td>  0.40649</td>\n",
        "      <td>  5.70</td>\n",
        "      <td>  1.35</td>\n",
        "      <td>  4.39</td>\n",
        "      <td> 1602.35</td>\n",
        "      <td>2010-06-19</td>\n",
        "      <td> 201006</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>74940</th>\n",
        "      <td> 1.5148</td>\n",
        "      <td> 0.9097</td>\n",
        "      <td> 0.4422</td>\n",
        "      <td>-0.630</td>\n",
        "      <td>  0.00039</td>\n",
        "      <td> 13.24</td>\n",
        "      <td>  9.54</td>\n",
        "      <td>  5.94</td>\n",
        "      <td> 2168.10</td>\n",
        "      <td>2008-11-30</td>\n",
        "      <td> 200811</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>46175</th>\n",
        "      <td>-1.0554</td>\n",
        "      <td>-0.0940</td>\n",
        "      <td> 2.3177</td>\n",
        "      <td> 1.542</td>\n",
        "      <td>  0.00509</td>\n",
        "      <td> 10.46</td>\n",
        "      <td>  7.49</td>\n",
        "      <td>  8.31</td>\n",
        "      <td> 3033.15</td>\n",
        "      <td>2006-07-19</td>\n",
        "      <td> 200607</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27821</th>\n",
        "      <td>-1.7693</td>\n",
        "      <td>-1.9125</td>\n",
        "      <td>-0.4074</td>\n",
        "      <td> 0.021</td>\n",
        "      <td> 20.33483</td>\n",
        "      <td>  1.45</td>\n",
        "      <td> -1.44</td>\n",
        "      <td>  8.60</td>\n",
        "      <td> 3139.00</td>\n",
        "      <td>2006-04-05</td>\n",
        "      <td> 200604</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>78632</th>\n",
        "      <td> 2.8675</td>\n",
        "      <td> 0.5733</td>\n",
        "      <td>-0.5305</td>\n",
        "      <td> 1.643</td>\n",
        "      <td>  0.00049</td>\n",
        "      <td> 13.00</td>\n",
        "      <td> 10.37</td>\n",
        "      <td>  9.69</td>\n",
        "      <td> 3536.85</td>\n",
        "      <td>2005-03-03</td>\n",
        "      <td> 200503</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14321</th>\n",
        "      <td>-2.0695</td>\n",
        "      <td>-0.5874</td>\n",
        "      <td>-2.3933</td>\n",
        "      <td> 0.991</td>\n",
        "      <td>  0.26132</td>\n",
        "      <td>  6.18</td>\n",
        "      <td>  3.72</td>\n",
        "      <td> 10.49</td>\n",
        "      <td> 3828.85</td>\n",
        "      <td>2004-05-15</td>\n",
        "      <td> 200405</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>96944</th>\n",
        "      <td> 1.3362</td>\n",
        "      <td> 0.1193</td>\n",
        "      <td>-3.0047</td>\n",
        "      <td> 1.542</td>\n",
        "      <td>  0.00965</td>\n",
        "      <td>  9.76</td>\n",
        "      <td>  7.35</td>\n",
        "      <td> 10.73</td>\n",
        "      <td> 3916.45</td>\n",
        "      <td>2004-02-17</td>\n",
        "      <td> 200402</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>49150</th>\n",
        "      <td> 0.0042</td>\n",
        "      <td>-1.6911</td>\n",
        "      <td> 2.8779</td>\n",
        "      <td> 1.713</td>\n",
        "      <td>  0.00031</td>\n",
        "      <td> 13.50</td>\n",
        "      <td> 11.12</td>\n",
        "      <td> 10.88</td>\n",
        "      <td> 3971.20</td>\n",
        "      <td>2003-12-24</td>\n",
        "      <td> 200312</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 417,
       "text": [
        "            x       y       z  color       lum  absmag  appmag  distly  \\\n",
        "index                                                                    \n",
        "1      0.0000  0.0000  0.0000  0.650   0.89130    4.85  -26.72    0.00   \n",
        "60456  0.8981 -0.9317 -0.0437  1.748   0.01000   15.45   11.01    4.22   \n",
        "61108  0.9649 -0.9404 -0.0162  0.946   0.40649    5.70    1.35    4.39   \n",
        "74940  1.5148  0.9097  0.4422 -0.630   0.00039   13.24    9.54    5.94   \n",
        "46175 -1.0554 -0.0940  2.3177  1.542   0.00509   10.46    7.49    8.31   \n",
        "27821 -1.7693 -1.9125 -0.4074  0.021  20.33483    1.45   -1.44    8.60   \n",
        "78632  2.8675  0.5733 -0.5305  1.643   0.00049   13.00   10.37    9.69   \n",
        "14321 -2.0695 -0.5874 -2.3933  0.991   0.26132    6.18    3.72   10.49   \n",
        "96944  1.3362  0.1193 -3.0047  1.542   0.00965    9.76    7.35   10.73   \n",
        "49150  0.0042 -1.6911  2.8779  1.713   0.00031   13.50   11.12   10.88   \n",
        "\n",
        "       distly_datetime       date  YearMonth  \n",
        "index                                         \n",
        "1                 0.00 2014-11-08     201411  \n",
        "60456          1540.30 2010-08-20     201008  \n",
        "61108          1602.35 2010-06-19     201006  \n",
        "74940          2168.10 2008-11-30     200811  \n",
        "46175          3033.15 2006-07-19     200607  \n",
        "27821          3139.00 2006-04-05     200604  \n",
        "78632          3536.85 2005-03-03     200503  \n",
        "14321          3828.85 2004-05-15     200405  \n",
        "96944          3916.45 2004-02-17     200402  \n",
        "49150          3971.20 2003-12-24     200312  "
       ]
      }
     ],
     "prompt_number": 417
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#headlines_data['date']  = pd.to_datetime(headlines_data['date'])\n",
      "headlines_data = pd.DataFrame.from_csv('/Users/williamliu/Desktop/amnh/headlines_clean.csv', sep=',')\n",
      "#headlines_data.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 418
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "headlines_data['date']  = pd.to_datetime(headlines_data['date'], format='%Y%m%d')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 419
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "headlines_data['YearMonth'] = headlines_data['date'].map(lambda x: 100*x.year + x.month)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 420
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#headlines_data.set_index('YearMonth', inplace=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 421
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "headlines_data.head(10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>date</th>\n",
        "      <th>headline</th>\n",
        "      <th>snippet</th>\n",
        "      <th>link</th>\n",
        "      <th>YearMonth</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>index</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td>1982-01-01</td>\n",
        "      <td> JUDGE FINDS BLACK CONCERN VIOLATED LAWS ON SEX...</td>\n",
        "      <td>   A Federal district judge has ruled that the ...</td>\n",
        "      <td> http://www.nytimes.com/1982/01/02/us/judge-fin...</td>\n",
        "      <td> 198201</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td>1982-02-01</td>\n",
        "      <td> DRAFT OF ORDER ON PRESS DISCLOSURES IS STUDIED...</td>\n",
        "      <td>   A ''working draft'' of a directive that woul...</td>\n",
        "      <td> http://www.nytimes.com/1982/02/02/us/draft-of-...</td>\n",
        "      <td> 198202</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td>1982-03-01</td>\n",
        "      <td>                                          BRIEFING</td>\n",
        "      <td>   Aquarium Casualties   A long campaign to sav...</td>\n",
        "      <td> http://www.nytimes.com/1982/03/02/us/briefing-...</td>\n",
        "      <td> 198203</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td>1982-04-01</td>\n",
        "      <td>   Defense Rests in Trial Over 1977 Nightclub Fire</td>\n",
        "      <td>   The defense has rested in a four-week civil ...</td>\n",
        "      <td> http://www.nytimes.com/1982/04/02/us/around-th...</td>\n",
        "      <td> 198204</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td>1982-05-01</td>\n",
        "      <td>    THAI DIPLOMAT HELD ON HEROIN CHARGE IN CHICAGO</td>\n",
        "      <td>   A Thai diplomat who is said to have operated...</td>\n",
        "      <td> http://www.nytimes.com/1982/05/02/us/thai-dipl...</td>\n",
        "      <td> 198205</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td>1982-06-01</td>\n",
        "      <td>                       High Court Bars Kleindienst</td>\n",
        "      <td>   Former Attorney General Richard G.  Kleindie...</td>\n",
        "      <td> http://www.nytimes.com/1982/06/02/us/high-cour...</td>\n",
        "      <td> 198206</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td>1982-07-01</td>\n",
        "      <td> NEW BUDGET FOR CALIFORNIA REFLECTS CUTBACK IN ...</td>\n",
        "      <td>   Four years after Californians rebelled and v...</td>\n",
        "      <td> http://www.nytimes.com/1982/07/02/us/new-budge...</td>\n",
        "      <td> 198207</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td>1982-08-01</td>\n",
        "      <td>  DISMISSAL SPLITS LANDMARK CHURCH IN RIGHTS FIGHT</td>\n",
        "      <td>   The padlocks were removed from the doors of ...</td>\n",
        "      <td> http://www.nytimes.com/1982/08/02/us/dismissal...</td>\n",
        "      <td> 198208</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td>1982-09-01</td>\n",
        "      <td>         U.S. Says Farm Income Is Down 24% in 1982</td>\n",
        "      <td>   Net income for American farmers is projected...</td>\n",
        "      <td> http://www.nytimes.com/1982/09/02/us/us-says-f...</td>\n",
        "      <td> 198209</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td>1982-10-01</td>\n",
        "      <td>                                AWAIT HOUSE ACTION</td>\n",
        "      <td>   The Senate, preparing to head home to the vo...</td>\n",
        "      <td> http://www.nytimes.com/1982/10/02/us/anticrime...</td>\n",
        "      <td> 198210</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 422,
       "text": [
        "            date                                           headline  \\\n",
        "index                                                                 \n",
        "1     1982-01-01  JUDGE FINDS BLACK CONCERN VIOLATED LAWS ON SEX...   \n",
        "2     1982-02-01  DRAFT OF ORDER ON PRESS DISCLOSURES IS STUDIED...   \n",
        "3     1982-03-01                                           BRIEFING   \n",
        "4     1982-04-01    Defense Rests in Trial Over 1977 Nightclub Fire   \n",
        "5     1982-05-01     THAI DIPLOMAT HELD ON HEROIN CHARGE IN CHICAGO   \n",
        "6     1982-06-01                        High Court Bars Kleindienst   \n",
        "7     1982-07-01  NEW BUDGET FOR CALIFORNIA REFLECTS CUTBACK IN ...   \n",
        "8     1982-08-01   DISMISSAL SPLITS LANDMARK CHURCH IN RIGHTS FIGHT   \n",
        "9     1982-09-01          U.S. Says Farm Income Is Down 24% in 1982   \n",
        "10    1982-10-01                                 AWAIT HOUSE ACTION   \n",
        "\n",
        "                                                 snippet  \\\n",
        "index                                                      \n",
        "1        A Federal district judge has ruled that the ...   \n",
        "2        A ''working draft'' of a directive that woul...   \n",
        "3        Aquarium Casualties   A long campaign to sav...   \n",
        "4        The defense has rested in a four-week civil ...   \n",
        "5        A Thai diplomat who is said to have operated...   \n",
        "6        Former Attorney General Richard G.  Kleindie...   \n",
        "7        Four years after Californians rebelled and v...   \n",
        "8        The padlocks were removed from the doors of ...   \n",
        "9        Net income for American farmers is projected...   \n",
        "10       The Senate, preparing to head home to the vo...   \n",
        "\n",
        "                                                    link  YearMonth  \n",
        "index                                                                \n",
        "1      http://www.nytimes.com/1982/01/02/us/judge-fin...     198201  \n",
        "2      http://www.nytimes.com/1982/02/02/us/draft-of-...     198202  \n",
        "3      http://www.nytimes.com/1982/03/02/us/briefing-...     198203  \n",
        "4      http://www.nytimes.com/1982/04/02/us/around-th...     198204  \n",
        "5      http://www.nytimes.com/1982/05/02/us/thai-dipl...     198205  \n",
        "6      http://www.nytimes.com/1982/06/02/us/high-cour...     198206  \n",
        "7      http://www.nytimes.com/1982/07/02/us/new-budge...     198207  \n",
        "8      http://www.nytimes.com/1982/08/02/us/dismissal...     198208  \n",
        "9      http://www.nytimes.com/1982/09/02/us/us-says-f...     198209  \n",
        "10     http://www.nytimes.com/1982/10/02/us/anticrime...     198210  "
       ]
      }
     ],
     "prompt_number": 422
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#full_data = pandas.concat([star_data_no_dupes, headlines_data])\n",
      "full_data = pandas.merge(star_data_no_dupes, headlines_data, on='YearMonth')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 423
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "full_data.head(20)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>x</th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>color</th>\n",
        "      <th>lum</th>\n",
        "      <th>absmag</th>\n",
        "      <th>appmag</th>\n",
        "      <th>distly</th>\n",
        "      <th>distly_datetime</th>\n",
        "      <th>date_x</th>\n",
        "      <th>YearMonth</th>\n",
        "      <th>date_y</th>\n",
        "      <th>headline</th>\n",
        "      <th>snippet</th>\n",
        "      <th>link</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0 </th>\n",
        "      <td> 2.8675</td>\n",
        "      <td> 0.5733</td>\n",
        "      <td>-0.5305</td>\n",
        "      <td> 1.643</td>\n",
        "      <td> 0.00049</td>\n",
        "      <td> 13.00</td>\n",
        "      <td> 10.37</td>\n",
        "      <td>  9.69</td>\n",
        "      <td> 3536.85</td>\n",
        "      <td>2005-03-03</td>\n",
        "      <td> 200503</td>\n",
        "      <td>2005-03-01</td>\n",
        "      <td>                          Eyeing the Next Einstein</td>\n",
        "      <td> In this lesson, students learn about the iconi...</td>\n",
        "      <td> http://learning.blogs.nytimes.com/2005/03/01/e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td>-2.0695</td>\n",
        "      <td>-0.5874</td>\n",
        "      <td>-2.3933</td>\n",
        "      <td> 0.991</td>\n",
        "      <td> 0.26132</td>\n",
        "      <td>  6.18</td>\n",
        "      <td>  3.72</td>\n",
        "      <td> 10.49</td>\n",
        "      <td> 3828.85</td>\n",
        "      <td>2004-05-15</td>\n",
        "      <td> 200405</td>\n",
        "      <td>2004-05-01</td>\n",
        "      <td> Airlines Gave F.B.I. Millions of Records on Tr...</td>\n",
        "      <td> In the days after the Sept. 11 terrorist attac...</td>\n",
        "      <td> http://www.nytimes.com/2004/05/01/us/airlines-...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td> 1.3362</td>\n",
        "      <td> 0.1193</td>\n",
        "      <td>-3.0047</td>\n",
        "      <td> 1.542</td>\n",
        "      <td> 0.00965</td>\n",
        "      <td>  9.76</td>\n",
        "      <td>  7.35</td>\n",
        "      <td> 10.73</td>\n",
        "      <td> 3916.45</td>\n",
        "      <td>2004-02-17</td>\n",
        "      <td> 200402</td>\n",
        "      <td>2004-02-01</td>\n",
        "      <td> Sharpton Makes His Stand To Win Voice at Conve...</td>\n",
        "      <td> The Rev. Al Sharpton was in prime form, whippi...</td>\n",
        "      <td> http://www.nytimes.com/2004/02/02/us/2004-camp...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td> 0.4638</td>\n",
        "      <td> 3.4336</td>\n",
        "      <td>-0.3529</td>\n",
        "      <td> 1.137</td>\n",
        "      <td> 0.07831</td>\n",
        "      <td>  7.49</td>\n",
        "      <td>  5.20</td>\n",
        "      <td> 11.35</td>\n",
        "      <td> 4142.75</td>\n",
        "      <td>2003-07-06</td>\n",
        "      <td> 200307</td>\n",
        "      <td>2003-07-01</td>\n",
        "      <td> National Briefing | Midwest: Illinois: Board A...</td>\n",
        "      <td> The Cook County Board of Commissioners approve...</td>\n",
        "      <td> http://www.nytimes.com/2003/07/02/us/national-...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td> 0.0402</td>\n",
        "      <td> 3.2054</td>\n",
        "      <td> 1.4424</td>\n",
        "      <td> 1.137</td>\n",
        "      <td> 0.00126</td>\n",
        "      <td> 11.97</td>\n",
        "      <td>  9.70</td>\n",
        "      <td> 11.46</td>\n",
        "      <td> 4182.90</td>\n",
        "      <td>2003-05-27</td>\n",
        "      <td> 200305</td>\n",
        "      <td>2003-05-01</td>\n",
        "      <td>                               Fit to Be Tied (In)</td>\n",
        "      <td> In this lesson, students examine promotional p...</td>\n",
        "      <td> http://learning.blogs.nytimes.com/2003/05/01/f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td>-1.5192</td>\n",
        "      <td> 3.0246</td>\n",
        "      <td>-1.1290</td>\n",
        "      <td> 1.478</td>\n",
        "      <td> 0.00574</td>\n",
        "      <td> 10.33</td>\n",
        "      <td>  8.09</td>\n",
        "      <td> 11.63</td>\n",
        "      <td> 4244.95</td>\n",
        "      <td>2003-03-26</td>\n",
        "      <td> 200303</td>\n",
        "      <td>2003-03-01</td>\n",
        "      <td> Tapes Show Desperation and Panic at Chicago Cl...</td>\n",
        "      <td> At first, faces fill the screen, some still gi...</td>\n",
        "      <td> http://www.nytimes.com/2003/03/01/us/tapes-sho...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td> 2.2186</td>\n",
        "      <td>-0.9782</td>\n",
        "      <td>-2.6965</td>\n",
        "      <td> 1.137</td>\n",
        "      <td> 0.13581</td>\n",
        "      <td>  6.89</td>\n",
        "      <td>  4.69</td>\n",
        "      <td> 11.82</td>\n",
        "      <td> 4314.30</td>\n",
        "      <td>2003-01-15</td>\n",
        "      <td> 200301</td>\n",
        "      <td>2003-01-01</td>\n",
        "      <td> National Briefing | West: California: Chickens...</td>\n",
        "      <td> State officials have ordered the destruction o...</td>\n",
        "      <td> http://www.nytimes.com/2003/01/01/us/national-...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td>-0.6254</td>\n",
        "      <td> 0.3655</td>\n",
        "      <td>-3.6455</td>\n",
        "      <td> 1.785</td>\n",
        "      <td> 0.00016</td>\n",
        "      <td> 14.25</td>\n",
        "      <td> 12.10</td>\n",
        "      <td> 12.12</td>\n",
        "      <td> 4423.80</td>\n",
        "      <td>2002-09-28</td>\n",
        "      <td> 200209</td>\n",
        "      <td>2002-09-01</td>\n",
        "      <td>       Gun Industry Is Gaining Immunity From Suits</td>\n",
        "      <td> A spate of government litigation against the n...</td>\n",
        "      <td> http://www.nytimes.com/2002/09/01/us/gun-indus...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td>-3.1570</td>\n",
        "      <td>-1.9985</td>\n",
        "      <td> 0.6842</td>\n",
        "      <td> 1.748</td>\n",
        "      <td> 0.00130</td>\n",
        "      <td> 11.94</td>\n",
        "      <td>  9.84</td>\n",
        "      <td> 12.38</td>\n",
        "      <td> 4518.70</td>\n",
        "      <td>2002-06-25</td>\n",
        "      <td> 200206</td>\n",
        "      <td>2002-06-01</td>\n",
        "      <td>       Agent Who Wrote Critical Letter May Testify</td>\n",
        "      <td> The nation will probably soon get to see and h...</td>\n",
        "      <td> http://www.nytimes.com/2002/06/01/us/agent-who...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td>-1.0571</td>\n",
        "      <td>-2.9872</td>\n",
        "      <td>-2.3036</td>\n",
        "      <td> 1.416</td>\n",
        "      <td> 0.00340</td>\n",
        "      <td> 10.89</td>\n",
        "      <td>  8.86</td>\n",
        "      <td> 12.77</td>\n",
        "      <td> 4661.05</td>\n",
        "      <td>2002-02-02</td>\n",
        "      <td> 200202</td>\n",
        "      <td>2002-02-01</td>\n",
        "      <td> Canada Wants Some Trucks Exempt From Border In...</td>\n",
        "      <td> At a time when the Bush administration is tryi...</td>\n",
        "      <td> http://www.nytimes.com/2002/02/01/us/nation-ch...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td>-1.0163</td>\n",
        "      <td> 3.8767</td>\n",
        "      <td>-0.0002</td>\n",
        "      <td> 1.542</td>\n",
        "      <td> 0.00182</td>\n",
        "      <td> 11.58</td>\n",
        "      <td>  9.59</td>\n",
        "      <td> 13.07</td>\n",
        "      <td> 4770.55</td>\n",
        "      <td>2001-10-16</td>\n",
        "      <td> 200110</td>\n",
        "      <td>2001-10-01</td>\n",
        "      <td> National Briefing | South: North Carolina: Ex-...</td>\n",
        "      <td> Erskine B. Bowles, the White House chief of st...</td>\n",
        "      <td> http://www.nytimes.com/2001/10/02/us/national-...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td>-3.4356</td>\n",
        "      <td>-2.2249</td>\n",
        "      <td>-0.4437</td>\n",
        "      <td> 1.713</td>\n",
        "      <td> 0.00047</td>\n",
        "      <td> 13.05</td>\n",
        "      <td> 11.12</td>\n",
        "      <td> 13.42</td>\n",
        "      <td> 4898.30</td>\n",
        "      <td>2001-06-10</td>\n",
        "      <td> 200106</td>\n",
        "      <td>2001-06-01</td>\n",
        "      <td> National Briefing | Washington: Rehnquist's Da...</td>\n",
        "      <td> President Bush nominated Janet Rehnquist, daug...</td>\n",
        "      <td> http://www.nytimes.com/2001/06/02/us/national-...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
        "      <td> 3.3075</td>\n",
        "      <td>-1.6456</td>\n",
        "      <td> 2.1031</td>\n",
        "      <td>-0.630</td>\n",
        "      <td> 0.00029</td>\n",
        "      <td> 13.58</td>\n",
        "      <td> 11.72</td>\n",
        "      <td> 13.86</td>\n",
        "      <td> 5058.90</td>\n",
        "      <td>2001-01-01</td>\n",
        "      <td> 200101</td>\n",
        "      <td>2001-01-01</td>\n",
        "      <td> Census Director Marvels at the New Portrait of...</td>\n",
        "      <td> KENNETH PREWITT is tired of the Beltway analys...</td>\n",
        "      <td> http://www.nytimes.com/2001/01/01/us/public-li...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
        "      <td> 1.0193</td>\n",
        "      <td>-0.2998</td>\n",
        "      <td>-4.2291</td>\n",
        "      <td> 1.542</td>\n",
        "      <td> 0.00556</td>\n",
        "      <td> 10.36</td>\n",
        "      <td>  8.56</td>\n",
        "      <td> 14.22</td>\n",
        "      <td> 5190.30</td>\n",
        "      <td>2000-08-22</td>\n",
        "      <td> 200008</td>\n",
        "      <td>2000-08-01</td>\n",
        "      <td>                                  Party's Schedule</td>\n",
        "      <td>  All times are Eastern daylight  8 p.m.    Hec...</td>\n",
        "      <td> http://www.nytimes.com/2000/08/02/us/the-repub...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
        "      <td>-2.5456</td>\n",
        "      <td>-0.8793</td>\n",
        "      <td>-3.4751</td>\n",
        "      <td>-0.630</td>\n",
        "      <td> 0.00021</td>\n",
        "      <td> 13.94</td>\n",
        "      <td> 12.16</td>\n",
        "      <td> 14.33</td>\n",
        "      <td> 5230.45</td>\n",
        "      <td>2000-07-13</td>\n",
        "      <td> 200007</td>\n",
        "      <td>2000-07-01</td>\n",
        "      <td>         Boom in Economy Skips Towns on the Plains</td>\n",
        "      <td> From above, the state of North Dakota is a qui...</td>\n",
        "      <td> http://www.nytimes.com/2000/07/02/us/boom-in-e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
        "      <td> 3.5079</td>\n",
        "      <td>-1.7457</td>\n",
        "      <td> 2.2303</td>\n",
        "      <td>-0.630</td>\n",
        "      <td> 0.00023</td>\n",
        "      <td> 13.80</td>\n",
        "      <td> 12.07</td>\n",
        "      <td> 14.70</td>\n",
        "      <td> 5365.50</td>\n",
        "      <td>2000-02-29</td>\n",
        "      <td> 200002</td>\n",
        "      <td>2000-02-01</td>\n",
        "      <td> Consecrations of U.S. Bishops by Episcopal Off...</td>\n",
        "      <td> In a highly unusual challenge to the Episcopal...</td>\n",
        "      <td> http://www.nytimes.com/2000/02/02/us/consecrat...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
        "      <td> 4.3082</td>\n",
        "      <td>-1.3168</td>\n",
        "      <td>-0.5352</td>\n",
        "      <td> 1.137</td>\n",
        "      <td> 0.00283</td>\n",
        "      <td> 11.10</td>\n",
        "      <td>  9.38</td>\n",
        "      <td> 14.79</td>\n",
        "      <td> 5398.35</td>\n",
        "      <td>2000-01-27</td>\n",
        "      <td> 200001</td>\n",
        "      <td>2000-01-01</td>\n",
        "      <td> THE YEAR 2000: Speeches, Star Power and a Whol...</td>\n",
        "      <td> All over the island, noise.     The first expl...</td>\n",
        "      <td> http://www.nytimes.com/2000/01/02/us/year-2000...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
        "      <td> 2.0240</td>\n",
        "      <td>-4.1479</td>\n",
        "      <td>-0.2303</td>\n",
        "      <td>-0.630</td>\n",
        "      <td> 0.00042</td>\n",
        "      <td> 13.18</td>\n",
        "      <td> 11.50</td>\n",
        "      <td> 15.06</td>\n",
        "      <td> 5496.90</td>\n",
        "      <td>1999-10-21</td>\n",
        "      <td> 199910</td>\n",
        "      <td>1999-10-01</td>\n",
        "      <td>    Rising Incomes Lift 1.1 Million Out of Poverty</td>\n",
        "      <td> Lifted by the strong economy, American househo...</td>\n",
        "      <td> http://www.nytimes.com/1999/10/01/us/rising-in...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
        "      <td> 1.4635</td>\n",
        "      <td> 1.8734</td>\n",
        "      <td>-4.0564</td>\n",
        "      <td> 1.748</td>\n",
        "      <td> 0.00148</td>\n",
        "      <td> 11.80</td>\n",
        "      <td> 10.16</td>\n",
        "      <td> 15.33</td>\n",
        "      <td> 5595.45</td>\n",
        "      <td>1999-07-14</td>\n",
        "      <td> 199907</td>\n",
        "      <td>1999-07-01</td>\n",
        "      <td> Miami Cubans Are Outraged At Treatment Of 6 Re...</td>\n",
        "      <td> Miami's large Cuban-American population was br...</td>\n",
        "      <td> http://www.nytimes.com/1999/07/01/us/miami-cub...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
        "      <td>-2.1437</td>\n",
        "      <td> 0.4363</td>\n",
        "      <td> 4.3088</td>\n",
        "      <td> 1.542</td>\n",
        "      <td> 0.00537</td>\n",
        "      <td> 10.40</td>\n",
        "      <td>  8.82</td>\n",
        "      <td> 15.75</td>\n",
        "      <td> 5748.75</td>\n",
        "      <td>1999-02-11</td>\n",
        "      <td> 199902</td>\n",
        "      <td>1999-02-01</td>\n",
        "      <td>              Explosion At Ford Plant Kills Worker</td>\n",
        "      <td> A thunderous explosion, followed by a fire tha...</td>\n",
        "      <td> http://www.nytimes.com/1999/02/02/us/explosion...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 424,
       "text": [
        "         x       y       z  color      lum  absmag  appmag  distly  \\\n",
        "0   2.8675  0.5733 -0.5305  1.643  0.00049   13.00   10.37    9.69   \n",
        "1  -2.0695 -0.5874 -2.3933  0.991  0.26132    6.18    3.72   10.49   \n",
        "2   1.3362  0.1193 -3.0047  1.542  0.00965    9.76    7.35   10.73   \n",
        "3   0.4638  3.4336 -0.3529  1.137  0.07831    7.49    5.20   11.35   \n",
        "4   0.0402  3.2054  1.4424  1.137  0.00126   11.97    9.70   11.46   \n",
        "5  -1.5192  3.0246 -1.1290  1.478  0.00574   10.33    8.09   11.63   \n",
        "6   2.2186 -0.9782 -2.6965  1.137  0.13581    6.89    4.69   11.82   \n",
        "7  -0.6254  0.3655 -3.6455  1.785  0.00016   14.25   12.10   12.12   \n",
        "8  -3.1570 -1.9985  0.6842  1.748  0.00130   11.94    9.84   12.38   \n",
        "9  -1.0571 -2.9872 -2.3036  1.416  0.00340   10.89    8.86   12.77   \n",
        "10 -1.0163  3.8767 -0.0002  1.542  0.00182   11.58    9.59   13.07   \n",
        "11 -3.4356 -2.2249 -0.4437  1.713  0.00047   13.05   11.12   13.42   \n",
        "12  3.3075 -1.6456  2.1031 -0.630  0.00029   13.58   11.72   13.86   \n",
        "13  1.0193 -0.2998 -4.2291  1.542  0.00556   10.36    8.56   14.22   \n",
        "14 -2.5456 -0.8793 -3.4751 -0.630  0.00021   13.94   12.16   14.33   \n",
        "15  3.5079 -1.7457  2.2303 -0.630  0.00023   13.80   12.07   14.70   \n",
        "16  4.3082 -1.3168 -0.5352  1.137  0.00283   11.10    9.38   14.79   \n",
        "17  2.0240 -4.1479 -0.2303 -0.630  0.00042   13.18   11.50   15.06   \n",
        "18  1.4635  1.8734 -4.0564  1.748  0.00148   11.80   10.16   15.33   \n",
        "19 -2.1437  0.4363  4.3088  1.542  0.00537   10.40    8.82   15.75   \n",
        "\n",
        "    distly_datetime     date_x  YearMonth     date_y  \\\n",
        "0           3536.85 2005-03-03     200503 2005-03-01   \n",
        "1           3828.85 2004-05-15     200405 2004-05-01   \n",
        "2           3916.45 2004-02-17     200402 2004-02-01   \n",
        "3           4142.75 2003-07-06     200307 2003-07-01   \n",
        "4           4182.90 2003-05-27     200305 2003-05-01   \n",
        "5           4244.95 2003-03-26     200303 2003-03-01   \n",
        "6           4314.30 2003-01-15     200301 2003-01-01   \n",
        "7           4423.80 2002-09-28     200209 2002-09-01   \n",
        "8           4518.70 2002-06-25     200206 2002-06-01   \n",
        "9           4661.05 2002-02-02     200202 2002-02-01   \n",
        "10          4770.55 2001-10-16     200110 2001-10-01   \n",
        "11          4898.30 2001-06-10     200106 2001-06-01   \n",
        "12          5058.90 2001-01-01     200101 2001-01-01   \n",
        "13          5190.30 2000-08-22     200008 2000-08-01   \n",
        "14          5230.45 2000-07-13     200007 2000-07-01   \n",
        "15          5365.50 2000-02-29     200002 2000-02-01   \n",
        "16          5398.35 2000-01-27     200001 2000-01-01   \n",
        "17          5496.90 1999-10-21     199910 1999-10-01   \n",
        "18          5595.45 1999-07-14     199907 1999-07-01   \n",
        "19          5748.75 1999-02-11     199902 1999-02-01   \n",
        "\n",
        "                                             headline  \\\n",
        "0                            Eyeing the Next Einstein   \n",
        "1   Airlines Gave F.B.I. Millions of Records on Tr...   \n",
        "2   Sharpton Makes His Stand To Win Voice at Conve...   \n",
        "3   National Briefing | Midwest: Illinois: Board A...   \n",
        "4                                 Fit to Be Tied (In)   \n",
        "5   Tapes Show Desperation and Panic at Chicago Cl...   \n",
        "6   National Briefing | West: California: Chickens...   \n",
        "7         Gun Industry Is Gaining Immunity From Suits   \n",
        "8         Agent Who Wrote Critical Letter May Testify   \n",
        "9   Canada Wants Some Trucks Exempt From Border In...   \n",
        "10  National Briefing | South: North Carolina: Ex-...   \n",
        "11  National Briefing | Washington: Rehnquist's Da...   \n",
        "12  Census Director Marvels at the New Portrait of...   \n",
        "13                                   Party's Schedule   \n",
        "14          Boom in Economy Skips Towns on the Plains   \n",
        "15  Consecrations of U.S. Bishops by Episcopal Off...   \n",
        "16  THE YEAR 2000: Speeches, Star Power and a Whol...   \n",
        "17     Rising Incomes Lift 1.1 Million Out of Poverty   \n",
        "18  Miami Cubans Are Outraged At Treatment Of 6 Re...   \n",
        "19               Explosion At Ford Plant Kills Worker   \n",
        "\n",
        "                                              snippet  \\\n",
        "0   In this lesson, students learn about the iconi...   \n",
        "1   In the days after the Sept. 11 terrorist attac...   \n",
        "2   The Rev. Al Sharpton was in prime form, whippi...   \n",
        "3   The Cook County Board of Commissioners approve...   \n",
        "4   In this lesson, students examine promotional p...   \n",
        "5   At first, faces fill the screen, some still gi...   \n",
        "6   State officials have ordered the destruction o...   \n",
        "7   A spate of government litigation against the n...   \n",
        "8   The nation will probably soon get to see and h...   \n",
        "9   At a time when the Bush administration is tryi...   \n",
        "10  Erskine B. Bowles, the White House chief of st...   \n",
        "11  President Bush nominated Janet Rehnquist, daug...   \n",
        "12  KENNETH PREWITT is tired of the Beltway analys...   \n",
        "13   All times are Eastern daylight  8 p.m.    Hec...   \n",
        "14  From above, the state of North Dakota is a qui...   \n",
        "15  In a highly unusual challenge to the Episcopal...   \n",
        "16  All over the island, noise.     The first expl...   \n",
        "17  Lifted by the strong economy, American househo...   \n",
        "18  Miami's large Cuban-American population was br...   \n",
        "19  A thunderous explosion, followed by a fire tha...   \n",
        "\n",
        "                                                 link  \n",
        "0   http://learning.blogs.nytimes.com/2005/03/01/e...  \n",
        "1   http://www.nytimes.com/2004/05/01/us/airlines-...  \n",
        "2   http://www.nytimes.com/2004/02/02/us/2004-camp...  \n",
        "3   http://www.nytimes.com/2003/07/02/us/national-...  \n",
        "4   http://learning.blogs.nytimes.com/2003/05/01/f...  \n",
        "5   http://www.nytimes.com/2003/03/01/us/tapes-sho...  \n",
        "6   http://www.nytimes.com/2003/01/01/us/national-...  \n",
        "7   http://www.nytimes.com/2002/09/01/us/gun-indus...  \n",
        "8   http://www.nytimes.com/2002/06/01/us/agent-who...  \n",
        "9   http://www.nytimes.com/2002/02/01/us/nation-ch...  \n",
        "10  http://www.nytimes.com/2001/10/02/us/national-...  \n",
        "11  http://www.nytimes.com/2001/06/02/us/national-...  \n",
        "12  http://www.nytimes.com/2001/01/01/us/public-li...  \n",
        "13  http://www.nytimes.com/2000/08/02/us/the-repub...  \n",
        "14  http://www.nytimes.com/2000/07/02/us/boom-in-e...  \n",
        "15  http://www.nytimes.com/2000/02/02/us/consecrat...  \n",
        "16  http://www.nytimes.com/2000/01/02/us/year-2000...  \n",
        "17  http://www.nytimes.com/1999/10/01/us/rising-in...  \n",
        "18  http://www.nytimes.com/1999/07/01/us/miami-cub...  \n",
        "19  http://www.nytimes.com/1999/02/02/us/explosion...  "
       ]
      }
     ],
     "prompt_number": 424
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "full_data.describe()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>x</th>\n",
        "      <th>y</th>\n",
        "      <th>z</th>\n",
        "      <th>color</th>\n",
        "      <th>lum</th>\n",
        "      <th>absmag</th>\n",
        "      <th>appmag</th>\n",
        "      <th>distly</th>\n",
        "      <th>distly_datetime</th>\n",
        "      <th>YearMonth</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>count</th>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td> 122.000000</td>\n",
        "      <td>   122.000000</td>\n",
        "      <td>    122.000000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>mean</th>\n",
        "      <td>  -2.449070</td>\n",
        "      <td>  -0.394039</td>\n",
        "      <td>  -0.717792</td>\n",
        "      <td>   0.925459</td>\n",
        "      <td>   0.932497</td>\n",
        "      <td>   8.981721</td>\n",
        "      <td>   8.630000</td>\n",
        "      <td>  35.765410</td>\n",
        "      <td> 13054.374590</td>\n",
        "      <td> 199347.598361</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>std</th>\n",
        "      <td>   9.429820</td>\n",
        "      <td>   7.749877</td>\n",
        "      <td>   7.709211</td>\n",
        "      <td>   0.803063</td>\n",
        "      <td>   4.533497</td>\n",
        "      <td>   3.194813</td>\n",
        "      <td>   2.829195</td>\n",
        "      <td>  31.569941</td>\n",
        "      <td> 11523.028622</td>\n",
        "      <td>    772.140512</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>min</th>\n",
        "      <td> -33.659700</td>\n",
        "      <td> -32.756500</td>\n",
        "      <td> -33.450000</td>\n",
        "      <td>  -0.630000</td>\n",
        "      <td>   0.000160</td>\n",
        "      <td>   0.580000</td>\n",
        "      <td>   0.030000</td>\n",
        "      <td>   9.690000</td>\n",
        "      <td>  3536.850000</td>\n",
        "      <td> 198202.000000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25%</th>\n",
        "      <td>  -4.654525</td>\n",
        "      <td>  -3.240575</td>\n",
        "      <td>  -3.917075</td>\n",
        "      <td>   0.540000</td>\n",
        "      <td>   0.001730</td>\n",
        "      <td>   6.230000</td>\n",
        "      <td>   6.350000</td>\n",
        "      <td>  18.590000</td>\n",
        "      <td>  6785.350000</td>\n",
        "      <td> 198610.250000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>50%</th>\n",
        "      <td>  -0.673900</td>\n",
        "      <td>  -0.200800</td>\n",
        "      <td>  -0.358450</td>\n",
        "      <td>   1.137000</td>\n",
        "      <td>   0.007415</td>\n",
        "      <td>  10.050000</td>\n",
        "      <td>   9.360000</td>\n",
        "      <td>  26.140000</td>\n",
        "      <td>  9541.100000</td>\n",
        "      <td> 199304.500000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>75%</th>\n",
        "      <td>   3.389400</td>\n",
        "      <td>   4.045300</td>\n",
        "      <td>   3.283900</td>\n",
        "      <td>   1.542000</td>\n",
        "      <td>   0.250625</td>\n",
        "      <td>  11.632500</td>\n",
        "      <td>  10.797500</td>\n",
        "      <td>  30.690000</td>\n",
        "      <td> 11201.850000</td>\n",
        "      <td> 200077.750000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>max</th>\n",
        "      <td>   9.184700</td>\n",
        "      <td>  19.792200</td>\n",
        "      <td>  31.953000</td>\n",
        "      <td>   1.822000</td>\n",
        "      <td>  45.424710</td>\n",
        "      <td>  14.250000</td>\n",
        "      <td>  12.480000</td>\n",
        "      <td> 110.850000</td>\n",
        "      <td> 40460.250000</td>\n",
        "      <td> 200511.000000</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 425,
       "text": [
        "                x           y           z       color         lum      absmag  \\\n",
        "count  122.000000  122.000000  122.000000  122.000000  122.000000  122.000000   \n",
        "mean    -2.449070   -0.394039   -0.717792    0.925459    0.932497    8.981721   \n",
        "std      9.429820    7.749877    7.709211    0.803063    4.533497    3.194813   \n",
        "min    -33.659700  -32.756500  -33.450000   -0.630000    0.000160    0.580000   \n",
        "25%     -4.654525   -3.240575   -3.917075    0.540000    0.001730    6.230000   \n",
        "50%     -0.673900   -0.200800   -0.358450    1.137000    0.007415   10.050000   \n",
        "75%      3.389400    4.045300    3.283900    1.542000    0.250625   11.632500   \n",
        "max      9.184700   19.792200   31.953000    1.822000   45.424710   14.250000   \n",
        "\n",
        "           appmag      distly  distly_datetime      YearMonth  \n",
        "count  122.000000  122.000000       122.000000     122.000000  \n",
        "mean     8.630000   35.765410     13054.374590  199347.598361  \n",
        "std      2.829195   31.569941     11523.028622     772.140512  \n",
        "min      0.030000    9.690000      3536.850000  198202.000000  \n",
        "25%      6.350000   18.590000      6785.350000  198610.250000  \n",
        "50%      9.360000   26.140000      9541.100000  199304.500000  \n",
        "75%     10.797500   30.690000     11201.850000  200077.750000  \n",
        "max     12.480000  110.850000     40460.250000  200511.000000  "
       ]
      }
     ],
     "prompt_number": 425
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "full_data.to_csv('amnh_data.csv')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 426
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 426
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}